orizu 0.3.4 → 0.3.6

package/src/skill-installer.ts

@@ -0,0 +1,260 @@
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'fs'
+import { dirname, isAbsolute, relative, resolve } from 'path'
+import { fileURLToPath } from 'url'
+import { homedir } from 'os'
+
+export const SKILL_NAME = 'orizu-cli'
+export const AGENTS_START_MARKER = '<!-- orizu-cli:start -->'
+export const AGENTS_END_MARKER = '<!-- orizu-cli:end -->'
+
+export const SKILL_INSTALL_TARGETS = [
+  'agent-user',
+  'codex-project',
+  'claude-user',
+  'claude-project',
+  'agents-md',
+] as const
+
+export type SkillInstallTarget = typeof SKILL_INSTALL_TARGETS[number]
+
+export interface SkillInstallOptions {
+  cwd?: string
+  homeDir?: string
+  overwrite?: boolean
+  dryRun?: boolean
+}
+
+export interface SkillInstallResult {
+  target: SkillInstallTarget
+  path: string
+  action: 'created' | 'updated' | 'would-create' | 'would-update'
+}
+
+function getCliRoot(): string {
+  return resolve(dirname(fileURLToPath(import.meta.url)), '..')
+}
+
+function isSkillDir(path: string): boolean {
+  return existsSync(resolve(path, 'SKILL.md'))
+}
+
+export function isSafeRelativeSkillPath(rel: string): boolean {
+  return rel !== '..' && !rel.startsWith('../') && !isAbsolute(rel)
+}
+
+export function formatMissingSkillSourceError(paths: string[]): string {
+  return `Unable to find bundled ${SKILL_NAME} skill. Reinstall the orizu CLI or run from the Orizu repository.\n`
+    + `Searched:\n- ${paths.join('\n- ')}`
+}
+
+export function resolveSkillSourceDir(): string {
+  const override = process.env.ORIZU_SKILL_SOURCE_DIR
+  if (override) {
+    const resolvedOverride = resolve(override)
+    if (!isSkillDir(resolvedOverride)) {
+      throw new Error(`ORIZU_SKILL_SOURCE_DIR is not an Orizu skill directory: ${resolvedOverride}`)
+    }
+    return resolvedOverride
+  }
+
+  const cliRoot = getCliRoot()
+  const packagedSkill = resolve(cliRoot, 'vendor', 'skills', SKILL_NAME)
+  if (isSkillDir(packagedSkill)) {
+    return packagedSkill
+  }
+
+  const repoSkill = resolve(cliRoot, '..', '..', 'skills', SKILL_NAME)
+  if (isSkillDir(repoSkill)) {
+    return repoSkill
+  }
+
+  const searchedPaths = [
+    `packaged skill: ${packagedSkill}`,
+    `repository fallback: ${repoSkill}`,
+  ]
+
+  throw new Error(formatMissingSkillSourceError(searchedPaths))
+}
+
+function resolveHome(options?: SkillInstallOptions): string {
+  return resolve(options?.homeDir || process.env.HOME || homedir())
+}
+
+function resolveCwd(options?: SkillInstallOptions): string {
+  return resolve(options?.cwd || process.cwd())
+}
+
+export function getSkillInstallPath(
+  target: SkillInstallTarget,
+  options?: SkillInstallOptions
+): string {
+  const home = resolveHome(options)
+  const cwd = resolveCwd(options)
+
+  if (target === 'agent-user') {
+    return resolve(home, '.agents', 'skills', SKILL_NAME)
+  }
+
+  if (target === 'codex-project') {
+    return resolve(cwd, '.codex', 'skills', SKILL_NAME)
+  }
+
+  if (target === 'claude-user') {
+    return resolve(home, '.claude', 'skills', SKILL_NAME)
+  }
+
+  if (target === 'claude-project') {
+    return resolve(cwd, '.claude', 'skills', SKILL_NAME)
+  }
+
+  return resolve(cwd, 'AGENTS.md')
+}
+
+function shouldCopySkillPath(sourceRoot: string, sourcePath: string): boolean {
+  const rel = relative(sourceRoot, sourcePath).replace(/\\/g, '/')
+  if (!rel) {
+    return true
+  }
+  if (!isSafeRelativeSkillPath(rel)) {
+    return false
+  }
+
+  const parts = rel.split('/')
+  return !parts.some(part =>
+    part === '.DS_Store' ||
+    part === '__pycache__' ||
+    part === '.pytest_cache'
+  ) && !rel.endsWith('.pyc') && !rel.endsWith('.pyo')
+}
+
+function copySkillTree(sourceDir: string, targetDir: string) {
+  rmSync(targetDir, { recursive: true, force: true })
+  mkdirSync(dirname(targetDir), { recursive: true })
+  cpSync(sourceDir, targetDir, {
+    recursive: true,
+    filter: sourcePath => shouldCopySkillPath(sourceDir, sourcePath),
+  })
+}
+
+function stripFrontmatter(markdown: string): string {
+  const match = markdown.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n/)
+  if (!match) {
+    return markdown
+  }
+
+  return markdown.slice(match[0].length).replace(/^[\r\n]+/, '')
+}
+
+function buildAgentsSection(sourceDir: string): string {
+  const skillMarkdown = readFileSync(resolve(sourceDir, 'SKILL.md'), 'utf8')
+  const body = stripFrontmatter(skillMarkdown).trimEnd()
+  return `${AGENTS_START_MARKER}\n${body}\n${AGENTS_END_MARKER}\n`
+}
+
+function installAgentsMd(
+  sourceDir: string,
+  targetPath: string,
+  options?: SkillInstallOptions
+): SkillInstallResult {
+  const section = buildAgentsSection(sourceDir)
+  const exists = existsSync(targetPath)
+  const existing = exists ? readFileSync(targetPath, 'utf8') : ''
+  const hasManagedSection = existing.includes(AGENTS_START_MARKER)
+    && existing.includes(AGENTS_END_MARKER)
+  const action = exists ? 'updated' : 'created'
+
+  if (options?.dryRun) {
+    return {
+      target: 'agents-md',
+      path: targetPath,
+      action: action === 'updated' ? 'would-update' : 'would-create',
+    }
+  }
+
+  if (hasManagedSection && !options?.overwrite) {
+    throw new Error(`${targetPath} already contains an Orizu CLI section. Pass --yes to replace it.`)
+  }
+
+  mkdirSync(dirname(targetPath), { recursive: true })
+  if (!exists) {
+    writeFileSync(targetPath, section, 'utf8')
+    return { target: 'agents-md', path: targetPath, action: 'created' }
+  }
+
+  if (hasManagedSection) {
+    const pattern = new RegExp(
+      `${escapeRegExp(AGENTS_START_MARKER)}[\\s\\S]*?${escapeRegExp(AGENTS_END_MARKER)}\\n?`
+    )
+    writeFileSync(targetPath, existing.replace(pattern, section), 'utf8')
+    return { target: 'agents-md', path: targetPath, action: 'updated' }
+  }
+
+  const existingPrefix = existing.trim().length === 0 ? '' : existing
+  const separator = existingPrefix.endsWith('\n') ? '\n' : '\n\n'
+  writeFileSync(targetPath, `${existingPrefix}${existingPrefix ? separator : ''}${section}`, 'utf8')
+  return { target: 'agents-md', path: targetPath, action: 'updated' }
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+export function targetNeedsOverwrite(
+  target: SkillInstallTarget,
+  options?: SkillInstallOptions
+): boolean {
+  const targetPath = getSkillInstallPath(target, options)
+  if (!existsSync(targetPath)) {
+    return false
+  }
+
+  if (target !== 'agents-md') {
+    return true
+  }
+
+  const existing = readFileSync(targetPath, 'utf8')
+  return existing.includes(AGENTS_START_MARKER) && existing.includes(AGENTS_END_MARKER)
+}
+
+export function installSkillTarget(
+  target: SkillInstallTarget,
+  options?: SkillInstallOptions
+): SkillInstallResult {
+  const sourceDir = resolveSkillSourceDir()
+  const targetPath = getSkillInstallPath(target, options)
+  const exists = existsSync(targetPath)
+
+  if (target === 'agents-md') {
+    return installAgentsMd(sourceDir, targetPath, options)
+  }
+
+  if (options?.dryRun) {
+    return {
+      target,
+      path: targetPath,
+      action: exists ? 'would-update' : 'would-create',
+    }
+  }
+
+  if (exists && !options?.overwrite) {
+    throw new Error(`${targetPath} already exists. Pass --yes to replace it.`)
+  }
+
+  copySkillTree(sourceDir, targetPath)
+  return {
+    target,
+    path: targetPath,
+    action: exists ? 'updated' : 'created',
+  }
+}
+
+export function isSkillInstallTarget(value: string): value is SkillInstallTarget {
+  return (SKILL_INSTALL_TARGETS as readonly string[]).includes(value)
+}

package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py

@@ -68,8 +68,22 @@ def main() -> None:
     parser.add_argument("--max-full-evals", type=positive_int)
     parser.add_argument("--reflection-model", default=TextGepaConfig.reflection_model)
     parser.add_argument("--reflection-temperature", type=float)
+    parser.add_argument("--reflection-max-tokens", type=positive_int)
     parser.add_argument("--reflection-prompt-template")
     parser.add_argument("--reflection-provider-settings")
+    parser.add_argument(
+        "--skip-perfect-parent-reflection",
+        dest="skip_perfect_parent_reflection",
+        action="store_true",
+        default=TextGepaConfig.skip_perfect_parent_reflection,
+        help="Skip reflection and child creation when the parent scores perfectly on the minibatch (default).",
+    )
+    parser.add_argument(
+        "--no-skip-perfect-parent-reflection",
+        dest="skip_perfect_parent_reflection",
+        action="store_false",
+        help="Reflect and create a child even when the parent scores perfectly on the minibatch.",
+    )
     parser.add_argument("--objective", default=TextGepaConfig.objective)
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--auto-promote", action="store_true")
@@ -94,8 +108,10 @@ def main() -> None:
         max_full_evals=args.max_full_evals,
         reflection_model=args.reflection_model,
         reflection_temperature=args.reflection_temperature,
+        reflection_max_tokens=args.reflection_max_tokens,
         reflection_prompt_template=read_text_arg(args.reflection_prompt_template),
         reflection_provider_settings=read_json_object_arg(args.reflection_provider_settings, "--reflection-provider-settings") or {},
+        skip_perfect_parent_reflection=args.skip_perfect_parent_reflection,
         objective=args.objective,
         seed=args.seed,
         auto_promote=args.auto_promote,

package/vendor/orizu-gepa-python/src/orizu_gepa/optimizer.py

@@ -179,7 +179,7 @@ class TextGepaConfig:
     max_candidate_proposals: int | None = None
     reflection_model: str = "anthropic/claude-opus-4-7"
     reflection_temperature: float | None = None
-    reflection_max_tokens: int = 4096
+    reflection_max_tokens: int | None = None
     reflection_prompt_template: str | None = None
     reflection_provider_settings: dict[str, Any] = field(default_factory=dict)
     objective: str = "Improve this text candidate to maximize evaluator score while preserving intended behavior."
@@ -189,6 +189,7 @@ class TextGepaConfig:
     fail_on_log_error: bool = True
     log_row_snapshots: bool = False
     cache_evaluations: bool = True
+    skip_perfect_parent_reflection: bool = True
 
 
 @dataclass(frozen=True)
@@ -319,6 +320,14 @@ def _is_better(challenger: float, incumbent: float, higher_is_better: bool) -> b
     return challenger > incumbent if higher_is_better else challenger < incumbent
 
 
+def _is_perfect_minibatch(results: list[RowEvaluation], higher_is_better: bool) -> bool:
+    if not results:
+        return False
+    if higher_is_better:
+        return all(result.score >= 1.0 for result in results)
+    return all(result.score <= 0.0 for result in results)
+
+
 class EvaluationCache:
     def __init__(self):
         self._values: dict[str, RowEvaluation] = {}
@@ -420,37 +429,12 @@ def _aggregate_scores(
     }
 
 
-def _remove_dominated_frontier_candidates(
-    frontier_by_row_id: dict[str, set[str]],
-    aggregate_scores: dict[str, float],
-    higher_is_better: bool = True,
-) -> dict[str, set[str]]:
-    remaining = {
-        candidate_id
-        for candidate_ids in frontier_by_row_id.values()
-        for candidate_id in candidate_ids
-    }
-    for candidate_id in sorted(
-        remaining,
-        key=lambda value: aggregate_scores.get(value, 0.0),
-        reverse=not higher_is_better,
-    ):
-        owned_rows = [
-            row_id
-            for row_id, candidate_ids in frontier_by_row_id.items()
-            if candidate_id in candidate_ids
-        ]
-        if owned_rows and all(
-            any(other_id != candidate_id and other_id in remaining for other_id in frontier_by_row_id[row_id])
-            for row_id in owned_rows
-        ):
-            remaining.discard(candidate_id)
-
-    return {
-        row_id: {candidate_id for candidate_id in candidate_ids if candidate_id in remaining}
-        for row_id, candidate_ids in frontier_by_row_id.items()
-        if any(candidate_id in remaining for candidate_id in candidate_ids)
-    }
+def _frontier_counts(frontier_by_row_id: dict[str, set[str]]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for candidate_ids in frontier_by_row_id.values():
+        for candidate_id in candidate_ids:
+            counts[candidate_id] = counts.get(candidate_id, 0) + 1
+    return counts
 
 
 def _select_parent_candidate_id(
@@ -479,15 +463,7 @@ def _select_parent_candidate_id(
             "candidate_selection_strategy must be one of: pareto, current_best, epsilon_greedy"
         )
 
-    frontier = _remove_dominated_frontier_candidates(
-        _frontier_by_row_id(val_scores_by_candidate, higher_is_better),
-        _aggregate_scores(val_scores_by_candidate),
-        higher_is_better,
-    )
-    frontier_counts: dict[str, int] = {}
-    for candidate_ids in frontier.values():
-        for candidate_id in candidate_ids:
-            frontier_counts[candidate_id] = frontier_counts.get(candidate_id, 0) + 1
+    frontier_counts = _frontier_counts(_frontier_by_row_id(val_scores_by_candidate, higher_is_better))
     sampling_list: list[str] = []
     for candidate_id, count in sorted(frontier_counts.items()):
         sampling_list.extend([candidate_id] * count)
@@ -505,15 +481,7 @@ def _parent_selection_payload(
     higher_is_better: bool = True,
 ) -> dict[str, Any]:
     aggregate_scores = _aggregate_scores(val_scores_by_candidate)
-    frontier = _remove_dominated_frontier_candidates(
-        _frontier_by_row_id(val_scores_by_candidate, higher_is_better),
-        aggregate_scores,
-        higher_is_better,
-    )
-    frontier_counts: dict[str, int] = {}
-    for candidate_ids in frontier.values():
-        for candidate_id in candidate_ids:
-            frontier_counts[candidate_id] = frontier_counts.get(candidate_id, 0) + 1
+    frontier_counts = _frontier_counts(_frontier_by_row_id(val_scores_by_candidate, higher_is_better))
     return {
         "strategy": config.candidate_selection_strategy,
         "selected_candidate_id": selected_candidate_id,
@@ -541,16 +509,23 @@ def _pareto_payload(
         }
 
     aggregate_scores = _aggregate_scores(val_scores_by_candidate)
-    best_candidate_id = max(aggregate_scores, key=aggregate_scores.get) if higher_is_better else min(aggregate_scores, key=aggregate_scores.get)
+    frontier_counts = _frontier_counts(raw_frontier)
+    if frontier_counts:
+        best_candidate_id = max(
+            sorted(frontier_counts),
+            key=lambda candidate_id: (
+                frontier_counts[candidate_id],
+                aggregate_scores.get(candidate_id, 0.0) if higher_is_better else -aggregate_scores.get(candidate_id, 0.0),
+            ),
+        )
+    else:
+        best_candidate_id = max(aggregate_scores, key=aggregate_scores.get) if higher_is_better else min(aggregate_scores, key=aggregate_scores.get)
     best_score = aggregate_scores[best_candidate_id]
     return best_candidate_id, best_score, {
         "frontier_by_row_id": frontier_by_row_id,
-        "frontier_candidate_ids": sorted({
-            candidate_id
-            for candidate_ids in raw_frontier.values()
-            for candidate_id in candidate_ids
-        }),
+        "frontier_candidate_ids": sorted(frontier_counts),
         "candidate_scores": aggregate_scores,
+        "frontier_counts": frontier_counts,
         "best_candidate_id": best_candidate_id,
         "best_score_mean": best_score,
         "best_candidate_preview": val_text_by_candidate[best_candidate_id][:1200],
@@ -853,6 +828,43 @@ def optimize_loaded_text_candidate(
                 candidate_id=parent_candidate_id,
             )
 
+            parent_is_perfect = _is_perfect_minibatch(parent_results, higher_is_better)
+            if config.skip_perfect_parent_reflection and parent_is_perfect:
+                event_sink.log_event(
+                    "reflection_skipped",
+                    {
+                        "reason": "parent_minibatch_perfect",
+                        "parent_candidate_id": parent_candidate_id,
+                        "parent_score_total": parent_total,
+                        "parent_score_mean": _mean(parent_results),
+                        "row_ids": [row.id for row in minibatch],
+                        "higher_is_better": higher_is_better,
+                        "decision_rule": "skip reflection when every parent minibatch row is already perfect",
+                    },
+                    event_layer="extension",
+                    iteration=iteration,
+                    candidate_id=parent_candidate_id,
+                )
+                event_sink.log_event("budget_updated", budget.to_payload(), event_layer="extension", iteration=iteration)
+                event_sink.log_event(
+                    "iteration_completed",
+                    {
+                        "parent_candidate_id": parent_candidate_id,
+                        "child_candidate_id": None,
+                        "parent_train_score_total": parent_total,
+                        "child_train_score_total": None,
+                        "child_validation_score": None,
+                        "best_candidate_id": best_candidate_id,
+                        "best_validation_score": best_score,
+                        "skipped_reflection": True,
+                        "skip_reason": "parent_minibatch_perfect",
+                        "budget": budget.to_payload(),
+                    },
+                    iteration=iteration,
+                    candidate_id=best_candidate_id,
+                )
+                continue
+
             event_sink.log_event(
                 "reflection_started",
                 {

package/vendor/orizu-gepa-python/src/orizu_gepa/reflection.py

@@ -62,12 +62,30 @@ def _add_temperature(payload: dict[str, Any], config: TextGepaConfig) -> dict[st
     return {**payload, "temperature": config.reflection_temperature}
 
 
+def _validate_reflection_max_tokens(value: int | None) -> None:
+    if value is not None and value <= 0:
+        raise RuntimeError("reflection_max_tokens must be positive")
+
+
+def _require_anthropic_reflection_max_tokens(value: int | None) -> int:
+    _validate_reflection_max_tokens(value)
+    if value is None:
+        raise RuntimeError("reflection_max_tokens is required for Anthropic reflection models")
+    return value
+
+
 def build_anthropic_reflection_payload(model: str, prompt: str, config: TextGepaConfig) -> dict[str, Any]:
-    payload = _merge_provider_settings({
+    max_tokens = _require_anthropic_reflection_max_tokens(config.reflection_max_tokens)
+    base_payload: dict[str, Any] = {
         "model": model,
-        "max_tokens": config.reflection_max_tokens,
+        "max_tokens": max_tokens,
         "messages": [{"role": "user", "content": prompt}],
-    }, config.reflection_provider_settings, reserved_keys={"model", "max_tokens", "messages"})
+    }
+    payload = _merge_provider_settings(
+        base_payload,
+        config.reflection_provider_settings,
+        reserved_keys={"model", "max_tokens", "messages"},
+    )
     payload = _add_temperature(payload, config)
     if "temperature" in payload and "thinking" in payload:
         raise RuntimeError("reflection_temperature cannot be combined with Anthropic thinking")
@@ -75,11 +93,18 @@ def build_anthropic_reflection_payload(model: str, prompt: str, config: TextGepa
 
 
 def build_openai_reflection_payload(model: str, prompt: str, config: TextGepaConfig) -> dict[str, Any]:
-    payload = _merge_provider_settings({
+    _validate_reflection_max_tokens(config.reflection_max_tokens)
+    base_payload: dict[str, Any] = {
         "model": model,
         "input": [{"role": "user", "content": prompt}],
-        "max_output_tokens": config.reflection_max_tokens,
-    }, config.reflection_provider_settings, reserved_keys={"model", "input", "max_output_tokens"})
+    }
+    if config.reflection_max_tokens is not None:
+        base_payload["max_output_tokens"] = config.reflection_max_tokens
+    payload = _merge_provider_settings(
+        base_payload,
+        config.reflection_provider_settings,
+        reserved_keys={"model", "input", "max_output_tokens"},
+    )
     return _add_temperature(payload, config)