orizu 0.3.6 → 0.3.7

package/dist/help.js

@@ -259,7 +259,7 @@ export const COMMAND_DOCS = [
     },
     {
         path: ['optimizations', 'run-gepa'],
-        usage: 'orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]',
+        usage: 'orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--num-threads auto|N] [--log-dir logs]',
         summary: 'Run the bundled GEPA-style optimizer locally and stream events.',
         group: 'Optimizations',
     },

package/dist/index.js

@@ -41,7 +41,7 @@ function printPreviewUsage() {
     printLine(`\nLocal app preview:\n\n  orizu apps preview --file <path> --input-schema <json-path> --output-schema <json-path> --sample-row <json-path> [--screenshot <png-path>] [--headed] [--keep-open] [--component <name>]`);
 }
 function printOptimizationUsage() {
-    printLine(`\nOptimization lifecycle commands:\n\n  orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n  orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs] [--no-skip-perfect-parent-reflection]\n  orizu optimizations export <run-id> [--out <path>] [--json]\n  orizu optimizations pause <run-id> [--reason <text>] [--json]\n  orizu optimizations resume <run-id> [--json]\n  orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations fail <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations cancel <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--json]`);
+    printLine(`\nOptimization lifecycle commands:\n\n  orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n  orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--num-threads auto|N] [--log-dir logs] [--no-skip-perfect-parent-reflection]\n  orizu optimizations export <run-id> [--out <path>] [--json]\n  orizu optimizations pause <run-id> [--reason <text>] [--json]\n  orizu optimizations resume <run-id> [--json]\n  orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations fail <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations cancel <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--json]`);
 }
 let cliArgs = process.argv.slice(2);
 let cliJsonOutput = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orizu",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "private": false,
   "type": "module",
   "bin": {

package/src/help.ts

@@ -284,7 +284,7 @@ export const COMMAND_DOCS: CliCommandDoc[] = [
   },
   {
     path: ['optimizations', 'run-gepa'],
-    usage: 'orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]',
+    usage: 'orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--num-threads auto|N] [--log-dir logs]',
     summary: 'Run the bundled GEPA-style optimizer locally and stream events.',
     group: 'Optimizations',
   },

package/src/index.ts

@@ -288,7 +288,7 @@ function printPreviewUsage() {
 }
 
 function printOptimizationUsage() {
-  printLine(`\nOptimization lifecycle commands:\n\n  orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n  orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs] [--no-skip-perfect-parent-reflection]\n  orizu optimizations export <run-id> [--out <path>] [--json]\n  orizu optimizations pause <run-id> [--reason <text>] [--json]\n  orizu optimizations resume <run-id> [--json]\n  orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations fail <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations cancel <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--json]`)
+  printLine(`\nOptimization lifecycle commands:\n\n  orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n  orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--num-threads auto|N] [--log-dir logs] [--no-skip-perfect-parent-reflection]\n  orizu optimizations export <run-id> [--out <path>] [--json]\n  orizu optimizations pause <run-id> [--reason <text>] [--json]\n  orizu optimizations resume <run-id> [--json]\n  orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations fail <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations cancel <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--json]`)
 }
 
 let cliArgs = process.argv.slice(2)

package/vendor/orizu-gepa-python/src/orizu_gepa/__init__.py

@@ -1,19 +1,23 @@
 from .optimizer import (
     Budget,
     DatasetRow,
+    NumThreadsPlan,
     PromptContext,
     RowEvaluation,
     TextGepaConfig,
     TextGepaResult,
     optimize_text_candidate,
+    resolve_num_threads,
 )
 
 __all__ = [
     "Budget",
     "DatasetRow",
+    "NumThreadsPlan",
     "PromptContext",
     "RowEvaluation",
     "TextGepaConfig",
     "TextGepaResult",
     "optimize_text_candidate",
+    "resolve_num_threads",
 ]

package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py

@@ -19,6 +19,12 @@ def positive_int(value: str) -> int:
     return parsed
 
 
+def num_threads_arg(value: str) -> int | str:
+    if value.strip().lower() == "auto":
+        return "auto"
+    return positive_int(value)
+
+
 def read_text_arg(value: str | None) -> str | None:
     if value is None:
         return None
@@ -54,9 +60,10 @@ def main() -> None:
     parser.add_argument("--split-set-id", required=True)
     parser.add_argument("--train-split", default="train")
     parser.add_argument("--val-split", default="validation")
-    parser.add_argument("--budget", default="light", choices=["auto", "light", "medium", "high"])
+    parser.add_argument("--budget", default="light", choices=["auto", "light", "medium", "heavy"])
     parser.add_argument("--max-iterations", type=positive_int, default=3)
     parser.add_argument("--minibatch-size", type=positive_int, default=3)
+    parser.add_argument("--num-threads", type=num_threads_arg, default=TextGepaConfig.num_threads)
     parser.add_argument(
         "--candidate-selection-strategy",
         default=TextGepaConfig.candidate_selection_strategy,
@@ -102,6 +109,7 @@ def main() -> None:
         budget=args.budget,
         max_iterations=args.max_iterations,
         minibatch_size=args.minibatch_size,
+        num_threads=args.num_threads,
         candidate_selection_strategy=args.candidate_selection_strategy,
         epsilon=args.epsilon,
         max_metric_calls=args.max_metric_calls,
@@ -153,6 +161,7 @@ def main() -> None:
         "dataset_size": len(trainset) + len(valset),
         "train_count": len(trainset),
         "validation_count": len(valset),
+        "num_threads": args.num_threads,
     }
 
     run_id = client.start_run(

package/vendor/orizu-gepa-python/src/orizu_gepa/optimizer.py

@@ -4,9 +4,13 @@ import dataclasses
 import hashlib
 import json
 import math
+import os
 import random
+import subprocess
+import sys
 import time
 import uuid
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from dataclasses import dataclass, field
 from typing import Any, Callable, Protocol
 
@@ -40,6 +44,31 @@ Return only the complete updated parameter text, exactly as it should be used by
 Do not include analysis, explanation, labels, XML tags, or a surrounding markdown code fence.
 Your response will be used verbatim as the next candidate."""
 
+AUTO_NUM_THREADS_HARD_CAP = 8
+AUTO_NUM_THREADS_WORKER_MEMORY_BYTES = 512 * 1024 * 1024
+AUTO_NUM_THREADS_MEMORY_RESERVE_BYTES = 1024 * 1024 * 1024
+AUTO_NUM_THREADS_FD_HEADROOM = 64
+AUTO_NUM_THREADS_FDS_PER_WORKER = 16
+
+
+@dataclass(frozen=True)
+class NumThreadsPlan:
+    requested: int | str
+    resolved: int
+    row_bound: int
+    cpu_bound: int
+    memory_bound: int | None
+    fd_bound: int | None
+    hard_cap: int
+    worker_memory_bytes: int
+    available_memory_bytes: int | None
+    total_memory_bytes: int | None
+    memory_reserve_bytes: int | None
+    limiting_factor: str
+
+    def to_payload(self) -> dict[str, Any]:
+        return dataclasses.asdict(self)
+
 
 @dataclass(frozen=True)
 class DatasetRow:
@@ -113,6 +142,34 @@ class ReflectionResult:
     candidate_text: str
 
 
+def _dspy_auto_metric_budget(
+    *,
+    num_components: int,
+    num_candidates: int,
+    valset_size: int,
+    minibatch_size: int = 35,
+    full_eval_steps: int = 5,
+) -> int:
+    if num_candidates <= 0:
+        raise ValueError("num_candidates must be > 0.")
+    if num_components < 0 or valset_size < 0 or minibatch_size < 0:
+        raise ValueError("num_components, valset_size, and minibatch_size must be >= 0.")
+    if full_eval_steps < 1:
+        raise ValueError("full_eval_steps must be >= 1.")
+
+    num_trials = int(max(2 * (num_components * 2) * math.log2(num_candidates), 1.5 * num_candidates))
+    total = valset_size
+    total += num_candidates * 5
+    total += num_trials * minibatch_size
+    if num_trials == 0:
+        return total
+
+    periodic_fulls = (num_trials + 1) // full_eval_steps + 1
+    extra_final = 1 if num_trials < full_eval_steps else 0
+    total += (periodic_fulls + extra_final) * valset_size
+    return total
+
+
 @dataclass
 class Budget:
     budget_kind: str
@@ -122,20 +179,35 @@ class Budget:
     used_candidate_proposals: int = 0
 
     @classmethod
-    def from_config(cls, config: "TextGepaConfig") -> "Budget":
+    def from_config(
+        cls,
+        config: "TextGepaConfig",
+        *,
+        trainset_size: int = 0,
+        valset_size: int = 0,
+        num_components: int = 1,
+    ) -> "Budget":
         if config.max_metric_calls is not None:
             return cls("max_metric_calls", config.max_metric_calls)
         if config.max_full_evals is not None:
-            return cls("max_full_evals", config.max_full_evals)
+            return cls("max_metric_calls", config.max_full_evals * (trainset_size + valset_size))
         if config.max_candidate_proposals is not None:
             return cls("max_candidate_proposals", config.max_candidate_proposals)
-        preset_metric_calls = {
-            "auto": 64,
-            "light": 48,
-            "medium": 96,
-            "high": 192,
-        }.get(config.budget, 48)
-        return cls("max_metric_calls", preset_metric_calls)
+        preset = "medium" if config.budget == "auto" else config.budget
+        preset_candidates = {
+            "light": 6,
+            "medium": 12,
+            "heavy": 18,
+            "high": 18,
+        }.get(preset, 6)
+        return cls(
+            "max_metric_calls",
+            _dspy_auto_metric_budget(
+                num_components=num_components,
+                num_candidates=preset_candidates,
+                valset_size=valset_size,
+            ),
+        )
 
     @property
     def used(self) -> int:
@@ -152,9 +224,6 @@ class Budget:
     def allows_iteration(self) -> bool:
         return self.remaining > 0
 
-    def allows_metric_call(self) -> bool:
-        return self.budget_kind != "max_metric_calls" or self.remaining > 0
-
     def to_payload(self) -> dict[str, Any]:
         return {
             "budget_kind": self.budget_kind,
@@ -172,6 +241,7 @@ class TextGepaConfig:
     budget: str = "light"
     max_iterations: int = 3
     minibatch_size: int = 3
+    num_threads: int | str = "auto"
     candidate_selection_strategy: str = "pareto"
     epsilon: float = 0.1
     max_metric_calls: int | None = None
@@ -316,6 +386,229 @@ def _metric_key(scorer_context: PromptContext) -> str:
     return scorer_context.metric_key or "score"
 
 
+def _positive_int_env(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        return default
+    return value if value > 0 else default
+
+
+def _proc_meminfo_bytes(key: str) -> int | None:
+    try:
+        with open("/proc/meminfo", encoding="utf-8") as handle:
+            for line in handle:
+                if not line.startswith(f"{key}:"):
+                    continue
+                parts = line.split()
+                if len(parts) >= 2:
+                    return int(parts[1]) * 1024
+    except (OSError, ValueError):
+        return None
+    return None
+
+
+def _sysconf_memory_bytes(pages_name: str) -> int | None:
+    try:
+        pages = os.sysconf(pages_name)
+        page_size = os.sysconf("SC_PAGE_SIZE")
+    except (AttributeError, OSError, ValueError):
+        return None
+    if not isinstance(pages, int) or not isinstance(page_size, int):
+        return None
+    if pages <= 0 or page_size <= 0:
+        return None
+    return pages * page_size
+
+
+def _darwin_available_memory_bytes() -> int | None:
+    if sys.platform != "darwin":
+        return None
+    try:
+        output = subprocess.check_output(["vm_stat"], text=True)
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+    lines = output.splitlines()
+    if not lines:
+        return None
+    page_size_marker = "page size of "
+    if page_size_marker not in lines[0]:
+        return None
+    try:
+        page_size = int(lines[0].split(page_size_marker, 1)[1].split()[0])
+    except (IndexError, ValueError):
+        return None
+
+    available_pages = 0
+    available_keys = {
+        "Pages free",
+        "Pages inactive",
+        "Pages speculative",
+        "Pages purgeable",
+    }
+    for line in lines[1:]:
+        key, separator, value = line.partition(":")
+        if separator != ":" or key not in available_keys:
+            continue
+        normalized_value = value.strip().rstrip(".").replace(",", "")
+        try:
+            available_pages += int(normalized_value)
+        except ValueError:
+            continue
+    return available_pages * page_size if available_pages > 0 else None
+
+
+def _available_memory_bytes() -> int | None:
+    return (
+        _proc_meminfo_bytes("MemAvailable")
+        or _darwin_available_memory_bytes()
+        or _sysconf_memory_bytes("SC_AVPHYS_PAGES")
+        or _total_memory_bytes()
+    )
+
+
+def _total_memory_bytes() -> int | None:
+    return _proc_meminfo_bytes("MemTotal") or _sysconf_memory_bytes("SC_PHYS_PAGES")
+
+
+def _open_file_soft_limit() -> int | None:
+    try:
+        import resource
+    except ImportError:
+        return None
+    try:
+        soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (OSError, ValueError):
+        return None
+    if soft_limit <= 0 or soft_limit == resource.RLIM_INFINITY:
+        return None
+    return int(soft_limit)
+
+
+def _parse_requested_num_threads(requested: int | str) -> int | str:
+    if isinstance(requested, int):
+        if requested <= 0:
+            raise ValueError("num_threads must be positive or 'auto'")
+        return requested
+    if isinstance(requested, str):
+        normalized = requested.strip().lower()
+        if normalized == "auto":
+            return "auto"
+        try:
+            parsed = int(normalized)
+        except ValueError as exc:
+            raise ValueError("num_threads must be positive or 'auto'") from exc
+        if parsed <= 0:
+            raise ValueError("num_threads must be positive or 'auto'")
+        return parsed
+    raise ValueError("num_threads must be positive or 'auto'")
+
+
+def resolve_num_threads(
+    requested: int | str,
+    *,
+    minibatch_size: int,
+    validation_count: int,
+    cpu_count: int | None = None,
+    available_memory_bytes: int | None = None,
+    total_memory_bytes: int | None = None,
+    fd_limit: int | None = None,
+    hard_cap: int | None = None,
+    worker_memory_bytes: int | None = None,
+) -> NumThreadsPlan:
+    """Resolve an explicit or automatic row-evaluation thread count for one GEPA run."""
+    parsed = _parse_requested_num_threads(requested)
+    row_bound = max(1, max(minibatch_size, validation_count))
+    detected_cpu_count = cpu_count if cpu_count is not None else os.cpu_count()
+    cpu_bound = max(1, detected_cpu_count or 1)
+    resolved_hard_cap = hard_cap or _positive_int_env(
+        "ORIZU_GEPA_AUTO_THREADS_MAX",
+        AUTO_NUM_THREADS_HARD_CAP,
+    )
+    resolved_worker_memory_bytes = worker_memory_bytes or (
+        _positive_int_env(
+            "ORIZU_GEPA_WORKER_MEMORY_MB",
+            AUTO_NUM_THREADS_WORKER_MEMORY_BYTES // (1024 * 1024),
+        ) * 1024 * 1024
+    )
+
+    detected_total_memory_bytes = (
+        total_memory_bytes if total_memory_bytes is not None else _total_memory_bytes()
+    )
+    detected_available_memory_bytes = (
+        available_memory_bytes
+        if available_memory_bytes is not None
+        else _available_memory_bytes()
+    )
+    memory_reserve_bytes = None
+    memory_bound = None
+    if detected_available_memory_bytes is not None:
+        total_reserve = (
+            detected_total_memory_bytes // 4
+            if detected_total_memory_bytes is not None
+            else 0
+        )
+        memory_reserve_bytes = max(AUTO_NUM_THREADS_MEMORY_RESERVE_BYTES, total_reserve)
+        usable_memory_bytes = max(0, detected_available_memory_bytes - memory_reserve_bytes)
+        memory_bound = max(1, usable_memory_bytes // max(1, resolved_worker_memory_bytes))
+
+    detected_fd_limit = fd_limit if fd_limit is not None else _open_file_soft_limit()
+    fd_bound = None
+    if detected_fd_limit is not None:
+        usable_fds = max(0, detected_fd_limit - AUTO_NUM_THREADS_FD_HEADROOM)
+        fd_bound = max(1, usable_fds // AUTO_NUM_THREADS_FDS_PER_WORKER)
+
+    if parsed != "auto":
+        return NumThreadsPlan(
+            requested=requested,
+            resolved=parsed,
+            row_bound=row_bound,
+            cpu_bound=cpu_bound,
+            memory_bound=memory_bound,
+            fd_bound=fd_bound,
+            hard_cap=resolved_hard_cap,
+            worker_memory_bytes=resolved_worker_memory_bytes,
+            available_memory_bytes=detected_available_memory_bytes,
+            total_memory_bytes=detected_total_memory_bytes,
+            memory_reserve_bytes=memory_reserve_bytes,
+            limiting_factor="explicit",
+        )
+
+    limits = {
+        "rows": row_bound,
+        "cpu": cpu_bound,
+        "hard_cap": resolved_hard_cap,
+    }
+    if memory_bound is not None:
+        limits["memory"] = memory_bound
+    if fd_bound is not None:
+        limits["file_descriptors"] = fd_bound
+    resolved = max(1, min(limits.values()))
+    limiting_factor = next(
+        name
+        for name in ("rows", "cpu", "memory", "file_descriptors", "hard_cap")
+        if limits.get(name) == resolved
+    )
+    return NumThreadsPlan(
+        requested=requested,
+        resolved=resolved,
+        row_bound=row_bound,
+        cpu_bound=cpu_bound,
+        memory_bound=memory_bound,
+        fd_bound=fd_bound,
+        hard_cap=resolved_hard_cap,
+        worker_memory_bytes=resolved_worker_memory_bytes,
+        available_memory_bytes=detected_available_memory_bytes,
+        total_memory_bytes=detected_total_memory_bytes,
+        memory_reserve_bytes=memory_reserve_bytes,
+        limiting_factor=limiting_factor,
+    )
+
+
 def _is_better(challenger: float, incumbent: float, higher_is_better: bool) -> bool:
     return challenger > incumbent if higher_is_better else challenger < incumbent
 
@@ -564,12 +857,13 @@ def evaluate_candidate(
     scorer_runner: ScorerRunner,
     budget: Budget,
     evaluation_cache: EvaluationCache | None = None,
+    num_threads: int = 1,
 ) -> list[RowEvaluation]:
-    evaluations: list[RowEvaluation] = []
-    for row in rows:
-        if not budget.allows_metric_call():
-            break
+    """Evaluate one candidate over rows, preserving row order while parallelizing uncached calls."""
+    evaluations_by_index: dict[int, RowEvaluation] = {}
+    pending: list[tuple[int, DatasetRow, str | None]] = []
 
+    for index, row in enumerate(rows):
         cache_key = None
         if evaluation_cache is not None:
             cache_key = evaluation_cache.key(
@@ -581,9 +875,12 @@ def evaluate_candidate(
             )
             cached = evaluation_cache.get(cache_key)
             if cached is not None:
-                evaluations.append(cached)
+                evaluations_by_index[index] = cached
                 continue
 
+        pending.append((index, row, cache_key))
+
+    def run_uncached_row(index: int, row: DatasetRow, cache_key: str | None) -> tuple[int, str | None, RowEvaluation]:
         started = time.time()
         candidate_result = candidate_runner(candidate_text, row, prompt_context, candidate_id)
         scorer_result = scorer_runner(row, candidate_result, scorer_context, candidate_id)
@@ -610,11 +907,57 @@ def evaluate_candidate(
             if candidate_result.cost_usd is not None or scorer_result.cost_usd is not None else None,
             error=candidate_result.error or scorer_result.error,
         )
-        evaluations.append(evaluation)
+        return index, cache_key, evaluation
+
+    worker_count = max(1, min(max(1, num_threads), len(pending) or 1))
+    if worker_count == 1:
+        completed = [run_uncached_row(index, row, cache_key) for index, row, cache_key in pending]
+    else:
+        completed = []
+        next_pending_index = 0
+        executor = ThreadPoolExecutor(max_workers=worker_count)
+        futures = set()
+
+        def submit_next_pending() -> bool:
+            nonlocal next_pending_index
+            if next_pending_index >= len(pending):
+                return False
+            index, row, cache_key = pending[next_pending_index]
+            next_pending_index += 1
+            futures.add(executor.submit(run_uncached_row, index, row, cache_key))
+            return True
+
+        try:
+            for _ in range(worker_count):
+                if not submit_next_pending():
+                    break
+            while futures:
+                done, _ = wait(futures, return_when=FIRST_COMPLETED)
+                batch = []
+                for future in done:
+                    futures.remove(future)
+                    batch.append(future.result())
+                completed.extend(batch)
+                for _ in batch:
+                    submit_next_pending()
+        except BaseException:
+            for future in futures:
+                future.cancel()
+            executor.shutdown(wait=True)
+            raise
+        else:
+            executor.shutdown(wait=True)
+
+    for index, cache_key, evaluation in completed:
+        evaluations_by_index[index] = evaluation
         if evaluation_cache is not None and cache_key is not None:
             evaluation_cache.set(cache_key, evaluation)
         budget.used_metric_calls += 1
-    return evaluations
+
+    return [
+        evaluations_by_index[index]
+        for index in sorted(evaluations_by_index)
+    ]
 
 
 def _completed_all_rows(results: list[RowEvaluation], rows: list[DatasetRow]) -> bool:
@@ -636,7 +979,17 @@ def optimize_loaded_text_candidate(
     local_logger: Any | None = None,
 ) -> TextGepaResult:
     seed_text = prompt_context.body or ""
-    budget = Budget.from_config(config)
+    budget = Budget.from_config(
+        config,
+        trainset_size=len(trainset),
+        valset_size=len(valset),
+        num_components=1,
+    )
+    num_threads_plan = resolve_num_threads(
+        config.num_threads,
+        minibatch_size=config.minibatch_size,
+        validation_count=len(valset),
+    )
     evaluation_cache = EvaluationCache() if config.cache_evaluations else None
     candidate_text_by_id = {"seed": seed_text}
     val_scores_by_candidate: dict[str, dict[str, float]] = {}
@@ -664,6 +1017,7 @@ def optimize_loaded_text_candidate(
             "train_count": len(trainset),
             "validation_count": len(valset),
             "config": dataclasses.asdict(config),
+            "num_threads": num_threads_plan.to_payload(),
             "budget": budget.to_payload(),
         })
 
@@ -684,6 +1038,7 @@ def optimize_loaded_text_candidate(
             scorer_runner=scorer_runner,
             budget=budget,
             evaluation_cache=evaluation_cache,
+            num_threads=num_threads_plan.resolved,
         )
         if local_logger is not None:
             local_logger.append_evaluations(
@@ -694,7 +1049,7 @@ def optimize_loaded_text_candidate(
                 results=seed_results,
             )
         if not _completed_all_rows(seed_results, valset):
-            raise RuntimeError("Budget exhausted before seed validation completed")
+            raise RuntimeError("Seed validation did not complete all requested rows")
         budget.used_full_evals += 1
         seed_score = _mean(seed_results)
         val_scores_by_candidate["seed"] = {result.row_id: result.score for result in seed_results}
@@ -725,6 +1080,20 @@ def optimize_loaded_text_candidate(
             candidate_id=best_candidate_id,
         )
         event_sink.log_event("budget_updated", budget.to_payload(), event_layer="extension")
+        if not budget.allows_iteration():
+            event_sink.log_event(
+                "budget_exhausted",
+                {
+                    "stage": "seed_val_set_completed",
+                    "requested_rows": len(valset),
+                    "completed_rows": len(seed_results),
+                    "budget": budget.to_payload(),
+                    "decision_rule": "budget is checked only after completing seed validation or a full iteration",
+                },
+                event_layer="extension",
+                candidate_id="seed",
+            )
+            budget_exhausted = True
 
         for iteration in range(1, config.max_iterations + 1):
             if not budget.allows_iteration():
@@ -787,6 +1156,7 @@ def optimize_loaded_text_candidate(
                 scorer_runner=scorer_runner,
                 budget=budget,
                 evaluation_cache=evaluation_cache,
+                num_threads=num_threads_plan.resolved,
             )
             if local_logger is not None:
                 local_logger.append_evaluations(
@@ -863,6 +1233,21 @@ def optimize_loaded_text_candidate(
                     iteration=iteration,
                     candidate_id=best_candidate_id,
                 )
+                if iteration < config.max_iterations and not budget.allows_iteration():
+                    event_sink.log_event(
+                        "budget_exhausted",
+                        {
+                            "stage": "iteration_completed",
+                            "completed_iteration": iteration,
+                            "budget": budget.to_payload(),
+                            "decision_rule": "budget is checked only between iterations",
+                        },
+                        event_layer="extension",
+                        iteration=iteration,
+                        candidate_id=best_candidate_id,
+                    )
+                    budget_exhausted = True
+                    break
                 continue
 
             event_sink.log_event(
@@ -946,6 +1331,7 @@ def optimize_loaded_text_candidate(
                 scorer_runner=scorer_runner,
                 budget=budget,
                 evaluation_cache=evaluation_cache,
+                num_threads=num_threads_plan.resolved,
             )
             if local_logger is not None:
                 local_logger.append_evaluations(
@@ -1028,6 +1414,7 @@ def optimize_loaded_text_candidate(
                     scorer_runner=scorer_runner,
                     budget=budget,
                     evaluation_cache=evaluation_cache,
+                    num_threads=num_threads_plan.resolved,
                 )
                 if local_logger is not None:
                     local_logger.append_evaluations(
@@ -1152,6 +1539,21 @@ def optimize_loaded_text_candidate(
                 iteration=iteration,
                 candidate_id=best_candidate_id,
             )
+            if iteration < config.max_iterations and not budget.allows_iteration():
+                event_sink.log_event(
+                    "budget_exhausted",
+                    {
+                        "stage": "iteration_completed",
+                        "completed_iteration": iteration,
+                        "budget": budget.to_payload(),
+                        "decision_rule": "budget is checked only between iterations",
+                    },
+                    event_layer="extension",
+                    iteration=iteration,
+                    candidate_id=best_candidate_id,
+                )
+                budget_exhausted = True
+                break
 
         if budget_exhausted:
             event_sink.log_event(

package/vendor/skills/orizu-cli/SKILL.md

@@ -148,6 +148,7 @@ Bundled `run-gepa` reflection behavior:
 - Reflection max-token limits are explicit config, not implicit defaults. Use `--reflection-max-tokens <n>` when a provider requires an output cap; this maps to Anthropic `max_tokens` and OpenAI `max_output_tokens`.
 - Anthropic native Messages reflection requires `max_tokens`, so `run-gepa` with an `anthropic/...` reflection model must receive `--reflection-max-tokens <n>` and should fail fast if it is omitted. OpenAI reflection may omit `max_output_tokens` unless the user explicitly wants a cap.
 - By default, `run-gepa` skips reflection and child creation when every row in the selected parent mini-batch is already perfect (`1.0` for higher-is-better scorers, `0.0` for lower-is-better scorers). This saves reflection tokens because the child cannot beat a saturated mini-batch. Use `--no-skip-perfect-parent-reflection` to force reflection anyway, or `--skip-perfect-parent-reflection` to make the default explicit in scripted runs.
+- By default, `run-gepa` sets `--num-threads auto` for row evaluations. Auto uses mini-batch size, validation-set size, CPU count, memory estimate, file-descriptor limit, and an 8-thread hard cap to avoid launching too many runner subprocesses at once. Use `--num-threads <n>` only when the runner/provider capacity is known.
 - `run-gepa` writes complete local logs by default under `logs/<optimization_run_id>`; override with `--log-dir <dir>` or disable with `--no-local-log`.
 - Server optimization events redact row snapshots and reflection prompts by default; the local log keeps full row inputs, outputs, scores, feedback, reflection prompts, and reflection responses for later agent analysis.
 - OpenAI example: `--reflection-model openai/gpt-5 --reflection-provider-settings '{"reasoning":{"effort":"medium","summary":"auto"}}'`.

package/vendor/skills/orizu-cli/references/cli-reference.md

@@ -232,12 +232,13 @@ For prompts, judges, runners, run submission, optimizer artifacts, live event lo
 Optimization trace commands:
 
 ```bash
-orizu optimizations run-gepa ... [--log-dir logs] [--no-local-log]
+orizu optimizations run-gepa ... [--num-threads auto|N] [--log-dir logs] [--no-local-log]
 orizu optimizations export <optimization-run-id> --out ./optimization.json
 orizu optimizations export <optimization-run-id> --json
 ```
 
 Behavior:
+- `run-gepa` defaults `--num-threads` to `auto`, resolving a conservative row-evaluation parallelism cap from mini-batch size, validation-set size, CPU count, memory estimate, file-descriptor limit, and an 8-thread hard cap.
 - `run-gepa` writes a complete local trace under `logs/<optimization_run_id>` by default.
 - The local trace is the best artifact for coding-agent analysis because it includes full rows, outputs, scores, feedback, scorer responses, reflection prompts, reflection responses, candidate text, and `result.json`.
 - `optimizations export` writes a portable JSON artifact from server data when the local log is unavailable or the run happened elsewhere.

package/vendor/skills/orizu-cli/references/optimization-reports.md

@@ -84,7 +84,7 @@ Report:
 - Acceptance rate by iteration. Many rejections suggest reflection is exploring but not finding hills; many acceptances suggest it is climbing a real signal.
 - Budget state. If budget was exhausted, the run paused before auto-promotion; manually decide whether to promote/finish. If max iterations was reached with budget remaining, the run likely converged under the configured search.
 - `scoreOverTime` / validation trajectory. Still climbing at the end suggests raising budget or iterations. Plateaued curves point toward scorer, feedback, or data work upstream.
-- Reflection model, inference model, scorer model, provider settings, minibatch size, budget preset/limit, candidate selection strategy, seed, and cache settings.
+- Reflection model, inference model, scorer model, provider settings, minibatch size, resolved thread count, budget preset/limit, candidate selection strategy, seed, and cache settings.
 
 ### 5. Recommendations
 
@@ -130,9 +130,9 @@ Set an explicit expectation before the run: for example, "if best kappa does not
 
 ### Budget Versus Iteration Cap
 
-Default budget presets are metric-call budgets: `auto = 64`, `light = 48`, `medium = 96`, and `high = 192`, unless `--max-metric-calls`, `--max-full-evals`, or `--max-candidate-proposals` overrides them.
+Default budget presets are DSPy-style metric-call budgets. `light`, `medium`, and `heavy` map to DSPy's `auto="light"`, `auto="medium"`, and `auto="heavy"` scales; `auto` is treated as the balanced `medium` preset. The final metric-call limit scales with validation-set size, rather than using fixed tiny constants. `--max-metric-calls` overrides this directly, while `--max-full-evals` is converted to a metric-call budget over `train + validation` rows.
 
-Each iteration costs the minibatch evaluation plus full validation calls for accepted candidates, so medium budget can be exhausted well before a high `--max-iterations` target. If a run ends as `budget_exhausted`, it pauses and does not auto-promote. If auto-promotion matters, either increase budget to fit the iteration target or lower `--max-iterations` to fit the budget.
+Budget is checked only between iterations. Once an iteration starts, `run-gepa` completes the parent minibatch, child minibatch, and full validation eval for an accepted child even if that overshoots the nominal budget. If a run ends as `budget_exhausted`, it pauses and does not auto-promote. If auto-promotion matters, either increase budget to fit the iteration target or lower `--max-iterations` to fit the budget.
 
 ### Reflection Signal Quality
 

package/vendor/skills/orizu-cli/references/prompt-control-plane.md

@@ -609,13 +609,15 @@ orizu --local optimizations run-gepa \
   --log-dir ./logs \
   --budget light \
   --max-iterations 3 \
-  --minibatch-size 3
+  --minibatch-size 3 \
+  --num-threads auto
 ```
 
 Useful GEPA flags:
 
-- `--budget auto|light|medium|high`, `--max-metric-calls <n>`, `--max-full-evals <n>`.
+- `--budget auto|light|medium|heavy`, `--max-metric-calls <n>`, `--max-full-evals <n>`.
 - `--minibatch-size <n>` defaults to 3.
+- `--num-threads auto|N` defaults to `auto`; auto caps row-evaluation concurrency from mini-batch size, validation-set size, CPU count, memory estimate, file-descriptor limit, and an 8-thread hard cap.
 - `--candidate-selection-strategy pareto|current_best|epsilon_greedy`; default is `pareto`.
 - `--reflection-model <provider/model>`, `--reflection-temperature <n>`, `--reflection-prompt-template <text|@file>`.
 - `--reflection-max-tokens <n>` is explicit provider config, not a global default. It maps to Anthropic `max_tokens` and OpenAI `max_output_tokens`; Anthropic native Messages reflection requires it, while OpenAI can omit it unless the user wants a cap.