orizu 0.5.19 → 0.5.21

package/vendor/orizu-gepa/src/orizu_gepa_connector/engine.py

@@ -0,0 +1,269 @@
+"""Official GEPA execution wiring for the Orizu adapter and callback bridge."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+import dataclasses
+import json
+import random
+from typing import Any
+
+from gepa import optimize
+from gepa.strategies.candidate_selector import EpsilonGreedyCandidateSelector
+from orizu_gepa.optimizer import TextGepaConfig
+
+from .callbacks import LifecycleHooks, OrizuCallback
+from .preflight import validate_seed_and_scorer
+from .stop_conditions import ProposalBudgetStopper
+
+
+def _row_id(row: Any, index: int) -> str:
+    value = getattr(row, "id", row.get("id") if isinstance(row, dict) else None)
+    if value is None:
+        raise ValueError(f"GEPA dataset row at index {index} is missing its id")
+    return str(value)
+
+
+class _RowIdentityLoader:
+    """Expose Orizu dataset ids to GEPA instead of its positional list ids."""
+
+    def __init__(self, rows: Sequence[Any]):
+        self._rows_by_id = {_row_id(row, index): row for index, row in enumerate(rows)}
+        if len(self._rows_by_id) != len(rows):
+            raise ValueError("GEPA datasets require unique DatasetRow ids")
+
+    def all_ids(self) -> list[str]:
+        return list(self._rows_by_id)
+
+    def fetch(self, row_ids: Sequence[str]) -> list[Any]:
+        return [self._rows_by_id[str(row_id)] for row_id in row_ids]
+
+    def __len__(self) -> int:
+        return len(self._rows_by_id)
+
+
+def _with_row_identities(rows: Sequence[Any]) -> Any:
+    """Keep caller-supplied loaders, otherwise prevent cross-split cache aliasing."""
+    if hasattr(rows, "all_ids") and hasattr(rows, "fetch"):
+        return rows
+    return _RowIdentityLoader(rows)
+
+
+class SeedValidationRefused(RuntimeError):
+    """Raised before a run is started when the scored seed has no gradient."""
+
+    def __init__(self, verdict: dict[str, Any], *, preflight_record_path: str | None = None):
+        self.verdict = verdict
+        self.preflight_record_path = preflight_record_path
+        message = (
+            f"{verdict['reason']}: preflight row evidence="
+            f"{json.dumps(verdict.get('row_evidence', []), ensure_ascii=False, default=str)}"
+        )
+        if preflight_record_path is not None:
+            message += f"; preflight record={preflight_record_path}"
+        super().__init__(message)
+
+
+def _preflight_row_evidence(rows: list[Any], scores: list[float], outputs: list[Any]) -> list[dict[str, Any]]:
+    """Keep the raw scorer evidence that explains a preflight refusal."""
+    evidence: list[dict[str, Any]] = []
+    for index, score in enumerate(scores):
+        output = outputs[index] if index < len(outputs) and isinstance(outputs[index], dict) else {}
+        row = rows[index] if index < len(rows) else None
+        scorer_error = output.get("scorer_error")
+        if scorer_error is None and output.get("error_source") == "scorer":
+            scorer_error = output.get("error")
+        item = {
+            "row_id": output.get("row_id", getattr(row, "id", None)),
+            "score": score,
+            "scorer_output": output.get("scorer_output", output.get("scorer_response")),
+            "scorer_error": scorer_error,
+        }
+        if output.get("error") is not None:
+            item["execution_error"] = output["error"]
+            item["execution_error_source"] = output.get("error_source")
+        evidence.append(item)
+    return evidence
+
+
+def _validate_seed_before_run(*, seed_candidate: dict[str, str], valset: list[Any], adapter: Any,
+                              hooks: LifecycleHooks, allow_degenerate_seed: bool) -> tuple[dict[str, Any], Any, list[Any]]:
+    """Cheap scorer preflight; GEPA owns the single full seed validation."""
+    # The approved cap bounds preflight spend. With a binary scorer and a seed
+    # that is correct 70% of the time, all three sampled rows can still score
+    # zero with probability 0.3**3 = 2.7%; do not silently change that policy.
+    preflight_rows = valset[:min(3, len(valset))]
+    seed_evaluation = adapter.evaluate(preflight_rows, seed_candidate, capture_traces=True)
+    scores = list(seed_evaluation.scores)
+    verdict = validate_seed_and_scorer(
+        scores=scores,
+        row_evidence=_preflight_row_evidence(preflight_rows, scores, list(seed_evaluation.outputs)),
+        allow_degenerate_seed=allow_degenerate_seed,
+        higher_is_better=getattr(getattr(adapter, "scorer_context", None), "higher_is_better", True),
+    )
+    verdict["preflight_metric_calls"] = getattr(adapter, "metric_calls_used", seed_evaluation.num_metric_calls)
+    hooks.emit("seed_validated", verdict)
+    if not verdict["allowed"]:
+        raise SeedValidationRefused(verdict)
+    return verdict, seed_evaluation, preflight_rows
+
+
+def validate_seed_before_run(*, seed_candidate: dict[str, str], valset: list[Any], adapter: Any,
+                             hooks: LifecycleHooks, allow_degenerate_seed: bool) -> dict[str, Any]:
+    """Validate the seed and return the durable public preflight verdict."""
+    verdict, _evaluation, _rows = _validate_seed_before_run(
+        seed_candidate=seed_candidate,
+        valset=valset,
+        adapter=adapter,
+        hooks=hooks,
+        allow_degenerate_seed=allow_degenerate_seed,
+    )
+    return verdict
+
+
+class _PreflightReusingAdapter:
+    """Serve GEPA's immediate seed evaluation from the completed preflight."""
+
+    def __init__(self, adapter: Any, seed_candidate: dict[str, str], rows: list[Any], evaluation: Any):
+        self._adapter = adapter
+        self._seed_candidate = seed_candidate
+        self._rows = rows
+        self._evaluation = evaluation
+        self._served_seed = False
+
+    def evaluate(self, batch: list[Any], candidate: dict[str, str], capture_traces: bool = False):
+        if (not self._served_seed and not capture_traces and candidate == self._seed_candidate
+                and list(batch) == self._rows):
+            self._served_seed = True
+            return self._evaluation
+        return self._adapter.evaluate(batch, candidate, capture_traces=capture_traces)
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._adapter, name)
+
+
+def run_official_gepa(
+    *,
+    seed_candidate: dict[str, str],
+    trainset: list[Any],
+    valset: list[Any],
+    adapter: Any,
+    callback: Any,
+    hooks: LifecycleHooks,
+    max_metric_calls: int | None = None,
+    stop_callbacks: Sequence[Callable[[Any], bool]] = (),
+    allow_degenerate_seed: bool = False,
+    reflection_lm: Callable[[str], str] | None = None,
+    custom_candidate_proposer: Callable[..., dict[str, str]] | None = None,
+    proposal_budget: Any | None = None,
+    seed_already_validated: bool = False,
+    config: TextGepaConfig | None = None,
+    module_selector: str = "round_robin",
+) -> Any:
+    """Run GEPA after the launch-time seed check and emit named lifecycle hooks.
+
+    The seed pass is deliberately outside ``optimize``: a bad scorer/seed is
+    refused before an optimization run exists or any configured GEPA budget is
+    consumed.  The actual GEPA loop remains responsible for every later
+    stopping boundary.
+    """
+    # Full-valset evaluations bypass GEPA's evaluation callbacks. Give the
+    # callback the real adapter so it can turn GEPA's positional score keys
+    # into the adapter's durable dataset row identities.
+    if isinstance(callback, OrizuCallback):
+        callback.evaluation_adapter = adapter
+        callback.validation_row_ids = [_row_id(row, index) for index, row in enumerate(valset)]
+
+    preflight_evaluation = None
+    preflight_rows: list[Any] = []
+    if not seed_already_validated:
+        _verdict, preflight_evaluation, preflight_rows = _validate_seed_before_run(
+            seed_candidate=seed_candidate,
+            valset=valset,
+            adapter=adapter,
+            hooks=hooks,
+            allow_degenerate_seed=allow_degenerate_seed,
+        )
+
+    # GEPA 0.1.4 reads this optional protocol attribute directly in its
+    # reflective proposer, even when a custom proposer owns the call. Our
+    # runner adapter supplies it; lightweight adapters used by the real GEPA
+    # boundary do not. Normalize the vendor's optional slot without replacing
+    # an adapter-owned proposer.
+    if custom_candidate_proposer is not None and not hasattr(adapter, "propose_new_texts"):
+        adapter.propose_new_texts = None
+
+    gepa_adapter = adapter
+    if custom_candidate_proposer is not None and preflight_evaluation is not None:
+        # A one-row low metric ceiling otherwise pays for this same seed row a
+        # second time inside GEPA before it reaches the custom proposal. The
+        # cached result is already a real evaluator result from this call's
+        # mandatory preflight; only the duplicate evaluator invocation is
+        # removed, never the separate proposal ledger.
+        gepa_adapter = _PreflightReusingAdapter(adapter, seed_candidate, preflight_rows, preflight_evaluation)
+    if custom_candidate_proposer is not None:
+        # GEPA refuses an adapter-owned proposer alongside its explicit custom
+        # proposer. The latter is the selected skilled-proposer path.
+        gepa_adapter.propose_new_texts = None
+
+    config = config or TextGepaConfig()
+    candidate_selection_strategy: Any = config.candidate_selection_strategy
+    # Official GEPA's string shortcut fixes epsilon at 0.1.  Use its public
+    # selector class so the legacy --epsilon contract remains observable.
+    if candidate_selection_strategy == "epsilon_greedy":
+        candidate_selection_strategy = EpsilonGreedyCandidateSelector(
+            epsilon=max(0.0, min(1.0, config.epsilon)),
+            rng=random.Random(config.seed),
+        )
+
+    owned_stop_callbacks = list(stop_callbacks)
+    if proposal_budget is not None:
+        owned_stop_callbacks.append(ProposalBudgetStopper(proposal_budget))
+
+    reflection_minibatch_size = config.minibatch_size
+    if custom_candidate_proposer is not None and max_metric_calls is not None:
+        # GEPA's metric stopper is evaluated at iteration boundaries.  Do not
+        # let its default three-row reflection batch overshoot a deliberately
+        # smaller evaluator-only ceiling before that boundary is observed.
+        reflection_minibatch_size = min(
+            reflection_minibatch_size,
+            max(1, max_metric_calls - len(valset)),
+        )
+
+    kwargs: dict[str, Any] = {
+        "seed_candidate": seed_candidate,
+        "trainset": trainset,
+        "valset": valset,
+        "adapter": gepa_adapter,
+        "callbacks": [callback],
+        "stop_callbacks": owned_stop_callbacks or None,
+        "max_metric_calls": max_metric_calls,
+        "candidate_selection_strategy": candidate_selection_strategy,
+        "skip_perfect_score": config.skip_perfect_parent_reflection,
+        "reflection_minibatch_size": reflection_minibatch_size,
+        "cache_evaluation": config.cache_evaluations,
+        "seed": config.seed,
+        "display_progress_bar": False,
+        "raise_on_exception": True,
+        "module_selector": module_selector,
+    }
+    if reflection_lm is not None:
+        kwargs["reflection_lm"] = reflection_lm
+        # The connector implements GEPA's component-aware strategy protocol;
+        # passing it as a bare callable would discard the selected component.
+        kwargs["reflection_strategy"] = reflection_lm
+    if custom_candidate_proposer is not None:
+        kwargs["custom_candidate_proposer"] = custom_candidate_proposer
+    kwargs["trainset"] = _with_row_identities(trainset)
+    kwargs["valset"] = _with_row_identities(valset)
+    result = optimize(**kwargs)
+    if custom_candidate_proposer is not None and preflight_evaluation is not None:
+        # GEPA's state correctly counts its seed evaluation, but this run
+        # served that evaluation from the already-accounted launch preflight.
+        # Its public metric result must consequently retain M1's convention:
+        # GEPA result counts exclude the one caller preflight, while the
+        # adapter's physical count is result + one.
+        total = getattr(result, "total_metric_calls", None)
+        if isinstance(total, int):
+            result = dataclasses.replace(result, total_metric_calls=max(0, total - len(preflight_rows)))
+    return result

package/vendor/orizu-gepa/src/orizu_gepa_connector/instruction_set_loader.py

@@ -0,0 +1,52 @@
+"""Offline reader for the ALI-1532 synced instruction-set layout."""
+
+import json
+from pathlib import Path
+
+
+class InstructionSetLoaderError(Exception):
+    def __init__(self, code: str):
+        self.code = code
+        super().__init__(code)
+
+
+def _inside(root: Path, candidate: Path) -> bool:
+    try:
+        candidate.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def load_instruction_set(directory: str, name: str, model_config_identity: str) -> dict[str, str]:
+    root = (Path(directory) / name).resolve()
+    if not root.is_dir():
+        raise InstructionSetLoaderError("instruction_set_not_synced")
+    manifest_path = root / "manifest.json"
+    if not manifest_path.is_file():
+        raise InstructionSetLoaderError("instruction_set_manifest_missing")
+    try:
+        manifest = json.loads(manifest_path.read_text())
+    except (OSError, json.JSONDecodeError) as error:
+        raise InstructionSetLoaderError("instruction_set_manifest_missing") from error
+    if manifest.get("manifestVersion") != 1:
+        raise InstructionSetLoaderError("instruction_set_manifest_version_unsupported")
+    if manifest.get("filteredTo") and model_config_identity not in manifest["filteredTo"]:
+        raise InstructionSetLoaderError("instruction_set_profile_not_synced")
+    production = next((profile.get("production") for profile in manifest.get("profiles", []) if profile.get("modelConfigIdentity") == model_config_identity), None)
+    material = production or manifest.get("default", {})
+    values: dict[str, str] = {}
+    for key in manifest.get("shape", []):
+        if key in material.get("pinnedComponents", {}):
+            raise InstructionSetLoaderError("instruction_set_component_unavailable")
+        relative = material.get("files", {}).get(key)
+        if not relative:
+            raise InstructionSetLoaderError("instruction_set_profile_key_missing")
+        path = (root / relative).resolve()
+        if not _inside(root, path):
+            raise InstructionSetLoaderError("instruction_set_path_unsafe")
+        try:
+            values[key] = path.read_text()
+        except OSError as error:
+            raise InstructionSetLoaderError("instruction_set_component_unreadable") from error
+    return values

package/vendor/orizu-gepa/src/orizu_gepa_connector/preflight.py

@@ -0,0 +1,64 @@
+"""Launch-time safety checks performed before an optimization run exists."""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+
+def validate_seed_and_scorer(
+    *,
+    scores: list[float],
+    row_evidence: list[dict[str, Any]] | None = None,
+    allow_degenerate_seed: bool,
+    higher_is_better: bool = True,
+) -> dict[str, Any]:
+    """Classify a sampled seed without treating a transient row failure as global.
+
+    A row that returns a score, including the worst score, proves the runner
+    and scorer pipeline executed.  Only a sample containing *no* such row is
+    an execution refusal.  Successful worst-score rows remain the separate,
+    deliberately bypassable ``degenerate_seed`` condition.
+    """
+    evidence = list(row_evidence or [])
+    successful_scores: list[float] = []
+    failures: dict[str, list[str | None]] = {}
+    for index, score in enumerate(scores):
+        row = evidence[index] if index < len(evidence) else {}
+        if row.get("scorer_error"):
+            reason = "scorer_execution_failed"
+        elif row.get("execution_error"):
+            source = row.get("execution_error_source")
+            reason = f"{source}_execution_failed" if source in {"candidate", "scorer"} else "runner_execution_failed"
+        else:
+            successful_scores.append(score)
+            continue
+        failures.setdefault(reason, []).append(row.get("row_id"))
+
+    warnings = [
+        {"reason": reason, "row_ids": row_ids}
+        for reason, row_ids in failures.items()
+    ]
+    if failures and not successful_scores and not allow_degenerate_seed:
+        # Every sampled row failed before yielding a usable score. Preserve a
+        # precise source when the failure mode is uniform, otherwise name the
+        # shared runner pipeline rather than pretending one source won.
+        reason = next(iter(failures)) if len(failures) == 1 else "runner_execution_failed"
+        return {"allowed": False, "reason": reason, "row_evidence": evidence}
+
+    worst_bound = all(math.isclose(score, 0.0, abs_tol=1e-9) for score in successful_scores)
+    # The adapter normalizes lower-is-better scores into GEPA's higher-is-
+    # better orientation. Legacy still refuses a uniform raw 0.0 lower-bound
+    # (normalized 1.0): it is a perfect seed or a silent contract mismatch,
+    # neither of which gives reflection a useful gradient.
+    lower_perfect_bound = not higher_is_better and all(
+        math.isclose(score, 1.0, abs_tol=1e-9) for score in successful_scores
+    )
+    degenerate = bool(successful_scores) and (worst_bound or lower_perfect_bound)
+    if degenerate and not allow_degenerate_seed:
+        verdict: dict[str, Any] = {"allowed": False, "reason": "degenerate_seed", "row_evidence": evidence}
+    else:
+        verdict = {"allowed": True, "reason": None, "row_evidence": evidence}
+    if warnings:
+        verdict["warnings"] = warnings
+    return verdict

package/vendor/orizu-gepa/src/orizu_gepa_connector/reflection.py

@@ -0,0 +1,186 @@
+"""GEPA reflection callable delegated to Orizu's existing provider layer."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import replace
+import json
+from typing import Any
+
+from orizu_gepa.optimizer import DEFAULT_REFLECTION_PROMPT_TEMPLATE, build_reflection_prompt
+from orizu_gepa.reflection import reflect_with_provider
+
+
+def provider_reflection_lm(parent_text: str, parent_results: list[Any], config: Any) -> str:
+    """Run the frozen provider layer and return its raw response.
+
+    ``reflect_with_provider`` owns retries, HTTP timeouts, token accounting and
+    candidate extraction.  The connector intentionally does not recreate any
+    of that transport policy.
+    """
+    result = reflect_with_provider(parent_text, parent_results, config)
+    return result.response
+
+
+class GepaReflectionLM:
+    """Reflection bridge that preserves provider failures before GEPA isolates them.
+
+    GEPA intentionally catches reflection failures per proposal.  That keeps an
+    optimization alive, but means the connector is the final durable boundary
+    for the actual exception.  This wrapper records it *before* re-raising to
+    GEPA's safe helper and exposes provider-measured token usage in GEPA's
+    ``TrackingLM``-shaped fields.
+    """
+
+    def __init__(
+        self,
+        *,
+        context_supplier: Callable[[], tuple[str | dict[str, str], list[Any]]],
+        config: Any,
+        failure_reporter: Callable[..., None] | None,
+        success_reporter: Callable[[str], None] | None,
+    ) -> None:
+        self._context_supplier = context_supplier
+        self._config = config
+        self._failure_reporter = failure_reporter
+        self._success_reporter = success_reporter
+        self._total_tokens_in = 0
+        self._total_tokens_out = 0
+        self._total_tokens = 0
+
+    @property
+    def total_cost(self) -> float:
+        return 0.0
+
+    @property
+    def total_tokens_in(self) -> int:
+        return self._total_tokens_in
+
+    @property
+    def total_tokens_out(self) -> int:
+        return self._total_tokens_out
+
+    @property
+    def total_tokens(self) -> int:
+        return self._total_tokens
+
+    def _account_usage(self, usage: Any) -> None:
+        if not isinstance(usage, dict):
+            raise RuntimeError("ALI_1505_PROVIDER_USAGE_MISSING: reflection transport returned no provider usage")
+        try:
+            input_tokens = int(usage["input_tokens"])
+            output_tokens = int(usage["output_tokens"])
+            total_tokens = int(usage["total_tokens"])
+        except (KeyError, TypeError, ValueError) as error:
+            raise RuntimeError("ALI_1505_PROVIDER_USAGE_INVALID: reflection transport returned invalid provider usage") from error
+        if min(input_tokens, output_tokens, total_tokens) < 0:
+            raise RuntimeError("ALI_1505_PROVIDER_USAGE_INVALID: reflection transport returned negative provider usage")
+        self._total_tokens_in += input_tokens
+        self._total_tokens_out += output_tokens
+        self._total_tokens += total_tokens
+
+    def __call__(self, gepa_prompt: str) -> str:
+        parent_candidate, parent_results = self._context_supplier()
+        parent_text = (
+            json.dumps(parent_candidate, sort_keys=True, separators=(",", ":"))
+            if isinstance(parent_candidate, dict) and len(parent_candidate) > 1
+            else next(iter(parent_candidate.values()))
+            if isinstance(parent_candidate, dict)
+            else parent_candidate
+        )
+        try:
+            # Prompt validation is part of the provider attempt. Keep it
+            # inside this boundary so GEPA cannot silently swallow its error.
+            provider_prompt = build_reflection_prompt(parent_text, parent_results, self._config)
+            result = reflect_with_provider(parent_text, parent_results, self._config)
+            response = result.response
+            provider_prompt = getattr(result, "prompt", provider_prompt)
+            self._account_usage(getattr(result, "usage", None))
+        except Exception as error:
+            billable_usage = getattr(error, "usage", None)
+            if billable_usage is not None:
+                self._account_usage(billable_usage)
+            if self._failure_reporter is not None:
+                self._failure_reporter(
+                    error=error,
+                    gepa_prompt=gepa_prompt,
+                    parent_text=parent_text,
+                    parent_results=parent_results,
+                )
+            raise
+        if self._success_reporter is not None:
+            self._success_reporter(str(provider_prompt))
+        return response
+
+    def reflect(
+        self,
+        candidate: dict[str, str],
+        reflective_dataset: Any,
+        components_to_update: list[str],
+    ) -> tuple[Any, "GepaReflectionLM"]:
+        """Implement GEPA's component-aware ReflectionLM protocol.
+
+        The callable protocol has no selected-component information.  GEPA's
+        newer protocol does, so tuples must use it rather than reverse-engineer
+        a key from a rendered vendor prompt.
+        """
+        from gepa.proposer.reflective_mutation.reflection_lm import ReflectionProposal
+
+        _, parent_results = self._context_supplier()
+        full_candidate = candidate
+        proposal = ReflectionProposal(new_texts={}, prompts={}, raw_lm_outputs={})
+        for component in components_to_update:
+            if component not in candidate:
+                continue
+            parent_text = candidate[component]
+            others = {key: value for key, value in full_candidate.items() if key != component}
+            config = self._config
+            if others:
+                template = config.reflection_prompt_template or DEFAULT_REFLECTION_PROMPT_TEMPLATE
+                config = replace(
+                    config,
+                    reflection_prompt_template=(
+                        f"{template}\n\nOther instruction-set components (read-only): "
+                        f"{json.dumps(others, sort_keys=True, ensure_ascii=False)}"
+                    ),
+                )
+            provider_prompt = ""
+            try:
+                provider_prompt = build_reflection_prompt(parent_text, parent_results, config)
+                result = reflect_with_provider(parent_text, parent_results, config)
+                self._account_usage(getattr(result, "usage", None))
+            except Exception as error:
+                billable_usage = getattr(error, "usage", None)
+                if billable_usage is not None:
+                    self._account_usage(billable_usage)
+                if self._failure_reporter is not None:
+                    self._failure_reporter(error=error, gepa_prompt=provider_prompt, parent_text=parent_text, parent_results=parent_results)
+                raise
+            proposal.new_texts[component] = result.response
+            proposal.prompts[component] = getattr(result, "prompt", provider_prompt)
+            proposal.raw_lm_outputs[component] = result.response
+            if self._success_reporter is not None:
+                self._success_reporter(str(proposal.prompts[component]))
+        return proposal, self
+
+
+def make_gepa_reflection_lm(
+    *,
+    context_supplier: Callable[[], tuple[str | dict[str, str], list[Any]]],
+    config: Any,
+    failure_reporter: Callable[..., None] | None = None,
+    success_reporter: Callable[[str], None] | None = None,
+) -> GepaReflectionLM:
+    """Adapt GEPA's ``Callable[[rendered_prompt], str]`` contract.
+
+    GEPA owns its rendered prompt. The frozen provider layer owns the
+    production prompt and retry policy. ``context_supplier`` is read at each
+    invocation, after GEPA's selected-parent minibatch evaluation, so a later
+    proposal cannot accidentally reflect on the seed's rows.
+    """
+    return GepaReflectionLM(
+        context_supplier=context_supplier,
+        config=config,
+        failure_reporter=failure_reporter,
+        success_reporter=success_reporter,
+    )