orizu 0.5.21 → 0.5.22

package/vendor/orizu-gepa/src/orizu_gepa_connector/skilled_proposer_bridge.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import atexit
 import dataclasses
+import hashlib
 import json
 import os
 import shutil
@@ -35,11 +36,16 @@ from orizu_gepa.optimizer import TextGepaConfig
 
 _SELECTION_ENV = "ORIZU_CANDIDATE_PROPOSER"
 _SELECTION_VALUE = "skilled-proposer"
+_CONFIG_ENV = "ORIZU_SKILLED_PROPOSER_CONFIG"
 _SUPPORTED_ROLES = {"system", "user", "assistant"}
 _ALLOWED_DSPY_REQUEST_FIELDS = frozenset({"temperature", "max_tokens", "cache"})
 _DspyBaseLM = dspy.BaseLM if dspy is not None else object
 
 
+class _ProposalObservabilityEventFailure(RuntimeError):
+    pass
+
+
 def _require_dspy() -> Any:
     if dspy is None:
         raise RuntimeError("ALI_1505_DSPY_UNAVAILABLE")
@@ -90,6 +96,8 @@ class ProposalObservability:
         self.durable_failure_root = durable_failure_root
         self.events: list[dict[str, Any]] = []
         self.budget = budget or ProposalCallBudget()
+        self._event_failure_count = 0
+        self._provider_failure_count = 0
         self._temporary_root: Path | None = None
 
     @classmethod
@@ -151,15 +159,50 @@ class ProposalObservability:
         detail: str,
         usage: dict[str, int] | None = None,
     ) -> None:
+        self._provider_failure_count += 1
         self.budget.record(usage)
         self._write_durable_failure(code=code, detail=detail, correlation_id=event.get("correlation_id"))
         self._append_event(event)
 
     def record_event_failure(self, error: Exception, *, correlation_id: str) -> None:
+        self._event_failure_count += 1
         self._write_durable_failure(
             code="proposal_observability_event_failed", detail=str(error), correlation_id=correlation_id,
         )
 
+    def record_proposer_failure(self, error: Exception) -> None:
+        self._write_durable_failure(code="proposal_generation_failed", detail=str(error))
+
+    def record_component_fallback(self, *, component: str, error: Exception) -> None:
+        event = {
+            "type": "proposal_component_fallback",
+            "component": component,
+            "policy": "keep",
+            "errorType": type(error).__name__,
+            "errorMessage": str(error),
+        }
+        try:
+            self._append_event(event)
+        except OSError as event_error:
+            self._write_durable_failure(
+                code="proposal_observability_event_failed", detail=str(event_error),
+            )
+            raise _ProposalObservabilityEventFailure(
+                "proposal_observability_event_failed",
+            ) from event_error
+
+    def write_proposer_identity(self, identity: dict[str, Any]) -> str:
+        root = self.event_log_root.parent
+        path = root / "proposer.json"
+        try:
+            root.mkdir(parents=True, exist_ok=True)
+            path.write_text(json.dumps(identity, sort_keys=True) + "\n", encoding="utf-8")
+        except OSError as error:
+            code = "skilled_proposer_identity_write_failed"
+            self._write_durable_failure(code=code, detail=str(error))
+            raise RuntimeError(code) from error
+        return str(path)
+
     def terminal_stats(self) -> dict[str, Any]:
         return {"call_count": self.budget.call_count, "usage": dict(self.budget.usage)}
 
@@ -182,6 +225,14 @@ class ProposalObservability:
     def has_safe_boundary_stop(self) -> bool:
         return self.budget.has_safe_boundary_stop()
 
+    @property
+    def provider_failure_count(self) -> int:
+        return self._provider_failure_count
+
+    @property
+    def event_failure_count(self) -> int:
+        return self._event_failure_count
+
 
 def _reject(message: str, *, code: str = "proposal_dspy_request_rejected") -> Any:
     return _require_dspy().LMError(message, code=code)
@@ -311,9 +362,11 @@ class OrizuSkilledProposerLM(_DspyBaseLM):
                 )
             except Exception as event_error:
                 self._observability.record_event_failure(event_error, correlation_id=correlation_id)
-                raise RuntimeError("proposal_observability_event_failed") from event_error
+                raise _ProposalObservabilityEventFailure(
+                    "proposal_observability_event_failed",
+                ) from event_error
             raise dspy_module.LMError(
-                f"proposal_provider_transport_failed: {error}",
+                "proposal_provider_transport_failed",
                 code="proposal_provider_transport_failed",
                 model=self.model,
                 provider="orizu",
@@ -337,7 +390,9 @@ class OrizuSkilledProposerLM(_DspyBaseLM):
             self._observability.record_success(success_event, usage)
         except Exception as error:
             self._observability.record_event_failure(error, correlation_id=correlation_id)
-            raise RuntimeError("proposal_observability_event_failed") from error
+            raise _ProposalObservabilityEventFailure(
+                "proposal_observability_event_failed",
+            ) from error
         return dspy_module.LMResponse.from_text(completion.text, model=self.model, usage=usage, cache_hit=False)
 
 
@@ -359,6 +414,268 @@ def make_skilled_proposer_bridge(
     )
 
 
+@dataclass(frozen=True)
+class _NormalizedText:
+    source: str
+    content: str
+    sha256: str
+    byte_length: int
+
+
+@dataclass(frozen=True)
+class _NormalizedSkill(_NormalizedText):
+    name: str
+    description: str | None
+
+
+@dataclass(frozen=True)
+class _SkilledProposerConfig:
+    config_file_sha256: str
+    implementation: str
+    package_version: str
+    config_sha256: str
+    skills: tuple[_NormalizedSkill, ...]
+    additional_instructions: _NormalizedText | None
+    base_instructions: _NormalizedText | None
+    max_words: int | None
+    max_tokens: int | None
+    max_examples: int | None
+    on_error: str
+
+    def identity(self) -> dict[str, Any]:
+        return {
+            "implementation": self.implementation,
+            "packageVersion": self.package_version,
+            "configSha256": self.config_sha256,
+            "skills": [{
+                "name": skill.name,
+                "source": skill.source,
+                "sha256": skill.sha256,
+                "byteLength": skill.byte_length,
+            } for skill in self.skills],
+            "additionalInstructionsSha256": (
+                self.additional_instructions.sha256 if self.additional_instructions else None
+            ),
+            "baseInstructionsSha256": self.base_instructions.sha256 if self.base_instructions else None,
+            "maxWords": self.max_words,
+            "maxTokens": self.max_tokens,
+            "maxExamples": self.max_examples,
+            "onError": self.on_error,
+        }
+
+
+def _config_error(code: str, field_name: str) -> Any:
+    raise RuntimeError(f"{code}: {field_name}")
+
+
+def _strict_object(value: Any, fields: set[str], field_name: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return _config_error("skilled_proposer_config_invalid_field", field_name)
+    unknown = sorted(set(value) - fields)
+    if unknown:
+        return _config_error(
+            "skilled_proposer_config_unknown_field",
+            f"{field_name + '.' if field_name else ''}{unknown[0]}",
+        )
+    missing = sorted(fields - set(value))
+    if missing:
+        return _config_error(
+            "skilled_proposer_config_missing_field",
+            f"{field_name + '.' if field_name else ''}{missing[0]}",
+        )
+    return value
+
+
+def _required_string(value: Any, field_name: str, *, allow_empty: bool = False) -> str:
+    if not isinstance(value, str) or (not allow_empty and not value.strip()):
+        return _config_error("skilled_proposer_config_invalid_field", field_name)
+    try:
+        value.encode("utf-8")
+    except UnicodeEncodeError:
+        return _config_error("skilled_proposer_config_invalid_unicode", field_name)
+    return value
+
+
+def _positive_int_or_none(value: Any, field_name: str) -> int | None:
+    if value is None:
+        return None
+    if type(value) is not int or value <= 0:
+        return _config_error("skilled_proposer_config_invalid_field", field_name)
+    return value
+
+
+def _normalized_text(value: Any, field_name: str) -> _NormalizedText:
+    raw = _strict_object(value, {"source", "content", "sha256", "byteLength"}, field_name)
+    source = _required_string(raw["source"], f"{field_name}.source")
+    content = _required_string(raw["content"], f"{field_name}.content")
+    digest = _required_string(raw["sha256"], f"{field_name}.sha256")
+    byte_length = raw["byteLength"]
+    if (len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest)
+            or type(byte_length) is not int or byte_length != len(content.encode("utf-8"))):
+        return _config_error("skilled_proposer_config_invalid_field", field_name)
+    if hashlib.sha256(content.encode()).hexdigest() != digest:
+        return _config_error("skilled_proposer_config_invalid_field", f"{field_name}.sha256")
+    return _NormalizedText(source, content, digest, byte_length)
+
+
+def _optional_normalized_text(value: Any, field_name: str) -> _NormalizedText | None:
+    return None if value is None else _normalized_text(value, field_name)
+
+
+def _normalized_text_payload(value: _NormalizedText | None) -> dict[str, Any] | None:
+    if value is None:
+        return None
+    return {
+        "source": value.source, "content": value.content,
+        "sha256": value.sha256, "byteLength": value.byte_length,
+    }
+
+
+def _parse_skilled_proposer_config(raw_payload: str) -> _SkilledProposerConfig:
+    try:
+        parsed = json.loads(raw_payload)
+    except (json.JSONDecodeError, TypeError) as error:
+        raise RuntimeError("skilled_proposer_config_invalid_json") from error
+    fields = {
+        "configFileSha256", "schemaVersion", "implementation", "packageVersion", "configSha256", "skills",
+        "additionalInstructions", "baseInstructions", "maxWords", "maxTokens", "maxExamples", "onError",
+    }
+    raw = _strict_object(parsed, fields, "")
+    if (type(raw["schemaVersion"]) is not int or raw["schemaVersion"] != 1
+            or raw["implementation"] != "cmpnd-ai/skilled-proposer"):
+        return _config_error("skilled_proposer_config_invalid_field", "schemaVersion or implementation")
+    package_version = _required_string(raw["packageVersion"], "packageVersion")
+    config_file_sha256 = _required_string(raw["configFileSha256"], "configFileSha256")
+    config_sha256 = _required_string(raw["configSha256"], "configSha256")
+    for field_name, digest in (("configFileSha256", config_file_sha256), ("configSha256", config_sha256)):
+        if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
+            return _config_error("skilled_proposer_config_invalid_field", field_name)
+    if not isinstance(raw["skills"], list):
+        return _config_error("skilled_proposer_config_invalid_field", "skills")
+    skills: list[_NormalizedSkill] = []
+    for index, value in enumerate(raw["skills"]):
+        field_name = f"skills[{index}]"
+        skill_raw = _strict_object(
+            value, {"name", "description", "source", "content", "sha256", "byteLength"}, field_name,
+        )
+        text = _normalized_text({key: skill_raw[key] for key in (
+            "source", "content", "sha256", "byteLength",
+        )}, field_name)
+        name = _required_string(skill_raw["name"], f"{field_name}.name")
+        description = skill_raw["description"]
+        if description is not None:
+            description = _required_string(description, f"{field_name}.description", allow_empty=True)
+        skills.append(_NormalizedSkill(
+            source=text.source, content=text.content, sha256=text.sha256, byte_length=text.byte_length,
+            name=name, description=description,
+        ))
+    if len({skill.name for skill in skills}) != len(skills):
+        return _config_error("skilled_proposer_config_invalid_field", "skills duplicate names")
+    on_error = raw["onError"]
+    if on_error not in ("keep", "raise"):
+        return _config_error("skilled_proposer_config_invalid_field", "onError")
+    result = _SkilledProposerConfig(
+        config_file_sha256=config_file_sha256,
+        implementation=raw["implementation"], package_version=package_version,
+        config_sha256=config_sha256, skills=tuple(skills),
+        additional_instructions=_optional_normalized_text(raw["additionalInstructions"], "additionalInstructions"),
+        base_instructions=_optional_normalized_text(raw["baseInstructions"], "baseInstructions"),
+        max_words=_positive_int_or_none(raw["maxWords"], "maxWords"),
+        max_tokens=_positive_int_or_none(raw["maxTokens"], "maxTokens"),
+        max_examples=_positive_int_or_none(raw["maxExamples"], "maxExamples"),
+        on_error=on_error,
+    )
+    effective = {
+        "configFileSha256": result.config_file_sha256,
+        "schemaVersion": 1,
+        "implementation": result.implementation,
+        "packageVersion": result.package_version,
+        "skills": [{
+            "name": skill.name, "description": skill.description, "source": skill.source,
+            "content": skill.content, "sha256": skill.sha256, "byteLength": skill.byte_length,
+        } for skill in result.skills],
+        "additionalInstructions": _normalized_text_payload(result.additional_instructions),
+        "baseInstructions": _normalized_text_payload(result.base_instructions),
+        "maxWords": result.max_words, "maxTokens": result.max_tokens,
+        "maxExamples": result.max_examples, "onError": result.on_error,
+    }
+    canonical = json.dumps(effective, ensure_ascii=False, separators=(",", ":"))
+    if hashlib.sha256(canonical.encode("utf-8")).hexdigest() != result.config_sha256:
+        return _config_error("skilled_proposer_config_hash_mismatch", "configSha256")
+    return result
+
+
+class _CompressionFailureObserver:
+    def __init__(self) -> None:
+        self.failure_count = 0
+        self.last_error: Exception | None = None
+
+    def on_module_start(self, **_unused: Any) -> None:
+        return
+
+    def on_module_end(self, *, exception: Exception | None = None, **_unused: Any) -> None:
+        if exception is not None:
+            self.failure_count += 1
+            self.last_error = exception
+
+
+class _ObservableConfiguredProposer:
+    def __init__(self, proposer: Any, observability: ProposalObservability, on_error: str) -> None:
+        self.proposer = proposer
+        self.observability = observability
+        self.on_error = on_error
+        self.compression_observer = _CompressionFailureObserver()
+        proposer.module.compress.callbacks.append(self.compression_observer)
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self.proposer, name)
+
+    def __call__(self, candidate: dict[str, str], reflective_dataset: Any,
+                 components_to_update: list[str]) -> dict[str, str]:
+        results: dict[str, str] = {}
+        for component in components_to_update:
+            compression_failures_before = self.compression_observer.failure_count
+            event_failures_before = self.observability.event_failure_count
+            failures_before = self.observability.provider_failure_count
+            try:
+                proposed = self.proposer(candidate, reflective_dataset, [component])
+            except Exception as error:
+                if self.observability.event_failure_count > event_failures_before:
+                    raise RuntimeError("proposal_observability_event_failed") from error
+                if isinstance(error, _ProposalObservabilityEventFailure):
+                    raise RuntimeError("proposal_observability_event_failed") from error
+                if isinstance(error, _require_dspy().LMError) and self.on_error == "keep":
+                    raise
+                if self.on_error == "raise":
+                    code = getattr(error, "code", None)
+                    if code != "proposal_provider_transport_failed":
+                        code = "proposal_generation_failed"
+                        self.observability.record_proposer_failure(error)
+                    raise RuntimeError(code or "proposal_observability_event_failed") from error
+                self.observability.record_component_fallback(component=component, error=error)
+                results[component] = candidate[component]
+                continue
+            if self.observability.event_failure_count > event_failures_before:
+                raise RuntimeError("proposal_observability_event_failed")
+            provider_failed = self.observability.provider_failure_count > failures_before
+            compression_failed = self.compression_observer.failure_count > compression_failures_before
+            if compression_failed or provider_failed:
+                compression_error = self.compression_observer.last_error if compression_failed else None
+                if isinstance(compression_error, _ProposalObservabilityEventFailure):
+                    raise RuntimeError("proposal_observability_event_failed") from compression_error
+                error = (_require_dspy().LMError(
+                    "proposal_compression_provider_failed: compression provider failure was suppressed upstream",
+                    code="proposal_compression_provider_failed",
+                ) if provider_failed else RuntimeError("proposal_compression_failed"))
+                if self.on_error == "raise":
+                    raise error from compression_error
+                self.observability.record_component_fallback(component=component, error=error)
+                results[component] = candidate[component]
+            else:
+                results.update(proposed)
+        return results
+
+
 def make_skilled_proposer_from_environment(
     *,
     config: TextGepaConfig,
@@ -369,7 +686,7 @@ def make_skilled_proposer_from_environment(
     """Create the opt-in proposer only for the exact selected environment value."""
     if os.environ.get(_SELECTION_ENV) != _SELECTION_VALUE:
         return None
-    from skilled_proposer import SkilledProposer
+    from skilled_proposer import Skill, SkilledProposer
 
     bridge = make_skilled_proposer_bridge(
         config=config,
@@ -378,4 +695,36 @@ def make_skilled_proposer_from_environment(
         endpoint_override=endpoint_override,
         ssl_cert_file=os.environ.get("SSL_CERT_FILE"),
     )
-    return SkilledProposer(prompt_model=bridge)
+    raw_proposer_config = os.environ.get(_CONFIG_ENV)
+    if raw_proposer_config is None:
+        return SkilledProposer(prompt_model=bridge)
+    if raw_proposer_config.startswith("@"):
+        payload_path = Path(raw_proposer_config[1:])
+        try:
+            raw_proposer_config = payload_path.read_text(encoding="utf-8")
+        except (OSError, UnicodeError) as error:
+            raise RuntimeError("skilled_proposer_config_payload_unreadable") from error
+        try:
+            payload_path.unlink()
+            if payload_path.parent.name.startswith("orizu-skilled-proposer-payload-"):
+                payload_path.parent.rmdir()
+        except OSError:
+            pass
+    proposer_config = _parse_skilled_proposer_config(raw_proposer_config)
+    proposer = SkilledProposer(
+        skills=[Skill(
+            name=skill.name, description=skill.description, content=skill.content,
+        ) for skill in proposer_config.skills],
+        additional_instructions=(
+            proposer_config.additional_instructions.content
+            if proposer_config.additional_instructions else None
+        ),
+        base_instructions=(proposer_config.base_instructions.content if proposer_config.base_instructions else None),
+        max_words=proposer_config.max_words,
+        max_tokens=proposer_config.max_tokens,
+        max_examples=proposer_config.max_examples,
+        on_error="raise",
+        prompt_model=bridge,
+    )
+    observability.write_proposer_identity(proposer_config.identity())
+    return _ObservableConfiguredProposer(proposer, observability, proposer_config.on_error)

package/vendor/skills/orizu-cli/SKILL.md

@@ -1,20 +1,20 @@
 ---
 name: orizu-cli
-description: Use when the user wants to improve an LLM application's performance in a measurable way, or mentions Orizu by name. Triggers include improving model or agent performance, collecting human feedback on model outputs, converting feedback into evals, building or crafting evals, running prompt optimization (including one-off prompt tweaks the user is willing to validate with evals), working with Orizu datasets, tasks, apps, prompts, judges, scorers, runners, score runs, or optimization runs, hill-climbing on metrics, or building "continually learning" agents. Orizu is a platform for building evals first, then improving LLM applications by optimizing against them. The CLI handles human feedback collection, eval creation, versioned prompt and scorer artifacts, local runner execution, score submission, and prompt optimization. Do NOT use for prompt advice when the user has explicitly said they don't want to set up evals.
+description: Use when the user mentions Orizu or wants to measurably improve an LLM application, model, or agent; collect human feedback on model outputs; convert feedback into or build evals; optimize instructions (including one-off changes they will validate with evals); work with Orizu datasets, tasks, apps, instruction sets, model-config profiles, components, judges, scorers, runners, score runs, optimization runs, GEPA, or hill-climbing; migrate a prompts-era setup; or build continually learning agents. Start instruction artifacts at `orizu instructions`. Orizu builds evals first, then improves applications against them; its CLI covers feedback collection, eval creation, versioned instructions and evaluator artifacts, local runner execution, score submission, and instruction optimization. Do NOT use for instruction advice when the user explicitly does not want to set up evals.
 ---
 
 # Orizu
 
-Orizu improves LLM applications by building evals first, then optimizing against them. The end-to-end loop has four steps: **Upload → Annotate → Judge → Optimize**. The CLI covers human-eval collection plus the Phase 0 prompt control plane: prompts, judges, scorers, runners, local runner execution, score submission, packaged GEPA-style text optimization, and live optimization logging.
+Orizu improves LLM applications by building evals first, then optimizing against them. The end-to-end loop has four steps: **Upload → Annotate → Judge → Optimize**. Start instruction work with `orizu instructions`: it is the single surface for instruction sets, model-config profiles, their components, and the set-wide default. The CLI also covers human-eval collection, judges, scorers, runners, local runner execution, score submission, packaged GEPA-style text optimization, and live optimization logging. Do not create or mutate standalone prompts; use read-only prompt compatibility only to find the instruction set that owns legacy material.
 
 For end-to-end methodology and rationale, read `references/primer.md`. The reference docs below cover specific stages in depth.
 
 ## When to step aside
 
 Skip this skill when:
-- The user wants a one-off prompt tweak they explicitly won't validate with evals.
+- The user wants a one-off instruction edit they explicitly won't validate with evals.
 - The user has already said they don't want to set up evals.
-- The request is generic prompt-engineering theory unrelated to a specific app.
+- The request is generic instruction-writing theory unrelated to a specific app.
 
 # Orizu CLI basics
 
@@ -54,6 +54,24 @@ Projects live inside teams. Resolve or create both before any workflow command.
 
 For team membership, app, dataset, and task command surfaces, see `references/cli-reference.md`.
 
+## Instruction artifacts: start here
+
+An instruction set is one customer-facing identity for an agent's instructions. Its manifest carries a `name`, optional `description`, fixed, ordered `shape`, and component values. Each model config has its own profile, and each profile owns its own component values; components are never shared between profiles or sets. The set's stable slug is its CLI reference even if the display name changes. Its default serves model configs that do not have a production profile version.
+
+Use one command surface throughout:
+
+```bash
+orizu instructions list --project <team>/<project>
+orizu instructions show <slug-or-exact-name> --project <team>/<project>
+# Human/local CLI only: a human runs create/push; hosted agents may run profiles new and sync.
+orizu instructions create ./orizu.instruction-set.json --project <team>/<project> --model-config <identity>
+orizu instructions push ./orizu.instruction-set.json --project <team>/<project> --set <slug-or-exact-name>
+orizu instructions profiles new <slug-or-exact-name> --project <team>/<project> --model-config <identity>
+orizu instructions sync <slug-or-exact-name> --out ./instructions --project <team>/<project>
+```
+
+For a prompts-era setup, read `references/instructions-after-prompts.md` before making changes. Legacy prompt read commands identify the owning instruction set and component key; continue from that set through `orizu instructions`.
+
 # Evals best practices
 
 Walk through the four steps in order — each depends on the output of the previous. Methodology and rationale: `references/primer.md`.
@@ -106,11 +124,11 @@ Deeper: `references/primer.md` (Step 2); `references/cli-reference.md` (apps + t
 
 ## 3. Judge — turn labels into automated evaluators
 
-Use the Phase 0 prompt control plane when the user wants judges stored in Orizu as versioned prompt artifacts, runner zips, scorer definitions, and submitted score results. Read **`references/prompt-control-plane.md`** before using those commands. When the user mentions prompt feedback, prompt comments, or asks what to focus on for the next prompt version, inspect comment threads with `orizu comments list --prompt <prompt-id-or-name> --project <team>/<project>` before editing or pushing.
+Use `orizu instructions` for the application instruction material being evaluated. Judges remain versioned evaluator artifacts with runner zips, scorer definitions, and submitted score results; read **`references/prompt-control-plane.md`** before using those compatible evaluator commands. When the user mentions legacy prompt feedback or comments, inspect the read-only threads with `orizu comments list --prompt <prompt-id-or-name> --project <team>/<project>`, find the owning instruction set, and revise that set through its manifest.
 
 Judge construction details are still useful when authoring the evaluator itself.
 
-A judge is the evaluator prompt artifact. A scorer is the metric contract that names what score is being produced and how it should be displayed, compared, and used in optimizations. For UI-visible prompt performance, register a scorer and submit score runs; do not rely only on raw `runs submit`.
+A judge is the evaluator artifact. A scorer is the metric contract that names what score is being produced and how it should be displayed, compared, and used in optimizations. For UI-visible instruction performance, register a scorer and submit score runs; do not rely only on raw `runs submit`.
 
 Set scorers are aggregate metric contracts. Use `orizu scorers exec` for builtin set metrics such as Cohen's kappa, accuracy, precision, recall, and F1; it consumes subject results or dependency row-scorer results, computes one aggregate, stores row evidence, and submits a canonical score run by default. If you already computed an aggregate locally, use `orizu scores submit --aggregate` with explicit `scoreValue`, `diagnostics`, `feedbackSummary`, and optional `rowEvidence`. `runners exec --scorer-version` remains a low-level row-runner compatibility command. Use a row-mode scorer, not a set scorer, for GEPA reflection because reflection needs per-row feedback.
 
@@ -131,7 +149,7 @@ Detailed walkthrough — code assertion patterns, LLM-judge prompt scaffold, tra
 
 ## 4. Optimize — hill-climb against validated judges
 
-Use the Phase 0 prompt control plane when the user wants an optimizer run to appear in Orizu, stream live events, or promote accepted candidates into the prompt timeline. For text-candidate GEPA-style optimization, prefer the bundled `orizu optimizations run-gepa` flow. Read **`references/prompt-control-plane.md`** before wiring those endpoints.
+Use the instruction control plane when the user wants an optimizer run to appear in Orizu, stream live events, or promote accepted candidates into the target instruction-set profile. For text-candidate GEPA-style optimization, prefer the bundled `orizu optimizations run-gepa` flow. Read **`references/prompt-control-plane.md`** before wiring those endpoints.
 
 GEPA details below are still useful for local optimizer implementation. DSPy is only relevant when the customer already uses DSPy or asks for an external DSPy GEPA implementation.
 
@@ -153,9 +171,9 @@ Local execution workflow:
 Bundled `run-gepa` reflection behavior:
 - **Scorer contract:** `run-gepa` sends the scorer runner a GEPA-shaped row (`{source_row, candidate_id, candidate_output, …}`), NOT the flat-row shape used by `runners exec --scorer-version`. A judge runner built for flat-row score runs will silently score every candidate 0 unless you pass `--scorer-input-contract flat_row` (add `--scorer-candidate-field <row-field>`, e.g. `draft`, when the judge reads the candidate output from a named row field). Do not hand-write an adapter runner — the flag applies the official adapter without changing the registered runner bytes, so it composes with `--scorer-runner-dir` content-sha verification. `run-gepa` validates the contract on the seed at launch and refuses a uniformly-worst-scoring seed with a diagnosis; `--allow-degenerate-seed` overrides only when the seed genuinely deserves the worst score everywhere. Both contracts: `references/prompt-control-plane.md` ("Scorer-Runner Input Contracts").
 - Budget controls are mutually exclusive. Use at most one of `--budget auto|light|medium|heavy`, `--max-metric-calls <n>`, `--max-full-evals <n>`, `--max-iterations <n>`, or `--max-candidate-proposals <n>`. The proposal cap is official-engine-only. If none is provided, `run-gepa` uses `--budget auto`, the balanced medium preset.
-- **Skilled proposer:** opt in with `--candidate-proposer skilled-proposer` on the official engine; the selected run prepares or reuses its managed Python environment automatically. Bound proposal work with `--proposal-max-tokens` and `--proposal-max-calls`, independently of `--max-metric-calls`. Read `references/prompt-control-plane.md` ("Skilled proposer") for validation and recovery; readiness is proven when optimizer execution begins without an `ALI_1505_*` environment diagnostic.
-- The reflective LM's final text is used verbatim as the next candidate prompt. It should return only the complete updated prompt body, not analysis, labels, XML tags, or markdown fences.
-- Keep provider-native reasoning controls separate from the prompt text with `--reflection-provider-settings <json|@file>`.
+- **Skilled proposer:** opt in with `--candidate-proposer skilled-proposer` on the official engine; the selected run prepares or reuses its managed Python environment automatically. Optionally add `--candidate-proposer-config @path` for explicit skills and guidance. The selection is incompatible with `--reflection-prompt-template`, even without a config. Bound aggregate proposal work with `--proposal-max-tokens` and `--proposal-max-calls`, independently of `--max-metric-calls`. Do not confuse that aggregate token budget with per-response `--reflection-max-tokens` or the config's proposed-instruction `maxTokens`. Read `references/prompt-control-plane.md` ("Skilled proposer") for the schema, validation, and managed-environment recovery; readiness is proven when optimizer execution begins without an `ALI_1505_*` environment diagnostic.
+- The reflective LM's final text becomes the selected component's next value. It should return only the complete updated component value, not analysis, labels, XML tags, or markdown fences.
+- Keep provider-native reasoning controls separate from the component text with `--reflection-provider-settings <json|@file>`.
 - Reflection max-token limits are explicit config, not implicit defaults. Use `--reflection-max-tokens <n>` when a provider requires an output cap; this maps to Anthropic `max_tokens` and OpenAI `max_output_tokens`.
 - Anthropic native Messages reflection requires `max_tokens`, so `run-gepa` with an `anthropic/...` reflection model must receive `--reflection-max-tokens <n>` and should fail fast if it is omitted. OpenAI reflection may omit `max_output_tokens` unless the user explicitly wants a cap.
 - Reflection HTTP calls retry transient provider failures by default (`--reflection-retry-attempts 3`, `--reflection-http-timeout-seconds 180`). Retryable failures include timeouts, connection reset/abort, HTTP 429, and common server/capacity statuses. If reflection still fails after retries, `run-gepa` logs `reflection_failed`, counts the proposal against budget, and continues to the next iteration.
@@ -172,7 +190,8 @@ Detailed walkthrough — GEPA mechanics, Orizu-tracked optimization, optional DS
 
 - `references/primer.md` — methodology end-to-end (read first when in doubt about *why*).
 - `references/cli-reference.md` — full CLI command surface.
-- `references/prompt-control-plane.md` — Phase 0 prompts, judges, scorers, runners, score submission, optimizer event logging, bundled GEPA, and promotion endpoints.
+- `references/prompt-control-plane.md` — instruction sets, judges, scorers, runners, score submission, optimizer event logging, bundled GEPA, and promotion endpoints.
+- `references/instructions-after-prompts.md` — finding the instruction set that owns prompts-era material and consolidating legacy one-component sets.
 - `references/building-apps.md` — labeler app contract, design principles, common patterns, offline smoke test.
 - `references/building-judges.md` — judge/scorer authoring + TPR/TNR validation.
 - `references/optimization-with-gepa.md` — GEPA optimization loop, with DSPy only as external context.
@@ -193,7 +212,7 @@ Detailed walkthrough — GEPA mechanics, Orizu-tracked optimization, optional DS
 - `--output-schema` JSON Schema validation surface is restricted to `type`, `required`, `properties`, `items`, `enum`.
 - Export defaults: `--format jsonl`, output `<taskId>.<format>`.
 - Task reports can be uploaded only after the task is `paused` or `completed`; rerun `tasks report set` or `tasks report upload` to replace the markdown report.
-- Prompt control-plane commands should use ids for dataset versions, split sets, prompt versions, scorer versions, runner versions, optimizer versions, and optimization runs.
+- Instruction and evaluation control-plane commands should use ids for dataset versions, split sets, component versions where compatibility commands require them, scorer versions, runner versions, optimizer versions, and optimization runs.
 - Optimization exports default to `<run-id>.optimization.json`; prefer the existing `logs/<run-id>` directory from `run-gepa` when it is available because it contains the full local trace without needing server rehydration.
 - When ending an optimization run, attach a markdown report with `--report-file` when possible. Use `references/optimization-reports.md` for what to include.
 - Use `orizu comments list --prompt <prompt-id-or-name> --project <team>/<project> [--label <label> | --version <id>] [--json]` to list prompt-level discussion threads with open/resolved status, selected text/line context, and replies.
@@ -201,8 +220,23 @@ Detailed walkthrough — GEPA mechanics, Orizu-tracked optimization, optional DS
 
 # Where things live
 
-- **Orizu is the source of truth for run metadata and pointers**: production/default prompt/scorer/runner pointers, version lineage, score runs, optimization runs, and report history all live in Orizu — query them with `orizu ...` commands (e.g. `orizu prompts list`, `orizu prompts versions`, `orizu runs ...`), never by reading repo files.
+- **Orizu is the source of truth for run metadata and pointers**: instruction-set defaults, profile production pointers, scorer/runner pointers, version lineage, score runs, optimization runs, and report history all live in Orizu — query them with `orizu ...` commands (for example, `orizu instructions list`, `orizu instructions show`, and `orizu runs ...`), never by reading repo files.
 - **The repo is the working set (files only)**: source, configs, `App.tsx`, schemas, and local traces under `logs/<run-id>`. It is the scratch space you edit and commit; it does NOT hold the canonical pointer/run state — that is in Orizu.
-- **Hosted sessions ship a pre-authenticated CLI**: inside a hosted agent sandbox the `orizu` CLI is already authenticated via `ORIZU_TOKEN_FILE` (with `ORIZU_BASE_URL` pointing at the API). You do NOT run `orizu login` — just call `orizu ...`. The bearer carries **full team-curator authority while your session is active** (ADR-008): anything a curator-role human can read or write in your team — create/read/update projects, datasets (+ rows/artifact bytes), prompts, scorers, runners/optimizers, apps, tasks, runs, score runs, reports, and promotion-manifest proposals — you can too, across the whole team. (The full per-role access matrix lives in the Orizu repo at `docs/authorization-matrix.md`.)
-- **Git-canonical artifacts: always write with `--session`, and "committed" means bytes-in-branch**: prompts/judges, scorers, runners, and optimizers are git-canonical (ADR-007) — their bytes must live as commits on your session branch, not only as Orizu version rows. From a hosted session, always pass `--session <session-id>` on artifact writes (`prompts push`, `judges push`, `scorers register`, `runners push`, `optimizers push`); the legacy sessionless create routes reject agent bearers with a 403 pointing you back at `--session`. The definition of done for any artifact you claim to have committed is that the bytes are actually in the branch: run `git ls-files <path>` (and check the commit) before reporting an artifact as committed — an Orizu version id alone is not proof the bytes exist anywhere.
-- **Human-only actions will 403**: promotion-manifest **approval / apply / reject**, production and default pointer flips, session/credential lifecycle (starting or ending hosted sessions, minting tokens, deploy keys), integration/connector credential reads (secrets reach your sandbox only as environment variables resolved at boot — never via API/DB), and team/member/installation/billing management. Do not attempt them from a hosted session — they return 403 by design. Prepare a promotion manifest and hand it to a human curator to approve and apply.
+- **Hosted sessions ship a pre-authenticated CLI**: inside a hosted agent sandbox the `orizu` CLI is already authenticated via `ORIZU_TOKEN_FILE` (with `ORIZU_BASE_URL` pointing at the API). You do NOT run `orizu login` — just call `orizu ...`. The bearer carries **team-curator authority while your session is active**, except for the explicit human-only carveouts below (ADR-008 and the ALI-1558 instruction-write restriction). It can read and sync instruction sets and can perform ordinary curator work across the team. (The full per-role access matrix lives in the Orizu repo at `docs/authorization-matrix.md`.)
+- **Hosted `--session` applies to git-canonical artifacts, not instruction sets**: instruction sets and profiles are control-plane canonical (ADR-023), and their CLI mutations do not accept `--session`. A hosted agent may inspect them, sync them, and prepare a complete manifest, but a human curator using the local CLI with a user token must run the six catalog mutations listed below and all pointer moves. Judges, scorers, runners, and optimizers remain git-canonical for teams with a workbench repository; their hosted writes must carry `--session <session-id>`, and their bytes must live as commits on the session branch. The remaining session-scoped legacy prompt compatibility is not a standalone instruction workflow. Before calling any artifact committed, run `git ls-files <path>` and inspect the commit; an Orizu version id alone does not prove the bytes exist in a branch.
+- **Human-only actions will 403**: a hosted agent must hand the following commands
+  to a human curator using the local CLI with a user token. Human/local CLI only:
+  `orizu instructions create`, `orizu instructions push`,
+  `orizu instructions shape add`, `orizu instructions shape remove`,
+  `orizu instructions archive`, and `orizu instructions restore`.
+  Human/local CLI only: The measured response for `orizu instructions create` and `orizu instructions push` is `Sessionless registration is not available to the hosted agent: this artifact class is git-canonical (ADR-007) and its bytes must be committed to the session branch. Re-run with --session <session-id>.` Those commands have no `--session` flag, so do not retry them.
+  Human/local CLI only: The measured response for `orizu instructions shape add` and `orizu instructions shape remove` is `Agents cannot change instruction-set shapes`.
+  Human/local CLI only: The measured response for `orizu instructions archive` and `orizu instructions restore` is `Agent sessions cannot perform this action`.
+  Promotion-manifest **approval / apply / reject**, production and default
+  pointer flips, session/credential lifecycle (starting or ending hosted
+  sessions, minting tokens, deploy keys), integration/connector credential
+  reads (secrets reach your sandbox only as environment variables resolved at
+  boot — never via API/DB), and team/member/installation/billing management
+  are also human-only. Do not attempt them from a hosted session. Prepare the
+  manifest or proposal and hand it to a human curator to run, approve, or
+  apply.

package/vendor/skills/orizu-cli/references/building-judges.md

@@ -1,6 +1,6 @@
 # Building Judges And Scorers
 
-How to turn human-labeled annotations into automated evaluators. Use this reference for authoring and validating the evaluator logic; use `prompt-control-plane.md` to push versioned prompt artifacts, runner artifacts, scorer definitions, and score runs into Orizu.
+How to turn human-labeled annotations into automated evaluators. Use this reference for authoring and validating the evaluator logic; use `prompt-control-plane.md` to push versioned judge artifacts, runner artifacts, scorer definitions, and score runs into Orizu.
 
 ## Inputs
 
@@ -138,7 +138,7 @@ Run the validated judge over future outputs to score them. Wire it into:
 - CI/CD on a curated test set (block merges that drop pass rate)
 - Online monitoring on sampled traffic (alert on regressions)
 - Optimization loop as a metric (`optimization-with-gepa.md`)
-- Orizu scorers and score runs (`prompt-control-plane.md`) so prompt detail, changelog, scorer detail, and optimization surfaces show comparable performance.
+- Orizu scorers and score runs (`prompt-control-plane.md`) so instruction-set detail, changelog, scorer detail, and optimization surfaces show comparable performance.
 
 ## Set scorers for judge-vs-gold agreement