orizu 0.5.20 → 0.5.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/archive-artifacts-cli.d.ts +50 -0
- package/dist/archive-artifacts-cli.d.ts.map +1 -0
- package/dist/artifact-archive.d.ts +48 -0
- package/dist/artifact-archive.d.ts.map +1 -0
- package/dist/artifact-pull.d.ts +22 -0
- package/dist/artifact-pull.d.ts.map +1 -0
- package/dist/banner.d.ts +3 -0
- package/dist/banner.d.ts.map +1 -0
- package/dist/connectors-cli.d.ts +31 -0
- package/dist/connectors-cli.d.ts.map +1 -0
- package/dist/credentials.d.ts +46 -0
- package/dist/credentials.d.ts.map +1 -0
- package/dist/csv.d.ts +49 -0
- package/dist/csv.d.ts.map +1 -0
- package/dist/dataset-download.d.ts +6 -0
- package/dist/dataset-download.d.ts.map +1 -0
- package/dist/dataset-edit-rows.d.ts +10 -0
- package/dist/dataset-edit-rows.d.ts.map +1 -0
- package/dist/dataset-upload-snapshot.d.ts +13 -0
- package/dist/dataset-upload-snapshot.d.ts.map +1 -0
- package/dist/daytona-slice-rehearsal.d.ts +89 -0
- package/dist/daytona-slice-rehearsal.d.ts.map +1 -0
- package/dist/daytona-slice.d.ts +106 -0
- package/dist/daytona-slice.d.ts.map +1 -0
- package/dist/deploy-key-merge-core.d.ts +121 -0
- package/dist/deploy-key-merge-core.d.ts.map +1 -0
- package/dist/diff-comments-cli.d.ts +12 -0
- package/dist/diff-comments-cli.d.ts.map +1 -0
- package/dist/diff-comments-cli.js +3 -0
- package/dist/egress-allowlist-cli.d.ts +23 -0
- package/dist/egress-allowlist-cli.d.ts.map +1 -0
- package/dist/egress-policy.d.ts +108 -0
- package/dist/egress-policy.d.ts.map +1 -0
- package/dist/error-response.d.ts +9 -0
- package/dist/error-response.d.ts.map +1 -0
- package/dist/file-parser.d.ts +5 -0
- package/dist/file-parser.d.ts.map +1 -0
- package/dist/gepa-engine-dispatch.d.ts +9 -0
- package/dist/gepa-engine-dispatch.d.ts.map +1 -0
- package/dist/gepa-engine-dispatch.js +48 -2
- package/dist/gepa-python-command.d.ts +6 -0
- package/dist/gepa-python-command.d.ts.map +1 -0
- package/dist/gepa-python-paths.d.ts +6 -0
- package/dist/gepa-python-paths.d.ts.map +1 -0
- package/dist/gepa-python-paths.js +6 -3
- package/dist/git-credential.d.ts +69 -0
- package/dist/git-credential.d.ts.map +1 -0
- package/dist/git-recovery-environment.d.ts +31 -0
- package/dist/git-recovery-environment.d.ts.map +1 -0
- package/dist/github-setup.d.ts +122 -0
- package/dist/github-setup.d.ts.map +1 -0
- package/dist/global-flags.d.ts +12 -0
- package/dist/global-flags.d.ts.map +1 -0
- package/dist/help.d.ts +46 -0
- package/dist/help.d.ts.map +1 -0
- package/dist/help.js +91 -23
- package/dist/hosted-boot.d.ts +273 -0
- package/dist/hosted-boot.d.ts.map +1 -0
- package/dist/hosted-bootstrap-rehearsal.d.ts +86 -0
- package/dist/hosted-bootstrap-rehearsal.d.ts.map +1 -0
- package/dist/hosted-bootstrap.d.ts +188 -0
- package/dist/hosted-bootstrap.d.ts.map +1 -0
- package/dist/hosted-broker-server.d.ts +50 -0
- package/dist/hosted-broker-server.d.ts.map +1 -0
- package/dist/hosted-git-http-server.d.ts +36 -0
- package/dist/hosted-git-http-server.d.ts.map +1 -0
- package/dist/hosted-harness-claude.d.ts +195 -0
- package/dist/hosted-harness-claude.d.ts.map +1 -0
- package/dist/hosted-harness-opencode.d.ts +227 -0
- package/dist/hosted-harness-opencode.d.ts.map +1 -0
- package/dist/hosted-harness.d.ts +118 -0
- package/dist/hosted-harness.d.ts.map +1 -0
- package/dist/hosted-harvest.d.ts +68 -0
- package/dist/hosted-harvest.d.ts.map +1 -0
- package/dist/hosted-headless.d.ts +72 -0
- package/dist/hosted-headless.d.ts.map +1 -0
- package/dist/hosted-loop-lifecycle.d.ts +206 -0
- package/dist/hosted-loop-lifecycle.d.ts.map +1 -0
- package/dist/hosted-loop.d.ts +170 -0
- package/dist/hosted-loop.d.ts.map +1 -0
- package/dist/hosted-run-event-sink.d.ts +222 -0
- package/dist/hosted-run-event-sink.d.ts.map +1 -0
- package/dist/hosted-runtime-assets.d.ts +186 -0
- package/dist/hosted-runtime-assets.d.ts.map +1 -0
- package/dist/hosted-session-cli.d.ts +258 -0
- package/dist/hosted-session-cli.d.ts.map +1 -0
- package/dist/hosted-skill-staging.d.ts +73 -0
- package/dist/hosted-skill-staging.d.ts.map +1 -0
- package/dist/http.d.ts +8 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +88 -92
- package/dist/instruction-set-gepa-launch.d.ts +3 -0
- package/dist/instruction-set-gepa-launch.d.ts.map +1 -0
- package/dist/instruction-set-gepa-launch.js +15 -0
- package/dist/instruction-set-loader.d.ts +6 -0
- package/dist/instruction-set-loader.d.ts.map +1 -0
- package/dist/instruction-set-loader.js +59 -0
- package/dist/instruction-set-manifest.d.ts +14 -0
- package/dist/instruction-set-manifest.d.ts.map +1 -0
- package/dist/instruction-set-manifest.js +162 -0
- package/dist/instruction-sets-cli.d.ts +40 -0
- package/dist/instruction-sets-cli.d.ts.map +1 -0
- package/dist/instruction-sets-cli.js +361 -0
- package/dist/json-response.d.ts +4 -0
- package/dist/json-response.d.ts.map +1 -0
- package/dist/jsonl-stream.d.ts +8 -0
- package/dist/jsonl-stream.d.ts.map +1 -0
- package/dist/kill-switch-cli.d.ts +30 -0
- package/dist/kill-switch-cli.d.ts.map +1 -0
- package/dist/loopback-serve.d.ts +25 -0
- package/dist/loopback-serve.d.ts.map +1 -0
- package/dist/manifests-cli.d.ts +22 -0
- package/dist/manifests-cli.d.ts.map +1 -0
- package/dist/markdown-report-input.d.ts +7 -0
- package/dist/markdown-report-input.d.ts.map +1 -0
- package/dist/merge-job.d.ts +155 -0
- package/dist/merge-job.d.ts.map +1 -0
- package/dist/model-configs-cli.d.ts +19 -0
- package/dist/model-configs-cli.d.ts.map +1 -0
- package/dist/model-configs-cli.js +104 -0
- package/dist/optimization-export-cli.d.ts +19 -0
- package/dist/optimization-export-cli.d.ts.map +1 -0
- package/dist/optimizations-list-cli.d.ts +20 -0
- package/dist/optimizations-list-cli.d.ts.map +1 -0
- package/dist/optimizations-promote-cli.d.ts +8 -0
- package/dist/optimizations-promote-cli.d.ts.map +1 -0
- package/dist/optimizations-promote-cli.js +34 -0
- package/dist/option-validation.d.ts +15 -0
- package/dist/option-validation.d.ts.map +1 -0
- package/dist/preview-runtime.d.ts +63 -0
- package/dist/preview-runtime.d.ts.map +1 -0
- package/dist/prompt-draft-push.d.ts +60 -0
- package/dist/prompt-draft-push.d.ts.map +1 -0
- package/dist/prompt-label-cli.d.ts +11 -0
- package/dist/prompt-label-cli.d.ts.map +1 -0
- package/dist/prompt-label-cli.js +29 -0
- package/dist/prompt-length-wire.d.ts +28 -0
- package/dist/prompt-length-wire.d.ts.map +1 -0
- package/dist/prompt-report-cli.d.ts +10 -0
- package/dist/prompt-report-cli.d.ts.map +1 -0
- package/dist/prompt-summary-table.d.ts +15 -0
- package/dist/prompt-summary-table.d.ts.map +1 -0
- package/dist/prompt-summary-table.js +6 -0
- package/dist/registration-tag-warning.d.ts +32 -0
- package/dist/registration-tag-warning.d.ts.map +1 -0
- package/dist/report-comments-cli.d.ts +16 -0
- package/dist/report-comments-cli.d.ts.map +1 -0
- package/dist/report-markdown-blankness.d.ts +2 -0
- package/dist/report-markdown-blankness.d.ts.map +1 -0
- package/dist/report-replacement-warning.d.ts +17 -0
- package/dist/report-replacement-warning.d.ts.map +1 -0
- package/dist/repository-backup-core.d.ts +225 -0
- package/dist/repository-backup-core.d.ts.map +1 -0
- package/dist/repository-recovery-manifest.d.ts +83 -0
- package/dist/repository-recovery-manifest.d.ts.map +1 -0
- package/dist/run-event-digest.d.ts +74 -0
- package/dist/run-event-digest.d.ts.map +1 -0
- package/dist/runner-dir-verify.d.ts +66 -0
- package/dist/runner-dir-verify.d.ts.map +1 -0
- package/dist/runner-instruction-set-contract.d.ts +24 -0
- package/dist/runner-instruction-set-contract.d.ts.map +1 -0
- package/dist/runner-instruction-set-contract.js +50 -0
- package/dist/sandbox-hygiene.d.ts +45 -0
- package/dist/sandbox-hygiene.d.ts.map +1 -0
- package/dist/sandbox-provider.d.ts +193 -0
- package/dist/sandbox-provider.d.ts.map +1 -0
- package/dist/scorer-draft-push.d.ts +50 -0
- package/dist/scorer-draft-push.d.ts.map +1 -0
- package/dist/scorers-cli.d.ts +20 -0
- package/dist/scorers-cli.d.ts.map +1 -0
- package/dist/scorers-cli.js +22 -0
- package/dist/scorers-verify-parity-bridge.d.ts +16 -0
- package/dist/scorers-verify-parity-bridge.d.ts.map +1 -0
- package/dist/scorers-verify-parity-bridge.js +90 -0
- package/dist/scorers-verify-parity.d.ts +81 -0
- package/dist/scorers-verify-parity.d.ts.map +1 -0
- package/dist/scorers-verify-parity.js +502 -0
- package/dist/scores-accept-cli.d.ts +21 -0
- package/dist/scores-accept-cli.d.ts.map +1 -0
- package/dist/secret-redaction.d.ts +47 -0
- package/dist/secret-redaction.d.ts.map +1 -0
- package/dist/session-timeline.d.ts +54 -0
- package/dist/session-timeline.d.ts.map +1 -0
- package/dist/setup-prompt.d.ts +5 -0
- package/dist/setup-prompt.d.ts.map +1 -0
- package/dist/skill-installer.d.ts +68 -0
- package/dist/skill-installer.d.ts.map +1 -0
- package/dist/skilled-proposer-launch.d.ts +7 -0
- package/dist/skilled-proposer-launch.d.ts.map +1 -0
- package/dist/skilled-proposer-launch.js +60 -0
- package/dist/smoke.d.ts +102 -0
- package/dist/smoke.d.ts.map +1 -0
- package/dist/start-intent.d.ts +12 -0
- package/dist/start-intent.d.ts.map +1 -0
- package/dist/task-assignment-manifest.d.ts +15 -0
- package/dist/task-assignment-manifest.d.ts.map +1 -0
- package/dist/task-create-error.d.ts +18 -0
- package/dist/task-create-error.d.ts.map +1 -0
- package/dist/team-connectors-cli.d.ts +29 -0
- package/dist/team-connectors-cli.d.ts.map +1 -0
- package/dist/types.d.ts +39 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/vercel-sandbox-provider.d.ts +156 -0
- package/dist/vercel-sandbox-provider.d.ts.map +1 -0
- package/dist/workbench-cli.d.ts +233 -0
- package/dist/workbench-cli.d.ts.map +1 -0
- package/dist/workspace-sync.d.ts +207 -0
- package/dist/workspace-sync.d.ts.map +1 -0
- package/dist/workspace.d.ts +92 -0
- package/dist/workspace.d.ts.map +1 -0
- package/dist/zip-draft-push.d.ts +55 -0
- package/dist/zip-draft-push.d.ts.map +1 -0
- package/package.json +14 -1
- package/requirements/README.md +138 -0
- package/requirements/skilled-proposer.in +66 -0
- package/requirements/skilled-proposer.lock +265 -0
- package/scripts/ensure-skilled-proposer-venv.mjs +5 -0
- package/scripts/refresh-skilled-proposer-lock.mjs +256 -0
- package/src/diff-comments-cli.ts +4 -0
- package/src/gepa-engine-dispatch.ts +42 -2
- package/src/gepa-python-paths.ts +6 -3
- package/src/help.ts +93 -23
- package/src/index.ts +99 -126
- package/src/instruction-set-gepa-launch.ts +28 -0
- package/src/instruction-set-loader.ts +37 -0
- package/src/instruction-set-manifest.ts +120 -0
- package/src/instruction-sets-cli.ts +306 -0
- package/src/model-configs-cli.ts +119 -0
- package/src/optimizations-promote-cli.ts +41 -0
- package/src/prompt-label-cli.ts +55 -0
- package/src/prompt-summary-table.ts +12 -0
- package/src/runner-instruction-set-contract.ts +64 -0
- package/src/scorers-cli.ts +40 -0
- package/src/scorers-verify-parity-bridge.ts +90 -0
- package/src/scorers-verify-parity.ts +564 -0
- package/src/skilled-proposer-launch.ts +89 -0
- package/src/skilled-proposer-venv-manager.mjs +757 -0
- package/tsconfig.json +2 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/__init__.py +2 -1
- package/vendor/orizu-gepa/src/orizu_gepa_connector/adapter.py +8 -2
- package/vendor/orizu-gepa/src/orizu_gepa_connector/callbacks.py +17 -3
- package/vendor/orizu-gepa/src/orizu_gepa_connector/engine.py +100 -8
- package/vendor/orizu-gepa/src/orizu_gepa_connector/instruction_set_loader.py +52 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/reflection.py +92 -12
- package/vendor/orizu-gepa/src/orizu_gepa_connector/runtime.py +100 -22
- package/vendor/orizu-gepa/src/orizu_gepa_connector/skilled_proposer_bridge.py +381 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/stop_conditions.py +17 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/translator.py +2 -1
- package/vendor/orizu-gepa-python/src/orizu_gepa/client.py +52 -7
- package/vendor/orizu-gepa-python/src/orizu_gepa/local_log.py +4 -1
- package/vendor/orizu-gepa-python/src/orizu_gepa/optimizer.py +10 -5
- package/vendor/orizu-gepa-python/src/orizu_gepa/reflection.py +323 -68
- package/vendor/orizu-gepa-python/src/orizu_gepa/runner.py +207 -13
- package/vendor/skills/orizu-cli/SKILL.md +2 -0
- package/vendor/skills/orizu-cli/references/cli-reference.md +85 -1
- package/vendor/skills/orizu-cli/references/migrate-existing-gepa-setup.md +303 -0
- package/vendor/skills/orizu-cli/references/optimization-with-gepa.md +25 -0
- package/vendor/skills/orizu-cli/references/prompt-control-plane.md +108 -8
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import os
|
|
7
|
+
from pathlib import Path
|
|
7
8
|
from typing import Any
|
|
8
9
|
from uuid import uuid4
|
|
9
10
|
|
|
@@ -102,11 +103,13 @@ class MandatoryEventSink:
|
|
|
102
103
|
|
|
103
104
|
|
|
104
105
|
def _required_environment() -> dict[str, str]:
|
|
105
|
-
names =
|
|
106
|
-
"ORIZU_PROJECT", "ORIZU_OPTIMIZER_VERSION_ID",
|
|
106
|
+
names = [
|
|
107
|
+
"ORIZU_PROJECT", "ORIZU_OPTIMIZER_VERSION_ID",
|
|
107
108
|
"ORIZU_DATASET_VERSION_ID", "ORIZU_SPLIT_SET_ID", "ORIZU_SCORER_VERSION_ID",
|
|
108
109
|
"ORIZU_RUNNER_VERSION_ID", "ORIZU_CANDIDATE_RUNNER_DIR", "ORIZU_SCORER_RUNNER_DIR",
|
|
109
|
-
|
|
110
|
+
]
|
|
111
|
+
if not os.environ.get("ORIZU_INSTRUCTION_SET_PROFILE_VERSION_ID"):
|
|
112
|
+
names.append("ORIZU_PROMPT_VERSION_ID")
|
|
110
113
|
missing = [name for name in names if not os.environ.get(name)]
|
|
111
114
|
if missing:
|
|
112
115
|
raise RuntimeError("missing connector environment: " + ", ".join(missing))
|
|
@@ -219,7 +222,7 @@ def build_config_from_environment() -> TextGepaConfig:
|
|
|
219
222
|
)
|
|
220
223
|
|
|
221
224
|
|
|
222
|
-
def resolved_budget(config: TextGepaConfig, *, trainset_size: int, valset_size: int):
|
|
225
|
+
def resolved_budget(config: TextGepaConfig, *, trainset_size: int, valset_size: int, num_components: int = 1):
|
|
223
226
|
"""Resolve the frozen legacy budget once for display *and* enforcement."""
|
|
224
227
|
from orizu_gepa.optimizer import Budget
|
|
225
228
|
|
|
@@ -227,7 +230,7 @@ def resolved_budget(config: TextGepaConfig, *, trainset_size: int, valset_size:
|
|
|
227
230
|
config,
|
|
228
231
|
trainset_size=trainset_size,
|
|
229
232
|
valset_size=valset_size,
|
|
230
|
-
num_components=
|
|
233
|
+
num_components=num_components,
|
|
231
234
|
)
|
|
232
235
|
|
|
233
236
|
|
|
@@ -242,12 +245,23 @@ def validate_launch_contract(
|
|
|
242
245
|
*,
|
|
243
246
|
candidate_runner_dir: str | None = None,
|
|
244
247
|
scorer_runner_dir: str | None = None,
|
|
248
|
+
pinned_components: dict[str, Any] | None = None,
|
|
249
|
+
instruction_set_shape: list[str] | None = None,
|
|
250
|
+
auto_promote: bool = False,
|
|
245
251
|
) -> None:
|
|
246
252
|
"""Refuse M1-unsupported topology before creating a run row."""
|
|
247
253
|
if _enabled("ORIZU_USE_MERGE") or os.environ.get("ORIZU_MAX_MERGE_INVOCATIONS"):
|
|
248
254
|
raise RuntimeError("merge is unsupported by the official GEPA connector (ALI-1506)")
|
|
249
|
-
|
|
250
|
-
|
|
255
|
+
pinned = pinned_components or {}
|
|
256
|
+
if pinned:
|
|
257
|
+
raise RuntimeError("instruction_set_pinned_components_unsupported: " + ", ".join(sorted(pinned)))
|
|
258
|
+
if instruction_set_shape is not None:
|
|
259
|
+
missing = [key for key in instruction_set_shape if key not in seed_candidate]
|
|
260
|
+
extra = [key for key in seed_candidate if key not in instruction_set_shape]
|
|
261
|
+
if missing or extra:
|
|
262
|
+
raise RuntimeError("instruction_set_tuple_shape_mismatch: " + ", ".join(missing or extra))
|
|
263
|
+
if auto_promote and len(seed_candidate) != 1:
|
|
264
|
+
raise RuntimeError("instruction_set_candidate_promotion_unsupported")
|
|
251
265
|
if os.environ.get("ORIZU_SAMPLING_STRATEGY") or os.environ.get("ORIZU_SELECTION_STRATEGY"):
|
|
252
266
|
raise RuntimeError("P×N sampling/selection is unsupported by the official GEPA connector (ALI-1507)")
|
|
253
267
|
if candidate_runner_dir is None or scorer_runner_dir is None:
|
|
@@ -399,13 +413,16 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
399
413
|
validation_split = os.environ.get("ORIZU_VALIDATION_SPLIT", "validation")
|
|
400
414
|
config = build_config_from_environment()
|
|
401
415
|
client = OrizuClient.from_env()
|
|
416
|
+
profile_version_id = os.environ.get("ORIZU_INSTRUCTION_SET_PROFILE_VERSION_ID")
|
|
402
417
|
prompt_context, trainset = client.fetch_exec_context(
|
|
403
|
-
prompt_version_id=
|
|
418
|
+
prompt_version_id=os.environ.get("ORIZU_PROMPT_VERSION_ID"), runner_version_id=env["ORIZU_RUNNER_VERSION_ID"],
|
|
404
419
|
dataset_version_id=env["ORIZU_DATASET_VERSION_ID"], split_set_id=env["ORIZU_SPLIT_SET_ID"], split=train_split,
|
|
420
|
+
instruction_set_profile_version_id=profile_version_id,
|
|
405
421
|
)
|
|
406
422
|
_, valset = client.fetch_exec_context(
|
|
407
|
-
prompt_version_id=
|
|
423
|
+
prompt_version_id=os.environ.get("ORIZU_PROMPT_VERSION_ID"), runner_version_id=env["ORIZU_RUNNER_VERSION_ID"],
|
|
408
424
|
dataset_version_id=env["ORIZU_DATASET_VERSION_ID"], split_set_id=env["ORIZU_SPLIT_SET_ID"], split=validation_split,
|
|
425
|
+
instruction_set_profile_version_id=profile_version_id,
|
|
409
426
|
)
|
|
410
427
|
scorer_context, _ = client.fetch_scorer_exec_context(
|
|
411
428
|
scorer_version_id=env["ORIZU_SCORER_VERSION_ID"], runner_version_id=os.environ.get("ORIZU_SCORER_RUNNER_VERSION_ID"),
|
|
@@ -423,11 +440,20 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
423
440
|
f"scorer-contract failure before launch for {env['ORIZU_SCORER_VERSION_ID']}: {error}"
|
|
424
441
|
) from error
|
|
425
442
|
hooks = LifecycleHooks()
|
|
426
|
-
|
|
443
|
+
instruction_set = prompt_context.instruction_set or {}
|
|
444
|
+
shape = instruction_set.get("shape") if isinstance(instruction_set.get("shape"), list) else None
|
|
445
|
+
components = instruction_set.get("components") if isinstance(instruction_set.get("components"), dict) else None
|
|
446
|
+
if profile_version_id and (not shape or not components):
|
|
447
|
+
raise RuntimeError("instruction_set_tuple_incomplete")
|
|
448
|
+
seed_candidate = ({key: components[key] for key in shape if isinstance(components.get(key), str)}
|
|
449
|
+
if profile_version_id and shape and components else {"prompt": prompt_context.body or ""})
|
|
427
450
|
validate_launch_contract(
|
|
428
451
|
seed_candidate,
|
|
429
452
|
candidate_runner_dir=env["ORIZU_CANDIDATE_RUNNER_DIR"],
|
|
430
453
|
scorer_runner_dir=env["ORIZU_SCORER_RUNNER_DIR"],
|
|
454
|
+
pinned_components=instruction_set.get("pinned_components") if profile_version_id and isinstance(instruction_set, dict) else None,
|
|
455
|
+
instruction_set_shape=shape if profile_version_id else None,
|
|
456
|
+
auto_promote=config.auto_promote,
|
|
431
457
|
)
|
|
432
458
|
preflight_adapter = RunnerEvaluationAdapter(
|
|
433
459
|
candidate_runner_dir=env["ORIZU_CANDIDATE_RUNNER_DIR"], scorer_runner_dir=env["ORIZU_SCORER_RUNNER_DIR"],
|
|
@@ -449,7 +475,7 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
449
475
|
except ScorerContractError as error:
|
|
450
476
|
raise RuntimeError(f"scorer-contract failure before launch for {env['ORIZU_SCORER_VERSION_ID']}: {error}") from error
|
|
451
477
|
preflight_metric_calls = int(preflight_verdict.get("preflight_metric_calls", 0))
|
|
452
|
-
budget = resolved_budget(config, trainset_size=len(trainset), valset_size=len(valset))
|
|
478
|
+
budget = resolved_budget(config, trainset_size=len(trainset), valset_size=len(valset), num_components=len(seed_candidate))
|
|
453
479
|
budget_kind, budget_limit = budget.budget_kind, budget.limit
|
|
454
480
|
metadata = {
|
|
455
481
|
**_metadata_from_environment(),
|
|
@@ -463,11 +489,23 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
463
489
|
"dataset_size": len(trainset) + len(valset), "train_count": len(trainset), "validation_count": len(valset),
|
|
464
490
|
"num_threads": preflight_adapter.num_threads_plan.to_payload(),
|
|
465
491
|
}
|
|
492
|
+
is_multi_component_instruction_set = bool(
|
|
493
|
+
profile_version_id and isinstance(shape, list) and len(shape) > 1
|
|
494
|
+
)
|
|
466
495
|
run_id = client.start_run(
|
|
467
496
|
project=env["ORIZU_PROJECT"], optimizer_version_id=env["ORIZU_OPTIMIZER_VERSION_ID"],
|
|
468
|
-
|
|
497
|
+
# A set of one is the legacy prompt subject. Only a genuine component
|
|
498
|
+
# map is attributed to the profile version instead of a prompt version.
|
|
499
|
+
prompt_version_id=(
|
|
500
|
+
None if is_multi_component_instruction_set else prompt_context.prompt_version_id
|
|
501
|
+
),
|
|
502
|
+
instruction_set_profile_version_id=(
|
|
503
|
+
profile_version_id if is_multi_component_instruction_set else None
|
|
504
|
+
),
|
|
505
|
+
scorer_version_id=scorer_context.scorer_version_id or env["ORIZU_SCORER_VERSION_ID"],
|
|
469
506
|
dataset_version_id=env["ORIZU_DATASET_VERSION_ID"], split_set_id=env["ORIZU_SPLIT_SET_ID"],
|
|
470
507
|
train_split=train_split, validation_split=validation_split, metadata=metadata,
|
|
508
|
+
model_config_settings_version_id=instruction_set.get("model_config_settings_version_id") if isinstance(instruction_set, dict) else None,
|
|
471
509
|
)
|
|
472
510
|
print(json.dumps({"optimization_run_id": run_id}), flush=True)
|
|
473
511
|
# The frozen sink treats ``None`` as no local artifact writer. Keep that
|
|
@@ -524,9 +562,40 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
524
562
|
proposal_count=lambda: callback.candidate_proposals,
|
|
525
563
|
))
|
|
526
564
|
try:
|
|
527
|
-
|
|
565
|
+
proposal_observability: Any | None = None
|
|
566
|
+
proposal_budget: Any | None = None
|
|
567
|
+
custom_candidate_proposer = None
|
|
568
|
+
if os.environ.get("ORIZU_CANDIDATE_PROPOSER") == "skilled-proposer":
|
|
569
|
+
# DSPy and skilled-proposer are installed only in the manager
|
|
570
|
+
# published opt-in venv. Keeping this import inside the exact
|
|
571
|
+
# selection branch preserves the established no-flag process.
|
|
572
|
+
from .skilled_proposer_bridge import (
|
|
573
|
+
ProposalCallBudget,
|
|
574
|
+
ProposalObservability,
|
|
575
|
+
make_skilled_proposer_from_environment,
|
|
576
|
+
)
|
|
577
|
+
artifact_root = (local_logger.directory if local_logger is not None
|
|
578
|
+
else Path.cwd() / ".orizu" / "proposal-observability" / run_id)
|
|
579
|
+
proposal_budget = ProposalCallBudget(
|
|
580
|
+
max_calls=_optional_int("ORIZU_PROPOSAL_MAX_CALLS"),
|
|
581
|
+
max_tokens=_optional_int("ORIZU_PROPOSAL_MAX_TOKENS"),
|
|
582
|
+
)
|
|
583
|
+
proposal_observability = ProposalObservability(
|
|
584
|
+
event_log_root=artifact_root / "proposal-observability",
|
|
585
|
+
durable_failure_root=artifact_root / "proposal-failures",
|
|
586
|
+
budget=proposal_budget,
|
|
587
|
+
)
|
|
588
|
+
custom_candidate_proposer = make_skilled_proposer_from_environment(
|
|
589
|
+
config=config,
|
|
590
|
+
observability=proposal_observability,
|
|
591
|
+
budget=proposal_budget,
|
|
592
|
+
)
|
|
593
|
+
if custom_candidate_proposer is None:
|
|
594
|
+
raise RuntimeError("ALI_1505_PROPOSER_FACTORY_REFUSED_SELECTED_VALUE")
|
|
595
|
+
|
|
596
|
+
reflection_lm = None if custom_candidate_proposer is not None else make_gepa_reflection_lm(
|
|
528
597
|
context_supplier=lambda: (
|
|
529
|
-
|
|
598
|
+
adapter.last_candidate or seed_candidate,
|
|
530
599
|
adapter.last_row_evaluations,
|
|
531
600
|
),
|
|
532
601
|
config=config,
|
|
@@ -541,7 +610,10 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
541
610
|
callback=callback, hooks=hooks, max_metric_calls=effective_metric_limit,
|
|
542
611
|
stop_callbacks=stoppers, allow_degenerate_seed=config.allow_degenerate_seed,
|
|
543
612
|
reflection_lm=reflection_lm, seed_already_validated=True,
|
|
544
|
-
|
|
613
|
+
custom_candidate_proposer=custom_candidate_proposer,
|
|
614
|
+
proposal_budget=proposal_budget,
|
|
615
|
+
config=config,
|
|
616
|
+
module_selector=os.environ.get("ORIZU_COMPONENT_SELECTOR", "round_robin").replace("-", "_"))
|
|
545
617
|
require_durable_logging(sink, callback)
|
|
546
618
|
best_id = str(result.best_idx)
|
|
547
619
|
best_score = float(result.val_aggregate_scores[result.best_idx])
|
|
@@ -588,7 +660,7 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
588
660
|
promoted_prompt_version_id = sink.promote_candidate(
|
|
589
661
|
candidate_id=best_id,
|
|
590
662
|
prompt_id=prompt_id,
|
|
591
|
-
parent_prompt_version_id=
|
|
663
|
+
parent_prompt_version_id=prompt_context.prompt_version_id,
|
|
592
664
|
body=next(iter(best_candidate.values())),
|
|
593
665
|
body_kind=prompt_context.body_kind,
|
|
594
666
|
provider_settings=prompt_context.provider_settings,
|
|
@@ -596,20 +668,26 @@ def run_from_environment() -> dict[str, Any]:
|
|
|
596
668
|
label=config.promotion_label,
|
|
597
669
|
)
|
|
598
670
|
if local_logger is not None:
|
|
671
|
+
best_candidate = result.candidates[result.best_idx]
|
|
599
672
|
local_logger.write_result(TextGepaResult(
|
|
600
673
|
run_id=run_id,
|
|
601
674
|
best_candidate_id=best_id,
|
|
602
|
-
best_candidate_text=
|
|
675
|
+
best_candidate_text=(best_candidate if len(best_candidate) > 1
|
|
676
|
+
else next(iter(best_candidate.values()))),
|
|
603
677
|
best_score=best_score,
|
|
604
678
|
seed_score=seed_score,
|
|
605
679
|
promoted_prompt_version_id=promoted_prompt_version_id,
|
|
606
680
|
budget=budget,
|
|
607
681
|
))
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
682
|
+
if proposal_observability is not None:
|
|
683
|
+
proposal_observability.write_terminal_lm_stats_artifact()
|
|
684
|
+
else:
|
|
685
|
+
local_logger.write_lm_stats(
|
|
686
|
+
total_cost=float(getattr(reflection_lm, "total_cost", 0.0)),
|
|
687
|
+
total_tokens_in=int(getattr(reflection_lm, "total_tokens_in", 0)),
|
|
688
|
+
total_tokens_out=int(getattr(reflection_lm, "total_tokens_out", 0)),
|
|
689
|
+
total_tokens=int(getattr(reflection_lm, "total_tokens", 0)),
|
|
690
|
+
)
|
|
613
691
|
if budget_exhausted:
|
|
614
692
|
client.update_run(
|
|
615
693
|
run_id,
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
"""Typed DSPy bridge for the opt-in skilled-proposer integration.
|
|
2
|
+
|
|
3
|
+
This module deliberately owns proposal-LM observability. It does not use
|
|
4
|
+
``dspy.LM`` or LiteLLM: every completion reaches the frozen provider transport
|
|
5
|
+
through ``complete_reflection_messages`` and is recorded before DSPy sees its
|
|
6
|
+
response.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import atexit
|
|
12
|
+
import dataclasses
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import shutil
|
|
16
|
+
import tempfile
|
|
17
|
+
import uuid
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import TYPE_CHECKING, Any
|
|
21
|
+
from urllib.parse import urlparse
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
import dspy
|
|
25
|
+
except ModuleNotFoundError as error:
|
|
26
|
+
if error.name != "dspy":
|
|
27
|
+
raise
|
|
28
|
+
dspy = None
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
import dspy as dspy_types
|
|
32
|
+
|
|
33
|
+
from orizu_gepa.reflection import ProviderCompletion, complete_reflection_messages
|
|
34
|
+
from orizu_gepa.optimizer import TextGepaConfig
|
|
35
|
+
|
|
36
|
+
_SELECTION_ENV = "ORIZU_CANDIDATE_PROPOSER"
|
|
37
|
+
_SELECTION_VALUE = "skilled-proposer"
|
|
38
|
+
_SUPPORTED_ROLES = {"system", "user", "assistant"}
|
|
39
|
+
_ALLOWED_DSPY_REQUEST_FIELDS = frozenset({"temperature", "max_tokens", "cache"})
|
|
40
|
+
_DspyBaseLM = dspy.BaseLM if dspy is not None else object
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _require_dspy() -> Any:
|
|
44
|
+
if dspy is None:
|
|
45
|
+
raise RuntimeError("ALI_1505_DSPY_UNAVAILABLE")
|
|
46
|
+
return dspy
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _usage_totals() -> dict[str, int]:
|
|
50
|
+
return {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _require_usage(usage: dict[str, int]) -> dict[str, int]:
|
|
54
|
+
try:
|
|
55
|
+
normalized = {name: int(usage[name]) for name in _usage_totals()}
|
|
56
|
+
except (KeyError, TypeError, ValueError) as error:
|
|
57
|
+
raise RuntimeError("ALI_1505_PROPOSAL_USAGE_INVALID") from error
|
|
58
|
+
if any(value < 0 for value in normalized.values()):
|
|
59
|
+
raise RuntimeError("ALI_1505_PROPOSAL_USAGE_INVALID")
|
|
60
|
+
return normalized
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class ProposalCallBudget:
|
|
65
|
+
"""The connector-owned proposal ledger; GEPA metric accounting is separate."""
|
|
66
|
+
|
|
67
|
+
max_calls: int | None = None
|
|
68
|
+
max_tokens: int | None = None
|
|
69
|
+
call_count: int = 0
|
|
70
|
+
usage: dict[str, int] = field(default_factory=_usage_totals)
|
|
71
|
+
|
|
72
|
+
def record(self, usage: dict[str, int] | None = None) -> None:
|
|
73
|
+
self.call_count += 1
|
|
74
|
+
if usage is None:
|
|
75
|
+
return
|
|
76
|
+
normalized = _require_usage(usage)
|
|
77
|
+
for name, value in normalized.items():
|
|
78
|
+
self.usage[name] += value
|
|
79
|
+
|
|
80
|
+
def has_safe_boundary_stop(self) -> bool:
|
|
81
|
+
return ((self.max_calls is not None and self.call_count >= self.max_calls) or
|
|
82
|
+
(self.max_tokens is not None and self.usage["total_tokens"] >= self.max_tokens))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class ProposalObservability:
|
|
86
|
+
"""Durable per-provider-call proposal evidence and aggregate raw usage."""
|
|
87
|
+
|
|
88
|
+
def __init__(self, *, event_log_root: Path, durable_failure_root: Path, budget: ProposalCallBudget | None = None):
|
|
89
|
+
self.event_log_root = event_log_root
|
|
90
|
+
self.durable_failure_root = durable_failure_root
|
|
91
|
+
self.events: list[dict[str, Any]] = []
|
|
92
|
+
self.budget = budget or ProposalCallBudget()
|
|
93
|
+
self._temporary_root: Path | None = None
|
|
94
|
+
|
|
95
|
+
@classmethod
|
|
96
|
+
def local_for_test(
|
|
97
|
+
cls,
|
|
98
|
+
*,
|
|
99
|
+
event_log_root: Path | None = None,
|
|
100
|
+
durable_failure_root: Path | None = None,
|
|
101
|
+
) -> "ProposalObservability":
|
|
102
|
+
root = Path(tempfile.mkdtemp(prefix="orizu-ali1505-proposal-events-"))
|
|
103
|
+
result = cls(
|
|
104
|
+
event_log_root=event_log_root or root / "events",
|
|
105
|
+
durable_failure_root=durable_failure_root or root / "durable-failures",
|
|
106
|
+
)
|
|
107
|
+
result._temporary_root = root
|
|
108
|
+
atexit.register(shutil.rmtree, root, ignore_errors=True)
|
|
109
|
+
return result
|
|
110
|
+
|
|
111
|
+
def bind_budget(self, budget: ProposalCallBudget) -> None:
|
|
112
|
+
self.budget = budget
|
|
113
|
+
|
|
114
|
+
def close(self) -> None:
|
|
115
|
+
if self._temporary_root is not None:
|
|
116
|
+
shutil.rmtree(self._temporary_root, ignore_errors=True)
|
|
117
|
+
self._temporary_root = None
|
|
118
|
+
|
|
119
|
+
def __del__(self) -> None:
|
|
120
|
+
self.close()
|
|
121
|
+
|
|
122
|
+
def _append_event(self, event: dict[str, Any]) -> None:
|
|
123
|
+
self.event_log_root.mkdir(parents=True, exist_ok=True)
|
|
124
|
+
with (self.event_log_root / "events.jsonl").open("a", encoding="utf-8") as handle:
|
|
125
|
+
handle.write(json.dumps(event, sort_keys=True) + "\n")
|
|
126
|
+
self.events.append(event)
|
|
127
|
+
|
|
128
|
+
def _write_durable_failure(self, *, code: str, detail: str, correlation_id: str | None = None) -> None:
|
|
129
|
+
try:
|
|
130
|
+
self.durable_failure_root.mkdir(parents=True, exist_ok=True)
|
|
131
|
+
payload: dict[str, Any] = {"source": "skilled_proposer", "code": code, "detail": detail}
|
|
132
|
+
if correlation_id is not None:
|
|
133
|
+
payload["correlation_id"] = correlation_id
|
|
134
|
+
(self.durable_failure_root / "latest.json").write_text(
|
|
135
|
+
json.dumps(payload, sort_keys=True) + "\n", encoding="utf-8",
|
|
136
|
+
)
|
|
137
|
+
except Exception:
|
|
138
|
+
# The original delivery failure remains the actionable error. A
|
|
139
|
+
# second durable-store failure cannot be made durable by retrying it.
|
|
140
|
+
return
|
|
141
|
+
|
|
142
|
+
def record_success(self, event: dict[str, Any], usage: dict[str, int]) -> None:
|
|
143
|
+
self.budget.record(usage)
|
|
144
|
+
self._append_event(event)
|
|
145
|
+
|
|
146
|
+
def record_provider_failure(
|
|
147
|
+
self,
|
|
148
|
+
event: dict[str, Any],
|
|
149
|
+
*,
|
|
150
|
+
code: str,
|
|
151
|
+
detail: str,
|
|
152
|
+
usage: dict[str, int] | None = None,
|
|
153
|
+
) -> None:
|
|
154
|
+
self.budget.record(usage)
|
|
155
|
+
self._write_durable_failure(code=code, detail=detail, correlation_id=event.get("correlation_id"))
|
|
156
|
+
self._append_event(event)
|
|
157
|
+
|
|
158
|
+
def record_event_failure(self, error: Exception, *, correlation_id: str) -> None:
|
|
159
|
+
self._write_durable_failure(
|
|
160
|
+
code="proposal_observability_event_failed", detail=str(error), correlation_id=correlation_id,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def terminal_stats(self) -> dict[str, Any]:
|
|
164
|
+
return {"call_count": self.budget.call_count, "usage": dict(self.budget.usage)}
|
|
165
|
+
|
|
166
|
+
def terminal_lm_stats(self) -> dict[str, int]:
|
|
167
|
+
return {
|
|
168
|
+
"total_tokens_in": self.budget.usage["input_tokens"],
|
|
169
|
+
"total_tokens_out": self.budget.usage["output_tokens"],
|
|
170
|
+
"total_tokens": self.budget.usage["total_tokens"],
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
def write_terminal_lm_stats_artifact(self) -> str:
|
|
174
|
+
self.event_log_root.mkdir(parents=True, exist_ok=True)
|
|
175
|
+
path = self.event_log_root / "lm_stats.json"
|
|
176
|
+
path.write_text(json.dumps(self.terminal_lm_stats(), sort_keys=True) + "\n", encoding="utf-8")
|
|
177
|
+
return str(path)
|
|
178
|
+
|
|
179
|
+
def read_durable_failure(self) -> dict[str, Any]:
|
|
180
|
+
return json.loads((self.durable_failure_root / "latest.json").read_text(encoding="utf-8"))
|
|
181
|
+
|
|
182
|
+
def has_safe_boundary_stop(self) -> bool:
|
|
183
|
+
return self.budget.has_safe_boundary_stop()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _reject(message: str, *, code: str = "proposal_dspy_request_rejected") -> Any:
|
|
187
|
+
return _require_dspy().LMError(message, code=code)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _provider_messages(request: "dspy_types.LMRequest") -> list[dict[str, str]]:
|
|
191
|
+
if request.tools:
|
|
192
|
+
raise _reject("proposal_dspy_request_rejected: tools are not supported")
|
|
193
|
+
messages: list[dict[str, str]] = []
|
|
194
|
+
for message in request.messages:
|
|
195
|
+
dumped = message.model_dump()
|
|
196
|
+
role = dumped.get("role")
|
|
197
|
+
if role not in _SUPPORTED_ROLES:
|
|
198
|
+
raise _reject(f"proposal_dspy_request_rejected: unsupported message role {role!r}")
|
|
199
|
+
text_parts: list[str] = []
|
|
200
|
+
for part in dumped.get("parts", []):
|
|
201
|
+
if part.get("type") != "text" or not isinstance(part.get("text"), str):
|
|
202
|
+
raise _reject("proposal_dspy_request_rejected: only text message parts are supported")
|
|
203
|
+
text_parts.append(part["text"])
|
|
204
|
+
if not text_parts:
|
|
205
|
+
raise _reject("proposal_dspy_request_rejected: text messages must contain at least one text part")
|
|
206
|
+
messages.append({"role": role, "content": "".join(text_parts)})
|
|
207
|
+
return messages
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _request_config(request: "dspy_types.LMRequest", base: TextGepaConfig) -> TextGepaConfig:
|
|
211
|
+
config = request.config
|
|
212
|
+
unsupported = {
|
|
213
|
+
name: value
|
|
214
|
+
for name, value in config.model_dump().items()
|
|
215
|
+
if name not in _ALLOWED_DSPY_REQUEST_FIELDS and value not in (None, {})
|
|
216
|
+
}
|
|
217
|
+
if unsupported:
|
|
218
|
+
raise _reject("proposal_dspy_request_rejected: unsupported DSPy generation configuration")
|
|
219
|
+
cache = config.cache
|
|
220
|
+
if cache is not None and (getattr(cache, "enabled", False) or getattr(cache, "rollout_id", None) is not None):
|
|
221
|
+
raise _reject("proposal_dspy_cache_rejected: proposal response caching is disabled", code="proposal_dspy_cache_rejected")
|
|
222
|
+
changes: dict[str, Any] = {}
|
|
223
|
+
if config.temperature is not None:
|
|
224
|
+
changes["reflection_temperature"] = config.temperature
|
|
225
|
+
if config.max_tokens is not None:
|
|
226
|
+
if config.max_tokens <= 0:
|
|
227
|
+
raise _reject("proposal_dspy_request_rejected: max_tokens must be positive")
|
|
228
|
+
changes["reflection_max_tokens"] = config.max_tokens
|
|
229
|
+
return dataclasses.replace(base, **changes) if changes else base
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _loopback_endpoint(endpoint: str) -> str:
|
|
233
|
+
parsed = urlparse(endpoint)
|
|
234
|
+
if parsed.scheme not in {"http", "https"} or parsed.hostname not in {"127.0.0.1", "::1", "localhost"}:
|
|
235
|
+
raise ValueError("ALI_1505_BRIDGE_ENDPOINT_OVERRIDE_NOT_LOOPBACK")
|
|
236
|
+
if parsed.username or parsed.password:
|
|
237
|
+
raise ValueError("ALI_1505_BRIDGE_ENDPOINT_OVERRIDE_INVALID")
|
|
238
|
+
return endpoint
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class OrizuSkilledProposerLM(_DspyBaseLM):
|
|
242
|
+
"""A typed DSPy LM whose only completion path is Orizu's provider transport."""
|
|
243
|
+
|
|
244
|
+
forward_contract = "typed_lm"
|
|
245
|
+
|
|
246
|
+
def __init__(
|
|
247
|
+
self,
|
|
248
|
+
*,
|
|
249
|
+
config: TextGepaConfig,
|
|
250
|
+
observability: ProposalObservability,
|
|
251
|
+
endpoint_override: str | None = None,
|
|
252
|
+
ssl_cert_file: str | None = None,
|
|
253
|
+
) -> None:
|
|
254
|
+
# Provider retries belong exclusively to the frozen transport. DSPy
|
|
255
|
+
# must not retry after a durable-event failure and create unrecorded
|
|
256
|
+
# duplicate proposal attempts.
|
|
257
|
+
_require_dspy().BaseLM.__init__(
|
|
258
|
+
self,
|
|
259
|
+
config.reflection_model,
|
|
260
|
+
temperature=config.reflection_temperature,
|
|
261
|
+
max_tokens=config.reflection_max_tokens,
|
|
262
|
+
cache=False,
|
|
263
|
+
num_retries=0,
|
|
264
|
+
)
|
|
265
|
+
self._config = config
|
|
266
|
+
self._observability = observability
|
|
267
|
+
self._endpoint_override = _loopback_endpoint(endpoint_override) if endpoint_override else None
|
|
268
|
+
self._ssl_cert_file = ssl_cert_file
|
|
269
|
+
|
|
270
|
+
def forward(self, request: "dspy_types.LMRequest") -> "dspy_types.LMResponse":
|
|
271
|
+
dspy_module = _require_dspy()
|
|
272
|
+
correlation_id = uuid.uuid4().hex
|
|
273
|
+
if request.model != self.model:
|
|
274
|
+
raise _reject(
|
|
275
|
+
"proposal_dspy_model_rejected: requested model differs from the configured reflection model",
|
|
276
|
+
code="proposal_dspy_model_rejected",
|
|
277
|
+
)
|
|
278
|
+
messages = _provider_messages(request)
|
|
279
|
+
config = _request_config(request, self._config)
|
|
280
|
+
try:
|
|
281
|
+
completion: ProviderCompletion = complete_reflection_messages(
|
|
282
|
+
model=self.model,
|
|
283
|
+
messages=messages,
|
|
284
|
+
config=config,
|
|
285
|
+
endpoint_override=self._endpoint_override,
|
|
286
|
+
ssl_cert_file=self._ssl_cert_file,
|
|
287
|
+
# The explicit loopback harness must never receive a real
|
|
288
|
+
# provider credential from this process.
|
|
289
|
+
api_key_override="loopback-test-key" if self._endpoint_override else None,
|
|
290
|
+
)
|
|
291
|
+
except Exception as error:
|
|
292
|
+
provider_code = getattr(error, "code", None)
|
|
293
|
+
failure_usage = getattr(error, "usage", None)
|
|
294
|
+
failure_event = {
|
|
295
|
+
"source": "skilled_proposer",
|
|
296
|
+
"status": "failure",
|
|
297
|
+
"attempt": 1,
|
|
298
|
+
"correlation_id": correlation_id,
|
|
299
|
+
"provider": "openai" if self.model.startswith("openai/") else "anthropic",
|
|
300
|
+
"cache_state": "disabled",
|
|
301
|
+
"failure_code": provider_code or "ALI_1505_PROPOSAL_TRANSPORT_FAILURE",
|
|
302
|
+
}
|
|
303
|
+
if failure_usage is not None:
|
|
304
|
+
failure_event["usage"] = failure_usage
|
|
305
|
+
try:
|
|
306
|
+
self._observability.record_provider_failure(
|
|
307
|
+
failure_event,
|
|
308
|
+
code="proposal_provider_transport_failed",
|
|
309
|
+
detail=str(error),
|
|
310
|
+
usage=failure_usage,
|
|
311
|
+
)
|
|
312
|
+
except Exception as event_error:
|
|
313
|
+
self._observability.record_event_failure(event_error, correlation_id=correlation_id)
|
|
314
|
+
raise RuntimeError("proposal_observability_event_failed") from event_error
|
|
315
|
+
raise dspy_module.LMError(
|
|
316
|
+
f"proposal_provider_transport_failed: {error}",
|
|
317
|
+
code="proposal_provider_transport_failed",
|
|
318
|
+
model=self.model,
|
|
319
|
+
provider="orizu",
|
|
320
|
+
provider_code=provider_code,
|
|
321
|
+
) from error
|
|
322
|
+
|
|
323
|
+
usage = _require_usage(completion.usage)
|
|
324
|
+
success_event = {
|
|
325
|
+
"source": "skilled_proposer",
|
|
326
|
+
"status": "success",
|
|
327
|
+
"attempt": 1,
|
|
328
|
+
"correlation_id": correlation_id,
|
|
329
|
+
"provider": completion.provider,
|
|
330
|
+
"model": self.model,
|
|
331
|
+
"request_id": completion.request_id,
|
|
332
|
+
"latency_ms": completion.latency_ms,
|
|
333
|
+
"usage": usage,
|
|
334
|
+
"cache_state": "disabled",
|
|
335
|
+
}
|
|
336
|
+
try:
|
|
337
|
+
self._observability.record_success(success_event, usage)
|
|
338
|
+
except Exception as error:
|
|
339
|
+
self._observability.record_event_failure(error, correlation_id=correlation_id)
|
|
340
|
+
raise RuntimeError("proposal_observability_event_failed") from error
|
|
341
|
+
return dspy_module.LMResponse.from_text(completion.text, model=self.model, usage=usage, cache_hit=False)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def make_skilled_proposer_bridge(
|
|
345
|
+
*,
|
|
346
|
+
config: TextGepaConfig,
|
|
347
|
+
observability: ProposalObservability,
|
|
348
|
+
budget: ProposalCallBudget,
|
|
349
|
+
endpoint_override: str | None = None,
|
|
350
|
+
ssl_cert_file: str | None = None,
|
|
351
|
+
) -> OrizuSkilledProposerLM:
|
|
352
|
+
"""Create the cache-disabled typed LM passed to unmodified SkilledProposer."""
|
|
353
|
+
observability.bind_budget(budget)
|
|
354
|
+
return OrizuSkilledProposerLM(
|
|
355
|
+
config=config,
|
|
356
|
+
observability=observability,
|
|
357
|
+
endpoint_override=endpoint_override,
|
|
358
|
+
ssl_cert_file=ssl_cert_file,
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def make_skilled_proposer_from_environment(
|
|
363
|
+
*,
|
|
364
|
+
config: TextGepaConfig,
|
|
365
|
+
observability: ProposalObservability,
|
|
366
|
+
budget: ProposalCallBudget | None = None,
|
|
367
|
+
endpoint_override: str | None = None,
|
|
368
|
+
) -> Any | None:
|
|
369
|
+
"""Create the opt-in proposer only for the exact selected environment value."""
|
|
370
|
+
if os.environ.get(_SELECTION_ENV) != _SELECTION_VALUE:
|
|
371
|
+
return None
|
|
372
|
+
from skilled_proposer import SkilledProposer
|
|
373
|
+
|
|
374
|
+
bridge = make_skilled_proposer_bridge(
|
|
375
|
+
config=config,
|
|
376
|
+
observability=observability,
|
|
377
|
+
budget=budget or ProposalCallBudget(),
|
|
378
|
+
endpoint_override=endpoint_override,
|
|
379
|
+
ssl_cert_file=os.environ.get("SSL_CERT_FILE"),
|
|
380
|
+
)
|
|
381
|
+
return SkilledProposer(prompt_model=bridge)
|
|
@@ -36,3 +36,20 @@ class MaxCandidateProposalsStopper:
|
|
|
36
36
|
|
|
37
37
|
def __call__(self, state):
|
|
38
38
|
return self.proposal_count() >= self.max_candidate_proposals
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ProposalBudgetStopper:
|
|
42
|
+
"""Stop only between GEPA iterations after an owned proposal budget trips.
|
|
43
|
+
|
|
44
|
+
A custom candidate proposer does not pass through GEPA's ``reflection_lm``
|
|
45
|
+
or its cost tracking. Its typed DSPy bridge instead records raw provider
|
|
46
|
+
usage in an Orizu-owned ledger. This stopper deliberately reads only that
|
|
47
|
+
ledger and never changes ``state.total_num_evals``: metric-call budgeting
|
|
48
|
+
remains evaluator-only.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(self, proposal_budget):
|
|
52
|
+
self.proposal_budget = proposal_budget
|
|
53
|
+
|
|
54
|
+
def __call__(self, state):
|
|
55
|
+
return self.proposal_budget.has_safe_boundary_stop() and state.i >= 0
|
|
@@ -98,9 +98,10 @@ def translate_callback(event_name: str, event: dict[str, Any], *, run_id: str, *
|
|
|
98
98
|
|
|
99
99
|
if event_name == "on_optimization_start":
|
|
100
100
|
event_type = "run_started"
|
|
101
|
+
payload.pop("components", None)
|
|
101
102
|
payload.update({
|
|
102
103
|
"seed_candidate_text": next(iter(components.values()), None),
|
|
103
|
-
"
|
|
104
|
+
"seed_components": components,
|
|
104
105
|
"trainset_size": event.get("trainset_size"),
|
|
105
106
|
"valset_size": event.get("valset_size"),
|
|
106
107
|
})
|