orizu 0.5.21 → 0.5.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -15
- package/dist/cleanup.d.ts +6 -0
- package/dist/cleanup.d.ts.map +1 -0
- package/dist/cleanup.js +27 -0
- package/dist/gepa-engine-dispatch.d.ts.map +1 -1
- package/dist/gepa-engine-dispatch.js +20 -1
- package/dist/gepa-python-paths.d.ts.map +1 -1
- package/dist/gepa-python-paths.js +25 -6
- package/dist/help.d.ts.map +1 -1
- package/dist/help.js +37 -15
- package/dist/hosted-commands.d.ts +13 -0
- package/dist/hosted-commands.d.ts.map +1 -0
- package/dist/hosted-commands.js +10 -0
- package/dist/hosted-gepa-cli.d.ts +11 -0
- package/dist/hosted-gepa-cli.d.ts.map +1 -0
- package/dist/hosted-gepa-cli.js +110 -0
- package/dist/hosted-gepa-numeric-values.d.ts +3 -0
- package/dist/hosted-gepa-numeric-values.d.ts.map +1 -0
- package/dist/hosted-gepa-numeric-values.js +49 -0
- package/dist/hosted-provider-settings.d.ts +3 -0
- package/dist/hosted-provider-settings.d.ts.map +1 -0
- package/dist/hosted-provider-settings.js +69 -0
- package/dist/hosted-session-cli.d.ts.map +1 -1
- package/dist/hosted-session-cli.js +6 -1
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +72 -74
- package/dist/instruction-set-loader.d.ts.map +1 -1
- package/dist/instruction-set-loader.js +57 -5
- package/dist/instruction-sets-cli.d.ts +5 -2
- package/dist/instruction-sets-cli.d.ts.map +1 -1
- package/dist/instruction-sets-cli.js +134 -24
- package/dist/optimizations-promote-cli.d.ts +3 -0
- package/dist/optimizations-promote-cli.d.ts.map +1 -1
- package/dist/optimizations-promote-cli.js +75 -7
- package/dist/runner-dir-verify.d.ts.map +1 -1
- package/dist/runner-dir-verify.js +4 -7
- package/dist/skilled-proposer-bake-report.d.ts +2 -0
- package/dist/skilled-proposer-bake-report.d.ts.map +1 -0
- package/dist/skilled-proposer-bake-report.js +1 -0
- package/dist/skilled-proposer-config.d.ts +2 -0
- package/dist/skilled-proposer-config.d.ts.map +1 -0
- package/dist/skilled-proposer-config.js +212 -0
- package/dist/skilled-proposer-launch.d.ts +26 -0
- package/dist/skilled-proposer-launch.d.ts.map +1 -1
- package/dist/skilled-proposer-launch.js +103 -6
- package/package.json +1 -1
- package/scripts/vendor-gepa-python.mjs +81 -12
- package/src/cleanup.ts +37 -0
- package/src/gepa-engine-dispatch.ts +21 -1
- package/src/gepa-python-paths.ts +24 -6
- package/src/help.ts +37 -15
- package/src/hosted-commands.ts +4 -0
- package/src/hosted-gepa-cli.ts +107 -0
- package/src/hosted-gepa-numeric-values.ts +60 -0
- package/src/hosted-provider-settings.ts +69 -0
- package/src/hosted-session-cli.ts +4 -1
- package/src/http.ts +8 -0
- package/src/index.ts +112 -89
- package/src/instruction-set-loader.ts +55 -5
- package/src/instruction-sets-cli.ts +173 -19
- package/src/optimizations-promote-cli.ts +85 -7
- package/src/runner-dir-verify.ts +11 -6
- package/src/skilled-proposer-bake-report.ts +1 -0
- package/src/skilled-proposer-config.ts +191 -0
- package/src/skilled-proposer-launch.ts +96 -7
- package/src/skilled-proposer-venv-manager.mjs +44 -27
- package/vendor/orizu-gepa/src/orizu_gepa_connector/engine.py +36 -3
- package/vendor/orizu-gepa/src/orizu_gepa_connector/instruction_set_loader.py +31 -3
- package/vendor/orizu-gepa/src/orizu_gepa_connector/runtime.py +7 -2
- package/vendor/orizu-gepa/src/orizu_gepa_connector/skilled_proposer_bridge.py +354 -5
- package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py +1 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/client.py +15 -0
- package/vendor/skills/orizu-cli/SKILL.md +94 -168
- package/vendor/skills/orizu-cli/references/assess-and-plan.md +107 -0
- package/vendor/skills/orizu-cli/references/building-judges.md +442 -180
- package/vendor/skills/orizu-cli/references/cli-reference.md +69 -39
- package/vendor/skills/orizu-cli/references/dataset-design.md +100 -0
- package/vendor/skills/orizu-cli/references/eval-strategy.md +64 -0
- package/vendor/skills/orizu-cli/references/instructions-after-prompts.md +115 -0
- package/vendor/skills/orizu-cli/references/migrate-existing-gepa-setup.md +150 -82
- package/vendor/skills/orizu-cli/references/optimization-reports.md +196 -104
- package/vendor/skills/orizu-cli/references/optimization-with-gepa.md +57 -37
- package/vendor/skills/orizu-cli/references/primer.md +43 -18
- package/vendor/skills/orizu-cli/references/prompt-control-plane.md +199 -131
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Instruction Control Plane
|
|
2
2
|
|
|
3
|
-
Use this reference for the Phase 0
|
|
3
|
+
Use this reference for the Phase 0 instruction, judge, scorer, runner, score, run, and optimization event control plane. Follow the [Authority map](../SKILL.md#authority-map) before executing its command sequences.
|
|
4
4
|
|
|
5
5
|
## Contents
|
|
6
6
|
|
|
@@ -16,14 +16,14 @@ Use this reference for the Phase 0 prompt, judge, scorer, runner, score, run, an
|
|
|
16
16
|
1. Verify auth: `orizu --local whoami`.
|
|
17
17
|
2. Export API context for scripts: `eval "$(orizu --local env --project <team>/<project>)"`.
|
|
18
18
|
3. Create immutable dataset versions before creating splits.
|
|
19
|
-
4. Push runners before
|
|
20
|
-
5. Register scorers after their backing prompt/runner exists, then bind headline/tracked scorers to
|
|
19
|
+
4. Push runners before instruction-set mutations, judges, or prompt-runner scorers; instruction components pin runner versions. Follow the Authority map after the runner version exists.
|
|
20
|
+
5. Register scorers after their backing judge prompt/runner exists, then bind headline/tracked scorers to instruction components when the UI should show those metrics.
|
|
21
21
|
6. Use `runners exec` to prove the runner contract locally before submitting runs or score runs.
|
|
22
22
|
7. For common text-candidate optimization, prefer `orizu optimizations run-gepa`; it starts the run and logs events for you.
|
|
23
23
|
8. After `run-gepa`, inspect `logs/<optimization_run_id>` first; it is the complete local trace for coding-agent analysis.
|
|
24
24
|
9. Use `orizu optimizations export <run-id> --out <run-id>.optimization.json` when the local log is missing or the run happened elsewhere.
|
|
25
|
-
10.
|
|
26
|
-
11.
|
|
25
|
+
10. Write and attach a markdown report for finished, failed, or cancelled runs; use `optimization-reports.md` for structure and diagnostic guidance.
|
|
26
|
+
11. Follow the [Authority map](../SKILL.md#authority-map). Simpler one-shot path: after the human accepts the report, a human curator runs `orizu optimizations promote <run-id> --candidate <id> --label production --project <team/project>`, materializing and labeling once. Equivalent two-stage path: an agent runs `orizu optimizations promote <run-id> --candidate <id> --project <team/project>` after acceptance. A human curator re-runs the same promotion as `orizu optimizations promote <run-id> --candidate <id> --label production --project <team/project>`. The idempotent finalizer finds the exact existing materialized version by run and candidate provenance, moves production, and creates no duplicate profile version or candidate-promoted event.
|
|
27
27
|
12. For custom optimizers, start an optimization run before local execution, then stream events into that run.
|
|
28
28
|
13. Use bare HTTP for optimization events; use `orizu log` only as a shell fallback.
|
|
29
29
|
14. Promote only accepted candidates; rejected candidates stay in optimization events.
|
|
@@ -32,59 +32,75 @@ Customer model-provider secrets stay local. Do not upload Anthropic/OpenAI/etc.
|
|
|
32
32
|
|
|
33
33
|
## Instruction sets
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
write them with
|
|
35
|
+
Work within the fixed, ordered component shape and model-config profiles defined
|
|
36
|
+
for instruction sets in Vocabulary. Existing prompts appear as one-component
|
|
37
|
+
sets. Inspect or write them with the customer-facing `orizu instructions`
|
|
38
|
+
namespace. Follow the [Authority map](../SKILL.md#authority-map) for mutation
|
|
39
|
+
custody before copying a command:
|
|
38
40
|
|
|
39
41
|
```bash
|
|
40
|
-
orizu
|
|
41
|
-
orizu
|
|
42
|
-
orizu
|
|
43
|
-
orizu
|
|
44
|
-
orizu
|
|
45
|
-
orizu
|
|
46
|
-
orizu
|
|
47
|
-
orizu
|
|
48
|
-
orizu
|
|
49
|
-
orizu
|
|
50
|
-
orizu
|
|
42
|
+
orizu instructions list --project core/evals --status active --json
|
|
43
|
+
orizu instructions show planner --project core/evals --status active --json
|
|
44
|
+
orizu instructions create ./orizu.instruction-set.json --project core/evals --model-config anthropic/claude-haiku
|
|
45
|
+
orizu instructions push ./orizu.instruction-set.json --project core/evals --set planner
|
|
46
|
+
orizu instructions profiles new planner --project core/evals --model-config anthropic/claude-haiku
|
|
47
|
+
orizu instructions profiles promote planner --project core/evals --model-config anthropic/claude-haiku --version 2
|
|
48
|
+
orizu instructions profiles rollback planner --project core/evals --model-config anthropic/claude-haiku --to 1
|
|
49
|
+
orizu instructions default show planner --project core/evals
|
|
50
|
+
orizu instructions default move planner --project core/evals --model-config anthropic/claude-haiku --version 2
|
|
51
|
+
orizu instructions shape add planner --project core/evals --key safety --from ./orizu.instruction-set.json
|
|
52
|
+
orizu instructions shape remove planner --project core/evals --key safety
|
|
53
|
+
orizu instructions archive planner --project core/evals --json
|
|
54
|
+
orizu instructions restore planner --project core/evals --json
|
|
51
55
|
```
|
|
52
56
|
|
|
53
|
-
The manifest contains `name`,
|
|
54
|
-
|
|
55
|
-
component
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
57
|
+
The manifest contains `name`, optional unversioned `description`, the ordered
|
|
58
|
+
component-key `shape`, and `components`; every shape key needs a set-wide
|
|
59
|
+
component (`key` plus `text` or a manifest-relative `path`). A component may
|
|
60
|
+
additionally specify `modelConfig` to override that key in a named profile.
|
|
61
|
+
Create materializes each named profile with its complete base-plus-override
|
|
62
|
+
component map. The default starts on the seed profile version. A profile with a
|
|
63
|
+
production version resolves to that component map; a profile without one
|
|
64
|
+
resolves to the default. Sets accept either their stable slug or exact display
|
|
65
|
+
name. Archiving changes list visibility only, so archived sets still resolve
|
|
66
|
+
and sync; use `--status archived` or `--status all` to find them before
|
|
67
|
+
restoring.
|
|
60
68
|
|
|
61
69
|
Shape changes create unpromoted profile heads but leave the default and
|
|
62
70
|
production pointers in place. The instruction set does not resolve for affected
|
|
63
71
|
model configs until those pointers move to their new shape-change versions; use
|
|
64
72
|
the follow-up commands printed by the text CLI.
|
|
65
73
|
|
|
66
|
-
Before
|
|
74
|
+
Before moving a default, `default show` lists the model configs whose
|
|
67
75
|
resolution would change. A move targets a version by model-config identity and
|
|
68
76
|
version number, and the database rejects an unsealed or non-commit-anchored
|
|
69
77
|
component map. Shape changes create a new `shape_change` version for every profile;
|
|
70
78
|
they do not repoint the default or any production label.
|
|
71
79
|
|
|
72
|
-
Sync an offline runner directory with `orizu
|
|
73
|
-
--out ./instructions --project core/evals`. The layout is
|
|
80
|
+
Sync an offline runner directory with `orizu instructions sync planner
|
|
81
|
+
--out ./instructions --project core/evals`. The layout root is the stable set
|
|
82
|
+
slug (`./instructions/planner/`), containing `manifest.json`,
|
|
74
83
|
`default/<key>.md`, and `profiles/<identity-slug>/<key>.md`; loaders are
|
|
75
|
-
`loadInstructionSet(dir,
|
|
76
|
-
`load_instruction_set(dir,
|
|
84
|
+
`loadInstructionSet(dir, setReference, modelConfigIdentity)` in TypeScript and
|
|
85
|
+
`load_instruction_set(dir, set_reference, model_config_identity)` in Python.
|
|
86
|
+
Prefer the stable slug; exact manifest names remain accepted for compatibility. A
|
|
77
87
|
Git-pinned component is recorded in the manifest but is intentionally not
|
|
78
88
|
downloaded; either loader raises `instruction_set_component_unavailable`. A
|
|
79
89
|
missing or unreadable local component file is distinct:
|
|
80
90
|
`instruction_set_component_unreadable`.
|
|
81
91
|
|
|
82
|
-
|
|
83
|
-
|
|
92
|
+
The sync manifest carries immutable `projectId` and `instructionSetId`; modern
|
|
93
|
+
sync replacement requires both identities as well as the slug to match.
|
|
94
|
+
Identity-less pre-slug trees are never auto-migrated: sync preserves them and
|
|
95
|
+
returns `instruction_set_sync_legacy_identity_required`, requiring a verified
|
|
96
|
+
clean output root or an explicit operator move.
|
|
97
|
+
|
|
98
|
+
Use `profiles new` to seed an additional model-config profile from the default.
|
|
99
|
+
Use `profiles
|
|
84
100
|
promote` to move only that profile's production label; use `profiles rollback
|
|
85
101
|
--to <n>` to create a new rollback version from the target version's component
|
|
86
|
-
|
|
87
|
-
|
|
102
|
+
map **and settings version** before moving it. Follow the Authority map for
|
|
103
|
+
promotion execution.
|
|
88
104
|
|
|
89
105
|
For a single-component instruction set that wraps an existing prompt, only a
|
|
90
106
|
promotion or rollback of the set's default profile also moves that prompt's
|
|
@@ -96,7 +112,7 @@ label` when a profile command performed this mirror.
|
|
|
96
112
|
Execution/privacy defaults:
|
|
97
113
|
|
|
98
114
|
- Runner subprocesses receive the input/output file-contract paths plus a small allowlist of provider/runtime environment variables. Orizu API tokens are not passed into runner processes. When an execution carries an instruction set, `ORIZU_INSTRUCTION_SET_DIR` is set explicitly to a fresh per-row synced layout; it is absent for legacy executions and is never inherited from the shell.
|
|
99
|
-
- `orizu optimizations run-gepa` redacts dataset row payloads and reflection text in logged events by default. Use `--log-row-snapshots` only when the customer explicitly wants raw row and
|
|
115
|
+
- `orizu optimizations run-gepa` redacts dataset row payloads and reflection text in logged events by default. Use `--log-row-snapshots` only when the customer explicitly wants raw row and component text in the optimization event stream.
|
|
100
116
|
- `run-gepa` still writes complete local traces under `logs/<optimization_run_id>` by default. Treat those logs as sensitive: they include row inputs, model outputs, scores, feedback, scorer responses, reflection prompts, reflection responses, and candidate text.
|
|
101
117
|
- Runner artifacts, runner output, score result uploads, and optimization event payloads are size-capped. If a run needs larger observability payloads, store the large artifact separately and log a pointer.
|
|
102
118
|
|
|
@@ -116,7 +132,7 @@ Required file: `manifest.json`.
|
|
|
116
132
|
}
|
|
117
133
|
```
|
|
118
134
|
|
|
119
|
-
For `runners exec`, the command must read input JSON from `ORIZU_RUNNER_INPUT_PATH` and write output JSON to `ORIZU_RUNNER_OUTPUT_PATH`. When present, `ORIZU_INSTRUCTION_SET_DIR` contains the same layout written by `
|
|
135
|
+
For `runners exec`, the command must read input JSON from `ORIZU_RUNNER_INPUT_PATH` and write output JSON to `ORIZU_RUNNER_OUTPUT_PATH`. When present, `ORIZU_INSTRUCTION_SET_DIR` contains the same layout written by `instructions sync`; use `loadInstructionSet`/`load_instruction_set` to read materialized component text. Pinned components are represented in the manifest and are intentionally not materialized as `.md` files.
|
|
120
136
|
|
|
121
137
|
Input shape:
|
|
122
138
|
|
|
@@ -237,36 +253,15 @@ seed before iterating. If the seed scores the worst possible value on every
|
|
|
237
253
|
validation row (0.0 for higher-is-better, 1.0 for lower-is-better), errors on
|
|
238
254
|
every row, or returns nothing parseable, the run fails loudly with a
|
|
239
255
|
contract-mismatch diagnosis instead of silently burning budget on zeroed
|
|
240
|
-
candidates. A uniformly-worst seed is almost always a harness bug, not
|
|
241
|
-
|
|
256
|
+
candidates. A uniformly-worst seed is almost always a harness bug, not bad
|
|
257
|
+
component values; override with `--allow-degenerate-seed` only when the seed genuinely
|
|
242
258
|
deserves the worst score everywhere.
|
|
243
259
|
|
|
244
|
-
###
|
|
260
|
+
### Judge Directory
|
|
245
261
|
|
|
246
262
|
Required files: `orizu.prompt.json` plus the body file referenced by `body_file`.
|
|
247
263
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
```json
|
|
251
|
-
{
|
|
252
|
-
"name": "hip-note-generator",
|
|
253
|
-
"role": "production_inference",
|
|
254
|
-
"description": "Generates a HIP note label.",
|
|
255
|
-
"body_file": "prompt.md",
|
|
256
|
-
"body_kind": "text",
|
|
257
|
-
"version_label": "v1",
|
|
258
|
-
"provider_settings": {
|
|
259
|
-
"model": "claude-sonnet-4-6",
|
|
260
|
-
"temperature": 0,
|
|
261
|
-
"max_tokens": 4096
|
|
262
|
-
},
|
|
263
|
-
"provenance": {
|
|
264
|
-
"kind": "coding-agent-edit"
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
```
|
|
268
|
-
|
|
269
|
-
Judge prompt:
|
|
264
|
+
Judge artifact:
|
|
270
265
|
|
|
271
266
|
```json
|
|
272
267
|
{
|
|
@@ -391,27 +386,19 @@ eval "$(orizu --local env --project <team>/<project>)"
|
|
|
391
386
|
### Dataset Versions And Splits
|
|
392
387
|
|
|
393
388
|
```bash
|
|
394
|
-
orizu --local datasets
|
|
389
|
+
orizu --local datasets upload \
|
|
390
|
+
--file ./dataset.jsonl \
|
|
395
391
|
--project <team>/<project> \
|
|
396
|
-
--
|
|
392
|
+
--name "<dataset-name>" \
|
|
393
|
+
--readme-file ./dataset-readme.md \
|
|
397
394
|
--json
|
|
398
395
|
```
|
|
399
396
|
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
Create ratio-based splits:
|
|
403
|
-
|
|
404
|
-
```bash
|
|
405
|
-
orizu --local datasets splits create <dataset-version-id> \
|
|
406
|
-
--name default \
|
|
407
|
-
--seed 42 \
|
|
408
|
-
--train 0.6 \
|
|
409
|
-
--validation 0.4 \
|
|
410
|
-
--test 0 \
|
|
411
|
-
--json
|
|
412
|
-
```
|
|
397
|
+
Upload creates the initial immutable `v1` snapshot. Capture its returned
|
|
398
|
+
`dataset_version_id`; do not immediately create a colliding `v1` version.
|
|
413
399
|
|
|
414
|
-
|
|
400
|
+
Review and commit a predefined split file with explicit train, validation, and
|
|
401
|
+
reserved final-held-out membership:
|
|
415
402
|
|
|
416
403
|
```json
|
|
417
404
|
{
|
|
@@ -421,13 +408,13 @@ Create predefined, Hugging Face-style splits:
|
|
|
421
408
|
"partitions": [
|
|
422
409
|
{ "name": "train", "row_ids": ["row-1", "row-2"] },
|
|
423
410
|
{ "name": "validation", "row_ids": ["row-3"] },
|
|
424
|
-
{ "name": "
|
|
411
|
+
{ "name": "final-held-out", "row_ids": ["row-4"] }
|
|
425
412
|
]
|
|
426
413
|
}
|
|
427
414
|
```
|
|
428
415
|
|
|
429
416
|
```bash
|
|
430
|
-
orizu --local datasets splits create <dataset-version-id> \
|
|
417
|
+
orizu --local datasets splits create <upload-returned-dataset-version-id> \
|
|
431
418
|
--from-file ./split.json \
|
|
432
419
|
--json
|
|
433
420
|
```
|
|
@@ -440,12 +427,13 @@ Returns `split_set_id`.
|
|
|
440
427
|
orizu --local runners push ./runner \
|
|
441
428
|
--project <team>/<project> \
|
|
442
429
|
--name hip-note-judge-runner \
|
|
443
|
-
--label default \
|
|
444
430
|
--json
|
|
445
431
|
```
|
|
446
432
|
|
|
447
433
|
Returns `runner_version_id`.
|
|
448
434
|
|
|
435
|
+
The agent push is unlabeled. No standalone runner-label command exists, so never label by repeating a mutable working directory. If the human accepts the exact version as default, the human curator materializes it with `orizu --local runners pull hip-note-judge-runner --project <team>/<project> --version <reviewed-runner-version-id> --out ./runner-default-handoff`, does not alter that fresh directory, then runs `orizu --local runners push ./runner-default-handoff --project <team>/<project> --name hip-note-judge-runner --label default --json`. Pull verifies the content-addressed archive, and deterministic push excludes the pull sidecar, so this resolves the reviewed version before moving the pointer.
|
|
436
|
+
|
|
449
437
|
Execute against a dataset split:
|
|
450
438
|
|
|
451
439
|
```bash
|
|
@@ -461,11 +449,15 @@ orizu --local runners exec \
|
|
|
461
449
|
|
|
462
450
|
Omit `--runner-dir` to download and materialize the pinned runner version from Orizu. `--out` may end in `.jsonl` or `.jsonl.gz`.
|
|
463
451
|
|
|
464
|
-
###
|
|
452
|
+
### Instructions And Judges
|
|
453
|
+
|
|
454
|
+
Prepare the complete instruction-set manifest and follow the Authority map for
|
|
455
|
+
the push; the judge remains a git-canonical artifact.
|
|
465
456
|
|
|
466
457
|
```bash
|
|
467
|
-
orizu --local
|
|
458
|
+
orizu --local instructions push ./orizu.instruction-set.json \
|
|
468
459
|
--project <team>/<project> \
|
|
460
|
+
--set <slug-or-exact-name> \
|
|
469
461
|
--runner-version <runner-version-id> \
|
|
470
462
|
--json
|
|
471
463
|
|
|
@@ -475,22 +467,27 @@ orizu --local judges push ./judge \
|
|
|
475
467
|
--json
|
|
476
468
|
```
|
|
477
469
|
|
|
478
|
-
|
|
470
|
+
Instruction push returns the updated set and profile versions. Judge push
|
|
471
|
+
returns `prompt_version_id` for the judge artifact used by scorer bindings.
|
|
479
472
|
|
|
480
473
|
List:
|
|
481
474
|
|
|
482
475
|
```bash
|
|
483
|
-
orizu --local
|
|
484
|
-
orizu --local
|
|
485
|
-
orizu --local
|
|
476
|
+
orizu --local instructions list --project <team>/<project>
|
|
477
|
+
orizu --local instructions list --project <team>/<project> --status archived
|
|
478
|
+
orizu --local instructions list --project <team>/<project> --status all
|
|
486
479
|
orizu --local judges list --project <team>/<project>
|
|
487
480
|
```
|
|
488
481
|
|
|
489
|
-
|
|
490
|
-
or `--status all` when you need archived artifacts.
|
|
482
|
+
Instruction and judge lists show active artifacts by default. Use `--status
|
|
483
|
+
archived` or `--status all` when you need archived artifacts. Existing prompts
|
|
484
|
+
appear in the instruction list as one-component sets; see
|
|
485
|
+
`instructions-after-prompts.md` for ownership output, deliberate standalone
|
|
486
|
+
mutation refusals, and manual consolidation.
|
|
491
487
|
|
|
492
|
-
|
|
493
|
-
`LINES`, `CHARS`, and `WORDS`, measured from the latest
|
|
488
|
+
Compatibility prompt and judge list tables include `ID`, `NAME`, `ROLE`,
|
|
489
|
+
`STATUS`, `TOKENS`, `LINES`, `CHARS`, and `WORDS`, measured from the latest
|
|
490
|
+
sealed version. `—`
|
|
494
491
|
means the canonical body could not be measured; zero remains `0`. The `~`
|
|
495
492
|
prefix on token counts means approximate: Orizu uses one fixed, model-agnostic
|
|
496
493
|
`gpt-tokenizer` encoding rather than claiming an exact count for every model.
|
|
@@ -504,13 +501,17 @@ first 500 sorted summaries are enriched per request, and canonical-body
|
|
|
504
501
|
resolution has a 15-second server budget. A summary skipped by either bound
|
|
505
502
|
carries null stats and `measurement_cap_exceeded`.
|
|
506
503
|
|
|
507
|
-
Archive or restore
|
|
504
|
+
Archive or restore an instruction set by stable slug or exact name, using the
|
|
505
|
+
executor selected by the Authority map:
|
|
508
506
|
|
|
509
507
|
```bash
|
|
510
|
-
orizu --local
|
|
511
|
-
orizu --local
|
|
508
|
+
orizu --local instructions archive <slug-or-exact-name> --project <team>/<project>
|
|
509
|
+
orizu --local instructions restore <slug-or-exact-name> --project <team>/<project>
|
|
512
510
|
```
|
|
513
511
|
|
|
512
|
+
Archive and restore affect visibility only. Archived sets still resolve and
|
|
513
|
+
sync, and can be inspected with `instructions show --status archived|all`.
|
|
514
|
+
|
|
514
515
|
List prompt comment threads for the latest version, or a specific label/version:
|
|
515
516
|
|
|
516
517
|
```bash
|
|
@@ -520,14 +521,19 @@ orizu --local comments list --prompt <prompt-id-or-name> \
|
|
|
520
521
|
[--json]
|
|
521
522
|
```
|
|
522
523
|
|
|
523
|
-
Human output shows the thread count, open/resolved counts, selected
|
|
524
|
+
Human output shows the thread count, open/resolved counts, selected component
|
|
525
|
+
text or source line, each top-level comment body, and replies. Use `--json` when
|
|
526
|
+
an agent or script needs structured `summary` and `comments` data. Check
|
|
527
|
+
unresolved comments before drafting or pushing the next instruction version.
|
|
524
528
|
|
|
525
|
-
|
|
529
|
+
After the promotion decision, move a profile's production pointer
|
|
530
|
+
through the Authority map:
|
|
526
531
|
|
|
527
532
|
```bash
|
|
528
|
-
orizu --local
|
|
533
|
+
orizu --local instructions profiles promote hip-note-judge \
|
|
529
534
|
--project <team>/<project> \
|
|
530
|
-
--
|
|
535
|
+
--model-config <identity> \
|
|
536
|
+
--version <n> \
|
|
531
537
|
--json
|
|
532
538
|
```
|
|
533
539
|
|
|
@@ -542,7 +548,6 @@ orizu --local scorers register \
|
|
|
542
548
|
--manifest ./scorer.manifest.json \
|
|
543
549
|
--prompt-version <judge-prompt-version-id> \
|
|
544
550
|
--runner-version <judge-runner-version-id> \
|
|
545
|
-
--label production \
|
|
546
551
|
--json
|
|
547
552
|
```
|
|
548
553
|
|
|
@@ -553,12 +558,10 @@ Inspect scorers:
|
|
|
553
558
|
```bash
|
|
554
559
|
orizu --local scorers list --project <team>/<project>
|
|
555
560
|
orizu --local scorers detail <scorer-id-or-name> --project <team>/<project> --json
|
|
556
|
-
orizu --local scorers labels set hip-note-judge-score production \
|
|
557
|
-
--project <team>/<project> \
|
|
558
|
-
--version <scorer-version-id> \
|
|
559
|
-
--json
|
|
560
561
|
```
|
|
561
562
|
|
|
563
|
+
Production scorer labels are pointer moves. The agent prepares the scorer name, version id, and evidence. After human acceptance, follow the [Authority map](../SKILL.md#authority-map): a human curator runs `orizu --local scorers labels set hip-note-judge-score production --version <scorer-version-id> --project <team>/<project> --json`.
|
|
564
|
+
|
|
562
565
|
Use scorer versions directly with the runner contract:
|
|
563
566
|
|
|
564
567
|
```bash
|
|
@@ -682,18 +685,25 @@ orizu --local scores submit ./candidate-scores.jsonl \
|
|
|
682
685
|
--json
|
|
683
686
|
```
|
|
684
687
|
|
|
685
|
-
Bind
|
|
688
|
+
Bind the headline scorer to the instruction component that owns the evaluated
|
|
689
|
+
text:
|
|
686
690
|
|
|
687
691
|
```bash
|
|
688
|
-
orizu --local
|
|
692
|
+
orizu --local instructions scorers set-headline <set> \
|
|
693
|
+
--key <component-key> \
|
|
689
694
|
--project <team>/<project> \
|
|
690
695
|
--scorer-version <scorer-version-id> \
|
|
691
696
|
--dataset-version <dataset-version-id> \
|
|
692
697
|
--split-set <split-set-id> \
|
|
693
698
|
--split validation \
|
|
694
699
|
--json
|
|
700
|
+
```
|
|
701
|
+
|
|
702
|
+
Add tracked scorers to the same set-and-component address:
|
|
695
703
|
|
|
696
|
-
|
|
704
|
+
```bash
|
|
705
|
+
orizu --local instructions scorers add <set> \
|
|
706
|
+
--key <component-key> \
|
|
697
707
|
--project <team>/<project> \
|
|
698
708
|
--scorer-version <scorer-version-id> \
|
|
699
709
|
--dataset-version <dataset-version-id> \
|
|
@@ -702,6 +712,12 @@ orizu --local prompts scorers add <prompt-id> \
|
|
|
702
712
|
--json
|
|
703
713
|
```
|
|
704
714
|
|
|
715
|
+
The legacy `prompts scorers set-headline` and `prompts scorers add` spellings
|
|
716
|
+
refuse before making a request. Their measured replacements are:
|
|
717
|
+
|
|
718
|
+
- `Use: orizu instructions scorers set-headline <set> --key <component-key> --scorer-version <id> --project <team/project>`
|
|
719
|
+
- `Use: orizu instructions scorers add <set> --key <component-key> --scorer-version <id> --project <team/project>`
|
|
720
|
+
|
|
705
721
|
### Runs
|
|
706
722
|
|
|
707
723
|
Submit local runner output:
|
|
@@ -736,11 +752,10 @@ orizu --local runs submit ./generator-results.jsonl \
|
|
|
736
752
|
orizu --local optimizers push ./optimizer \
|
|
737
753
|
--project <team>/<project> \
|
|
738
754
|
--name hip-gepa-optimizer \
|
|
739
|
-
--label gepa-v1 \
|
|
740
755
|
--json
|
|
741
756
|
```
|
|
742
757
|
|
|
743
|
-
Returns `optimizer_version_id`.
|
|
758
|
+
Returns `optimizer_version_id`; the agent-created version is unlabeled. If a human curator accepts that exact version as `gepa-v1`, the human pulls it with `orizu --local optimizers pull hip-gepa-optimizer --project <team>/<project> --version <optimizer-version-id> --out ./optimizer-label-handoff`, leaves that fresh directory unchanged, then runs `orizu --local optimizers push ./optimizer-label-handoff --project <team>/<project> --name hip-gepa-optimizer --label gepa-v1 --json`.
|
|
744
759
|
|
|
745
760
|
Start a local optimization run before the optimizer process begins:
|
|
746
761
|
|
|
@@ -772,7 +787,9 @@ For bundled text-candidate GEPA-style optimization, let the CLI manage start/eve
|
|
|
772
787
|
orizu --local optimizations run-gepa \
|
|
773
788
|
--project <team>/<project> \
|
|
774
789
|
--optimizer-version-id <optimizer-version-id> \
|
|
775
|
-
--
|
|
790
|
+
--instruction-set <slug-or-exact-name> \
|
|
791
|
+
--model-config <identity> \
|
|
792
|
+
--component-selector all \
|
|
776
793
|
--runner-version-id <runner-version-id> \
|
|
777
794
|
--candidate-runner-dir ./candidate-runner \
|
|
778
795
|
--scorer-version-id <row-scorer-version-id> \
|
|
@@ -790,10 +807,21 @@ orizu --local optimizations run-gepa \
|
|
|
790
807
|
|
|
791
808
|
Useful GEPA flags:
|
|
792
809
|
|
|
810
|
+
- `--instruction-set <slug-or-exact-name> --model-config <identity>` selects a
|
|
811
|
+
resolving instruction-set profile. `--component-selector round-robin|all`
|
|
812
|
+
controls which components GEPA proposes changing; `all` lets it optimize the
|
|
813
|
+
complete component map. `--candidate-version-id` remains a compatibility
|
|
814
|
+
input for a single legacy prompt version and is mutually exclusive with the
|
|
815
|
+
instruction-set selector.
|
|
793
816
|
- `--engine official|legacy` defaults to `official`, which runs the bundled official GEPA connector. Use `legacy` only as the frozen recovery hatch; it keeps the historical Python loop and does not support `--max-candidate-proposals` or the skilled proposer.
|
|
794
817
|
- `--scorer-input-contract gepa|flat_row` selects the scorer-runner input shape. Use `flat_row` to reuse a judge runner written for `runners exec --scorer-version` without a hand-written adapter; add `--scorer-candidate-field <row-field>` when that judge reads the candidate output from a specific row field (e.g. `draft`). Passing a candidate field under the `gepa` contract is refused at launch, not silently ignored. See "Scorer-Runner Input Contracts" above.
|
|
795
818
|
- `--allow-degenerate-seed` opts out of the launch-time refusal when the seed scores the worst possible value on every validation row. Leave it off by default — a uniformly-worst seed is almost always a scorer contract mismatch.
|
|
796
819
|
- Budget controls are mutually exclusive: choose at most one of `--budget auto|light|medium|heavy`, `--max-metric-calls <n>`, `--max-full-evals <n>`, `--max-iterations <n>`, or `--max-candidate-proposals <n>`. With none provided, `run-gepa` defaults to `--budget auto`, the balanced medium preset. `--max-candidate-proposals` is available only with `--engine official`.
|
|
820
|
+
- Hosted optimization (`--hosted`) is launched only by a human/PAT caller on
|
|
821
|
+
an enabled team. It accepts a named `--budget` preset only, does not require
|
|
822
|
+
candidate/scorer runner directories or local logs, and sends no local runner
|
|
823
|
+
bytes or provider credentials. Reuse `--launch-intent-id <uuid>` after a lost
|
|
824
|
+
response; use a new intent when the project or job specification changes.
|
|
797
825
|
- `--minibatch-size <n>` defaults to 3.
|
|
798
826
|
- `--num-threads auto|N` defaults to `auto`; auto caps row-evaluation concurrency from mini-batch size, validation-set size, 2x CPU count, memory estimate, file-descriptor limit, and a 64-thread default ceiling. Set `ORIZU_GEPA_AUTO_THREADS_MAX` or use `--num-threads <n>` only when the runner/provider capacity is known.
|
|
799
827
|
- `--candidate-selection-strategy pareto|current_best|epsilon_greedy`; default is `pareto`. With `epsilon_greedy`, `--epsilon <n>` controls the random-selection probability (default `0.1`, clamped to `0`–`1`).
|
|
@@ -801,9 +829,9 @@ Useful GEPA flags:
|
|
|
801
829
|
- `--reflection-model <provider/model>`, `--reflection-temperature <n>`, `--reflection-prompt-template <text|@file>`.
|
|
802
830
|
- `--reflection-max-tokens <n>` is explicit provider config, not a global default. It maps to Anthropic `max_tokens` and OpenAI `max_output_tokens`; Anthropic native Messages reflection requires it, while OpenAI can omit it unless the user wants a cap.
|
|
803
831
|
- `--reflection-retry-attempts` and `--reflection-http-timeout-seconds` tune transient reflection-provider retries. Exhausted retryable failures log `reflection_failed`, count against candidate-proposal budget, and continue with the next iteration.
|
|
804
|
-
- `--reflection-provider-settings <json|@file>` passes provider-native reflection settings separately from the
|
|
832
|
+
- `--reflection-provider-settings <json|@file>` passes provider-native reflection settings separately from the component text. Anthropic example: `{"thinking":{"type":"adaptive"},"output_config":{"effort":"medium"}}`. OpenAI example: `{"reasoning":{"effort":"medium","summary":"auto"}}`.
|
|
805
833
|
- `--disable-evaluation-cache` turns off candidate/row/scorer cache reuse.
|
|
806
|
-
- `--auto-promote --promotion-label <label>`
|
|
834
|
+
- `--auto-promote --promotion-label <label>` can move a label to the best candidate at the end. Omit both flags: run without auto-promotion, write the report, obtain the human promotion decision, then route the manual promotion command through the Authority map.
|
|
807
835
|
- `--log-row-snapshots` includes raw row and reflection text in events; leave off by default.
|
|
808
836
|
- `--log-dir <dir>` controls the local log root; default is `logs`.
|
|
809
837
|
- `--no-local-log` disables local trace files. Use this only when the environment must not persist raw rows or reflection context.
|
|
@@ -812,6 +840,10 @@ Useful GEPA flags:
|
|
|
812
840
|
|
|
813
841
|
`--candidate-proposer skilled-proposer` selects the skilled proposer; omission keeps the default official proposer. The flag is official-engine-only and accepts that one value. A missing value is refused before launch as `--candidate-proposer requires a value that does not start with --`; another value is refused as `--candidate-proposer must be skilled-proposer`; selecting it with `--engine legacy` is refused with `--candidate-proposer is supported by the official GEPA engine only`.
|
|
814
842
|
|
|
843
|
+
`--candidate-proposer-config <@file|path>` is optional and requires that selection on the official engine. Its strict schema-v1 object requires `schemaVersion: 1` and `skills: []`; each ordered skill is exactly either `{ "path": "..." }` (with optional `name`/`description` overrides) or `{ "name": "...", "description": "...", "inline": "..." }`. Optional fields are `additionalInstructionsFile`, `baseInstructionsFile`, positive-integer-or-null `maxWords`, `maxTokens`, and `maxExamples`, plus `onError: "keep"|"raise"`. Paths are config-file-relative; directories read only `SKILL.md`. Missing, unreadable, empty, malformed, ambiguous, duplicate-name, and unknown-field inputs fail before launch. `baseInstructionsFile` is advanced use: it replaces upstream's built-in anti-overfitting meta-prompt.
|
|
844
|
+
|
|
845
|
+
`--reflection-prompt-template` is incompatible whenever skilled-proposer is selected, with or without a proposer config; it configures the ordinary GEPA reflection path and would otherwise be silently ignored. The three token controls have different scopes: `--reflection-max-tokens` caps one provider response, config `maxTokens` caps a proposed instruction, and `--proposal-max-tokens` caps aggregate proposal-provider usage.
|
|
846
|
+
|
|
815
847
|
`--proposal-max-calls <positive-integer>` and `--proposal-max-tokens <positive-integer>` default to unset and are valid only with the skilled proposer. A missing operand names that flag; using either ceiling without the selection produces `--proposal-max-calls and --proposal-max-tokens require --candidate-proposer skilled-proposer`; the legacy-engine error names the individual flag. Integer and positive-value validation happens in the connector after launch and names `ORIZU_PROPOSAL_MAX_CALLS` or `ORIZU_PROPOSAL_MAX_TOKENS`.
|
|
816
848
|
|
|
817
849
|
The call ceiling counts each transport-bearing proposer bridge call after its success or failure is recorded; provider retries inside one bridge call remain one counted call. The token ceiling uses accumulated provider-derived `total_tokens`, including recorded usage from unsuccessful completions. Both ceilings stop new proposal work at the next safe boundary, so an in-flight call can meet or pass a ceiling. They govern only the skilled proposer and remain independent of overall GEPA controls such as `--max-metric-calls`; set at least one proposal ceiling and one overall GEPA ceiling, then start the run only when both limits match the intended experiment.
|
|
@@ -854,10 +886,11 @@ orizu --local optimizations export <optimization-run-id> \
|
|
|
854
886
|
|
|
855
887
|
Use export when the local log is unavailable, the run happened on another machine, or a coding agent needs a portable single JSON artifact. The export fetches all optimization events, derives seed vs best, Pareto frontier, score-over-time, candidates, iterations, minibatch rows, and validation rows, and rehydrates row inputs from the dataset version artifact when possible. Server events may not contain row snapshots or reflection prompts unless the run used `--log-row-snapshots`; reflection responses are included for bundled `run-gepa` runs.
|
|
856
888
|
|
|
857
|
-
Both diff-comment read routes (`/api/optimization-runs/<run-id>/diff-comments`
|
|
858
|
-
`/api/prompts/<prompt-id>/diff-comments`) return `componentKey` for
|
|
859
|
-
comments. Preserve it on writes and use it in grouping keys:
|
|
860
|
-
from different instruction-set components are distinct
|
|
889
|
+
Both diff-comment read routes (`/api/optimization-runs/<run-id>/diff-comments`
|
|
890
|
+
and `/api/prompts/<prompt-id>/diff-comments`) return `componentKey` for
|
|
891
|
+
multi-component comments. Preserve it on writes and use it in grouping keys:
|
|
892
|
+
equal line ranges from different instruction-set components are distinct
|
|
893
|
+
comments.
|
|
861
894
|
|
|
862
895
|
The v1 export keeps the run row's `best_candidate_id` in
|
|
863
896
|
`summary.bestCandidateId` when event derivation rejects that identifier as an
|
|
@@ -866,20 +899,46 @@ identifier. If neither source names a best candidate, the field is `null`.
|
|
|
866
899
|
|
|
867
900
|
Lifecycle controls:
|
|
868
901
|
|
|
902
|
+
Agent-run pause and resume controls:
|
|
903
|
+
|
|
869
904
|
```bash
|
|
870
905
|
orizu --local optimizations pause <optimization-run-id> --reason "inspect candidate"
|
|
871
906
|
orizu --local optimizations resume <optimization-run-id>
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
907
|
+
```
|
|
908
|
+
|
|
909
|
+
**Multi-component instruction-set run.** The agent materializes the candidate
|
|
910
|
+
without a label, then finishes without putting the returned profile version ID
|
|
911
|
+
in the prompt-only result field:
|
|
912
|
+
|
|
913
|
+
```bash
|
|
914
|
+
orizu --local optimizations promote <optimization-run-id> --candidate <candidate-id>
|
|
915
|
+
orizu --local optimizations finish <optimization-run-id> --best-candidate <candidate-id> --report-file ./reports/<optimization-run-id>.md
|
|
916
|
+
```
|
|
917
|
+
|
|
918
|
+
**Plain-prompt run.** A human curator runs the promotion as `orizu --local optimizations promote <optimization-run-id> --candidate <candidate-id> --label production`; this is the one allowed live promotion.
|
|
919
|
+
|
|
920
|
+
After the human returns `promptVersionId`, the agent records it on finish:
|
|
921
|
+
|
|
922
|
+
```bash
|
|
923
|
+
orizu --local optimizations finish <optimization-run-id> --best-candidate <candidate-id> --result-prompt-version <prompt-version-id> --report-file ./reports/<optimization-run-id>.md
|
|
924
|
+
```
|
|
925
|
+
|
|
926
|
+
**Set-of-one instruction-set run.** The agent must never call `optimizations promote`; follow `optimization-reports.md`'s provenance
|
|
927
|
+
stop/doctrine and finish without a result:
|
|
928
|
+
|
|
929
|
+
```bash
|
|
930
|
+
orizu --local optimizations finish <optimization-run-id> --best-candidate <candidate-id> --report-file ./reports/<optimization-run-id>.md
|
|
931
|
+
```
|
|
932
|
+
|
|
933
|
+
Agent-run failure and cancellation controls:
|
|
934
|
+
|
|
935
|
+
```bash
|
|
877
936
|
orizu --local optimizations fail <optimization-run-id> --reason "provider outage" --report-file ./reports/<optimization-run-id>.md
|
|
878
937
|
orizu --local optimizations cancel <optimization-run-id> --reason "user stopped" --report "## Cancelled\n\nStopped after manual inspection."
|
|
879
938
|
```
|
|
880
939
|
|
|
881
940
|
`pause` and `cancel` store `metadata.reason`. `fail` stores `metadata.failure_reason`.
|
|
882
|
-
`finish` marks the run `succeeded
|
|
941
|
+
`finish` marks the run `succeeded` and attaches its report; it does not authorize promotion. The three subject branches above are normative; see `optimization-reports.md` for seed-selected and no-valid-candidate branches. Finish the report first, then obtain the human decision before materialization or a serving-pointer move.
|
|
883
942
|
Use `--report-file <path>` or `--report <markdown|@file>` on `finish`, `fail`, or `cancel` to attach the markdown report shown in the optimization detail Report tab. Prefer generating this from the local GEPA logs (`result.json`, `evaluations.jsonl`, `reflections.jsonl`, and `events.jsonl`) after the run ends. Report structure and interpretation rules: `optimization-reports.md`.
|
|
884
943
|
|
|
885
944
|
## Optimization Event Logging
|
|
@@ -966,8 +1025,7 @@ curl -sS -X POST "$ORIZU_API_URL/api/cli/optimization-runs/$ORIZU_RUN_ID/promote
|
|
|
966
1025
|
"temperature": 0,
|
|
967
1026
|
"max_tokens": 4096
|
|
968
1027
|
},
|
|
969
|
-
"runnerVersionId": "runner-version-uuid"
|
|
970
|
-
"label": "production"
|
|
1028
|
+
"runnerVersionId": "runner-version-uuid"
|
|
971
1029
|
}
|
|
972
1030
|
```
|
|
973
1031
|
|
|
@@ -981,16 +1039,26 @@ Response:
|
|
|
981
1039
|
|
|
982
1040
|
## End-to-End Flow
|
|
983
1041
|
|
|
1042
|
+
Use a JSON array for `dataset.json` in this piped form. JSONL uploads use the
|
|
1043
|
+
chunked uploader, which writes progress lines to stdout before its final JSON
|
|
1044
|
+
document even with `--json`, so piping that stdout directly to `jq` is invalid.
|
|
1045
|
+
|
|
984
1046
|
```bash
|
|
985
1047
|
eval "$(orizu --local env --project hip/judge-optimization)"
|
|
986
1048
|
|
|
987
|
-
DATASET_VERSION_ID="$(orizu --local datasets
|
|
988
|
-
SPLIT_SET_ID="$(orizu --local datasets splits create "$DATASET_VERSION_ID" --
|
|
1049
|
+
DATASET_VERSION_ID="$(orizu --local datasets upload --file ./dataset.json --project hip/judge-optimization --name hip-note-judge-labels --readme-file ./dataset-readme.md --json | jq -r .dataset_version_id)"
|
|
1050
|
+
SPLIT_SET_ID="$(orizu --local datasets splits create "$DATASET_VERSION_ID" --from-file ./split.json --json | jq -r .split_set_id)"
|
|
989
1051
|
|
|
990
|
-
RUNNER_VERSION_ID="$(orizu --local runners push ./runner --project hip/judge-optimization --name hip-note-judge-runner --
|
|
1052
|
+
RUNNER_VERSION_ID="$(orizu --local runners push ./runner --project hip/judge-optimization --name hip-note-judge-runner --json | jq -r .runner_version_id)"
|
|
991
1053
|
JUDGE_VERSION_ID="$(orizu --local judges push ./judge --project hip/judge-optimization --runner-version "$RUNNER_VERSION_ID" --json | jq -r .prompt_version_id)"
|
|
992
|
-
SCORER_VERSION_ID="$(orizu --local scorers register --project hip/judge-optimization --name hip-note-judge-score --manifest ./scorer.manifest.json --prompt-version "$JUDGE_VERSION_ID" --runner-version "$RUNNER_VERSION_ID" --
|
|
993
|
-
|
|
1054
|
+
SCORER_VERSION_ID="$(orizu --local scorers register --project hip/judge-optimization --name hip-note-judge-score --manifest ./scorer.manifest.json --prompt-version "$JUDGE_VERSION_ID" --runner-version "$RUNNER_VERSION_ID" --json | jq -r .scorer_version_id)"
|
|
1055
|
+
```
|
|
1056
|
+
|
|
1057
|
+
Human curator hand-off through the [Authority map](../SKILL.md#authority-map) (do not run as the agent): no standalone runner-label command exists, so the human first runs `orizu --local runners pull hip-note-judge-runner --project hip/judge-optimization --version "$RUNNER_VERSION_ID" --out ./runner-default-handoff`, leaves that fresh exact-version directory unchanged, then runs `orizu --local runners push ./runner-default-handoff --project hip/judge-optimization --name hip-note-judge-runner --label default --json`. The human curator then runs `orizu --local scorers labels set hip-note-judge-score production --version "$SCORER_VERSION_ID" --project hip/judge-optimization --json`.
|
|
1058
|
+
The agent continues with an unlabeled optimizer version:
|
|
1059
|
+
|
|
1060
|
+
```bash
|
|
1061
|
+
OPTIMIZER_VERSION_ID="$(orizu --local optimizers push ./optimizer --project hip/judge-optimization --name hip-gepa-optimizer --json | jq -r .optimizer_version_id)"
|
|
994
1062
|
OPTIMIZATION_RUN_ID="$(orizu --local optimizations start --project hip/judge-optimization --optimizer-version "$OPTIMIZER_VERSION_ID" --prompt-version "$JUDGE_VERSION_ID" --selection-scorer "$SCORER_VERSION_ID" --reflection-scorer "$SCORER_VERSION_ID" --dataset-version "$DATASET_VERSION_ID" --split-set "$SPLIT_SET_ID" --json | jq -r .optimization_run_id)"
|
|
995
1063
|
export ORIZU_RUN_ID="$OPTIMIZATION_RUN_ID"
|
|
996
1064
|
|
|
@@ -1019,11 +1087,11 @@ orizu --local scores submit ./judge-results.jsonl \
|
|
|
1019
1087
|
--split-set "$SPLIT_SET_ID" \
|
|
1020
1088
|
--split validation
|
|
1021
1089
|
|
|
1022
|
-
# After the local optimizer has logged events and
|
|
1090
|
+
# After the local optimizer has logged events: finish and attach the report, obtain human acceptance, then hand the one-shot promotion to a human curator.
|
|
1023
1091
|
orizu --local optimizations finish "$OPTIMIZATION_RUN_ID" \
|
|
1024
1092
|
--best-score 0.82 \
|
|
1025
1093
|
--best-candidate candidate-7 \
|
|
1026
|
-
--
|
|
1094
|
+
--report-file "./reports/$OPTIMIZATION_RUN_ID.md"
|
|
1027
1095
|
```
|
|
1028
1096
|
|
|
1029
1097
|
## Notes And Limits
|