orizu 0.5.21 → 0.5.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -15
- package/dist/cleanup.d.ts +6 -0
- package/dist/cleanup.d.ts.map +1 -0
- package/dist/cleanup.js +27 -0
- package/dist/gepa-engine-dispatch.d.ts.map +1 -1
- package/dist/gepa-engine-dispatch.js +20 -1
- package/dist/gepa-python-paths.d.ts.map +1 -1
- package/dist/gepa-python-paths.js +25 -6
- package/dist/help.d.ts.map +1 -1
- package/dist/help.js +37 -15
- package/dist/hosted-commands.d.ts +13 -0
- package/dist/hosted-commands.d.ts.map +1 -0
- package/dist/hosted-commands.js +10 -0
- package/dist/hosted-gepa-cli.d.ts +11 -0
- package/dist/hosted-gepa-cli.d.ts.map +1 -0
- package/dist/hosted-gepa-cli.js +110 -0
- package/dist/hosted-gepa-numeric-values.d.ts +3 -0
- package/dist/hosted-gepa-numeric-values.d.ts.map +1 -0
- package/dist/hosted-gepa-numeric-values.js +49 -0
- package/dist/hosted-provider-settings.d.ts +3 -0
- package/dist/hosted-provider-settings.d.ts.map +1 -0
- package/dist/hosted-provider-settings.js +69 -0
- package/dist/hosted-session-cli.d.ts.map +1 -1
- package/dist/hosted-session-cli.js +6 -1
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +72 -74
- package/dist/instruction-set-loader.d.ts.map +1 -1
- package/dist/instruction-set-loader.js +57 -5
- package/dist/instruction-sets-cli.d.ts +5 -2
- package/dist/instruction-sets-cli.d.ts.map +1 -1
- package/dist/instruction-sets-cli.js +134 -24
- package/dist/optimizations-promote-cli.d.ts +3 -0
- package/dist/optimizations-promote-cli.d.ts.map +1 -1
- package/dist/optimizations-promote-cli.js +75 -7
- package/dist/runner-dir-verify.d.ts.map +1 -1
- package/dist/runner-dir-verify.js +4 -7
- package/dist/skilled-proposer-bake-report.d.ts +2 -0
- package/dist/skilled-proposer-bake-report.d.ts.map +1 -0
- package/dist/skilled-proposer-bake-report.js +1 -0
- package/dist/skilled-proposer-config.d.ts +2 -0
- package/dist/skilled-proposer-config.d.ts.map +1 -0
- package/dist/skilled-proposer-config.js +212 -0
- package/dist/skilled-proposer-launch.d.ts +26 -0
- package/dist/skilled-proposer-launch.d.ts.map +1 -1
- package/dist/skilled-proposer-launch.js +103 -6
- package/package.json +1 -1
- package/scripts/vendor-gepa-python.mjs +81 -12
- package/src/cleanup.ts +37 -0
- package/src/gepa-engine-dispatch.ts +21 -1
- package/src/gepa-python-paths.ts +24 -6
- package/src/help.ts +37 -15
- package/src/hosted-commands.ts +4 -0
- package/src/hosted-gepa-cli.ts +107 -0
- package/src/hosted-gepa-numeric-values.ts +60 -0
- package/src/hosted-provider-settings.ts +69 -0
- package/src/hosted-session-cli.ts +4 -1
- package/src/http.ts +8 -0
- package/src/index.ts +112 -89
- package/src/instruction-set-loader.ts +55 -5
- package/src/instruction-sets-cli.ts +173 -19
- package/src/optimizations-promote-cli.ts +85 -7
- package/src/runner-dir-verify.ts +11 -6
- package/src/skilled-proposer-bake-report.ts +1 -0
- package/src/skilled-proposer-config.ts +191 -0
- package/src/skilled-proposer-launch.ts +96 -7
- package/src/skilled-proposer-venv-manager.mjs +44 -27
- package/vendor/orizu-gepa/src/orizu_gepa_connector/engine.py +36 -3
- package/vendor/orizu-gepa/src/orizu_gepa_connector/instruction_set_loader.py +31 -3
- package/vendor/orizu-gepa/src/orizu_gepa_connector/runtime.py +7 -2
- package/vendor/orizu-gepa/src/orizu_gepa_connector/skilled_proposer_bridge.py +354 -5
- package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py +1 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/client.py +15 -0
- package/vendor/skills/orizu-cli/SKILL.md +94 -168
- package/vendor/skills/orizu-cli/references/assess-and-plan.md +107 -0
- package/vendor/skills/orizu-cli/references/building-judges.md +442 -180
- package/vendor/skills/orizu-cli/references/cli-reference.md +69 -39
- package/vendor/skills/orizu-cli/references/dataset-design.md +100 -0
- package/vendor/skills/orizu-cli/references/eval-strategy.md +64 -0
- package/vendor/skills/orizu-cli/references/instructions-after-prompts.md +115 -0
- package/vendor/skills/orizu-cli/references/migrate-existing-gepa-setup.md +150 -82
- package/vendor/skills/orizu-cli/references/optimization-reports.md +196 -104
- package/vendor/skills/orizu-cli/references/optimization-with-gepa.md +57 -37
- package/vendor/skills/orizu-cli/references/primer.md +43 -18
- package/vendor/skills/orizu-cli/references/prompt-control-plane.md +199 -131
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ Orizu is a platform that helps you build continually learning agents and other L
|
|
|
9
9
|
There are 3 key pillars to Orizu:
|
|
10
10
|
1. **Human evaluation:** We consider human evaluation to be the gold standard, particularly when the humans involved are people with deep context and understanding of the output being evaluated. This is often the cross-functional team behind a product (product managers, engineers, designers, content writers, researchers, etc) or external experts (lawyers, engineers, writers, etc). Any good evaluation system relies on the presence of such high quality judgment to compare to. We provide tools that simplify the process of collecting the feedback from this audience.
|
|
11
11
|
2. **Auto evaluation:** Ultimately, human experts can only review so much. As your data scales, you need to encode their knowledge, their opinions, their taste and judgment into auto-evaluations. We automate this for you using the data we help you collect above.
|
|
12
|
-
3. **Optimization:** We turn your evals into an asset for you by building an automated hill-climbing workflow. We've started with
|
|
12
|
+
3. **Optimization:** We turn your evals into an asset for you by building an automated hill-climbing workflow. We've started with instruction optimization because multiple studies and our own experience with our customers has shown strong measurable lifts with even relatively small datasets. Over time, we'll introduce additional capabilities, including optimizing your skills and other contextual information, and even model weights if your model provider allows finetuning.
|
|
13
13
|
|
|
14
14
|
## CLI and Agent Skills
|
|
15
15
|
|
|
@@ -28,24 +28,52 @@ By default, the CLI talks to `https://orizu.ai`.
|
|
|
28
28
|
## Updating bundled GEPA
|
|
29
29
|
|
|
30
30
|
The CLI ships a verified, source-only copy of official GEPA so optimization
|
|
31
|
-
users never need `pip` or a network connection.
|
|
32
|
-
|
|
33
|
-
`scripts/vendor-gepa-python.mjs`, then re-runs the refresh command with the
|
|
34
|
-
matching downloaded wheel:
|
|
31
|
+
users never need `pip` or a network connection. From the repository root,
|
|
32
|
+
derive both mandatory pins independently from the downloaded wheel:
|
|
35
33
|
|
|
36
34
|
```bash
|
|
37
|
-
|
|
35
|
+
python3 - /path/to/gepa-<version>-py3-none-any.whl <version> <<'PY'
|
|
36
|
+
import hashlib, json, sys, zipfile
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
|
|
39
|
+
wheel, version = Path(sys.argv[1]), sys.argv[2]
|
|
40
|
+
license_member = f'gepa-{version}.dist-info/licenses/LICENSE'
|
|
41
|
+
files = {}
|
|
42
|
+
with zipfile.ZipFile(wheel) as archive:
|
|
43
|
+
for member in archive.infolist():
|
|
44
|
+
if member.is_dir():
|
|
45
|
+
continue
|
|
46
|
+
if member.filename.startswith('gepa/'):
|
|
47
|
+
path = f'src/{member.filename}'
|
|
48
|
+
elif member.filename == license_member:
|
|
49
|
+
path = 'LICENSE'
|
|
50
|
+
else:
|
|
51
|
+
continue
|
|
52
|
+
files[path] = hashlib.sha256(archive.read(member)).hexdigest()
|
|
53
|
+
entries = sorted(files.items())
|
|
54
|
+
print('GEPA_WHEEL_SHA256=' + hashlib.sha256(wheel.read_bytes()).hexdigest())
|
|
55
|
+
print('GEPA_TREE_SHA256=' + hashlib.sha256(json.dumps(entries, separators=(',', ':')).encode()).hexdigest())
|
|
56
|
+
PY
|
|
38
57
|
```
|
|
39
58
|
|
|
40
|
-
|
|
41
|
-
`
|
|
42
|
-
|
|
59
|
+
Update `GEPA_VERSION`, `GEPA_RELEASE_COMMIT`, `GEPA_WHEEL_FILENAME`,
|
|
60
|
+
`GEPA_WHEEL_SHA256`, `GEPA_TREE_SHA256`, and the extractor's license member in
|
|
61
|
+
`packages/cli/scripts/vendor-gepa-python.mjs`. Both SHA-256 constants must
|
|
62
|
+
change together before running the matching refresh command:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
node packages/cli/scripts/vendor-gepa-python.mjs --refresh --archive /path/to/gepa-<version>-py3-none-any.whl
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Commit the refreshed `packages/cli/gepa-python-source.zip`, then run
|
|
69
|
+
`bun test test/cli-gepa-packaging.test.ts test/cli/provision-snapshot.test.ts`.
|
|
70
|
+
These gates cover both pins. `npm pack` never downloads GEPA.
|
|
43
71
|
|
|
44
72
|
For a development checkout, materialize that same verified source before local
|
|
45
73
|
work that imports GEPA:
|
|
46
74
|
|
|
47
75
|
```bash
|
|
48
|
-
node scripts/vendor-gepa-python.mjs
|
|
76
|
+
node packages/cli/scripts/vendor-gepa-python.mjs
|
|
49
77
|
```
|
|
50
78
|
|
|
51
79
|
## Install
|
|
@@ -109,6 +137,9 @@ Agents are first-class users of this CLI: **every command supports `--json`**
|
|
|
109
137
|
(as a prefix, `orizu --json teams list`, or a trailing flag) and emits a single
|
|
110
138
|
machine-readable JSON document instead of formatted text. Discover the full
|
|
111
139
|
command surface, including global options, with `orizu capabilities --json`.
|
|
140
|
+
Hosted agents may read and sync instruction sets and prepare complete manifests. A local coding agent using the user's token runs instruction create, push, shape,
|
|
141
|
+
archive, and restore itself after plan ratification. It gathers only read-only pointer-resolution and binding evidence; a human curator executes every production/default pointer command on every surface.
|
|
142
|
+
In a hosted session, a human curator using the local CLI must run both categories.
|
|
112
143
|
|
|
113
144
|
## Authentication
|
|
114
145
|
|
|
@@ -186,19 +217,19 @@ orizu logout
|
|
|
186
217
|
<td><code>orizu projects create --name "Support QA" --team ops-eval</code></td>
|
|
187
218
|
</tr>
|
|
188
219
|
<tr>
|
|
189
|
-
<td colspan="2"><strong>
|
|
220
|
+
<td colspan="2"><strong>Instruction sets and prompt reads</strong></td>
|
|
190
221
|
</tr>
|
|
191
222
|
<tr>
|
|
192
223
|
<td>List prompts</td>
|
|
193
224
|
<td><code>orizu prompts list --project ops-eval/support-qa [--status active|archived|all]</code></td>
|
|
194
225
|
</tr>
|
|
195
226
|
<tr>
|
|
196
|
-
<td>Archive or restore
|
|
197
|
-
<td><code>orizu
|
|
227
|
+
<td>Archive or restore an instruction set (local agent or human; hosted human only)</td>
|
|
228
|
+
<td><code>orizu instructions archive <slug-or-exact-name> --project ops-eval/support-qa</code><br><code>orizu instructions restore <slug-or-exact-name> --project ops-eval/support-qa</code></td>
|
|
198
229
|
</tr>
|
|
199
230
|
<tr>
|
|
200
|
-
<td>Push
|
|
201
|
-
<td><code>orizu
|
|
231
|
+
<td>Push an instruction-set manifest (local agent or human; hosted human only)</td>
|
|
232
|
+
<td><code>orizu instructions push ./orizu.instruction-set.json --project ops-eval/support-qa --set <slug-or-exact-name></code></td>
|
|
202
233
|
</tr>
|
|
203
234
|
<tr>
|
|
204
235
|
<td>Upload a report for an existing prompt version</td>
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type Cleanup = () => void;
|
|
2
|
+
/** Attempt every disposer even when an earlier cleanup fails. */
|
|
3
|
+
export declare function cleanupAll(cleanups: readonly Cleanup[], message: string): void;
|
|
4
|
+
/** Preserve the operation failure together with every cleanup failure. */
|
|
5
|
+
export declare function throwWithCleanup(operationError: unknown, cleanups: readonly Cleanup[], message: string): never;
|
|
6
|
+
//# sourceMappingURL=cleanup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cleanup.d.ts","sourceRoot":"","sources":["../src/cleanup.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,OAAO,GAAG,MAAM,IAAI,CAAA;AAchC,iEAAiE;AACjE,wBAAgB,UAAU,CACxB,QAAQ,EAAE,SAAS,OAAO,EAAE,EAC5B,OAAO,EAAE,MAAM,GACd,IAAI,CAKN;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAC9B,cAAc,EAAE,OAAO,EACvB,QAAQ,EAAE,SAAS,OAAO,EAAE,EAC5B,OAAO,EAAE,MAAM,GACd,KAAK,CAMP"}
|
package/dist/cleanup.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
function collectCleanupFailures(cleanups) {
|
|
2
|
+
const failures = [];
|
|
3
|
+
for (const cleanup of cleanups) {
|
|
4
|
+
try {
|
|
5
|
+
cleanup();
|
|
6
|
+
}
|
|
7
|
+
catch (error) {
|
|
8
|
+
failures.push(error);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
return failures;
|
|
12
|
+
}
|
|
13
|
+
/** Attempt every disposer even when an earlier cleanup fails. */
|
|
14
|
+
export function cleanupAll(cleanups, message) {
|
|
15
|
+
const failures = collectCleanupFailures(cleanups);
|
|
16
|
+
if (failures.length > 0) {
|
|
17
|
+
throw new AggregateError(failures, message);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** Preserve the operation failure together with every cleanup failure. */
|
|
21
|
+
export function throwWithCleanup(operationError, cleanups, message) {
|
|
22
|
+
const cleanupFailures = collectCleanupFailures(cleanups);
|
|
23
|
+
if (cleanupFailures.length > 0) {
|
|
24
|
+
throw new AggregateError([operationError, ...cleanupFailures], message);
|
|
25
|
+
}
|
|
26
|
+
throw operationError;
|
|
27
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gepa-engine-dispatch.d.ts","sourceRoot":"","sources":["../src/gepa-engine-dispatch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"gepa-engine-dispatch.d.ts","sourceRoot":"","sources":["../src/gepa-engine-dispatch.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,QAAQ,CAAA;AAE9C,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,UAAU,CAAA;IAClB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,WAAW,EAAE,MAAM,CAAC,UAAU,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;CACf;AAyUD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,UAAU,GAAG,kBAAkB,CAmBtH"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from 'fs';
|
|
2
|
+
import { normalizedSkilledProposerConfig } from './skilled-proposer-config.js';
|
|
2
3
|
const VALUE_OPTIONS = [
|
|
3
4
|
{ flag: '--optimizer-version-id', environment: 'ORIZU_OPTIMIZER_VERSION_ID' },
|
|
4
5
|
{ flag: '--candidate-version-id', environment: 'ORIZU_PROMPT_VERSION_ID' },
|
|
@@ -148,6 +149,7 @@ const CONTROLLED_ENVIRONMENT_NAMES = new Set([
|
|
|
148
149
|
// translation prevents an inherited environment from selecting a proposer
|
|
149
150
|
// on an otherwise byte-for-byte normal official-GEPA launch.
|
|
150
151
|
'ORIZU_CANDIDATE_PROPOSER',
|
|
152
|
+
'ORIZU_SKILLED_PROPOSER_CONFIG',
|
|
151
153
|
'ORIZU_COMPONENT_SELECTOR',
|
|
152
154
|
]);
|
|
153
155
|
function optionValueOrThrow(args, index, flag, allowEmpty = false) {
|
|
@@ -173,6 +175,7 @@ function translateOfficialOptions(args, project, environment) {
|
|
|
173
175
|
}
|
|
174
176
|
connectorEnvironment.ORIZU_PROJECT = project;
|
|
175
177
|
let metadata;
|
|
178
|
+
let candidateProposerConfig;
|
|
176
179
|
let hasInstructionSet = false;
|
|
177
180
|
let hasCandidateVersion = false;
|
|
178
181
|
const budgetFlags = [];
|
|
@@ -205,6 +208,12 @@ function translateOfficialOptions(args, project, environment) {
|
|
|
205
208
|
index = parsed.nextIndex;
|
|
206
209
|
continue;
|
|
207
210
|
}
|
|
211
|
+
if (flag === '--candidate-proposer-config') {
|
|
212
|
+
const parsed = optionValueOrThrow(args, index, flag);
|
|
213
|
+
candidateProposerConfig = parsed.value;
|
|
214
|
+
index = parsed.nextIndex;
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
208
217
|
const option = VALUE_OPTIONS_BY_FLAG.get(flag);
|
|
209
218
|
if (option) {
|
|
210
219
|
const parsed = optionValueOrThrow(args, index, flag, option.allowEmpty);
|
|
@@ -266,6 +275,16 @@ function translateOfficialOptions(args, project, environment) {
|
|
|
266
275
|
&& connectorEnvironment.ORIZU_CANDIDATE_PROPOSER !== 'skilled-proposer') {
|
|
267
276
|
throw new Error('--proposal-max-calls and --proposal-max-tokens require --candidate-proposer skilled-proposer');
|
|
268
277
|
}
|
|
278
|
+
if (candidateProposerConfig !== undefined) {
|
|
279
|
+
if (connectorEnvironment.ORIZU_CANDIDATE_PROPOSER !== 'skilled-proposer') {
|
|
280
|
+
throw new Error('--candidate-proposer-config requires --candidate-proposer skilled-proposer');
|
|
281
|
+
}
|
|
282
|
+
connectorEnvironment.ORIZU_SKILLED_PROPOSER_CONFIG = normalizedSkilledProposerConfig(candidateProposerConfig);
|
|
283
|
+
}
|
|
284
|
+
if (connectorEnvironment.ORIZU_CANDIDATE_PROPOSER === 'skilled-proposer'
|
|
285
|
+
&& connectorEnvironment.ORIZU_REFLECTION_PROMPT_TEMPLATE !== undefined) {
|
|
286
|
+
throw new Error('--reflection-prompt-template is incompatible with --candidate-proposer skilled-proposer');
|
|
287
|
+
}
|
|
269
288
|
const reflectionModel = connectorEnvironment.ORIZU_REFLECTION_MODEL ?? 'anthropic/claude-opus-4-7';
|
|
270
289
|
if (!reflectionModel.startsWith('openai/') && connectorEnvironment.ORIZU_REFLECTION_MAX_TOKENS === undefined) {
|
|
271
290
|
throw new Error('--reflection-max-tokens is required for Anthropic reflection models');
|
|
@@ -289,7 +308,7 @@ function validatedMetadata(raw) {
|
|
|
289
308
|
export function dispatchGepaEngine(args, project, environment) {
|
|
290
309
|
const selected = removeEngine(args);
|
|
291
310
|
if (selected.engine === 'legacy') {
|
|
292
|
-
for (const flag of ['--instruction-set', '--model-config', '--component-selector', '--max-candidate-proposals', '--candidate-proposer', '--proposal-max-calls', '--proposal-max-tokens']) {
|
|
311
|
+
for (const flag of ['--instruction-set', '--model-config', '--component-selector', '--max-candidate-proposals', '--candidate-proposer', '--candidate-proposer-config', '--proposal-max-calls', '--proposal-max-tokens']) {
|
|
293
312
|
if (selected.args.some(argument => argument === flag || argument.startsWith(`${flag}=`))) {
|
|
294
313
|
throw new Error(`${flag} is supported by the official GEPA engine only`);
|
|
295
314
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gepa-python-paths.d.ts","sourceRoot":"","sources":["../src/gepa-python-paths.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"gepa-python-paths.d.ts","sourceRoot":"","sources":["../src/gepa-python-paths.ts"],"names":[],"mappings":"AAaA,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAoB,GACtD,MAAM,GAAG,IAAI,CAEf;AAED,wBAAgB,0BAA0B,IAAI,MAAM,GAAG,IAAI,CAM1D;AAED,wBAAgB,mCAAmC,IAAI,MAAM,GAAG,IAAI,CAMnE;AAED,wBAAgB,6BAA6B,IAAI,MAAM,GAAG,IAAI,CA0B7D;AAED,wBAAgB,wBAAwB,CAAC,kBAAkB,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CASzF"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { existsSync } from 'fs';
|
|
2
|
+
import { execFileSync } from 'child_process';
|
|
2
3
|
import { fileURLToPath } from 'url';
|
|
3
4
|
import { isOrizuSourceCheckout } from '../scripts/orizu-source-checkout.mjs';
|
|
4
5
|
function prefersLiveSource() {
|
|
@@ -23,13 +24,31 @@ export function bundledOrizuGepaConnectorPythonPath() {
|
|
|
23
24
|
return resolveGepaPythonPath(candidates);
|
|
24
25
|
}
|
|
25
26
|
export function bundledOfficialGepaPythonPath() {
|
|
26
|
-
const sourceArchive = fileURLToPath(new URL('../gepa-python-source.zip', import.meta.url));
|
|
27
27
|
const vendor = fileURLToPath(new URL('../vendor/gepa-python/src', import.meta.url));
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
28
|
+
if (prefersLiveSource()) {
|
|
29
|
+
const requiredVendorFiles = [
|
|
30
|
+
fileURLToPath(new URL('../vendor/gepa-python/manifest.json', import.meta.url)),
|
|
31
|
+
fileURLToPath(new URL('../vendor/gepa-python/src/gepa/__init__.py', import.meta.url)),
|
|
32
|
+
fileURLToPath(new URL('../vendor/gepa-python/src/gepa/adapters/optimize_anything_adapter/optimize_anything_adapter.py', import.meta.url)),
|
|
33
|
+
];
|
|
34
|
+
if (!requiredVendorFiles.every(existsSync)) {
|
|
35
|
+
const vendoringScript = fileURLToPath(new URL('../scripts/vendor-gepa-python.mjs', import.meta.url));
|
|
36
|
+
try {
|
|
37
|
+
execFileSync(process.execPath, [vendoringScript], { stdio: 'pipe' });
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
const details = error instanceof Error ? error.message : String(error);
|
|
41
|
+
throw new Error(`Unable to materialize bundled official GEPA: ${details}`);
|
|
42
|
+
}
|
|
43
|
+
if (!requiredVendorFiles.every(existsSync)) {
|
|
44
|
+
throw new Error('Unable to materialize bundled official GEPA: extracted tree is incomplete');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// zipimport cannot resolve GEPA's namespace-package adapters on every
|
|
49
|
+
// supported Python minor. Both source and published CLIs execute only the
|
|
50
|
+
// verified tree materialized from the pinned wheel.
|
|
51
|
+
return resolveGepaPythonPath([vendor]);
|
|
33
52
|
}
|
|
34
53
|
export function getGepaPythonPathEntries(existingPythonPath) {
|
|
35
54
|
const pythonPathEntries = [
|
package/dist/help.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"help.d.ts","sourceRoot":"","sources":["../src/help.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,CAAA;IACpB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;CACpB;AAoCD,eAAO,MAAM,YAAY,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"help.d.ts","sourceRoot":"","sources":["../src/help.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,CAAA;IACpB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;CACpB;AAoCD,eAAO,MAAM,YAAY,EAAE,aAAa,EA6nCvC,CAAA;AAUD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,aAAa,GAAG,SAAS,CAMxE;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,aAAa,EAAE,CAWrE;AAOD,wBAAgB,cAAc,IAAI,MAAM,CA0CvC;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAoBtD;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,aAAa,GAAG,MAAM,CA4B5D;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAgBxD;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;EAoB9C"}
|
package/dist/help.js
CHANGED
|
@@ -64,6 +64,40 @@ export const COMMAND_DOCS = [
|
|
|
64
64
|
},
|
|
65
65
|
{ path: ['instructions', 'archive'], aliases: [['instruction-sets', 'archive']], usage: 'orizu instructions archive <slug-or-exact-name> --project <team/project> [--json]', summary: 'Archive an instruction set without changing resolution or sync.', group: 'Instruction sets', options: [{ name: '--project <team/project>', help: 'Project slug.', required: true }, { name: '--json', help: 'Print JSON.' }], examples: ['orizu instructions archive planner --project core/evals'] },
|
|
66
66
|
{ path: ['instructions', 'restore'], aliases: [['instruction-sets', 'restore']], usage: 'orizu instructions restore <slug-or-exact-name> --project <team/project> [--json]', summary: 'Restore an archived instruction set.', group: 'Instruction sets', options: [{ name: '--project <team/project>', help: 'Project slug.', required: true }, { name: '--json', help: 'Print JSON.' }], examples: ['orizu instructions restore planner --project core/evals'] },
|
|
67
|
+
{
|
|
68
|
+
path: ['instructions', 'scorers', 'set-headline'],
|
|
69
|
+
aliases: [['instruction-sets', 'scorers', 'set-headline']],
|
|
70
|
+
usage: 'orizu instructions scorers set-headline <set> --key <component-key> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
|
|
71
|
+
summary: 'Bind the headline scorer for one instruction-set component.',
|
|
72
|
+
group: 'Instruction sets',
|
|
73
|
+
options: [
|
|
74
|
+
{ name: '--key <component-key>', help: 'Instruction-set component key.', required: true },
|
|
75
|
+
{ name: '--scorer-version <id>', help: 'Scorer version to bind.', required: true },
|
|
76
|
+
{ name: '--dataset-version <id>', help: 'Default dataset version.' },
|
|
77
|
+
{ name: '--split-set <id>', help: 'Default dataset split set.' },
|
|
78
|
+
{ name: '--split <name>', help: 'Default split name.' },
|
|
79
|
+
{ name: '--project <team/project>', help: 'Project slug.' },
|
|
80
|
+
{ name: '--json', help: 'Print JSON.' },
|
|
81
|
+
],
|
|
82
|
+
examples: ['orizu instructions scorers set-headline planner --key system --scorer-version <id> --project core/evals'],
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
path: ['instructions', 'scorers', 'add'],
|
|
86
|
+
aliases: [['instruction-sets', 'scorers', 'add']],
|
|
87
|
+
usage: 'orizu instructions scorers add <set> --key <component-key> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
|
|
88
|
+
summary: 'Track an additional scorer for one instruction-set component.',
|
|
89
|
+
group: 'Instruction sets',
|
|
90
|
+
options: [
|
|
91
|
+
{ name: '--key <component-key>', help: 'Instruction-set component key.', required: true },
|
|
92
|
+
{ name: '--scorer-version <id>', help: 'Scorer version to bind.', required: true },
|
|
93
|
+
{ name: '--dataset-version <id>', help: 'Default dataset version.' },
|
|
94
|
+
{ name: '--split-set <id>', help: 'Default dataset split set.' },
|
|
95
|
+
{ name: '--split <name>', help: 'Default split name.' },
|
|
96
|
+
{ name: '--project <team/project>', help: 'Project slug.' },
|
|
97
|
+
{ name: '--json', help: 'Print JSON.' },
|
|
98
|
+
],
|
|
99
|
+
examples: ['orizu instructions scorers add planner --key system --scorer-version <id> --project core/evals'],
|
|
100
|
+
},
|
|
67
101
|
{
|
|
68
102
|
path: ['login'],
|
|
69
103
|
usage: 'orizu login [--no-prompt-if-logged-in]',
|
|
@@ -94,7 +128,7 @@ export const COMMAND_DOCS = [
|
|
|
94
128
|
},
|
|
95
129
|
{
|
|
96
130
|
path: ['log'],
|
|
97
|
-
usage: 'orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json',
|
|
131
|
+
usage: 'orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json [--event-id <id>] [--event-layer <layer>] [--optimizer-family <family>] [--iteration <n>] [--candidate-id <id>] [--parent-candidate-id <id>] [--child-candidate-id <id>]',
|
|
98
132
|
summary: 'Submit an optimization event payload.',
|
|
99
133
|
group: 'Optimizations',
|
|
100
134
|
},
|
|
@@ -388,18 +422,6 @@ export const COMMAND_DOCS = [
|
|
|
388
422
|
summary: 'Move or set a non-production prompt label; use instructions profiles promote for production.',
|
|
389
423
|
group: 'Prompts and judges',
|
|
390
424
|
},
|
|
391
|
-
{
|
|
392
|
-
path: ['prompts', 'scorers', 'set-headline'],
|
|
393
|
-
usage: 'orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
|
|
394
|
-
summary: 'Bind the headline scorer for a prompt.',
|
|
395
|
-
group: 'Scorers and runners',
|
|
396
|
-
},
|
|
397
|
-
{
|
|
398
|
-
path: ['prompts', 'scorers', 'add'],
|
|
399
|
-
usage: 'orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
|
|
400
|
-
summary: 'Track an additional scorer for a prompt.',
|
|
401
|
-
group: 'Scorers and runners',
|
|
402
|
-
},
|
|
403
425
|
{
|
|
404
426
|
path: ['judges', 'list'],
|
|
405
427
|
usage: 'orizu judges list --project <team/project> [--status active|archived|all]',
|
|
@@ -553,8 +575,8 @@ export const COMMAND_DOCS = [
|
|
|
553
575
|
},
|
|
554
576
|
{
|
|
555
577
|
path: ['optimizations', 'run-gepa'],
|
|
556
|
-
usage: 'orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> (--candidate-version-id <id> | --instruction-set <name> --model-config <identity> [--component-selector round-robin|all]) --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> [--engine official|legacy] [--scorer-input-contract gepa|flat_row] [--scorer-candidate-field <row-field>] [--allow-degenerate-seed] --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--budget auto|light|medium|heavy | --max-metric-calls <n> | --max-full-evals <n> | --max-iterations <n> | --max-candidate-proposals <n>] [--num-threads auto|N] [--reflection-retry-attempts N] [--reflection-http-timeout-seconds N] [--log-dir logs]',
|
|
557
|
-
summary: 'Run official GEPA locally and stream events
|
|
578
|
+
usage: 'orizu optimizations run-gepa [--hosted [--launch-intent-id <uuid>]] --project <team/project> --optimizer-version-id <id> (--candidate-version-id <id> | --instruction-set <name> --model-config <identity> [--component-selector round-robin|all]) --runner-version-id <id> [--candidate-runner-dir <dir>] --scorer-version-id <id> --scorer-runner-version-id <id> [--scorer-runner-dir <dir>] [--engine official|legacy] [--candidate-proposer skilled-proposer [--candidate-proposer-config <@file|path>] [--proposal-max-calls <n>] [--proposal-max-tokens <n>]] [--scorer-input-contract gepa|flat_row] [--scorer-candidate-field <row-field>] [--allow-degenerate-seed] [--seed <n>] --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--minibatch-size <n>] [--candidate-selection-strategy pareto|current_best|epsilon_greedy] [--epsilon <n>] [--objective <text>] [--budget auto|light|medium|heavy | --max-metric-calls <n> | --max-full-evals <n> | --max-iterations <n> | --max-candidate-proposals <n>] [--num-threads auto|N] [--reflection-model <provider/model>] [--reflection-temperature <n>] [--reflection-prompt-template <text|@file>] [--reflection-max-tokens N] [--reflection-provider-settings <json|@file>] [--reflection-retry-attempts N] [--reflection-http-timeout-seconds N] [--skip-perfect-parent-reflection|--no-skip-perfect-parent-reflection] [--disable-evaluation-cache] [--auto-promote] [--promotion-label <label>] [--log-row-snapshots] [--metadata <json>] [--python <command>] [--log-dir logs] [--no-local-log] [--json]',
|
|
579
|
+
summary: 'Run official GEPA locally and stream events, or use --hosted for a queued Orizu launch. Hosted mode does not require --candidate-runner-dir or --scorer-runner-dir, accepts only a named --budget preset, and supports response-loss recovery with --launch-intent-id. Local runner version ids materialize registered bytes, while optional runner dirs are additionally byte-verified before execution. Pass --engine legacy only for the frozen safety hatch. Skilled-proposer is official-only, its config requires that selection, and the selection cannot be combined with --reflection-prompt-template. --instruction-set requires --model-config and cannot be combined with --candidate-version-id; round-robin updates one component-map entry per round and all updates every component. --max-candidate-proposals is official-engine-only. Validates the scorer contract on the seed before iterating; judge runners built for flat-row score runs need --scorer-input-contract flat_row (ALI-1158).',
|
|
558
580
|
group: 'Optimizations',
|
|
559
581
|
},
|
|
560
582
|
{
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
interface HostedCommandContext {
|
|
2
|
+
environment: NodeJS.ProcessEnv;
|
|
3
|
+
getArg: (name: string) => string | null;
|
|
4
|
+
json: boolean;
|
|
5
|
+
printJson: (value: Record<string, unknown>) => void;
|
|
6
|
+
printLine: (value?: string) => void;
|
|
7
|
+
printErr: (value: string) => void;
|
|
8
|
+
resolveProjectSlug: (projectArg: string | null) => Promise<string>;
|
|
9
|
+
setExitCode: (code: number) => void;
|
|
10
|
+
}
|
|
11
|
+
export declare function dispatchHostedCommands(args: string[], context: HostedCommandContext): Promise<boolean>;
|
|
12
|
+
export {};
|
|
13
|
+
//# sourceMappingURL=hosted-commands.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hosted-commands.d.ts","sourceRoot":"","sources":["../src/hosted-commands.ts"],"names":[],"mappings":"AAEA,UAAU,oBAAoB;IAAG,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC;IAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAAC,kBAAkB,EAAE,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAAC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE;AAC/V,wBAAsB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAwqB"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { runHostedGepaOptimization } from './hosted-gepa-cli.js';
|
|
2
|
+
import { hostedCommand } from './hosted-session-cli.js';
|
|
3
|
+
export async function dispatchHostedCommands(args, context) { const [command, subcommand] = args; if (command === 'optimizations' && subcommand === 'run-gepa' && args.includes('--hosted')) {
|
|
4
|
+
const project = context.getArg('--project') || await context.resolveProjectSlug(null);
|
|
5
|
+
await runHostedGepaOptimization({ args: args.slice(2), project, environment: context.environment, json: context.json, printJson: context.printJson, printLine: context.printLine });
|
|
6
|
+
return true;
|
|
7
|
+
} if (command === 'internal' || (command === 'session' && subcommand === 'start' && args.includes('--hosted'))) {
|
|
8
|
+
context.setExitCode(await hostedCommand(args, { json: context.json, print: context.printLine, printErr: context.printErr }));
|
|
9
|
+
return true;
|
|
10
|
+
} return false; }
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
interface HostedGepaOptions {
|
|
2
|
+
args: string[];
|
|
3
|
+
project: string;
|
|
4
|
+
environment: NodeJS.ProcessEnv;
|
|
5
|
+
json: boolean;
|
|
6
|
+
printJson: (value: Record<string, unknown>) => void;
|
|
7
|
+
printLine: (value: string) => void;
|
|
8
|
+
}
|
|
9
|
+
export declare function runHostedGepaOptimization(options: HostedGepaOptions): Promise<void>;
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=hosted-gepa-cli.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hosted-gepa-cli.d.ts","sourceRoot":"","sources":["../src/hosted-gepa-cli.ts"],"names":[],"mappings":"AAQA,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAC,UAAU,CAAA;IAC9B,IAAI,EAAE,OAAO,CAAA;IACb,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IACnD,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;CACnC;AAmBD,wBAAsB,yBAAyB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwEzF"}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { dispatchGepaEngine } from './gepa-engine-dispatch.js';
|
|
3
|
+
import { parseHostedGepaNumericValue } from './hosted-gepa-numeric-values.js';
|
|
4
|
+
import { authedFetch } from './http.js';
|
|
5
|
+
import { hostedProviderSettingsError } from './hosted-provider-settings.js';
|
|
6
|
+
import { parseJsonResponse } from './json-response.js';
|
|
7
|
+
function argumentValue(args, name) {
|
|
8
|
+
const index = args.indexOf(name);
|
|
9
|
+
if (index >= 0 && index + 1 < args.length)
|
|
10
|
+
return args[index + 1];
|
|
11
|
+
return args.find(argument => argument.startsWith(`${name}=`))?.slice(name.length + 1) ?? null;
|
|
12
|
+
}
|
|
13
|
+
function dispatchArgs(args) {
|
|
14
|
+
const filtered = [];
|
|
15
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
16
|
+
if (args[index] === '--hosted')
|
|
17
|
+
continue;
|
|
18
|
+
if (args[index] === '--launch-intent-id') {
|
|
19
|
+
index += 1;
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
22
|
+
if (args[index].startsWith('--launch-intent-id='))
|
|
23
|
+
continue;
|
|
24
|
+
filtered.push(args[index]);
|
|
25
|
+
}
|
|
26
|
+
return filtered;
|
|
27
|
+
}
|
|
28
|
+
export async function runHostedGepaOptimization(options) {
|
|
29
|
+
const hasLaunchIntent = options.args.some(argument => argument === '--launch-intent-id' || argument.startsWith('--launch-intent-id='));
|
|
30
|
+
const explicitLaunchIntent = argumentValue(options.args, '--launch-intent-id');
|
|
31
|
+
if (hasLaunchIntent && (!explicitLaunchIntent || explicitLaunchIntent.startsWith('-')
|
|
32
|
+
|| !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(explicitLaunchIntent))) {
|
|
33
|
+
throw new Error('--launch-intent-id must be a UUIDv4 value when provided.');
|
|
34
|
+
}
|
|
35
|
+
const args = dispatchArgs(options.args);
|
|
36
|
+
if (['--max-iterations', '--max-full-evals', '--max-metric-calls', '--max-candidate-proposals']
|
|
37
|
+
.some(flag => args.some(argument => argument === flag || argument.startsWith(`${flag}=`)))) {
|
|
38
|
+
throw new Error('Hosted optimization requires a named --budget preset; numeric budget controls are not accepted.');
|
|
39
|
+
}
|
|
40
|
+
const dispatch = dispatchGepaEngine(args, options.project, options.environment);
|
|
41
|
+
if (dispatch.engine !== 'official')
|
|
42
|
+
throw new Error('Hosted optimization requires the official GEPA engine.');
|
|
43
|
+
const env = dispatch.environment;
|
|
44
|
+
const required = [
|
|
45
|
+
['ORIZU_OPTIMIZER_VERSION_ID', '--optimizer-version-id'], ['ORIZU_RUNNER_VERSION_ID', '--runner-version-id'],
|
|
46
|
+
['ORIZU_SCORER_VERSION_ID', '--scorer-version-id'], ['ORIZU_SCORER_RUNNER_VERSION_ID', '--scorer-runner-version-id'],
|
|
47
|
+
['ORIZU_DATASET_VERSION_ID', '--dataset-version-id'], ['ORIZU_SPLIT_SET_ID', '--split-set-id'],
|
|
48
|
+
];
|
|
49
|
+
for (const [name, flag] of required)
|
|
50
|
+
if (!env[name])
|
|
51
|
+
throw new Error(`${flag} is required for hosted optimization.`);
|
|
52
|
+
const providerSettings = JSON.parse(env.ORIZU_REFLECTION_PROVIDER_SETTINGS ?? '{}');
|
|
53
|
+
const providerSettingsError = hostedProviderSettingsError(providerSettings, env.ORIZU_REFLECTION_MODEL ?? 'anthropic/', env.ORIZU_REFLECTION_TEMPERATURE);
|
|
54
|
+
if (providerSettingsError)
|
|
55
|
+
throw new Error(providerSettingsError);
|
|
56
|
+
const optional = {};
|
|
57
|
+
const fields = [
|
|
58
|
+
['candidateProposer', 'ORIZU_CANDIDATE_PROPOSER', 'string'], ['candidateProposerConfig', 'ORIZU_SKILLED_PROPOSER_CONFIG', 'json'],
|
|
59
|
+
['proposalMaxCalls', 'ORIZU_PROPOSAL_MAX_CALLS', 'number'], ['proposalMaxTokens', 'ORIZU_PROPOSAL_MAX_TOKENS', 'number'],
|
|
60
|
+
['minibatchSize', 'ORIZU_MINIBATCH_SIZE', 'number'], ['numThreads', 'ORIZU_NUM_THREADS', 'number'],
|
|
61
|
+
['candidateSelectionStrategy', 'ORIZU_CANDIDATE_SELECTION_STRATEGY', 'string'], ['epsilon', 'ORIZU_EPSILON', 'number'],
|
|
62
|
+
['reflectionModel', 'ORIZU_REFLECTION_MODEL', 'string'], ['reflectionTemperature', 'ORIZU_REFLECTION_TEMPERATURE', 'number'],
|
|
63
|
+
['reflectionMaxTokens', 'ORIZU_REFLECTION_MAX_TOKENS', 'number'], ['reflectionRetryAttempts', 'ORIZU_REFLECTION_RETRY_ATTEMPTS', 'number'],
|
|
64
|
+
['reflectionHttpTimeoutSeconds', 'ORIZU_REFLECTION_HTTP_TIMEOUT_SECONDS', 'number'],
|
|
65
|
+
['reflectionPromptTemplate', 'ORIZU_REFLECTION_PROMPT_TEMPLATE', 'string'],
|
|
66
|
+
['reflectionProviderSettings', 'ORIZU_REFLECTION_PROVIDER_SETTINGS', 'json'], ['objective', 'ORIZU_OBJECTIVE', 'string'],
|
|
67
|
+
['seed', 'ORIZU_SEED', 'number'], ['disableEvaluationCache', 'ORIZU_DISABLE_EVALUATION_CACHE', 'boolean'],
|
|
68
|
+
['allowDegenerateSeed', 'ORIZU_ALLOW_DEGENERATE_SEED', 'boolean'],
|
|
69
|
+
['skipPerfectParentReflection', 'ORIZU_SKIP_PERFECT_PARENT_REFLECTION', 'boolean'],
|
|
70
|
+
];
|
|
71
|
+
for (const [field, name, kind] of fields) {
|
|
72
|
+
if (env[name] === undefined)
|
|
73
|
+
continue;
|
|
74
|
+
optional[field] = kind === 'number' ? parseHostedGepaNumericValue(field, env[name])
|
|
75
|
+
: kind === 'json' ? JSON.parse(env[name]) : kind === 'boolean' ? env[name] === '1' : env[name];
|
|
76
|
+
}
|
|
77
|
+
const launchIntentId = explicitLaunchIntent || randomUUID();
|
|
78
|
+
const body = {
|
|
79
|
+
hosted: true, launchIntentId, optimizerVersionId: env.ORIZU_OPTIMIZER_VERSION_ID,
|
|
80
|
+
promptVersionIds: env.ORIZU_PROMPT_VERSION_ID ? [env.ORIZU_PROMPT_VERSION_ID] : [],
|
|
81
|
+
...(env.ORIZU_INSTRUCTION_SET_NAME ? { instructionSetName: env.ORIZU_INSTRUCTION_SET_NAME,
|
|
82
|
+
modelConfigIdentity: env.ORIZU_MODEL_CONFIG_IDENTITY, componentSelector: env.ORIZU_COMPONENT_SELECTOR } : {}),
|
|
83
|
+
runnerVersionId: env.ORIZU_RUNNER_VERSION_ID,
|
|
84
|
+
scorers: [{ scorerVersionId: env.ORIZU_SCORER_VERSION_ID,
|
|
85
|
+
scorerRunnerVersionId: env.ORIZU_SCORER_RUNNER_VERSION_ID, role: 'selection', scorerConfig: {} }],
|
|
86
|
+
datasetVersionId: env.ORIZU_DATASET_VERSION_ID, splitSetId: env.ORIZU_SPLIT_SET_ID,
|
|
87
|
+
trainSplitName: env.ORIZU_TRAIN_SPLIT ?? 'train', validationSplitName: env.ORIZU_VALIDATION_SPLIT ?? 'validation',
|
|
88
|
+
engine: 'official', scorerInputContract: env.ORIZU_SCORER_INPUT_CONTRACT ?? 'gepa',
|
|
89
|
+
scorerCandidateField: env.ORIZU_SCORER_CANDIDATE_FIELD ?? null, budget: env.ORIZU_BUDGET ?? 'auto', ...optional,
|
|
90
|
+
};
|
|
91
|
+
const response = await authedFetch(`/api/cli/optimization-runs?project=${encodeURIComponent(options.project)}`, {
|
|
92
|
+
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body),
|
|
93
|
+
});
|
|
94
|
+
const data = await parseJsonResponse(response, 'Hosted optimization launch');
|
|
95
|
+
if (!response.ok) {
|
|
96
|
+
if (options.json) {
|
|
97
|
+
options.printJson(data);
|
|
98
|
+
process.exitCode = 1;
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
throw new Error(JSON.stringify(data));
|
|
102
|
+
}
|
|
103
|
+
if (options.json) {
|
|
104
|
+
options.printJson(data);
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
options.printLine('Queued hosted optimization.');
|
|
108
|
+
options.printLine(`Launch intent: ${launchIntentId}`);
|
|
109
|
+
options.printLine(String(data.monitorUrl));
|
|
110
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hosted-gepa-numeric-values.d.ts","sourceRoot":"","sources":["../src/hosted-gepa-numeric-values.ts"],"names":[],"mappings":"AAyCA,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAS5F;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,IAAI,CAOrF"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
const isFiniteNumber = (value) => typeof value === 'number' && Number.isFinite(value);
|
|
2
|
+
const isInteger = (value) => isFiniteNumber(value) && Number.isInteger(value);
|
|
3
|
+
const isPositiveInteger = (value) => isInteger(value) && value > 0;
|
|
4
|
+
const rules = [
|
|
5
|
+
...[
|
|
6
|
+
['proposalMaxCalls', '--proposal-max-calls'],
|
|
7
|
+
['proposalMaxTokens', '--proposal-max-tokens'],
|
|
8
|
+
['maxIterations', '--max-iterations'],
|
|
9
|
+
['maxCandidateProposals', '--max-candidate-proposals'],
|
|
10
|
+
['maxFullEvals', '--max-full-evals'],
|
|
11
|
+
['maxMetricCalls', '--max-metric-calls'],
|
|
12
|
+
['minibatchSize', '--minibatch-size'],
|
|
13
|
+
['reflectionMaxTokens', '--reflection-max-tokens'],
|
|
14
|
+
['reflectionRetryAttempts', '--reflection-retry-attempts'],
|
|
15
|
+
['reflectionHttpTimeoutSeconds', '--reflection-http-timeout-seconds'],
|
|
16
|
+
].map(([field, flag]) => ({ field, flag, grammar: 'positive integer', accepts: isPositiveInteger })),
|
|
17
|
+
{
|
|
18
|
+
field: 'numThreads', flag: '--num-threads', grammar: 'auto or positive integer',
|
|
19
|
+
accepts: value => value === 'auto' || isPositiveInteger(value),
|
|
20
|
+
},
|
|
21
|
+
...[
|
|
22
|
+
['epsilon', '--epsilon'],
|
|
23
|
+
['reflectionTemperature', '--reflection-temperature'],
|
|
24
|
+
].map(([field, flag]) => ({ field, flag, grammar: 'finite number', accepts: isFiniteNumber })),
|
|
25
|
+
{ field: 'seed', flag: '--seed', grammar: 'integer', accepts: isInteger },
|
|
26
|
+
];
|
|
27
|
+
const integerPattern = /^[+-]?\d+$/;
|
|
28
|
+
const floatPattern = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?$/i;
|
|
29
|
+
export function parseHostedGepaNumericValue(field, rawValue) {
|
|
30
|
+
const rule = rules.find(candidate => candidate.field === field);
|
|
31
|
+
if (!rule)
|
|
32
|
+
throw new Error(`Unknown hosted numeric field ${field}`);
|
|
33
|
+
const trimmed = rawValue.trim();
|
|
34
|
+
if (field === 'numThreads' && trimmed.toLowerCase() === 'auto')
|
|
35
|
+
return 'auto';
|
|
36
|
+
const lexicalPattern = rule.grammar === 'finite number' ? floatPattern : integerPattern;
|
|
37
|
+
const parsed = lexicalPattern.test(trimmed) ? Number(trimmed) : Number.NaN;
|
|
38
|
+
if (!rule.accepts(parsed))
|
|
39
|
+
throw new Error(`${rule.flag} must be ${rule.grammar}`);
|
|
40
|
+
return parsed;
|
|
41
|
+
}
|
|
42
|
+
export function hostedGepaNumericError(values) {
|
|
43
|
+
for (const rule of rules) {
|
|
44
|
+
if (values[rule.field] !== undefined && !rule.accepts(values[rule.field])) {
|
|
45
|
+
return `${rule.field} must be ${rule.grammar}`;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hosted-provider-settings.d.ts","sourceRoot":"","sources":["../src/hosted-provider-settings.ts"],"names":[],"mappings":"AAqDA,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAqC;AAE9G,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,OAAO,EAAE,KAAK,SAAe,EAAE,WAAW,CAAC,EAAE,OAAO,GAC1D,MAAM,GAAG,IAAI,CAWf"}
|