orizu 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/index.js +115 -2
- package/package.json +1 -1
- package/src/index.ts +197 -2
- package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py +46 -13
- package/vendor/orizu-gepa-python/src/orizu_gepa/client.py +46 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/local_log.py +123 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/optimizer.py +46 -1
package/README.md
CHANGED
|
@@ -101,6 +101,17 @@ orizu logout
|
|
|
101
101
|
<td>Create a project</td>
|
|
102
102
|
<td><code>orizu projects create --name "Support QA" --team ops-eval</code></td>
|
|
103
103
|
</tr>
|
|
104
|
+
<tr>
|
|
105
|
+
<td colspan="2"><strong>Prompts</strong></td>
|
|
106
|
+
</tr>
|
|
107
|
+
<tr>
|
|
108
|
+
<td>List prompts</td>
|
|
109
|
+
<td><code>orizu prompts list --project ops-eval/support-qa</code></td>
|
|
110
|
+
</tr>
|
|
111
|
+
<tr>
|
|
112
|
+
<td>Show prompt comments</td>
|
|
113
|
+
<td><code>orizu prompts comments <promptIdOrName> --project ops-eval/support-qa</code></td>
|
|
114
|
+
</tr>
|
|
104
115
|
<tr>
|
|
105
116
|
<td colspan="2"><strong>Datasets</strong></td>
|
|
106
117
|
</tr>
|
package/dist/index.js
CHANGED
|
@@ -30,10 +30,10 @@ function printVersion() {
|
|
|
30
30
|
printLine(`orizu ${getCliVersion()}`);
|
|
31
31
|
}
|
|
32
32
|
function printUsage() {
|
|
33
|
-
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`);
|
|
33
|
+
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <id>] [--json]\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`);
|
|
34
34
|
}
|
|
35
35
|
function printOptimizationUsage() {
|
|
36
|
-
printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`);
|
|
36
|
+
printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]\n orizu optimizations export <run-id> [--out <path>] [--json]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`);
|
|
37
37
|
}
|
|
38
38
|
let cliArgs = process.argv.slice(2);
|
|
39
39
|
function getArg(name) {
|
|
@@ -480,6 +480,63 @@ function printTaskStatusSummary(data) {
|
|
|
480
480
|
});
|
|
481
481
|
}
|
|
482
482
|
}
|
|
483
|
+
function compactTerminalText(value, maxLength = 160) {
|
|
484
|
+
const compact = sanitizeTerminalText(value).replace(/\s+/g, ' ').trim();
|
|
485
|
+
if (compact.length <= maxLength) {
|
|
486
|
+
return compact;
|
|
487
|
+
}
|
|
488
|
+
return `${compact.slice(0, Math.max(0, maxLength - 1)).trim()}…`;
|
|
489
|
+
}
|
|
490
|
+
function printIndentedBody(value, indent) {
|
|
491
|
+
const lines = sanitizeTerminalText(value || '').split(/\r?\n/);
|
|
492
|
+
for (const line of lines) {
|
|
493
|
+
printLine(`${indent}${line}`);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
function formatPromptCommentAnchor(anchor) {
|
|
497
|
+
if (!anchor) {
|
|
498
|
+
return null;
|
|
499
|
+
}
|
|
500
|
+
const lineLabel = anchor.startLine && anchor.endLine
|
|
501
|
+
? anchor.startLine === anchor.endLine
|
|
502
|
+
? `line ${anchor.startLine}`
|
|
503
|
+
: `lines ${anchor.startLine}-${anchor.endLine}`
|
|
504
|
+
: null;
|
|
505
|
+
const selectedText = anchor.text ? `"${compactTerminalText(anchor.text, 180)}"` : null;
|
|
506
|
+
if (lineLabel && selectedText) {
|
|
507
|
+
return `${lineLabel}: ${selectedText}`;
|
|
508
|
+
}
|
|
509
|
+
return lineLabel || selectedText;
|
|
510
|
+
}
|
|
511
|
+
function printPromptComments(data) {
|
|
512
|
+
const versionLabel = data.version.versionLabel || (data.version.versionNumber ? `v${data.version.versionNumber}` : data.version.id);
|
|
513
|
+
printLine(`Prompt: ${sanitizeTerminalText(data.prompt.name)} (${sanitizeTerminalText(data.prompt.id)})`);
|
|
514
|
+
printLine(`Version: ${sanitizeTerminalText(String(versionLabel))} (${sanitizeTerminalText(data.version.id)})`);
|
|
515
|
+
printLine(`Comments: ${data.summary.threadCount} thread${data.summary.threadCount === 1 ? '' : 's'} ` +
|
|
516
|
+
`(${data.summary.openThreadCount} open, ${data.summary.resolvedThreadCount} resolved, ${data.summary.replyCount} replies)`);
|
|
517
|
+
if (data.comments.length === 0) {
|
|
518
|
+
printLine('\nNo comments found for this prompt version.');
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
for (const [index, thread] of data.comments.entries()) {
|
|
522
|
+
const authorName = thread.author?.name || 'Unknown';
|
|
523
|
+
const anchor = formatPromptCommentAnchor(thread.anchor);
|
|
524
|
+
printLine('');
|
|
525
|
+
printLine(`${index + 1}. [${thread.status}] ${sanitizeTerminalText(authorName)} · ${sanitizeTerminalText(thread.createdAt)}`);
|
|
526
|
+
if (anchor) {
|
|
527
|
+
printLine(` Selection: ${anchor}`);
|
|
528
|
+
}
|
|
529
|
+
printIndentedBody(thread.body, ' ');
|
|
530
|
+
if (thread.replies.length > 0) {
|
|
531
|
+
printLine(` Replies (${thread.replies.length})`);
|
|
532
|
+
for (const reply of thread.replies) {
|
|
533
|
+
const replyAuthor = reply.author?.name || 'Unknown';
|
|
534
|
+
printLine(` - ${sanitizeTerminalText(replyAuthor)} · ${sanitizeTerminalText(reply.createdAt)}`);
|
|
535
|
+
printIndentedBody(reply.body, ' ');
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
483
540
|
const DEFAULT_AUTH_CALLBACK_PORT = 43123;
|
|
484
541
|
function resolveAuthCallbackPort() {
|
|
485
542
|
const envPort = process.env.ORIZU_AUTH_PORT;
|
|
@@ -788,6 +845,33 @@ async function listJudges() {
|
|
|
788
845
|
const data = await parseJsonResponse(response, 'Judges list');
|
|
789
846
|
printPromptSummaries(data.judges, 'No judges found.');
|
|
790
847
|
}
|
|
848
|
+
async function listPromptComments() {
|
|
849
|
+
const promptRef = getPositionalArg(2);
|
|
850
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
851
|
+
const label = getArg('--label');
|
|
852
|
+
const version = getArg('--version');
|
|
853
|
+
if (!promptRef) {
|
|
854
|
+
throw new Error('Usage: orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <version-id>] [--json]');
|
|
855
|
+
}
|
|
856
|
+
if (label && version) {
|
|
857
|
+
throw new Error('Use either --label or --version, not both');
|
|
858
|
+
}
|
|
859
|
+
const params = new URLSearchParams({ project });
|
|
860
|
+
if (label)
|
|
861
|
+
params.set('label', label);
|
|
862
|
+
if (version)
|
|
863
|
+
params.set('version', version);
|
|
864
|
+
const response = await authedFetch(`/api/cli/prompts/${encodeURIComponent(promptRef)}/comments?${params.toString()}`);
|
|
865
|
+
if (!response.ok) {
|
|
866
|
+
throw new Error(`Failed to fetch prompt comments: ${await response.text()}`);
|
|
867
|
+
}
|
|
868
|
+
const data = await parseJsonResponse(response, 'Prompt comments');
|
|
869
|
+
if (hasJsonFlag()) {
|
|
870
|
+
printJson(data);
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
printPromptComments(data);
|
|
874
|
+
}
|
|
791
875
|
async function listScorers() {
|
|
792
876
|
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
793
877
|
const response = await authedFetch(`/api/cli/scorers?project=${encodeURIComponent(project)}`);
|
|
@@ -1473,6 +1557,27 @@ async function updateOptimizationRunLifecycle(action) {
|
|
|
1473
1557
|
printLine(`Cancelled optimization run ${id}`);
|
|
1474
1558
|
}
|
|
1475
1559
|
}
|
|
1560
|
+
async function exportOptimizationRun() {
|
|
1561
|
+
const runId = getPositionalArg(2) || getArg('--run-id');
|
|
1562
|
+
const outPathArg = getArg('--out');
|
|
1563
|
+
if (!runId) {
|
|
1564
|
+
throw new Error('Usage: orizu optimizations export <run-id> [--out <path>] [--json]');
|
|
1565
|
+
}
|
|
1566
|
+
const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}/export`);
|
|
1567
|
+
if (!response.ok) {
|
|
1568
|
+
throw new Error(`Failed to export optimization run: ${await response.text()}`);
|
|
1569
|
+
}
|
|
1570
|
+
const data = await parseJsonResponse(response, 'Optimization export');
|
|
1571
|
+
if (hasJsonFlag() && !outPathArg) {
|
|
1572
|
+
printJson(data);
|
|
1573
|
+
return;
|
|
1574
|
+
}
|
|
1575
|
+
const filename = outPathArg
|
|
1576
|
+
? expandHomePath(outPathArg)
|
|
1577
|
+
: `${runId}.optimization.json`;
|
|
1578
|
+
writeTextFileEnsuringDir(filename, `${JSON.stringify(data, null, 2)}\n`);
|
|
1579
|
+
printLine(`Saved optimization export to ${sanitizeTerminalText(filename)}`);
|
|
1580
|
+
}
|
|
1476
1581
|
function removeFlagWithValue(args, flag) {
|
|
1477
1582
|
const filtered = [];
|
|
1478
1583
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -2937,6 +3042,10 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
2937
3042
|
await listPrompts();
|
|
2938
3043
|
return;
|
|
2939
3044
|
}
|
|
3045
|
+
if (command === 'prompts' && subcommand === 'comments') {
|
|
3046
|
+
await listPromptComments();
|
|
3047
|
+
return;
|
|
3048
|
+
}
|
|
2940
3049
|
if (command === 'prompts' && subcommand === 'pull') {
|
|
2941
3050
|
await pullPromptArtifact('prompt');
|
|
2942
3051
|
return;
|
|
@@ -3013,6 +3122,10 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
3013
3122
|
await runGepaOptimization();
|
|
3014
3123
|
return;
|
|
3015
3124
|
}
|
|
3125
|
+
if (command === 'optimizations' && subcommand === 'export') {
|
|
3126
|
+
await exportOptimizationRun();
|
|
3127
|
+
return;
|
|
3128
|
+
}
|
|
3016
3129
|
if (command === 'optimizations' &&
|
|
3017
3130
|
(subcommand === 'pause' ||
|
|
3018
3131
|
subcommand === 'resume' ||
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -99,6 +99,62 @@ interface PromptSummary {
|
|
|
99
99
|
description?: string | null
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
interface PromptCommentAuthor {
|
|
103
|
+
name: string
|
|
104
|
+
initials?: string
|
|
105
|
+
color?: string
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
interface PromptCommentAnchor {
|
|
109
|
+
text: string | null
|
|
110
|
+
startLine: number | null
|
|
111
|
+
endLine: number | null
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
interface PromptCommentReply {
|
|
115
|
+
id: string
|
|
116
|
+
body: string
|
|
117
|
+
author: PromptCommentAuthor
|
|
118
|
+
createdAt: string
|
|
119
|
+
updatedAt: string
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
interface PromptCommentThread {
|
|
123
|
+
id: string
|
|
124
|
+
status: 'open' | 'resolved'
|
|
125
|
+
body: string
|
|
126
|
+
anchor: PromptCommentAnchor | null
|
|
127
|
+
author: PromptCommentAuthor
|
|
128
|
+
createdAt: string
|
|
129
|
+
updatedAt: string
|
|
130
|
+
resolvedAt: string | null
|
|
131
|
+
resolvedByUserId: string | null
|
|
132
|
+
replyCount: number
|
|
133
|
+
replies: PromptCommentReply[]
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
interface PromptCommentsPayload {
|
|
137
|
+
prompt: {
|
|
138
|
+
id: string
|
|
139
|
+
name: string
|
|
140
|
+
role: string
|
|
141
|
+
description?: string | null
|
|
142
|
+
}
|
|
143
|
+
version: {
|
|
144
|
+
id: string
|
|
145
|
+
versionNumber?: number
|
|
146
|
+
versionLabel?: string | null
|
|
147
|
+
status?: string | null
|
|
148
|
+
}
|
|
149
|
+
summary: {
|
|
150
|
+
threadCount: number
|
|
151
|
+
openThreadCount: number
|
|
152
|
+
resolvedThreadCount: number
|
|
153
|
+
replyCount: number
|
|
154
|
+
}
|
|
155
|
+
comments: PromptCommentThread[]
|
|
156
|
+
}
|
|
157
|
+
|
|
102
158
|
interface ScorerSummary {
|
|
103
159
|
id: string
|
|
104
160
|
name: string
|
|
@@ -200,11 +256,11 @@ function printVersion() {
|
|
|
200
256
|
}
|
|
201
257
|
|
|
202
258
|
function printUsage() {
|
|
203
|
-
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`)
|
|
259
|
+
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <id>] [--json]\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`)
|
|
204
260
|
}
|
|
205
261
|
|
|
206
262
|
function printOptimizationUsage() {
|
|
207
|
-
printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`)
|
|
263
|
+
printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]\n orizu optimizations export <run-id> [--out <path>] [--json]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`)
|
|
208
264
|
}
|
|
209
265
|
|
|
210
266
|
let cliArgs = process.argv.slice(2)
|
|
@@ -835,6 +891,78 @@ function printTaskStatusSummary(data: TaskStatusPayload) {
|
|
|
835
891
|
}
|
|
836
892
|
}
|
|
837
893
|
|
|
894
|
+
function compactTerminalText(value: string, maxLength = 160): string {
|
|
895
|
+
const compact = sanitizeTerminalText(value).replace(/\s+/g, ' ').trim()
|
|
896
|
+
if (compact.length <= maxLength) {
|
|
897
|
+
return compact
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
return `${compact.slice(0, Math.max(0, maxLength - 1)).trim()}…`
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
function printIndentedBody(value: string, indent: string) {
|
|
904
|
+
const lines = sanitizeTerminalText(value || '').split(/\r?\n/)
|
|
905
|
+
for (const line of lines) {
|
|
906
|
+
printLine(`${indent}${line}`)
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
function formatPromptCommentAnchor(anchor: PromptCommentAnchor | null): string | null {
|
|
911
|
+
if (!anchor) {
|
|
912
|
+
return null
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
const lineLabel = anchor.startLine && anchor.endLine
|
|
916
|
+
? anchor.startLine === anchor.endLine
|
|
917
|
+
? `line ${anchor.startLine}`
|
|
918
|
+
: `lines ${anchor.startLine}-${anchor.endLine}`
|
|
919
|
+
: null
|
|
920
|
+
const selectedText = anchor.text ? `"${compactTerminalText(anchor.text, 180)}"` : null
|
|
921
|
+
|
|
922
|
+
if (lineLabel && selectedText) {
|
|
923
|
+
return `${lineLabel}: ${selectedText}`
|
|
924
|
+
}
|
|
925
|
+
return lineLabel || selectedText
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
function printPromptComments(data: PromptCommentsPayload) {
|
|
929
|
+
const versionLabel = data.version.versionLabel || (
|
|
930
|
+
data.version.versionNumber ? `v${data.version.versionNumber}` : data.version.id
|
|
931
|
+
)
|
|
932
|
+
|
|
933
|
+
printLine(`Prompt: ${sanitizeTerminalText(data.prompt.name)} (${sanitizeTerminalText(data.prompt.id)})`)
|
|
934
|
+
printLine(`Version: ${sanitizeTerminalText(String(versionLabel))} (${sanitizeTerminalText(data.version.id)})`)
|
|
935
|
+
printLine(
|
|
936
|
+
`Comments: ${data.summary.threadCount} thread${data.summary.threadCount === 1 ? '' : 's'} ` +
|
|
937
|
+
`(${data.summary.openThreadCount} open, ${data.summary.resolvedThreadCount} resolved, ${data.summary.replyCount} replies)`
|
|
938
|
+
)
|
|
939
|
+
|
|
940
|
+
if (data.comments.length === 0) {
|
|
941
|
+
printLine('\nNo comments found for this prompt version.')
|
|
942
|
+
return
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
for (const [index, thread] of data.comments.entries()) {
|
|
946
|
+
const authorName = thread.author?.name || 'Unknown'
|
|
947
|
+
const anchor = formatPromptCommentAnchor(thread.anchor)
|
|
948
|
+
printLine('')
|
|
949
|
+
printLine(`${index + 1}. [${thread.status}] ${sanitizeTerminalText(authorName)} · ${sanitizeTerminalText(thread.createdAt)}`)
|
|
950
|
+
if (anchor) {
|
|
951
|
+
printLine(` Selection: ${anchor}`)
|
|
952
|
+
}
|
|
953
|
+
printIndentedBody(thread.body, ' ')
|
|
954
|
+
|
|
955
|
+
if (thread.replies.length > 0) {
|
|
956
|
+
printLine(` Replies (${thread.replies.length})`)
|
|
957
|
+
for (const reply of thread.replies) {
|
|
958
|
+
const replyAuthor = reply.author?.name || 'Unknown'
|
|
959
|
+
printLine(` - ${sanitizeTerminalText(replyAuthor)} · ${sanitizeTerminalText(reply.createdAt)}`)
|
|
960
|
+
printIndentedBody(reply.body, ' ')
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
838
966
|
const DEFAULT_AUTH_CALLBACK_PORT = 43123
|
|
839
967
|
|
|
840
968
|
function resolveAuthCallbackPort(): number {
|
|
@@ -1185,6 +1313,37 @@ async function listJudges() {
|
|
|
1185
1313
|
printPromptSummaries(data.judges, 'No judges found.')
|
|
1186
1314
|
}
|
|
1187
1315
|
|
|
1316
|
+
async function listPromptComments() {
|
|
1317
|
+
const promptRef = getPositionalArg(2)
|
|
1318
|
+
const project = getArg('--project') || await resolveProjectSlug(null)
|
|
1319
|
+
const label = getArg('--label')
|
|
1320
|
+
const version = getArg('--version')
|
|
1321
|
+
|
|
1322
|
+
if (!promptRef) {
|
|
1323
|
+
throw new Error('Usage: orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <version-id>] [--json]')
|
|
1324
|
+
}
|
|
1325
|
+
if (label && version) {
|
|
1326
|
+
throw new Error('Use either --label or --version, not both')
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
const params = new URLSearchParams({ project })
|
|
1330
|
+
if (label) params.set('label', label)
|
|
1331
|
+
if (version) params.set('version', version)
|
|
1332
|
+
|
|
1333
|
+
const response = await authedFetch(`/api/cli/prompts/${encodeURIComponent(promptRef)}/comments?${params.toString()}`)
|
|
1334
|
+
if (!response.ok) {
|
|
1335
|
+
throw new Error(`Failed to fetch prompt comments: ${await response.text()}`)
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
const data = await parseJsonResponse<PromptCommentsPayload>(response, 'Prompt comments')
|
|
1339
|
+
if (hasJsonFlag()) {
|
|
1340
|
+
printJson(data as unknown as Record<string, unknown>)
|
|
1341
|
+
return
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
printPromptComments(data)
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1188
1347
|
async function listScorers() {
|
|
1189
1348
|
const project = getArg('--project') || await resolveProjectSlug(null)
|
|
1190
1349
|
const response = await authedFetch(`/api/cli/scorers?project=${encodeURIComponent(project)}`)
|
|
@@ -2070,6 +2229,32 @@ async function updateOptimizationRunLifecycle(action: OptimizationLifecycleActio
|
|
|
2070
2229
|
}
|
|
2071
2230
|
}
|
|
2072
2231
|
|
|
2232
|
+
async function exportOptimizationRun() {
|
|
2233
|
+
const runId = getPositionalArg(2) || getArg('--run-id')
|
|
2234
|
+
const outPathArg = getArg('--out')
|
|
2235
|
+
|
|
2236
|
+
if (!runId) {
|
|
2237
|
+
throw new Error('Usage: orizu optimizations export <run-id> [--out <path>] [--json]')
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}/export`)
|
|
2241
|
+
if (!response.ok) {
|
|
2242
|
+
throw new Error(`Failed to export optimization run: ${await response.text()}`)
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
const data = await parseJsonResponse<Record<string, unknown>>(response, 'Optimization export')
|
|
2246
|
+
if (hasJsonFlag() && !outPathArg) {
|
|
2247
|
+
printJson(data)
|
|
2248
|
+
return
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
const filename = outPathArg
|
|
2252
|
+
? expandHomePath(outPathArg)
|
|
2253
|
+
: `${runId}.optimization.json`
|
|
2254
|
+
writeTextFileEnsuringDir(filename, `${JSON.stringify(data, null, 2)}\n`)
|
|
2255
|
+
printLine(`Saved optimization export to ${sanitizeTerminalText(filename)}`)
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2073
2258
|
function removeFlagWithValue(args: string[], flag: string): string[] {
|
|
2074
2259
|
const filtered: string[] = []
|
|
2075
2260
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -3930,6 +4115,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
3930
4115
|
return
|
|
3931
4116
|
}
|
|
3932
4117
|
|
|
4118
|
+
if (command === 'prompts' && subcommand === 'comments') {
|
|
4119
|
+
await listPromptComments()
|
|
4120
|
+
return
|
|
4121
|
+
}
|
|
4122
|
+
|
|
3933
4123
|
if (command === 'prompts' && subcommand === 'pull') {
|
|
3934
4124
|
await pullPromptArtifact('prompt')
|
|
3935
4125
|
return
|
|
@@ -4025,6 +4215,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
4025
4215
|
return
|
|
4026
4216
|
}
|
|
4027
4217
|
|
|
4218
|
+
if (command === 'optimizations' && subcommand === 'export') {
|
|
4219
|
+
await exportOptimizationRun()
|
|
4220
|
+
return
|
|
4221
|
+
}
|
|
4222
|
+
|
|
4028
4223
|
if (
|
|
4029
4224
|
command === 'optimizations' &&
|
|
4030
4225
|
(
|
|
@@ -6,6 +6,7 @@ from pathlib import Path
|
|
|
6
6
|
from typing import Any
|
|
7
7
|
|
|
8
8
|
from .client import OrizuClient, OrizuEventSink
|
|
9
|
+
from .local_log import LocalOptimizationLogger
|
|
9
10
|
from .optimizer import TextGepaConfig, optimize_loaded_text_candidate
|
|
10
11
|
from .reflection import reflect_with_provider
|
|
11
12
|
from .runner import make_candidate_runner, make_scorer_runner
|
|
@@ -74,6 +75,8 @@ def main() -> None:
|
|
|
74
75
|
parser.add_argument("--auto-promote", action="store_true")
|
|
75
76
|
parser.add_argument("--promotion-label")
|
|
76
77
|
parser.add_argument("--log-row-snapshots", action="store_true")
|
|
78
|
+
parser.add_argument("--log-dir", default="logs")
|
|
79
|
+
parser.add_argument("--no-local-log", action="store_true")
|
|
77
80
|
parser.add_argument("--metadata", default="{}")
|
|
78
81
|
args = parser.parse_args()
|
|
79
82
|
|
|
@@ -123,6 +126,19 @@ def main() -> None:
|
|
|
123
126
|
split=args.val_split,
|
|
124
127
|
)
|
|
125
128
|
|
|
129
|
+
run_metadata = {
|
|
130
|
+
**metadata,
|
|
131
|
+
"optimizer_package": "orizu-gepa-python",
|
|
132
|
+
"optimizer_family": "gepa",
|
|
133
|
+
"mode": "text-candidate",
|
|
134
|
+
"inference_lm": prompt_context.provider_settings.get("model"),
|
|
135
|
+
"reflection_lm": config.reflection_model,
|
|
136
|
+
"scorer_lm": scorer_context.provider_settings.get("model"),
|
|
137
|
+
"dataset_size": len(trainset) + len(valset),
|
|
138
|
+
"train_count": len(trainset),
|
|
139
|
+
"validation_count": len(valset),
|
|
140
|
+
}
|
|
141
|
+
|
|
126
142
|
run_id = client.start_run(
|
|
127
143
|
project=args.project,
|
|
128
144
|
optimizer_version_id=args.optimizer_version_id,
|
|
@@ -132,21 +148,29 @@ def main() -> None:
|
|
|
132
148
|
split_set_id=args.split_set_id,
|
|
133
149
|
train_split=args.train_split,
|
|
134
150
|
validation_split=args.val_split,
|
|
135
|
-
metadata=
|
|
136
|
-
**metadata,
|
|
137
|
-
"optimizer_package": "orizu-gepa-python",
|
|
138
|
-
"optimizer_family": "gepa",
|
|
139
|
-
"mode": "text-candidate",
|
|
140
|
-
"inference_lm": prompt_context.provider_settings.get("model"),
|
|
141
|
-
"reflection_lm": config.reflection_model,
|
|
142
|
-
"scorer_lm": scorer_context.provider_settings.get("model"),
|
|
143
|
-
"dataset_size": len(trainset) + len(valset),
|
|
144
|
-
"train_count": len(trainset),
|
|
145
|
-
"validation_count": len(valset),
|
|
146
|
-
},
|
|
151
|
+
metadata=run_metadata,
|
|
147
152
|
)
|
|
148
153
|
print(f"[orizu-gepa] started optimization run {run_id}", flush=True)
|
|
149
154
|
|
|
155
|
+
local_logger = None
|
|
156
|
+
if not args.no_local_log:
|
|
157
|
+
local_logger = LocalOptimizationLogger.create(args.log_dir, run_id)
|
|
158
|
+
local_logger.write_context(
|
|
159
|
+
project=args.project,
|
|
160
|
+
run_id=run_id,
|
|
161
|
+
args={
|
|
162
|
+
key: value
|
|
163
|
+
for key, value in vars(args).items()
|
|
164
|
+
if key not in {"metadata"}
|
|
165
|
+
},
|
|
166
|
+
prompt_context=prompt_context,
|
|
167
|
+
scorer_context=scorer_context,
|
|
168
|
+
trainset=trainset,
|
|
169
|
+
valset=valset,
|
|
170
|
+
metadata=run_metadata,
|
|
171
|
+
)
|
|
172
|
+
print(f"[orizu-gepa] local log: {local_logger.path}", flush=True)
|
|
173
|
+
|
|
150
174
|
result = optimize_loaded_text_candidate(
|
|
151
175
|
run_id=run_id,
|
|
152
176
|
prompt_context=prompt_context,
|
|
@@ -156,9 +180,17 @@ def main() -> None:
|
|
|
156
180
|
candidate_runner=make_candidate_runner(Path(args.candidate_runner_dir), run_id),
|
|
157
181
|
scorer_runner=make_scorer_runner(Path(args.scorer_runner_dir), run_id),
|
|
158
182
|
reflector=reflect_with_provider,
|
|
159
|
-
event_sink=OrizuEventSink(
|
|
183
|
+
event_sink=OrizuEventSink(
|
|
184
|
+
client,
|
|
185
|
+
run_id,
|
|
186
|
+
fail_on_log_error=config.fail_on_log_error,
|
|
187
|
+
local_logger=local_logger,
|
|
188
|
+
),
|
|
160
189
|
config=config,
|
|
190
|
+
local_logger=local_logger,
|
|
161
191
|
)
|
|
192
|
+
if local_logger is not None:
|
|
193
|
+
local_logger.write_result(result)
|
|
162
194
|
print(json.dumps({
|
|
163
195
|
"optimization_run_id": result.run_id,
|
|
164
196
|
"best_candidate_id": result.best_candidate_id,
|
|
@@ -166,6 +198,7 @@ def main() -> None:
|
|
|
166
198
|
"seed_score": result.seed_score,
|
|
167
199
|
"promoted_prompt_version_id": result.promoted_prompt_version_id,
|
|
168
200
|
"budget": result.budget.to_payload(),
|
|
201
|
+
"local_log_dir": local_logger.path if local_logger is not None else None,
|
|
169
202
|
}))
|
|
170
203
|
|
|
171
204
|
|
|
@@ -255,12 +255,14 @@ class OrizuEventSink:
|
|
|
255
255
|
*,
|
|
256
256
|
fail_on_log_error: bool = True,
|
|
257
257
|
max_log_retries: int = 2,
|
|
258
|
+
local_logger: Any | None = None,
|
|
258
259
|
):
|
|
259
260
|
self.client = client
|
|
260
261
|
self.run_id = run_id
|
|
261
262
|
self.sequence = 0
|
|
262
263
|
self.fail_on_log_error = fail_on_log_error
|
|
263
264
|
self.max_log_retries = max(0, max_log_retries)
|
|
265
|
+
self.local_logger = local_logger
|
|
264
266
|
|
|
265
267
|
def log_event(
|
|
266
268
|
self,
|
|
@@ -275,6 +277,20 @@ class OrizuEventSink:
|
|
|
275
277
|
child_candidate_id: str | None = None,
|
|
276
278
|
) -> None:
|
|
277
279
|
self.sequence += 1
|
|
280
|
+
local_event = {
|
|
281
|
+
"run_id": self.run_id,
|
|
282
|
+
"sequence": self.sequence,
|
|
283
|
+
"event_type": event_type,
|
|
284
|
+
"event_layer": event_layer,
|
|
285
|
+
"optimizer_family": optimizer_family,
|
|
286
|
+
"iteration": iteration,
|
|
287
|
+
"candidate_id": candidate_id,
|
|
288
|
+
"parent_candidate_id": parent_candidate_id,
|
|
289
|
+
"child_candidate_id": child_candidate_id,
|
|
290
|
+
"payload": payload or {},
|
|
291
|
+
}
|
|
292
|
+
if self.local_logger is not None:
|
|
293
|
+
self.local_logger.append_event(local_event)
|
|
278
294
|
last_error: Exception | None = None
|
|
279
295
|
for attempt in range(self.max_log_retries + 1):
|
|
280
296
|
try:
|
|
@@ -308,7 +324,37 @@ class OrizuEventSink:
|
|
|
308
324
|
prompt_version_id = self.client.promote_candidate(self.run_id, **kwargs)
|
|
309
325
|
# Promotion writes a system event at max(sequence)+1 in Postgres.
|
|
310
326
|
self.sequence += 1
|
|
327
|
+
if self.local_logger is not None:
|
|
328
|
+
self.local_logger.append_event({
|
|
329
|
+
"run_id": self.run_id,
|
|
330
|
+
"sequence": self.sequence,
|
|
331
|
+
"event_type": "candidate_promoted",
|
|
332
|
+
"event_layer": "system",
|
|
333
|
+
"optimizer_family": "gepa",
|
|
334
|
+
"iteration": None,
|
|
335
|
+
"candidate_id": kwargs.get("candidate_id"),
|
|
336
|
+
"parent_candidate_id": None,
|
|
337
|
+
"child_candidate_id": None,
|
|
338
|
+
"payload": {
|
|
339
|
+
**kwargs,
|
|
340
|
+
"prompt_version_id": prompt_version_id,
|
|
341
|
+
},
|
|
342
|
+
})
|
|
311
343
|
return prompt_version_id
|
|
312
344
|
|
|
313
345
|
def finish_run(self, **kwargs: Any) -> None:
|
|
346
|
+
if self.local_logger is not None:
|
|
347
|
+
self.sequence += 1
|
|
348
|
+
self.local_logger.append_event({
|
|
349
|
+
"run_id": self.run_id,
|
|
350
|
+
"sequence": self.sequence,
|
|
351
|
+
"event_type": "run_status_updated",
|
|
352
|
+
"event_layer": "system",
|
|
353
|
+
"optimizer_family": "gepa",
|
|
354
|
+
"iteration": None,
|
|
355
|
+
"candidate_id": kwargs.get("best_candidate_id"),
|
|
356
|
+
"parent_candidate_id": None,
|
|
357
|
+
"child_candidate_id": None,
|
|
358
|
+
"payload": kwargs,
|
|
359
|
+
})
|
|
314
360
|
self.client.update_run(self.run_id, **kwargs)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from .optimizer import DatasetRow, PromptContext, RowEvaluation, TextGepaResult
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _json_default(value: Any) -> Any:
|
|
12
|
+
if dataclasses.is_dataclass(value):
|
|
13
|
+
return dataclasses.asdict(value)
|
|
14
|
+
return str(value)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
18
|
+
path.write_text(
|
|
19
|
+
json.dumps(payload, indent=2, ensure_ascii=False, default=_json_default) + "\n",
|
|
20
|
+
encoding="utf-8",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _append_jsonl(path: Path, payload: dict[str, Any]) -> None:
|
|
25
|
+
with path.open("a", encoding="utf-8") as handle:
|
|
26
|
+
handle.write(json.dumps(payload, ensure_ascii=False, default=_json_default) + "\n")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class LocalOptimizationLogger:
|
|
30
|
+
def __init__(self, directory: Path):
|
|
31
|
+
self.directory = directory
|
|
32
|
+
self.directory.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def create(cls, root: str | Path, run_id: str) -> "LocalOptimizationLogger":
|
|
36
|
+
return cls(Path(root) / run_id)
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def path(self) -> str:
|
|
40
|
+
return str(self.directory)
|
|
41
|
+
|
|
42
|
+
def write_context(
|
|
43
|
+
self,
|
|
44
|
+
*,
|
|
45
|
+
project: str,
|
|
46
|
+
run_id: str,
|
|
47
|
+
args: dict[str, Any],
|
|
48
|
+
prompt_context: PromptContext,
|
|
49
|
+
scorer_context: PromptContext,
|
|
50
|
+
trainset: list[DatasetRow],
|
|
51
|
+
valset: list[DatasetRow],
|
|
52
|
+
metadata: dict[str, Any],
|
|
53
|
+
) -> None:
|
|
54
|
+
_write_json(self.directory / "run.json", {
|
|
55
|
+
"schema_version": "orizu.optimization-local-log.v1",
|
|
56
|
+
"optimization_run_id": run_id,
|
|
57
|
+
"project": project,
|
|
58
|
+
"args": args,
|
|
59
|
+
"metadata": metadata,
|
|
60
|
+
})
|
|
61
|
+
_write_json(self.directory / "prompt_context.json", dataclasses.asdict(prompt_context))
|
|
62
|
+
_write_json(self.directory / "scorer_context.json", dataclasses.asdict(scorer_context))
|
|
63
|
+
_write_json(self.directory / "trainset.json", {
|
|
64
|
+
"rows": [dataclasses.asdict(row) for row in trainset],
|
|
65
|
+
})
|
|
66
|
+
_write_json(self.directory / "valset.json", {
|
|
67
|
+
"rows": [dataclasses.asdict(row) for row in valset],
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
def append_event(self, event: dict[str, Any]) -> None:
|
|
71
|
+
_append_jsonl(self.directory / "events.jsonl", event)
|
|
72
|
+
|
|
73
|
+
def append_evaluations(
|
|
74
|
+
self,
|
|
75
|
+
*,
|
|
76
|
+
stage: str,
|
|
77
|
+
split: str,
|
|
78
|
+
iteration: int | None,
|
|
79
|
+
candidate_id: str,
|
|
80
|
+
parent_candidate_id: str | None = None,
|
|
81
|
+
results: list[RowEvaluation],
|
|
82
|
+
) -> None:
|
|
83
|
+
for result in results:
|
|
84
|
+
_append_jsonl(self.directory / "evaluations.jsonl", {
|
|
85
|
+
"stage": stage,
|
|
86
|
+
"split": split,
|
|
87
|
+
"iteration": iteration,
|
|
88
|
+
"candidate_id": candidate_id,
|
|
89
|
+
"parent_candidate_id": parent_candidate_id,
|
|
90
|
+
**result.to_payload(),
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
def append_reflection(
|
|
94
|
+
self,
|
|
95
|
+
*,
|
|
96
|
+
iteration: int,
|
|
97
|
+
parent_candidate_id: str,
|
|
98
|
+
child_candidate_id: str,
|
|
99
|
+
row_ids: list[str],
|
|
100
|
+
prompt: str,
|
|
101
|
+
response: str,
|
|
102
|
+
candidate_text: str,
|
|
103
|
+
) -> None:
|
|
104
|
+
_append_jsonl(self.directory / "reflections.jsonl", {
|
|
105
|
+
"iteration": iteration,
|
|
106
|
+
"parent_candidate_id": parent_candidate_id,
|
|
107
|
+
"child_candidate_id": child_candidate_id,
|
|
108
|
+
"row_ids": row_ids,
|
|
109
|
+
"prompt": prompt,
|
|
110
|
+
"response": response,
|
|
111
|
+
"candidate_text": candidate_text,
|
|
112
|
+
})
|
|
113
|
+
|
|
114
|
+
def write_result(self, result: TextGepaResult) -> None:
|
|
115
|
+
_write_json(self.directory / "result.json", {
|
|
116
|
+
"optimization_run_id": result.run_id,
|
|
117
|
+
"best_candidate_id": result.best_candidate_id,
|
|
118
|
+
"best_candidate_text": result.best_candidate_text,
|
|
119
|
+
"best_score": result.best_score,
|
|
120
|
+
"seed_score": result.seed_score,
|
|
121
|
+
"promoted_prompt_version_id": result.promoted_prompt_version_id,
|
|
122
|
+
"budget": result.budget.to_payload(),
|
|
123
|
+
})
|
|
@@ -658,6 +658,7 @@ def optimize_loaded_text_candidate(
|
|
|
658
658
|
reflector: Reflector,
|
|
659
659
|
event_sink: EventSink,
|
|
660
660
|
config: TextGepaConfig,
|
|
661
|
+
local_logger: Any | None = None,
|
|
661
662
|
) -> TextGepaResult:
|
|
662
663
|
seed_text = prompt_context.body or ""
|
|
663
664
|
budget = Budget.from_config(config)
|
|
@@ -709,6 +710,14 @@ def optimize_loaded_text_candidate(
|
|
|
709
710
|
budget=budget,
|
|
710
711
|
evaluation_cache=evaluation_cache,
|
|
711
712
|
)
|
|
713
|
+
if local_logger is not None:
|
|
714
|
+
local_logger.append_evaluations(
|
|
715
|
+
stage="seed_val_set",
|
|
716
|
+
split="validation",
|
|
717
|
+
iteration=None,
|
|
718
|
+
candidate_id="seed",
|
|
719
|
+
results=seed_results,
|
|
720
|
+
)
|
|
712
721
|
if not _completed_all_rows(seed_results, valset):
|
|
713
722
|
raise RuntimeError("Budget exhausted before seed validation completed")
|
|
714
723
|
budget.used_full_evals += 1
|
|
@@ -804,6 +813,14 @@ def optimize_loaded_text_candidate(
|
|
|
804
813
|
budget=budget,
|
|
805
814
|
evaluation_cache=evaluation_cache,
|
|
806
815
|
)
|
|
816
|
+
if local_logger is not None:
|
|
817
|
+
local_logger.append_evaluations(
|
|
818
|
+
stage="parent_minibatch",
|
|
819
|
+
split="train",
|
|
820
|
+
iteration=iteration,
|
|
821
|
+
candidate_id=parent_candidate_id,
|
|
822
|
+
results=parent_results,
|
|
823
|
+
)
|
|
807
824
|
if not _completed_all_rows(parent_results, minibatch):
|
|
808
825
|
event_sink.log_event(
|
|
809
826
|
"budget_exhausted",
|
|
@@ -852,11 +869,21 @@ def optimize_loaded_text_candidate(
|
|
|
852
869
|
child_text = reflection.candidate_text
|
|
853
870
|
candidate_text_by_id[child_id] = child_text
|
|
854
871
|
budget.used_candidate_proposals += 1
|
|
872
|
+
if local_logger is not None:
|
|
873
|
+
local_logger.append_reflection(
|
|
874
|
+
iteration=iteration,
|
|
875
|
+
parent_candidate_id=parent_candidate_id,
|
|
876
|
+
child_candidate_id=child_id,
|
|
877
|
+
row_ids=[row.id for row in minibatch],
|
|
878
|
+
prompt=reflection.prompt,
|
|
879
|
+
response=reflection.response,
|
|
880
|
+
candidate_text=reflection.candidate_text,
|
|
881
|
+
)
|
|
855
882
|
event_sink.log_event(
|
|
856
883
|
"reflection_completed",
|
|
857
884
|
{
|
|
858
885
|
**_text_log_fields("prompt", reflection.prompt, include_text=config.log_row_snapshots),
|
|
859
|
-
|
|
886
|
+
"response": reflection.response,
|
|
860
887
|
"child_candidate_id": child_id,
|
|
861
888
|
"candidate_text": child_text,
|
|
862
889
|
},
|
|
@@ -908,6 +935,15 @@ def optimize_loaded_text_candidate(
|
|
|
908
935
|
budget=budget,
|
|
909
936
|
evaluation_cache=evaluation_cache,
|
|
910
937
|
)
|
|
938
|
+
if local_logger is not None:
|
|
939
|
+
local_logger.append_evaluations(
|
|
940
|
+
stage="child_minibatch",
|
|
941
|
+
split="train",
|
|
942
|
+
iteration=iteration,
|
|
943
|
+
candidate_id=child_id,
|
|
944
|
+
parent_candidate_id=parent_candidate_id,
|
|
945
|
+
results=child_results,
|
|
946
|
+
)
|
|
911
947
|
if not _completed_all_rows(child_results, minibatch):
|
|
912
948
|
event_sink.log_event(
|
|
913
949
|
"budget_exhausted",
|
|
@@ -981,6 +1017,15 @@ def optimize_loaded_text_candidate(
|
|
|
981
1017
|
budget=budget,
|
|
982
1018
|
evaluation_cache=evaluation_cache,
|
|
983
1019
|
)
|
|
1020
|
+
if local_logger is not None:
|
|
1021
|
+
local_logger.append_evaluations(
|
|
1022
|
+
stage="child_val_set",
|
|
1023
|
+
split="validation",
|
|
1024
|
+
iteration=iteration,
|
|
1025
|
+
candidate_id=child_id,
|
|
1026
|
+
parent_candidate_id=parent_candidate_id,
|
|
1027
|
+
results=child_val_results,
|
|
1028
|
+
)
|
|
984
1029
|
if not _completed_all_rows(child_val_results, valset):
|
|
985
1030
|
event_sink.log_event(
|
|
986
1031
|
"budget_exhausted",
|