orizu 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -101,6 +101,17 @@ orizu logout
101
101
  <td>Create a project</td>
102
102
  <td><code>orizu projects create --name "Support QA" --team ops-eval</code></td>
103
103
  </tr>
104
+ <tr>
105
+ <td colspan="2"><strong>Prompts</strong></td>
106
+ </tr>
107
+ <tr>
108
+ <td>List prompts</td>
109
+ <td><code>orizu prompts list --project ops-eval/support-qa</code></td>
110
+ </tr>
111
+ <tr>
112
+ <td>Show prompt comments</td>
113
+ <td><code>orizu prompts comments &lt;promptIdOrName&gt; --project ops-eval/support-qa</code></td>
114
+ </tr>
104
115
  <tr>
105
116
  <td colspan="2"><strong>Datasets</strong></td>
106
117
  </tr>
package/dist/index.js CHANGED
@@ -30,10 +30,10 @@ function printVersion() {
30
30
  printLine(`orizu ${getCliVersion()}`);
31
31
  }
32
32
  function printUsage() {
33
- printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`);
33
+ printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <id>] [--json]\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`);
34
34
  }
35
35
  function printOptimizationUsage() {
36
- printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`);
36
+ printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]\n orizu optimizations export <run-id> [--out <path>] [--json]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`);
37
37
  }
38
38
  let cliArgs = process.argv.slice(2);
39
39
  function getArg(name) {
@@ -480,6 +480,63 @@ function printTaskStatusSummary(data) {
480
480
  });
481
481
  }
482
482
  }
483
+ function compactTerminalText(value, maxLength = 160) {
484
+ const compact = sanitizeTerminalText(value).replace(/\s+/g, ' ').trim();
485
+ if (compact.length <= maxLength) {
486
+ return compact;
487
+ }
488
+ return `${compact.slice(0, Math.max(0, maxLength - 1)).trim()}…`;
489
+ }
490
+ function printIndentedBody(value, indent) {
491
+ const lines = sanitizeTerminalText(value || '').split(/\r?\n/);
492
+ for (const line of lines) {
493
+ printLine(`${indent}${line}`);
494
+ }
495
+ }
496
+ function formatPromptCommentAnchor(anchor) {
497
+ if (!anchor) {
498
+ return null;
499
+ }
500
+ const lineLabel = anchor.startLine && anchor.endLine
501
+ ? anchor.startLine === anchor.endLine
502
+ ? `line ${anchor.startLine}`
503
+ : `lines ${anchor.startLine}-${anchor.endLine}`
504
+ : null;
505
+ const selectedText = anchor.text ? `"${compactTerminalText(anchor.text, 180)}"` : null;
506
+ if (lineLabel && selectedText) {
507
+ return `${lineLabel}: ${selectedText}`;
508
+ }
509
+ return lineLabel || selectedText;
510
+ }
511
+ function printPromptComments(data) {
512
+ const versionLabel = data.version.versionLabel || (data.version.versionNumber ? `v${data.version.versionNumber}` : data.version.id);
513
+ printLine(`Prompt: ${sanitizeTerminalText(data.prompt.name)} (${sanitizeTerminalText(data.prompt.id)})`);
514
+ printLine(`Version: ${sanitizeTerminalText(String(versionLabel))} (${sanitizeTerminalText(data.version.id)})`);
515
+ printLine(`Comments: ${data.summary.threadCount} thread${data.summary.threadCount === 1 ? '' : 's'} ` +
516
+ `(${data.summary.openThreadCount} open, ${data.summary.resolvedThreadCount} resolved, ${data.summary.replyCount} replies)`);
517
+ if (data.comments.length === 0) {
518
+ printLine('\nNo comments found for this prompt version.');
519
+ return;
520
+ }
521
+ for (const [index, thread] of data.comments.entries()) {
522
+ const authorName = thread.author?.name || 'Unknown';
523
+ const anchor = formatPromptCommentAnchor(thread.anchor);
524
+ printLine('');
525
+ printLine(`${index + 1}. [${thread.status}] ${sanitizeTerminalText(authorName)} · ${sanitizeTerminalText(thread.createdAt)}`);
526
+ if (anchor) {
527
+ printLine(` Selection: ${anchor}`);
528
+ }
529
+ printIndentedBody(thread.body, ' ');
530
+ if (thread.replies.length > 0) {
531
+ printLine(` Replies (${thread.replies.length})`);
532
+ for (const reply of thread.replies) {
533
+ const replyAuthor = reply.author?.name || 'Unknown';
534
+ printLine(` - ${sanitizeTerminalText(replyAuthor)} · ${sanitizeTerminalText(reply.createdAt)}`);
535
+ printIndentedBody(reply.body, ' ');
536
+ }
537
+ }
538
+ }
539
+ }
483
540
  const DEFAULT_AUTH_CALLBACK_PORT = 43123;
484
541
  function resolveAuthCallbackPort() {
485
542
  const envPort = process.env.ORIZU_AUTH_PORT;
@@ -788,6 +845,33 @@ async function listJudges() {
788
845
  const data = await parseJsonResponse(response, 'Judges list');
789
846
  printPromptSummaries(data.judges, 'No judges found.');
790
847
  }
848
+ async function listPromptComments() {
849
+ const promptRef = getPositionalArg(2);
850
+ const project = getArg('--project') || await resolveProjectSlug(null);
851
+ const label = getArg('--label');
852
+ const version = getArg('--version');
853
+ if (!promptRef) {
854
+ throw new Error('Usage: orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <version-id>] [--json]');
855
+ }
856
+ if (label && version) {
857
+ throw new Error('Use either --label or --version, not both');
858
+ }
859
+ const params = new URLSearchParams({ project });
860
+ if (label)
861
+ params.set('label', label);
862
+ if (version)
863
+ params.set('version', version);
864
+ const response = await authedFetch(`/api/cli/prompts/${encodeURIComponent(promptRef)}/comments?${params.toString()}`);
865
+ if (!response.ok) {
866
+ throw new Error(`Failed to fetch prompt comments: ${await response.text()}`);
867
+ }
868
+ const data = await parseJsonResponse(response, 'Prompt comments');
869
+ if (hasJsonFlag()) {
870
+ printJson(data);
871
+ return;
872
+ }
873
+ printPromptComments(data);
874
+ }
791
875
  async function listScorers() {
792
876
  const project = getArg('--project') || await resolveProjectSlug(null);
793
877
  const response = await authedFetch(`/api/cli/scorers?project=${encodeURIComponent(project)}`);
@@ -1473,6 +1557,27 @@ async function updateOptimizationRunLifecycle(action) {
1473
1557
  printLine(`Cancelled optimization run ${id}`);
1474
1558
  }
1475
1559
  }
1560
+ async function exportOptimizationRun() {
1561
+ const runId = getPositionalArg(2) || getArg('--run-id');
1562
+ const outPathArg = getArg('--out');
1563
+ if (!runId) {
1564
+ throw new Error('Usage: orizu optimizations export <run-id> [--out <path>] [--json]');
1565
+ }
1566
+ const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}/export`);
1567
+ if (!response.ok) {
1568
+ throw new Error(`Failed to export optimization run: ${await response.text()}`);
1569
+ }
1570
+ const data = await parseJsonResponse(response, 'Optimization export');
1571
+ if (hasJsonFlag() && !outPathArg) {
1572
+ printJson(data);
1573
+ return;
1574
+ }
1575
+ const filename = outPathArg
1576
+ ? expandHomePath(outPathArg)
1577
+ : `${runId}.optimization.json`;
1578
+ writeTextFileEnsuringDir(filename, `${JSON.stringify(data, null, 2)}\n`);
1579
+ printLine(`Saved optimization export to ${sanitizeTerminalText(filename)}`);
1580
+ }
1476
1581
  function removeFlagWithValue(args, flag) {
1477
1582
  const filtered = [];
1478
1583
  for (let index = 0; index < args.length; index += 1) {
@@ -2937,6 +3042,10 @@ export async function main(rawArgs = process.argv.slice(2)) {
2937
3042
  await listPrompts();
2938
3043
  return;
2939
3044
  }
3045
+ if (command === 'prompts' && subcommand === 'comments') {
3046
+ await listPromptComments();
3047
+ return;
3048
+ }
2940
3049
  if (command === 'prompts' && subcommand === 'pull') {
2941
3050
  await pullPromptArtifact('prompt');
2942
3051
  return;
@@ -3013,6 +3122,10 @@ export async function main(rawArgs = process.argv.slice(2)) {
3013
3122
  await runGepaOptimization();
3014
3123
  return;
3015
3124
  }
3125
+ if (command === 'optimizations' && subcommand === 'export') {
3126
+ await exportOptimizationRun();
3127
+ return;
3128
+ }
3016
3129
  if (command === 'optimizations' &&
3017
3130
  (subcommand === 'pause' ||
3018
3131
  subcommand === 'resume' ||
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "orizu",
3
- "version": "0.3.2",
3
+ "version": "0.3.3",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
package/src/index.ts CHANGED
@@ -99,6 +99,62 @@ interface PromptSummary {
99
99
  description?: string | null
100
100
  }
101
101
 
102
+ interface PromptCommentAuthor {
103
+ name: string
104
+ initials?: string
105
+ color?: string
106
+ }
107
+
108
+ interface PromptCommentAnchor {
109
+ text: string | null
110
+ startLine: number | null
111
+ endLine: number | null
112
+ }
113
+
114
+ interface PromptCommentReply {
115
+ id: string
116
+ body: string
117
+ author: PromptCommentAuthor
118
+ createdAt: string
119
+ updatedAt: string
120
+ }
121
+
122
+ interface PromptCommentThread {
123
+ id: string
124
+ status: 'open' | 'resolved'
125
+ body: string
126
+ anchor: PromptCommentAnchor | null
127
+ author: PromptCommentAuthor
128
+ createdAt: string
129
+ updatedAt: string
130
+ resolvedAt: string | null
131
+ resolvedByUserId: string | null
132
+ replyCount: number
133
+ replies: PromptCommentReply[]
134
+ }
135
+
136
+ interface PromptCommentsPayload {
137
+ prompt: {
138
+ id: string
139
+ name: string
140
+ role: string
141
+ description?: string | null
142
+ }
143
+ version: {
144
+ id: string
145
+ versionNumber?: number
146
+ versionLabel?: string | null
147
+ status?: string | null
148
+ }
149
+ summary: {
150
+ threadCount: number
151
+ openThreadCount: number
152
+ resolvedThreadCount: number
153
+ replyCount: number
154
+ }
155
+ comments: PromptCommentThread[]
156
+ }
157
+
102
158
  interface ScorerSummary {
103
159
  id: string
104
160
  name: string
@@ -200,11 +256,11 @@ function printVersion() {
200
256
  }
201
257
 
202
258
  function printUsage() {
203
- printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`)
259
+ printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <id>] [--json]\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`)
204
260
  }
205
261
 
206
262
  function printOptimizationUsage() {
207
- printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`)
263
+ printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]\n orizu optimizations export <run-id> [--out <path>] [--json]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`)
208
264
  }
209
265
 
210
266
  let cliArgs = process.argv.slice(2)
@@ -835,6 +891,78 @@ function printTaskStatusSummary(data: TaskStatusPayload) {
835
891
  }
836
892
  }
837
893
 
894
+ function compactTerminalText(value: string, maxLength = 160): string {
895
+ const compact = sanitizeTerminalText(value).replace(/\s+/g, ' ').trim()
896
+ if (compact.length <= maxLength) {
897
+ return compact
898
+ }
899
+
900
+ return `${compact.slice(0, Math.max(0, maxLength - 1)).trim()}…`
901
+ }
902
+
903
+ function printIndentedBody(value: string, indent: string) {
904
+ const lines = sanitizeTerminalText(value || '').split(/\r?\n/)
905
+ for (const line of lines) {
906
+ printLine(`${indent}${line}`)
907
+ }
908
+ }
909
+
910
+ function formatPromptCommentAnchor(anchor: PromptCommentAnchor | null): string | null {
911
+ if (!anchor) {
912
+ return null
913
+ }
914
+
915
+ const lineLabel = anchor.startLine && anchor.endLine
916
+ ? anchor.startLine === anchor.endLine
917
+ ? `line ${anchor.startLine}`
918
+ : `lines ${anchor.startLine}-${anchor.endLine}`
919
+ : null
920
+ const selectedText = anchor.text ? `"${compactTerminalText(anchor.text, 180)}"` : null
921
+
922
+ if (lineLabel && selectedText) {
923
+ return `${lineLabel}: ${selectedText}`
924
+ }
925
+ return lineLabel || selectedText
926
+ }
927
+
928
+ function printPromptComments(data: PromptCommentsPayload) {
929
+ const versionLabel = data.version.versionLabel || (
930
+ data.version.versionNumber ? `v${data.version.versionNumber}` : data.version.id
931
+ )
932
+
933
+ printLine(`Prompt: ${sanitizeTerminalText(data.prompt.name)} (${sanitizeTerminalText(data.prompt.id)})`)
934
+ printLine(`Version: ${sanitizeTerminalText(String(versionLabel))} (${sanitizeTerminalText(data.version.id)})`)
935
+ printLine(
936
+ `Comments: ${data.summary.threadCount} thread${data.summary.threadCount === 1 ? '' : 's'} ` +
937
+ `(${data.summary.openThreadCount} open, ${data.summary.resolvedThreadCount} resolved, ${data.summary.replyCount} replies)`
938
+ )
939
+
940
+ if (data.comments.length === 0) {
941
+ printLine('\nNo comments found for this prompt version.')
942
+ return
943
+ }
944
+
945
+ for (const [index, thread] of data.comments.entries()) {
946
+ const authorName = thread.author?.name || 'Unknown'
947
+ const anchor = formatPromptCommentAnchor(thread.anchor)
948
+ printLine('')
949
+ printLine(`${index + 1}. [${thread.status}] ${sanitizeTerminalText(authorName)} · ${sanitizeTerminalText(thread.createdAt)}`)
950
+ if (anchor) {
951
+ printLine(` Selection: ${anchor}`)
952
+ }
953
+ printIndentedBody(thread.body, ' ')
954
+
955
+ if (thread.replies.length > 0) {
956
+ printLine(` Replies (${thread.replies.length})`)
957
+ for (const reply of thread.replies) {
958
+ const replyAuthor = reply.author?.name || 'Unknown'
959
+ printLine(` - ${sanitizeTerminalText(replyAuthor)} · ${sanitizeTerminalText(reply.createdAt)}`)
960
+ printIndentedBody(reply.body, ' ')
961
+ }
962
+ }
963
+ }
964
+ }
965
+
838
966
  const DEFAULT_AUTH_CALLBACK_PORT = 43123
839
967
 
840
968
  function resolveAuthCallbackPort(): number {
@@ -1185,6 +1313,37 @@ async function listJudges() {
1185
1313
  printPromptSummaries(data.judges, 'No judges found.')
1186
1314
  }
1187
1315
 
1316
+ async function listPromptComments() {
1317
+ const promptRef = getPositionalArg(2)
1318
+ const project = getArg('--project') || await resolveProjectSlug(null)
1319
+ const label = getArg('--label')
1320
+ const version = getArg('--version')
1321
+
1322
+ if (!promptRef) {
1323
+ throw new Error('Usage: orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <version-id>] [--json]')
1324
+ }
1325
+ if (label && version) {
1326
+ throw new Error('Use either --label or --version, not both')
1327
+ }
1328
+
1329
+ const params = new URLSearchParams({ project })
1330
+ if (label) params.set('label', label)
1331
+ if (version) params.set('version', version)
1332
+
1333
+ const response = await authedFetch(`/api/cli/prompts/${encodeURIComponent(promptRef)}/comments?${params.toString()}`)
1334
+ if (!response.ok) {
1335
+ throw new Error(`Failed to fetch prompt comments: ${await response.text()}`)
1336
+ }
1337
+
1338
+ const data = await parseJsonResponse<PromptCommentsPayload>(response, 'Prompt comments')
1339
+ if (hasJsonFlag()) {
1340
+ printJson(data as unknown as Record<string, unknown>)
1341
+ return
1342
+ }
1343
+
1344
+ printPromptComments(data)
1345
+ }
1346
+
1188
1347
  async function listScorers() {
1189
1348
  const project = getArg('--project') || await resolveProjectSlug(null)
1190
1349
  const response = await authedFetch(`/api/cli/scorers?project=${encodeURIComponent(project)}`)
@@ -2070,6 +2229,32 @@ async function updateOptimizationRunLifecycle(action: OptimizationLifecycleActio
2070
2229
  }
2071
2230
  }
2072
2231
 
2232
+ async function exportOptimizationRun() {
2233
+ const runId = getPositionalArg(2) || getArg('--run-id')
2234
+ const outPathArg = getArg('--out')
2235
+
2236
+ if (!runId) {
2237
+ throw new Error('Usage: orizu optimizations export <run-id> [--out <path>] [--json]')
2238
+ }
2239
+
2240
+ const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}/export`)
2241
+ if (!response.ok) {
2242
+ throw new Error(`Failed to export optimization run: ${await response.text()}`)
2243
+ }
2244
+
2245
+ const data = await parseJsonResponse<Record<string, unknown>>(response, 'Optimization export')
2246
+ if (hasJsonFlag() && !outPathArg) {
2247
+ printJson(data)
2248
+ return
2249
+ }
2250
+
2251
+ const filename = outPathArg
2252
+ ? expandHomePath(outPathArg)
2253
+ : `${runId}.optimization.json`
2254
+ writeTextFileEnsuringDir(filename, `${JSON.stringify(data, null, 2)}\n`)
2255
+ printLine(`Saved optimization export to ${sanitizeTerminalText(filename)}`)
2256
+ }
2257
+
2073
2258
  function removeFlagWithValue(args: string[], flag: string): string[] {
2074
2259
  const filtered: string[] = []
2075
2260
  for (let index = 0; index < args.length; index += 1) {
@@ -3930,6 +4115,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
3930
4115
  return
3931
4116
  }
3932
4117
 
4118
+ if (command === 'prompts' && subcommand === 'comments') {
4119
+ await listPromptComments()
4120
+ return
4121
+ }
4122
+
3933
4123
  if (command === 'prompts' && subcommand === 'pull') {
3934
4124
  await pullPromptArtifact('prompt')
3935
4125
  return
@@ -4025,6 +4215,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
4025
4215
  return
4026
4216
  }
4027
4217
 
4218
+ if (command === 'optimizations' && subcommand === 'export') {
4219
+ await exportOptimizationRun()
4220
+ return
4221
+ }
4222
+
4028
4223
  if (
4029
4224
  command === 'optimizations' &&
4030
4225
  (
@@ -6,6 +6,7 @@ from pathlib import Path
6
6
  from typing import Any
7
7
 
8
8
  from .client import OrizuClient, OrizuEventSink
9
+ from .local_log import LocalOptimizationLogger
9
10
  from .optimizer import TextGepaConfig, optimize_loaded_text_candidate
10
11
  from .reflection import reflect_with_provider
11
12
  from .runner import make_candidate_runner, make_scorer_runner
@@ -74,6 +75,8 @@ def main() -> None:
74
75
  parser.add_argument("--auto-promote", action="store_true")
75
76
  parser.add_argument("--promotion-label")
76
77
  parser.add_argument("--log-row-snapshots", action="store_true")
78
+ parser.add_argument("--log-dir", default="logs")
79
+ parser.add_argument("--no-local-log", action="store_true")
77
80
  parser.add_argument("--metadata", default="{}")
78
81
  args = parser.parse_args()
79
82
 
@@ -123,6 +126,19 @@ def main() -> None:
123
126
  split=args.val_split,
124
127
  )
125
128
 
129
+ run_metadata = {
130
+ **metadata,
131
+ "optimizer_package": "orizu-gepa-python",
132
+ "optimizer_family": "gepa",
133
+ "mode": "text-candidate",
134
+ "inference_lm": prompt_context.provider_settings.get("model"),
135
+ "reflection_lm": config.reflection_model,
136
+ "scorer_lm": scorer_context.provider_settings.get("model"),
137
+ "dataset_size": len(trainset) + len(valset),
138
+ "train_count": len(trainset),
139
+ "validation_count": len(valset),
140
+ }
141
+
126
142
  run_id = client.start_run(
127
143
  project=args.project,
128
144
  optimizer_version_id=args.optimizer_version_id,
@@ -132,21 +148,29 @@ def main() -> None:
132
148
  split_set_id=args.split_set_id,
133
149
  train_split=args.train_split,
134
150
  validation_split=args.val_split,
135
- metadata={
136
- **metadata,
137
- "optimizer_package": "orizu-gepa-python",
138
- "optimizer_family": "gepa",
139
- "mode": "text-candidate",
140
- "inference_lm": prompt_context.provider_settings.get("model"),
141
- "reflection_lm": config.reflection_model,
142
- "scorer_lm": scorer_context.provider_settings.get("model"),
143
- "dataset_size": len(trainset) + len(valset),
144
- "train_count": len(trainset),
145
- "validation_count": len(valset),
146
- },
151
+ metadata=run_metadata,
147
152
  )
148
153
  print(f"[orizu-gepa] started optimization run {run_id}", flush=True)
149
154
 
155
+ local_logger = None
156
+ if not args.no_local_log:
157
+ local_logger = LocalOptimizationLogger.create(args.log_dir, run_id)
158
+ local_logger.write_context(
159
+ project=args.project,
160
+ run_id=run_id,
161
+ args={
162
+ key: value
163
+ for key, value in vars(args).items()
164
+ if key not in {"metadata"}
165
+ },
166
+ prompt_context=prompt_context,
167
+ scorer_context=scorer_context,
168
+ trainset=trainset,
169
+ valset=valset,
170
+ metadata=run_metadata,
171
+ )
172
+ print(f"[orizu-gepa] local log: {local_logger.path}", flush=True)
173
+
150
174
  result = optimize_loaded_text_candidate(
151
175
  run_id=run_id,
152
176
  prompt_context=prompt_context,
@@ -156,9 +180,17 @@ def main() -> None:
156
180
  candidate_runner=make_candidate_runner(Path(args.candidate_runner_dir), run_id),
157
181
  scorer_runner=make_scorer_runner(Path(args.scorer_runner_dir), run_id),
158
182
  reflector=reflect_with_provider,
159
- event_sink=OrizuEventSink(client, run_id, fail_on_log_error=config.fail_on_log_error),
183
+ event_sink=OrizuEventSink(
184
+ client,
185
+ run_id,
186
+ fail_on_log_error=config.fail_on_log_error,
187
+ local_logger=local_logger,
188
+ ),
160
189
  config=config,
190
+ local_logger=local_logger,
161
191
  )
192
+ if local_logger is not None:
193
+ local_logger.write_result(result)
162
194
  print(json.dumps({
163
195
  "optimization_run_id": result.run_id,
164
196
  "best_candidate_id": result.best_candidate_id,
@@ -166,6 +198,7 @@ def main() -> None:
166
198
  "seed_score": result.seed_score,
167
199
  "promoted_prompt_version_id": result.promoted_prompt_version_id,
168
200
  "budget": result.budget.to_payload(),
201
+ "local_log_dir": local_logger.path if local_logger is not None else None,
169
202
  }))
170
203
 
171
204
 
@@ -255,12 +255,14 @@ class OrizuEventSink:
255
255
  *,
256
256
  fail_on_log_error: bool = True,
257
257
  max_log_retries: int = 2,
258
+ local_logger: Any | None = None,
258
259
  ):
259
260
  self.client = client
260
261
  self.run_id = run_id
261
262
  self.sequence = 0
262
263
  self.fail_on_log_error = fail_on_log_error
263
264
  self.max_log_retries = max(0, max_log_retries)
265
+ self.local_logger = local_logger
264
266
 
265
267
  def log_event(
266
268
  self,
@@ -275,6 +277,20 @@ class OrizuEventSink:
275
277
  child_candidate_id: str | None = None,
276
278
  ) -> None:
277
279
  self.sequence += 1
280
+ local_event = {
281
+ "run_id": self.run_id,
282
+ "sequence": self.sequence,
283
+ "event_type": event_type,
284
+ "event_layer": event_layer,
285
+ "optimizer_family": optimizer_family,
286
+ "iteration": iteration,
287
+ "candidate_id": candidate_id,
288
+ "parent_candidate_id": parent_candidate_id,
289
+ "child_candidate_id": child_candidate_id,
290
+ "payload": payload or {},
291
+ }
292
+ if self.local_logger is not None:
293
+ self.local_logger.append_event(local_event)
278
294
  last_error: Exception | None = None
279
295
  for attempt in range(self.max_log_retries + 1):
280
296
  try:
@@ -308,7 +324,37 @@ class OrizuEventSink:
308
324
  prompt_version_id = self.client.promote_candidate(self.run_id, **kwargs)
309
325
  # Promotion writes a system event at max(sequence)+1 in Postgres.
310
326
  self.sequence += 1
327
+ if self.local_logger is not None:
328
+ self.local_logger.append_event({
329
+ "run_id": self.run_id,
330
+ "sequence": self.sequence,
331
+ "event_type": "candidate_promoted",
332
+ "event_layer": "system",
333
+ "optimizer_family": "gepa",
334
+ "iteration": None,
335
+ "candidate_id": kwargs.get("candidate_id"),
336
+ "parent_candidate_id": None,
337
+ "child_candidate_id": None,
338
+ "payload": {
339
+ **kwargs,
340
+ "prompt_version_id": prompt_version_id,
341
+ },
342
+ })
311
343
  return prompt_version_id
312
344
 
313
345
  def finish_run(self, **kwargs: Any) -> None:
346
+ if self.local_logger is not None:
347
+ self.sequence += 1
348
+ self.local_logger.append_event({
349
+ "run_id": self.run_id,
350
+ "sequence": self.sequence,
351
+ "event_type": "run_status_updated",
352
+ "event_layer": "system",
353
+ "optimizer_family": "gepa",
354
+ "iteration": None,
355
+ "candidate_id": kwargs.get("best_candidate_id"),
356
+ "parent_candidate_id": None,
357
+ "child_candidate_id": None,
358
+ "payload": kwargs,
359
+ })
314
360
  self.client.update_run(self.run_id, **kwargs)
@@ -0,0 +1,123 @@
1
+ from __future__ import annotations
2
+
3
+ import dataclasses
4
+ import json
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from .optimizer import DatasetRow, PromptContext, RowEvaluation, TextGepaResult
9
+
10
+
11
+ def _json_default(value: Any) -> Any:
12
+ if dataclasses.is_dataclass(value):
13
+ return dataclasses.asdict(value)
14
+ return str(value)
15
+
16
+
17
+ def _write_json(path: Path, payload: dict[str, Any]) -> None:
18
+ path.write_text(
19
+ json.dumps(payload, indent=2, ensure_ascii=False, default=_json_default) + "\n",
20
+ encoding="utf-8",
21
+ )
22
+
23
+
24
+ def _append_jsonl(path: Path, payload: dict[str, Any]) -> None:
25
+ with path.open("a", encoding="utf-8") as handle:
26
+ handle.write(json.dumps(payload, ensure_ascii=False, default=_json_default) + "\n")
27
+
28
+
29
+ class LocalOptimizationLogger:
30
+ def __init__(self, directory: Path):
31
+ self.directory = directory
32
+ self.directory.mkdir(parents=True, exist_ok=True)
33
+
34
+ @classmethod
35
+ def create(cls, root: str | Path, run_id: str) -> "LocalOptimizationLogger":
36
+ return cls(Path(root) / run_id)
37
+
38
+ @property
39
+ def path(self) -> str:
40
+ return str(self.directory)
41
+
42
+ def write_context(
43
+ self,
44
+ *,
45
+ project: str,
46
+ run_id: str,
47
+ args: dict[str, Any],
48
+ prompt_context: PromptContext,
49
+ scorer_context: PromptContext,
50
+ trainset: list[DatasetRow],
51
+ valset: list[DatasetRow],
52
+ metadata: dict[str, Any],
53
+ ) -> None:
54
+ _write_json(self.directory / "run.json", {
55
+ "schema_version": "orizu.optimization-local-log.v1",
56
+ "optimization_run_id": run_id,
57
+ "project": project,
58
+ "args": args,
59
+ "metadata": metadata,
60
+ })
61
+ _write_json(self.directory / "prompt_context.json", dataclasses.asdict(prompt_context))
62
+ _write_json(self.directory / "scorer_context.json", dataclasses.asdict(scorer_context))
63
+ _write_json(self.directory / "trainset.json", {
64
+ "rows": [dataclasses.asdict(row) for row in trainset],
65
+ })
66
+ _write_json(self.directory / "valset.json", {
67
+ "rows": [dataclasses.asdict(row) for row in valset],
68
+ })
69
+
70
+ def append_event(self, event: dict[str, Any]) -> None:
71
+ _append_jsonl(self.directory / "events.jsonl", event)
72
+
73
+ def append_evaluations(
74
+ self,
75
+ *,
76
+ stage: str,
77
+ split: str,
78
+ iteration: int | None,
79
+ candidate_id: str,
80
+ parent_candidate_id: str | None = None,
81
+ results: list[RowEvaluation],
82
+ ) -> None:
83
+ for result in results:
84
+ _append_jsonl(self.directory / "evaluations.jsonl", {
85
+ "stage": stage,
86
+ "split": split,
87
+ "iteration": iteration,
88
+ "candidate_id": candidate_id,
89
+ "parent_candidate_id": parent_candidate_id,
90
+ **result.to_payload(),
91
+ })
92
+
93
+ def append_reflection(
94
+ self,
95
+ *,
96
+ iteration: int,
97
+ parent_candidate_id: str,
98
+ child_candidate_id: str,
99
+ row_ids: list[str],
100
+ prompt: str,
101
+ response: str,
102
+ candidate_text: str,
103
+ ) -> None:
104
+ _append_jsonl(self.directory / "reflections.jsonl", {
105
+ "iteration": iteration,
106
+ "parent_candidate_id": parent_candidate_id,
107
+ "child_candidate_id": child_candidate_id,
108
+ "row_ids": row_ids,
109
+ "prompt": prompt,
110
+ "response": response,
111
+ "candidate_text": candidate_text,
112
+ })
113
+
114
+ def write_result(self, result: TextGepaResult) -> None:
115
+ _write_json(self.directory / "result.json", {
116
+ "optimization_run_id": result.run_id,
117
+ "best_candidate_id": result.best_candidate_id,
118
+ "best_candidate_text": result.best_candidate_text,
119
+ "best_score": result.best_score,
120
+ "seed_score": result.seed_score,
121
+ "promoted_prompt_version_id": result.promoted_prompt_version_id,
122
+ "budget": result.budget.to_payload(),
123
+ })
@@ -658,6 +658,7 @@ def optimize_loaded_text_candidate(
658
658
  reflector: Reflector,
659
659
  event_sink: EventSink,
660
660
  config: TextGepaConfig,
661
+ local_logger: Any | None = None,
661
662
  ) -> TextGepaResult:
662
663
  seed_text = prompt_context.body or ""
663
664
  budget = Budget.from_config(config)
@@ -709,6 +710,14 @@ def optimize_loaded_text_candidate(
709
710
  budget=budget,
710
711
  evaluation_cache=evaluation_cache,
711
712
  )
713
+ if local_logger is not None:
714
+ local_logger.append_evaluations(
715
+ stage="seed_val_set",
716
+ split="validation",
717
+ iteration=None,
718
+ candidate_id="seed",
719
+ results=seed_results,
720
+ )
712
721
  if not _completed_all_rows(seed_results, valset):
713
722
  raise RuntimeError("Budget exhausted before seed validation completed")
714
723
  budget.used_full_evals += 1
@@ -804,6 +813,14 @@ def optimize_loaded_text_candidate(
804
813
  budget=budget,
805
814
  evaluation_cache=evaluation_cache,
806
815
  )
816
+ if local_logger is not None:
817
+ local_logger.append_evaluations(
818
+ stage="parent_minibatch",
819
+ split="train",
820
+ iteration=iteration,
821
+ candidate_id=parent_candidate_id,
822
+ results=parent_results,
823
+ )
807
824
  if not _completed_all_rows(parent_results, minibatch):
808
825
  event_sink.log_event(
809
826
  "budget_exhausted",
@@ -852,11 +869,21 @@ def optimize_loaded_text_candidate(
852
869
  child_text = reflection.candidate_text
853
870
  candidate_text_by_id[child_id] = child_text
854
871
  budget.used_candidate_proposals += 1
872
+ if local_logger is not None:
873
+ local_logger.append_reflection(
874
+ iteration=iteration,
875
+ parent_candidate_id=parent_candidate_id,
876
+ child_candidate_id=child_id,
877
+ row_ids=[row.id for row in minibatch],
878
+ prompt=reflection.prompt,
879
+ response=reflection.response,
880
+ candidate_text=reflection.candidate_text,
881
+ )
855
882
  event_sink.log_event(
856
883
  "reflection_completed",
857
884
  {
858
885
  **_text_log_fields("prompt", reflection.prompt, include_text=config.log_row_snapshots),
859
- **_text_log_fields("response", reflection.response, include_text=config.log_row_snapshots),
886
+ "response": reflection.response,
860
887
  "child_candidate_id": child_id,
861
888
  "candidate_text": child_text,
862
889
  },
@@ -908,6 +935,15 @@ def optimize_loaded_text_candidate(
908
935
  budget=budget,
909
936
  evaluation_cache=evaluation_cache,
910
937
  )
938
+ if local_logger is not None:
939
+ local_logger.append_evaluations(
940
+ stage="child_minibatch",
941
+ split="train",
942
+ iteration=iteration,
943
+ candidate_id=child_id,
944
+ parent_candidate_id=parent_candidate_id,
945
+ results=child_results,
946
+ )
911
947
  if not _completed_all_rows(child_results, minibatch):
912
948
  event_sink.log_event(
913
949
  "budget_exhausted",
@@ -981,6 +1017,15 @@ def optimize_loaded_text_candidate(
981
1017
  budget=budget,
982
1018
  evaluation_cache=evaluation_cache,
983
1019
  )
1020
+ if local_logger is not None:
1021
+ local_logger.append_evaluations(
1022
+ stage="child_val_set",
1023
+ split="validation",
1024
+ iteration=iteration,
1025
+ candidate_id=child_id,
1026
+ parent_candidate_id=parent_candidate_id,
1027
+ results=child_val_results,
1028
+ )
984
1029
  if not _completed_all_rows(child_val_results, valset):
985
1030
  event_sink.log_event(
986
1031
  "budget_exhausted",