orizu 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/index.js +171 -11
- package/package.json +12 -2
- package/scripts/vendor-gepa-python.mjs +54 -0
- package/src/index.ts +203 -4
- package/vendor/orizu-gepa-python/manifest.json +9 -0
- package/vendor/orizu-gepa-python/pyproject.toml +16 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/__init__.py +19 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py +206 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/client.py +360 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/local_log.py +123 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/optimizer.py +1276 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/reflection.py +177 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/runner.py +207 -0
package/src/index.ts
CHANGED
|
@@ -99,6 +99,62 @@ interface PromptSummary {
|
|
|
99
99
|
description?: string | null
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
interface PromptCommentAuthor {
|
|
103
|
+
name: string
|
|
104
|
+
initials?: string
|
|
105
|
+
color?: string
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
interface PromptCommentAnchor {
|
|
109
|
+
text: string | null
|
|
110
|
+
startLine: number | null
|
|
111
|
+
endLine: number | null
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
interface PromptCommentReply {
|
|
115
|
+
id: string
|
|
116
|
+
body: string
|
|
117
|
+
author: PromptCommentAuthor
|
|
118
|
+
createdAt: string
|
|
119
|
+
updatedAt: string
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
interface PromptCommentThread {
|
|
123
|
+
id: string
|
|
124
|
+
status: 'open' | 'resolved'
|
|
125
|
+
body: string
|
|
126
|
+
anchor: PromptCommentAnchor | null
|
|
127
|
+
author: PromptCommentAuthor
|
|
128
|
+
createdAt: string
|
|
129
|
+
updatedAt: string
|
|
130
|
+
resolvedAt: string | null
|
|
131
|
+
resolvedByUserId: string | null
|
|
132
|
+
replyCount: number
|
|
133
|
+
replies: PromptCommentReply[]
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
interface PromptCommentsPayload {
|
|
137
|
+
prompt: {
|
|
138
|
+
id: string
|
|
139
|
+
name: string
|
|
140
|
+
role: string
|
|
141
|
+
description?: string | null
|
|
142
|
+
}
|
|
143
|
+
version: {
|
|
144
|
+
id: string
|
|
145
|
+
versionNumber?: number
|
|
146
|
+
versionLabel?: string | null
|
|
147
|
+
status?: string | null
|
|
148
|
+
}
|
|
149
|
+
summary: {
|
|
150
|
+
threadCount: number
|
|
151
|
+
openThreadCount: number
|
|
152
|
+
resolvedThreadCount: number
|
|
153
|
+
replyCount: number
|
|
154
|
+
}
|
|
155
|
+
comments: PromptCommentThread[]
|
|
156
|
+
}
|
|
157
|
+
|
|
102
158
|
interface ScorerSummary {
|
|
103
159
|
id: string
|
|
104
160
|
name: string
|
|
@@ -200,11 +256,11 @@ function printVersion() {
|
|
|
200
256
|
}
|
|
201
257
|
|
|
202
258
|
function printUsage() {
|
|
203
|
-
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`)
|
|
259
|
+
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <id>] [--json]\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`)
|
|
204
260
|
}
|
|
205
261
|
|
|
206
262
|
function printOptimizationUsage() {
|
|
207
|
-
printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`)
|
|
263
|
+
printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]\n orizu optimizations export <run-id> [--out <path>] [--json]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`)
|
|
208
264
|
}
|
|
209
265
|
|
|
210
266
|
let cliArgs = process.argv.slice(2)
|
|
@@ -835,6 +891,78 @@ function printTaskStatusSummary(data: TaskStatusPayload) {
|
|
|
835
891
|
}
|
|
836
892
|
}
|
|
837
893
|
|
|
894
|
+
function compactTerminalText(value: string, maxLength = 160): string {
|
|
895
|
+
const compact = sanitizeTerminalText(value).replace(/\s+/g, ' ').trim()
|
|
896
|
+
if (compact.length <= maxLength) {
|
|
897
|
+
return compact
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
return `${compact.slice(0, Math.max(0, maxLength - 1)).trim()}…`
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
function printIndentedBody(value: string, indent: string) {
|
|
904
|
+
const lines = sanitizeTerminalText(value || '').split(/\r?\n/)
|
|
905
|
+
for (const line of lines) {
|
|
906
|
+
printLine(`${indent}${line}`)
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
function formatPromptCommentAnchor(anchor: PromptCommentAnchor | null): string | null {
|
|
911
|
+
if (!anchor) {
|
|
912
|
+
return null
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
const lineLabel = anchor.startLine && anchor.endLine
|
|
916
|
+
? anchor.startLine === anchor.endLine
|
|
917
|
+
? `line ${anchor.startLine}`
|
|
918
|
+
: `lines ${anchor.startLine}-${anchor.endLine}`
|
|
919
|
+
: null
|
|
920
|
+
const selectedText = anchor.text ? `"${compactTerminalText(anchor.text, 180)}"` : null
|
|
921
|
+
|
|
922
|
+
if (lineLabel && selectedText) {
|
|
923
|
+
return `${lineLabel}: ${selectedText}`
|
|
924
|
+
}
|
|
925
|
+
return lineLabel || selectedText
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
function printPromptComments(data: PromptCommentsPayload) {
|
|
929
|
+
const versionLabel = data.version.versionLabel || (
|
|
930
|
+
data.version.versionNumber ? `v${data.version.versionNumber}` : data.version.id
|
|
931
|
+
)
|
|
932
|
+
|
|
933
|
+
printLine(`Prompt: ${sanitizeTerminalText(data.prompt.name)} (${sanitizeTerminalText(data.prompt.id)})`)
|
|
934
|
+
printLine(`Version: ${sanitizeTerminalText(String(versionLabel))} (${sanitizeTerminalText(data.version.id)})`)
|
|
935
|
+
printLine(
|
|
936
|
+
`Comments: ${data.summary.threadCount} thread${data.summary.threadCount === 1 ? '' : 's'} ` +
|
|
937
|
+
`(${data.summary.openThreadCount} open, ${data.summary.resolvedThreadCount} resolved, ${data.summary.replyCount} replies)`
|
|
938
|
+
)
|
|
939
|
+
|
|
940
|
+
if (data.comments.length === 0) {
|
|
941
|
+
printLine('\nNo comments found for this prompt version.')
|
|
942
|
+
return
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
for (const [index, thread] of data.comments.entries()) {
|
|
946
|
+
const authorName = thread.author?.name || 'Unknown'
|
|
947
|
+
const anchor = formatPromptCommentAnchor(thread.anchor)
|
|
948
|
+
printLine('')
|
|
949
|
+
printLine(`${index + 1}. [${thread.status}] ${sanitizeTerminalText(authorName)} · ${sanitizeTerminalText(thread.createdAt)}`)
|
|
950
|
+
if (anchor) {
|
|
951
|
+
printLine(` Selection: ${anchor}`)
|
|
952
|
+
}
|
|
953
|
+
printIndentedBody(thread.body, ' ')
|
|
954
|
+
|
|
955
|
+
if (thread.replies.length > 0) {
|
|
956
|
+
printLine(` Replies (${thread.replies.length})`)
|
|
957
|
+
for (const reply of thread.replies) {
|
|
958
|
+
const replyAuthor = reply.author?.name || 'Unknown'
|
|
959
|
+
printLine(` - ${sanitizeTerminalText(replyAuthor)} · ${sanitizeTerminalText(reply.createdAt)}`)
|
|
960
|
+
printIndentedBody(reply.body, ' ')
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
838
966
|
const DEFAULT_AUTH_CALLBACK_PORT = 43123
|
|
839
967
|
|
|
840
968
|
function resolveAuthCallbackPort(): number {
|
|
@@ -1185,6 +1313,37 @@ async function listJudges() {
|
|
|
1185
1313
|
printPromptSummaries(data.judges, 'No judges found.')
|
|
1186
1314
|
}
|
|
1187
1315
|
|
|
1316
|
+
async function listPromptComments() {
|
|
1317
|
+
const promptRef = getPositionalArg(2)
|
|
1318
|
+
const project = getArg('--project') || await resolveProjectSlug(null)
|
|
1319
|
+
const label = getArg('--label')
|
|
1320
|
+
const version = getArg('--version')
|
|
1321
|
+
|
|
1322
|
+
if (!promptRef) {
|
|
1323
|
+
throw new Error('Usage: orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <version-id>] [--json]')
|
|
1324
|
+
}
|
|
1325
|
+
if (label && version) {
|
|
1326
|
+
throw new Error('Use either --label or --version, not both')
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
const params = new URLSearchParams({ project })
|
|
1330
|
+
if (label) params.set('label', label)
|
|
1331
|
+
if (version) params.set('version', version)
|
|
1332
|
+
|
|
1333
|
+
const response = await authedFetch(`/api/cli/prompts/${encodeURIComponent(promptRef)}/comments?${params.toString()}`)
|
|
1334
|
+
if (!response.ok) {
|
|
1335
|
+
throw new Error(`Failed to fetch prompt comments: ${await response.text()}`)
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
const data = await parseJsonResponse<PromptCommentsPayload>(response, 'Prompt comments')
|
|
1339
|
+
if (hasJsonFlag()) {
|
|
1340
|
+
printJson(data as unknown as Record<string, unknown>)
|
|
1341
|
+
return
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
printPromptComments(data)
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1188
1347
|
async function listScorers() {
|
|
1189
1348
|
const project = getArg('--project') || await resolveProjectSlug(null)
|
|
1190
1349
|
const response = await authedFetch(`/api/cli/scorers?project=${encodeURIComponent(project)}`)
|
|
@@ -2070,6 +2229,32 @@ async function updateOptimizationRunLifecycle(action: OptimizationLifecycleActio
|
|
|
2070
2229
|
}
|
|
2071
2230
|
}
|
|
2072
2231
|
|
|
2232
|
+
async function exportOptimizationRun() {
|
|
2233
|
+
const runId = getPositionalArg(2) || getArg('--run-id')
|
|
2234
|
+
const outPathArg = getArg('--out')
|
|
2235
|
+
|
|
2236
|
+
if (!runId) {
|
|
2237
|
+
throw new Error('Usage: orizu optimizations export <run-id> [--out <path>] [--json]')
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}/export`)
|
|
2241
|
+
if (!response.ok) {
|
|
2242
|
+
throw new Error(`Failed to export optimization run: ${await response.text()}`)
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
const data = await parseJsonResponse<Record<string, unknown>>(response, 'Optimization export')
|
|
2246
|
+
if (hasJsonFlag() && !outPathArg) {
|
|
2247
|
+
printJson(data)
|
|
2248
|
+
return
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
const filename = outPathArg
|
|
2252
|
+
? expandHomePath(outPathArg)
|
|
2253
|
+
: `${runId}.optimization.json`
|
|
2254
|
+
writeTextFileEnsuringDir(filename, `${JSON.stringify(data, null, 2)}\n`)
|
|
2255
|
+
printLine(`Saved optimization export to ${sanitizeTerminalText(filename)}`)
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2073
2258
|
function removeFlagWithValue(args: string[], flag: string): string[] {
|
|
2074
2259
|
const filtered: string[] = []
|
|
2075
2260
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -2083,8 +2268,12 @@ function removeFlagWithValue(args: string[], flag: string): string[] {
|
|
|
2083
2268
|
}
|
|
2084
2269
|
|
|
2085
2270
|
function bundledOrizuGepaPythonPath(): string | null {
|
|
2086
|
-
const
|
|
2087
|
-
|
|
2271
|
+
const candidates = [
|
|
2272
|
+
fileURLToPath(new URL('../vendor/orizu-gepa-python/src', import.meta.url)),
|
|
2273
|
+
fileURLToPath(new URL('../../orizu-gepa-python/src', import.meta.url)),
|
|
2274
|
+
]
|
|
2275
|
+
|
|
2276
|
+
return candidates.find(candidate => existsSync(candidate)) ?? null
|
|
2088
2277
|
}
|
|
2089
2278
|
|
|
2090
2279
|
async function runGepaOptimization() {
|
|
@@ -3926,6 +4115,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
3926
4115
|
return
|
|
3927
4116
|
}
|
|
3928
4117
|
|
|
4118
|
+
if (command === 'prompts' && subcommand === 'comments') {
|
|
4119
|
+
await listPromptComments()
|
|
4120
|
+
return
|
|
4121
|
+
}
|
|
4122
|
+
|
|
3929
4123
|
if (command === 'prompts' && subcommand === 'pull') {
|
|
3930
4124
|
await pullPromptArtifact('prompt')
|
|
3931
4125
|
return
|
|
@@ -4021,6 +4215,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
4021
4215
|
return
|
|
4022
4216
|
}
|
|
4023
4217
|
|
|
4218
|
+
if (command === 'optimizations' && subcommand === 'export') {
|
|
4219
|
+
await exportOptimizationRun()
|
|
4220
|
+
return
|
|
4221
|
+
}
|
|
4222
|
+
|
|
4024
4223
|
if (
|
|
4025
4224
|
command === 'optimizations' &&
|
|
4026
4225
|
(
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "orizu.optimizer.v1",
|
|
3
|
+
"name": "orizu-gepa-text",
|
|
4
|
+
"description": "Orizu-owned text candidate optimizer with GEPA-style reflection and mandatory live event logging.",
|
|
5
|
+
"language": "python",
|
|
6
|
+
"command": ["python3", "-m", "orizu_gepa.cli"],
|
|
7
|
+
"optimizer_family": "gepa",
|
|
8
|
+
"candidate_kind": "text"
|
|
9
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "orizu-gepa"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Orizu-owned text prompt optimizer with GEPA-style reflection and mandatory live logging."
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = []
|
|
11
|
+
|
|
12
|
+
[project.scripts]
|
|
13
|
+
orizu-gepa = "orizu_gepa.cli:main"
|
|
14
|
+
|
|
15
|
+
[tool.setuptools.packages.find]
|
|
16
|
+
where = ["src"]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from .optimizer import (
|
|
2
|
+
Budget,
|
|
3
|
+
DatasetRow,
|
|
4
|
+
PromptContext,
|
|
5
|
+
RowEvaluation,
|
|
6
|
+
TextGepaConfig,
|
|
7
|
+
TextGepaResult,
|
|
8
|
+
optimize_text_candidate,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Budget",
|
|
13
|
+
"DatasetRow",
|
|
14
|
+
"PromptContext",
|
|
15
|
+
"RowEvaluation",
|
|
16
|
+
"TextGepaConfig",
|
|
17
|
+
"TextGepaResult",
|
|
18
|
+
"optimize_text_candidate",
|
|
19
|
+
]
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from .client import OrizuClient, OrizuEventSink
|
|
9
|
+
from .local_log import LocalOptimizationLogger
|
|
10
|
+
from .optimizer import TextGepaConfig, optimize_loaded_text_candidate
|
|
11
|
+
from .reflection import reflect_with_provider
|
|
12
|
+
from .runner import make_candidate_runner, make_scorer_runner
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def positive_int(value: str) -> int:
|
|
16
|
+
parsed = int(value)
|
|
17
|
+
if parsed <= 0:
|
|
18
|
+
raise argparse.ArgumentTypeError("must be positive")
|
|
19
|
+
return parsed
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def read_text_arg(value: str | None) -> str | None:
|
|
23
|
+
if value is None:
|
|
24
|
+
return None
|
|
25
|
+
if value.startswith("@"):
|
|
26
|
+
return Path(value[1:]).read_text()
|
|
27
|
+
path = Path(value)
|
|
28
|
+
if path.exists():
|
|
29
|
+
return path.read_text()
|
|
30
|
+
return value
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def read_json_object_arg(value: str | None, flag_name: str) -> dict[str, Any] | None:
|
|
34
|
+
text = read_text_arg(value)
|
|
35
|
+
if text is None:
|
|
36
|
+
return None
|
|
37
|
+
parsed = json.loads(text)
|
|
38
|
+
if not isinstance(parsed, dict):
|
|
39
|
+
raise RuntimeError(f"{flag_name} must be a JSON object")
|
|
40
|
+
return parsed
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def main() -> None:
|
|
44
|
+
parser = argparse.ArgumentParser(prog="orizu-gepa")
|
|
45
|
+
parser.add_argument("--project", required=True)
|
|
46
|
+
parser.add_argument("--optimizer-version-id", required=True)
|
|
47
|
+
parser.add_argument("--candidate-version-id", required=True)
|
|
48
|
+
parser.add_argument("--runner-version-id", required=True)
|
|
49
|
+
parser.add_argument("--candidate-runner-dir", required=True)
|
|
50
|
+
parser.add_argument("--scorer-version-id", required=True)
|
|
51
|
+
parser.add_argument("--scorer-runner-version-id", required=True)
|
|
52
|
+
parser.add_argument("--scorer-runner-dir", required=True)
|
|
53
|
+
parser.add_argument("--dataset-version-id", required=True)
|
|
54
|
+
parser.add_argument("--split-set-id", required=True)
|
|
55
|
+
parser.add_argument("--train-split", default="train")
|
|
56
|
+
parser.add_argument("--val-split", default="validation")
|
|
57
|
+
parser.add_argument("--budget", default="light", choices=["auto", "light", "medium", "high"])
|
|
58
|
+
parser.add_argument("--max-iterations", type=positive_int, default=3)
|
|
59
|
+
parser.add_argument("--minibatch-size", type=positive_int, default=3)
|
|
60
|
+
parser.add_argument(
|
|
61
|
+
"--candidate-selection-strategy",
|
|
62
|
+
default=TextGepaConfig.candidate_selection_strategy,
|
|
63
|
+
choices=["pareto", "current_best", "epsilon_greedy"],
|
|
64
|
+
)
|
|
65
|
+
parser.add_argument("--epsilon", type=float, default=TextGepaConfig.epsilon)
|
|
66
|
+
parser.add_argument("--disable-evaluation-cache", action="store_true")
|
|
67
|
+
parser.add_argument("--max-metric-calls", type=positive_int)
|
|
68
|
+
parser.add_argument("--max-full-evals", type=positive_int)
|
|
69
|
+
parser.add_argument("--reflection-model", default=TextGepaConfig.reflection_model)
|
|
70
|
+
parser.add_argument("--reflection-temperature", type=float)
|
|
71
|
+
parser.add_argument("--reflection-prompt-template")
|
|
72
|
+
parser.add_argument("--reflection-provider-settings")
|
|
73
|
+
parser.add_argument("--objective", default=TextGepaConfig.objective)
|
|
74
|
+
parser.add_argument("--seed", type=int, default=0)
|
|
75
|
+
parser.add_argument("--auto-promote", action="store_true")
|
|
76
|
+
parser.add_argument("--promotion-label")
|
|
77
|
+
parser.add_argument("--log-row-snapshots", action="store_true")
|
|
78
|
+
parser.add_argument("--log-dir", default="logs")
|
|
79
|
+
parser.add_argument("--no-local-log", action="store_true")
|
|
80
|
+
parser.add_argument("--metadata", default="{}")
|
|
81
|
+
args = parser.parse_args()
|
|
82
|
+
|
|
83
|
+
metadata = json.loads(args.metadata)
|
|
84
|
+
if not isinstance(metadata, dict):
|
|
85
|
+
raise RuntimeError("--metadata must be a JSON object")
|
|
86
|
+
|
|
87
|
+
config = TextGepaConfig(
|
|
88
|
+
budget=args.budget,
|
|
89
|
+
max_iterations=args.max_iterations,
|
|
90
|
+
minibatch_size=args.minibatch_size,
|
|
91
|
+
candidate_selection_strategy=args.candidate_selection_strategy,
|
|
92
|
+
epsilon=args.epsilon,
|
|
93
|
+
max_metric_calls=args.max_metric_calls,
|
|
94
|
+
max_full_evals=args.max_full_evals,
|
|
95
|
+
reflection_model=args.reflection_model,
|
|
96
|
+
reflection_temperature=args.reflection_temperature,
|
|
97
|
+
reflection_prompt_template=read_text_arg(args.reflection_prompt_template),
|
|
98
|
+
reflection_provider_settings=read_json_object_arg(args.reflection_provider_settings, "--reflection-provider-settings") or {},
|
|
99
|
+
objective=args.objective,
|
|
100
|
+
seed=args.seed,
|
|
101
|
+
auto_promote=args.auto_promote,
|
|
102
|
+
promotion_label=args.promotion_label,
|
|
103
|
+
cache_evaluations=not args.disable_evaluation_cache,
|
|
104
|
+
log_row_snapshots=args.log_row_snapshots,
|
|
105
|
+
)
|
|
106
|
+
client = OrizuClient.from_env()
|
|
107
|
+
prompt_context, trainset = client.fetch_exec_context(
|
|
108
|
+
prompt_version_id=args.candidate_version_id,
|
|
109
|
+
runner_version_id=args.runner_version_id,
|
|
110
|
+
dataset_version_id=args.dataset_version_id,
|
|
111
|
+
split_set_id=args.split_set_id,
|
|
112
|
+
split=args.train_split,
|
|
113
|
+
)
|
|
114
|
+
_, valset = client.fetch_exec_context(
|
|
115
|
+
prompt_version_id=args.candidate_version_id,
|
|
116
|
+
runner_version_id=args.runner_version_id,
|
|
117
|
+
dataset_version_id=args.dataset_version_id,
|
|
118
|
+
split_set_id=args.split_set_id,
|
|
119
|
+
split=args.val_split,
|
|
120
|
+
)
|
|
121
|
+
scorer_context, _ = client.fetch_scorer_exec_context(
|
|
122
|
+
scorer_version_id=args.scorer_version_id,
|
|
123
|
+
runner_version_id=args.scorer_runner_version_id,
|
|
124
|
+
dataset_version_id=args.dataset_version_id,
|
|
125
|
+
split_set_id=args.split_set_id,
|
|
126
|
+
split=args.val_split,
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
run_metadata = {
|
|
130
|
+
**metadata,
|
|
131
|
+
"optimizer_package": "orizu-gepa-python",
|
|
132
|
+
"optimizer_family": "gepa",
|
|
133
|
+
"mode": "text-candidate",
|
|
134
|
+
"inference_lm": prompt_context.provider_settings.get("model"),
|
|
135
|
+
"reflection_lm": config.reflection_model,
|
|
136
|
+
"scorer_lm": scorer_context.provider_settings.get("model"),
|
|
137
|
+
"dataset_size": len(trainset) + len(valset),
|
|
138
|
+
"train_count": len(trainset),
|
|
139
|
+
"validation_count": len(valset),
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
run_id = client.start_run(
|
|
143
|
+
project=args.project,
|
|
144
|
+
optimizer_version_id=args.optimizer_version_id,
|
|
145
|
+
prompt_version_id=args.candidate_version_id,
|
|
146
|
+
scorer_version_id=args.scorer_version_id,
|
|
147
|
+
dataset_version_id=args.dataset_version_id,
|
|
148
|
+
split_set_id=args.split_set_id,
|
|
149
|
+
train_split=args.train_split,
|
|
150
|
+
validation_split=args.val_split,
|
|
151
|
+
metadata=run_metadata,
|
|
152
|
+
)
|
|
153
|
+
print(f"[orizu-gepa] started optimization run {run_id}", flush=True)
|
|
154
|
+
|
|
155
|
+
local_logger = None
|
|
156
|
+
if not args.no_local_log:
|
|
157
|
+
local_logger = LocalOptimizationLogger.create(args.log_dir, run_id)
|
|
158
|
+
local_logger.write_context(
|
|
159
|
+
project=args.project,
|
|
160
|
+
run_id=run_id,
|
|
161
|
+
args={
|
|
162
|
+
key: value
|
|
163
|
+
for key, value in vars(args).items()
|
|
164
|
+
if key not in {"metadata"}
|
|
165
|
+
},
|
|
166
|
+
prompt_context=prompt_context,
|
|
167
|
+
scorer_context=scorer_context,
|
|
168
|
+
trainset=trainset,
|
|
169
|
+
valset=valset,
|
|
170
|
+
metadata=run_metadata,
|
|
171
|
+
)
|
|
172
|
+
print(f"[orizu-gepa] local log: {local_logger.path}", flush=True)
|
|
173
|
+
|
|
174
|
+
result = optimize_loaded_text_candidate(
|
|
175
|
+
run_id=run_id,
|
|
176
|
+
prompt_context=prompt_context,
|
|
177
|
+
scorer_context=scorer_context,
|
|
178
|
+
trainset=trainset,
|
|
179
|
+
valset=valset,
|
|
180
|
+
candidate_runner=make_candidate_runner(Path(args.candidate_runner_dir), run_id),
|
|
181
|
+
scorer_runner=make_scorer_runner(Path(args.scorer_runner_dir), run_id),
|
|
182
|
+
reflector=reflect_with_provider,
|
|
183
|
+
event_sink=OrizuEventSink(
|
|
184
|
+
client,
|
|
185
|
+
run_id,
|
|
186
|
+
fail_on_log_error=config.fail_on_log_error,
|
|
187
|
+
local_logger=local_logger,
|
|
188
|
+
),
|
|
189
|
+
config=config,
|
|
190
|
+
local_logger=local_logger,
|
|
191
|
+
)
|
|
192
|
+
if local_logger is not None:
|
|
193
|
+
local_logger.write_result(result)
|
|
194
|
+
print(json.dumps({
|
|
195
|
+
"optimization_run_id": result.run_id,
|
|
196
|
+
"best_candidate_id": result.best_candidate_id,
|
|
197
|
+
"best_score": result.best_score,
|
|
198
|
+
"seed_score": result.seed_score,
|
|
199
|
+
"promoted_prompt_version_id": result.promoted_prompt_version_id,
|
|
200
|
+
"budget": result.budget.to_payload(),
|
|
201
|
+
"local_log_dir": local_logger.path if local_logger is not None else None,
|
|
202
|
+
}))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
if __name__ == "__main__":
|
|
206
|
+
main()
|