orizu 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1418 -13
- package/package.json +1 -1
- package/src/index.ts +1849 -22
package/dist/index.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { createHash, randomBytes } from 'crypto';
|
|
3
|
-
import { basename, extname } from 'path';
|
|
4
|
-
import {
|
|
2
|
+
import { createHash, randomBytes, randomUUID } from 'crypto';
|
|
3
|
+
import { basename, delimiter, dirname, extname, isAbsolute, join, normalize } from 'path';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
5
5
|
import { createServer } from 'http';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { gunzipSync, gzipSync } from 'zlib';
|
|
7
|
+
import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync, } from 'fs';
|
|
8
|
+
import { spawn, spawnSync } from 'child_process';
|
|
9
|
+
import { tmpdir } from 'os';
|
|
8
10
|
import { createInterface } from 'readline/promises';
|
|
9
11
|
import { stdin as input, stdout as output } from 'process';
|
|
10
12
|
import { clearServerCredentials, getServerCredentials, saveServerCredentials } from './credentials.js';
|
|
@@ -28,7 +30,10 @@ function printVersion() {
|
|
|
28
30
|
printLine(`orizu ${getCliVersion()}`);
|
|
29
31
|
}
|
|
30
32
|
function printUsage() {
|
|
31
|
-
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login\n orizu logout\n orizu whoami\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`);
|
|
33
|
+
printLine(`orizu global options:\n\n --local Use http://localhost:3000\n --server <url> Use a specific server origin (for example: https://preview.example.com)\n --version, -v Print the orizu CLI version\n\norizu commands:\n\n orizu login [--no-prompt-if-logged-in]\n orizu logout\n orizu whoami\n orizu env [--project <team/project>] [--project-id <projectId>]\n orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n orizu teams list\n orizu teams create [--name <name>]\n orizu teams members list [--team <teamSlug>]\n orizu teams members add --email <email> [--team <teamSlug>]\n orizu teams members remove --email <email> [--team <teamSlug>]\n orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n orizu projects list [--team <teamSlug>]\n orizu projects create --name <name> [--team <teamSlug>]\n orizu prompts list --project <team/project>\n orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu scorers list --project <team/project>\n orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n orizu judges list --project <team/project>\n orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n orizu apps list [--project <team/project>]\n orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n orizu apps detail --app <appId> [--project <team/project>] [--json]\n orizu tasks list [--project <team/project>]\n orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n orizu tasks status --task <taskId> [--json]\n orizu tasks pause --task <taskId>\n orizu tasks unpause --task <taskId>\n orizu datasets upload --file <path> [--project <team/project>] [--name <name>]\n orizu datasets push <path> [--project <team/project>] [--name <name>] [--json]\n orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]\n orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`);
|
|
34
|
+
}
|
|
35
|
+
function printOptimizationUsage() {
|
|
36
|
+
printLine(`\nOptimization lifecycle commands:\n\n orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation]\n orizu optimizations pause <run-id> [--reason <text>] [--json]\n orizu optimizations resume <run-id> [--json]\n orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--metadata <json|@file>] [--json]\n orizu optimizations fail <run-id> [--reason <text>] [--metadata <json|@file>] [--json]\n orizu optimizations cancel <run-id> [--reason <text>] [--json]`);
|
|
32
37
|
}
|
|
33
38
|
let cliArgs = process.argv.slice(2);
|
|
34
39
|
function getArg(name) {
|
|
@@ -137,6 +142,79 @@ export async function parseJsonResponse(response, context) {
|
|
|
137
142
|
`Body preview: ${sanitizeTerminalText(rawBody.slice(0, 180))}`);
|
|
138
143
|
}
|
|
139
144
|
}
|
|
145
|
+
function shellQuote(value) {
|
|
146
|
+
return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
|
147
|
+
}
|
|
148
|
+
function getStoredAuthTokenForBaseUrl(baseUrl) {
|
|
149
|
+
const credentials = getServerCredentials(baseUrl);
|
|
150
|
+
if (!credentials) {
|
|
151
|
+
throw new Error(`Not logged in for ${baseUrl}. Run \`orizu login --server ${baseUrl}\` (or \`--local\`) first.`);
|
|
152
|
+
}
|
|
153
|
+
return 'accessToken' in credentials ? credentials.accessToken : credentials.apiKey;
|
|
154
|
+
}
|
|
155
|
+
function printPromptSummaries(items, emptyMessage) {
|
|
156
|
+
if (items.length === 0) {
|
|
157
|
+
printLine(emptyMessage);
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
const rows = items.map(item => ({
|
|
161
|
+
id: sanitizeTerminalText(item.id),
|
|
162
|
+
name: sanitizeTerminalText(item.name),
|
|
163
|
+
role: sanitizeTerminalText(item.role),
|
|
164
|
+
}));
|
|
165
|
+
const idWidth = Math.max('ID'.length, ...rows.map(row => row.id.length));
|
|
166
|
+
const nameWidth = Math.max('NAME'.length, ...rows.map(row => row.name.length));
|
|
167
|
+
const roleWidth = Math.max('ROLE'.length, ...rows.map(row => row.role.length));
|
|
168
|
+
printLine(`${'ID'.padEnd(idWidth)} ${'NAME'.padEnd(nameWidth)} ${'ROLE'.padEnd(roleWidth)}`);
|
|
169
|
+
printLine(`${'-'.repeat(idWidth)} ${'-'.repeat(nameWidth)} ${'-'.repeat(roleWidth)}`);
|
|
170
|
+
rows.forEach(row => {
|
|
171
|
+
printLine(`${row.id.padEnd(idWidth)} ${row.name.padEnd(nameWidth)} ${row.role.padEnd(roleWidth)}`);
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
function printScorerSummaries(items) {
|
|
175
|
+
if (items.length === 0) {
|
|
176
|
+
printLine('No scorers found.');
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
const rows = items.map(item => ({
|
|
180
|
+
id: sanitizeTerminalText(item.id),
|
|
181
|
+
name: sanitizeTerminalText(item.name),
|
|
182
|
+
mode: sanitizeTerminalText(item.mode),
|
|
183
|
+
metric: sanitizeTerminalText(item.metricLabel),
|
|
184
|
+
implementation: sanitizeTerminalText(item.implementationKind),
|
|
185
|
+
}));
|
|
186
|
+
const idWidth = Math.max('ID'.length, ...rows.map(row => row.id.length));
|
|
187
|
+
const nameWidth = Math.max('NAME'.length, ...rows.map(row => row.name.length));
|
|
188
|
+
const modeWidth = Math.max('MODE'.length, ...rows.map(row => row.mode.length));
|
|
189
|
+
const metricWidth = Math.max('METRIC'.length, ...rows.map(row => row.metric.length));
|
|
190
|
+
printLine(`${'ID'.padEnd(idWidth)} ${'NAME'.padEnd(nameWidth)} ${'MODE'.padEnd(modeWidth)} ${'METRIC'.padEnd(metricWidth)} IMPLEMENTATION`);
|
|
191
|
+
printLine(`${'-'.repeat(idWidth)} ${'-'.repeat(nameWidth)} ${'-'.repeat(modeWidth)} ${'-'.repeat(metricWidth)} ${'-'.repeat('IMPLEMENTATION'.length)}`);
|
|
192
|
+
rows.forEach(row => {
|
|
193
|
+
printLine(`${row.id.padEnd(idWidth)} ${row.name.padEnd(nameWidth)} ${row.mode.padEnd(modeWidth)} ${row.metric.padEnd(metricWidth)} ${row.implementation}`);
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
function readJsonObjectArg(valueArg, label) {
|
|
197
|
+
if (!valueArg) {
|
|
198
|
+
return {};
|
|
199
|
+
}
|
|
200
|
+
const trimmed = valueArg.trim();
|
|
201
|
+
const raw = trimmed.startsWith('{')
|
|
202
|
+
? trimmed
|
|
203
|
+
: readSourceFile(trimmed.startsWith('@') ? trimmed.slice(1) : trimmed);
|
|
204
|
+
try {
|
|
205
|
+
const parsed = JSON.parse(raw);
|
|
206
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
207
|
+
throw new Error('JSON root must be an object');
|
|
208
|
+
}
|
|
209
|
+
return parsed;
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
throw new Error(`Invalid ${label} '${valueArg}': ${error?.message || String(error)}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function readJsonPayloadArg(pathArg) {
|
|
216
|
+
return readJsonObjectArg(pathArg, 'event payload');
|
|
217
|
+
}
|
|
140
218
|
async function promptSelect(title, items, label, options) {
|
|
141
219
|
if (items.length === 0) {
|
|
142
220
|
throw new Error(`No options available for ${title.toLowerCase()}`);
|
|
@@ -217,7 +295,8 @@ async function fetchTeamMembers(teamSlug) {
|
|
|
217
295
|
const data = await parseJsonResponse(response, 'Team members list');
|
|
218
296
|
return data.members;
|
|
219
297
|
}
|
|
220
|
-
async function
|
|
298
|
+
async function resolveProjectSelection(projectArg) {
|
|
299
|
+
projectArg = projectArg || process.env.ORIZU_PROJECT || null;
|
|
221
300
|
const teams = await fetchTeams();
|
|
222
301
|
if (teams.length === 0) {
|
|
223
302
|
throw new Error('No accessible teams found for this user.');
|
|
@@ -226,7 +305,7 @@ async function resolveProjectSlug(projectArg) {
|
|
|
226
305
|
const team = await promptSelect('Select a team', teams, teamOption => `${teamOption.name} (${teamOption.slug})`, { forcePrompt: true });
|
|
227
306
|
const projects = await fetchProjects(team.slug);
|
|
228
307
|
const project = await promptSelect(`Select a project in ${team.slug}`, projects, projectOption => `${projectOption.name} (${projectOption.teamSlug}/${projectOption.slug})`, { forcePrompt: true });
|
|
229
|
-
return
|
|
308
|
+
return project;
|
|
230
309
|
}
|
|
231
310
|
const segments = projectArg.split('/');
|
|
232
311
|
if (segments.length !== 2 || !segments[0] || !segments[1]) {
|
|
@@ -239,16 +318,20 @@ async function resolveProjectSlug(projectArg) {
|
|
|
239
318
|
const selectedTeam = await promptSelect('Select a team', teams, team => `${team.name} (${team.slug})`);
|
|
240
319
|
const projects = await fetchProjects(selectedTeam.slug);
|
|
241
320
|
const selectedProject = await promptSelect(`Select a project in ${selectedTeam.slug}`, projects, project => `${project.name} (${project.teamSlug}/${project.slug})`);
|
|
242
|
-
return
|
|
321
|
+
return selectedProject;
|
|
243
322
|
}
|
|
244
323
|
const projects = await fetchProjects(matchedTeam.slug);
|
|
245
324
|
const matchedProject = projects.find(project => project.slug === projectSlug);
|
|
246
325
|
if (!matchedProject) {
|
|
247
326
|
console.error(`Project '${sanitizeTerminalText(projectSlug)}' not found in team '${sanitizeTerminalText(matchedTeam.slug)}'.`);
|
|
248
327
|
const selectedProject = await promptSelect(`Select a project in ${matchedTeam.slug}`, projects, project => `${project.name} (${project.teamSlug}/${project.slug})`);
|
|
249
|
-
return
|
|
328
|
+
return selectedProject;
|
|
250
329
|
}
|
|
251
|
-
return
|
|
330
|
+
return matchedProject;
|
|
331
|
+
}
|
|
332
|
+
async function resolveProjectSlug(projectArg) {
|
|
333
|
+
const project = await resolveProjectSelection(projectArg);
|
|
334
|
+
return `${project.teamSlug}/${project.slug}`;
|
|
252
335
|
}
|
|
253
336
|
async function selectTaskIdInteractively() {
|
|
254
337
|
const team = await promptSelect('Select a team', await fetchTeams(), item => `${item.name} (${item.slug})`, { forcePrompt: true });
|
|
@@ -517,8 +600,12 @@ function renderCliAuthBrowserPage(status) {
|
|
|
517
600
|
</html>`;
|
|
518
601
|
}
|
|
519
602
|
async function login() {
|
|
520
|
-
const baseUrl = resolveLoginBaseUrl();
|
|
603
|
+
const baseUrl = hasArg('--no-prompt-if-logged-in') ? getBaseUrl() : resolveLoginBaseUrl();
|
|
521
604
|
assertSecureTokenTransport(baseUrl);
|
|
605
|
+
if (hasArg('--no-prompt-if-logged-in') && getServerCredentials(baseUrl)) {
|
|
606
|
+
printLine(`Already logged in to ${sanitizeTerminalText(baseUrl)}.`);
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
522
609
|
const codeVerifier = createCodeVerifier();
|
|
523
610
|
const codeChallenge = createCodeChallenge(codeVerifier);
|
|
524
611
|
const callbackPort = resolveAuthCallbackPort();
|
|
@@ -638,6 +725,801 @@ async function logout() {
|
|
|
638
725
|
}
|
|
639
726
|
printLine(`Logged out from ${sanitizeTerminalText(baseUrl)}.`);
|
|
640
727
|
}
|
|
728
|
+
async function printEnv() {
|
|
729
|
+
const baseUrl = getBaseUrl();
|
|
730
|
+
const token = getStoredAuthTokenForBaseUrl(baseUrl);
|
|
731
|
+
const projectArg = getArg('--project');
|
|
732
|
+
const project = projectArg ? await resolveProjectSelection(projectArg) : null;
|
|
733
|
+
const projectId = project?.id || getArg('--project-id') || process.env.ORIZU_PROJECT_ID || '';
|
|
734
|
+
const projectSlug = project ? `${project.teamSlug}/${project.slug}` : process.env.ORIZU_PROJECT || '';
|
|
735
|
+
printLine(`export ORIZU_API_URL=${shellQuote(baseUrl)}`);
|
|
736
|
+
printLine(`export ORIZU_TOKEN=${shellQuote(token)}`);
|
|
737
|
+
printLine(`export ORIZU_PROJECT_ID=${shellQuote(projectId)}`);
|
|
738
|
+
if (projectSlug) {
|
|
739
|
+
printLine(`export ORIZU_PROJECT=${shellQuote(projectSlug)}`);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
async function logOptimizationEvent() {
|
|
743
|
+
const eventType = cliArgs[1];
|
|
744
|
+
const runId = getArg('--run-id');
|
|
745
|
+
const sequenceArg = getArg('--sequence');
|
|
746
|
+
const payload = readJsonPayloadArg(getArg('--payload'));
|
|
747
|
+
const sequence = sequenceArg ? Number(sequenceArg) : Number.NaN;
|
|
748
|
+
if (!eventType || eventType.startsWith('--') || !runId || !Number.isInteger(sequence) || sequence <= 0) {
|
|
749
|
+
throw new Error('Usage: orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json');
|
|
750
|
+
}
|
|
751
|
+
const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}/events`, {
|
|
752
|
+
method: 'POST',
|
|
753
|
+
headers: { 'Content-Type': 'application/json' },
|
|
754
|
+
body: JSON.stringify({
|
|
755
|
+
eventId: getArg('--event-id') || randomUUID(),
|
|
756
|
+
sequence,
|
|
757
|
+
eventType,
|
|
758
|
+
eventLayer: getArg('--event-layer') || 'core',
|
|
759
|
+
optimizerFamily: getArg('--optimizer-family'),
|
|
760
|
+
iteration: getArg('--iteration') ? Number(getArg('--iteration')) : undefined,
|
|
761
|
+
candidateId: getArg('--candidate-id'),
|
|
762
|
+
parentCandidateId: getArg('--parent-candidate-id'),
|
|
763
|
+
childCandidateId: getArg('--child-candidate-id'),
|
|
764
|
+
payload,
|
|
765
|
+
}),
|
|
766
|
+
});
|
|
767
|
+
if (!response.ok) {
|
|
768
|
+
throw new Error(`Failed to log event: ${await response.text()}`);
|
|
769
|
+
}
|
|
770
|
+
const data = await parseJsonResponse(response, 'Optimization event log');
|
|
771
|
+
printLine(`Logged event ${sanitizeTerminalText(data.eventId)}`);
|
|
772
|
+
}
|
|
773
|
+
async function listPrompts() {
|
|
774
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
775
|
+
const response = await authedFetch(`/api/cli/prompts?project=${encodeURIComponent(project)}`);
|
|
776
|
+
if (!response.ok) {
|
|
777
|
+
throw new Error(`Failed to fetch prompts: ${await response.text()}`);
|
|
778
|
+
}
|
|
779
|
+
const data = await parseJsonResponse(response, 'Prompts list');
|
|
780
|
+
printPromptSummaries(data.prompts, 'No prompts found.');
|
|
781
|
+
}
|
|
782
|
+
async function listJudges() {
|
|
783
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
784
|
+
const response = await authedFetch(`/api/cli/judges?project=${encodeURIComponent(project)}`);
|
|
785
|
+
if (!response.ok) {
|
|
786
|
+
throw new Error(`Failed to fetch judges: ${await response.text()}`);
|
|
787
|
+
}
|
|
788
|
+
const data = await parseJsonResponse(response, 'Judges list');
|
|
789
|
+
printPromptSummaries(data.judges, 'No judges found.');
|
|
790
|
+
}
|
|
791
|
+
async function listScorers() {
|
|
792
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
793
|
+
const response = await authedFetch(`/api/cli/scorers?project=${encodeURIComponent(project)}`);
|
|
794
|
+
if (!response.ok) {
|
|
795
|
+
throw new Error(`Failed to fetch scorers: ${await response.text()}`);
|
|
796
|
+
}
|
|
797
|
+
const data = await parseJsonResponse(response, 'Scorers list');
|
|
798
|
+
printScorerSummaries(data.scorers);
|
|
799
|
+
}
|
|
800
|
+
async function registerScorer() {
|
|
801
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
802
|
+
const name = getArg('--name');
|
|
803
|
+
const manifestPath = getArg('--manifest');
|
|
804
|
+
if (!name || !manifestPath) {
|
|
805
|
+
throw new Error('Usage: orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]');
|
|
806
|
+
}
|
|
807
|
+
const manifest = readJsonFile(manifestPath);
|
|
808
|
+
const response = await authedFetch(`/api/cli/scorers?project=${encodeURIComponent(project)}`, {
|
|
809
|
+
method: 'POST',
|
|
810
|
+
headers: { 'Content-Type': 'application/json' },
|
|
811
|
+
body: JSON.stringify({
|
|
812
|
+
name,
|
|
813
|
+
manifest,
|
|
814
|
+
promptVersionId: getArg('--prompt-version') || undefined,
|
|
815
|
+
runnerVersionId: getArg('--runner-version') || undefined,
|
|
816
|
+
label: getArg('--label') || undefined,
|
|
817
|
+
}),
|
|
818
|
+
});
|
|
819
|
+
if (!response.ok) {
|
|
820
|
+
throw new Error(`Failed to register scorer: ${await response.text()}`);
|
|
821
|
+
}
|
|
822
|
+
const data = await parseJsonResponse(response, 'Scorer register');
|
|
823
|
+
if (hasJsonFlag()) {
|
|
824
|
+
printJson(data);
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
printLine(`Registered scorer ${sanitizeTerminalText(name)} (${sanitizeTerminalText(String(data.scorer_version_id || 'unknown version'))})`);
|
|
828
|
+
}
|
|
829
|
+
async function showScorerDetail() {
|
|
830
|
+
const scorerId = getPositionalArg(2);
|
|
831
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
832
|
+
if (!scorerId) {
|
|
833
|
+
throw new Error('Usage: orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]');
|
|
834
|
+
}
|
|
835
|
+
const response = await authedFetch(`/api/cli/scorers/${encodeURIComponent(scorerId)}?project=${encodeURIComponent(project)}`);
|
|
836
|
+
if (!response.ok) {
|
|
837
|
+
throw new Error(`Failed to fetch scorer: ${await response.text()}`);
|
|
838
|
+
}
|
|
839
|
+
const data = await parseJsonResponse(response, 'Scorer detail');
|
|
840
|
+
if (hasJsonFlag()) {
|
|
841
|
+
printJson(data);
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
const scorer = data.scorer;
|
|
845
|
+
printLine(`${sanitizeTerminalText(String(scorer?.name || scorerId))}`);
|
|
846
|
+
printLine(`Metric: ${sanitizeTerminalText(String(scorer?.metricLabel || 'Score'))}`);
|
|
847
|
+
}
|
|
848
|
+
async function setScorerLabel() {
|
|
849
|
+
const scorerName = getPositionalArg(3);
|
|
850
|
+
const label = getPositionalArg(4);
|
|
851
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
852
|
+
const scorerVersionId = getArg('--version');
|
|
853
|
+
if (!scorerName || !label || !scorerVersionId) {
|
|
854
|
+
throw new Error('Usage: orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]');
|
|
855
|
+
}
|
|
856
|
+
const response = await authedFetch(`/api/cli/scorers/labels?project=${encodeURIComponent(project)}`, {
|
|
857
|
+
method: 'POST',
|
|
858
|
+
headers: { 'Content-Type': 'application/json' },
|
|
859
|
+
body: JSON.stringify({ scorerName, label, scorerVersionId }),
|
|
860
|
+
});
|
|
861
|
+
if (!response.ok) {
|
|
862
|
+
throw new Error(`Failed to set scorer label: ${await response.text()}`);
|
|
863
|
+
}
|
|
864
|
+
const data = await parseJsonResponse(response, 'Scorer label set');
|
|
865
|
+
if (hasJsonFlag()) {
|
|
866
|
+
printJson(data);
|
|
867
|
+
return;
|
|
868
|
+
}
|
|
869
|
+
printLine(`Moved ${sanitizeTerminalText(label)} to ${sanitizeTerminalText(scorerVersionId)}`);
|
|
870
|
+
}
|
|
871
|
+
async function pushRunnerArtifact(kind) {
|
|
872
|
+
const artifactDir = getPositionalArg(2);
|
|
873
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
874
|
+
const name = getArg('--name');
|
|
875
|
+
const label = getArg('--label') || undefined;
|
|
876
|
+
if (!artifactDir) {
|
|
877
|
+
throw new Error(`Usage: orizu ${kind === 'runner' ? 'runners' : 'optimizers'} push <dir> --project <team/project> [--name <name>] [--label <label>] [--json]`);
|
|
878
|
+
}
|
|
879
|
+
const manifest = readManifestFile(artifactDir);
|
|
880
|
+
const artifactName = name || (typeof manifest.name === 'string' ? manifest.name : basename(expandHomePath(artifactDir)));
|
|
881
|
+
const description = typeof manifest.description === 'string' ? manifest.description : undefined;
|
|
882
|
+
const { zipBase64, contentSha256 } = zipDirectoryToBase64(artifactDir);
|
|
883
|
+
const endpoint = kind === 'runner' ? 'runners' : 'optimizers';
|
|
884
|
+
const response = await authedFetch(`/api/cli/${endpoint}?project=${encodeURIComponent(project)}`, {
|
|
885
|
+
method: 'POST',
|
|
886
|
+
headers: { 'Content-Type': 'application/json' },
|
|
887
|
+
body: JSON.stringify({
|
|
888
|
+
name: artifactName,
|
|
889
|
+
description,
|
|
890
|
+
label,
|
|
891
|
+
manifest,
|
|
892
|
+
zipBase64,
|
|
893
|
+
contentSha256,
|
|
894
|
+
}),
|
|
895
|
+
});
|
|
896
|
+
if (!response.ok) {
|
|
897
|
+
throw new Error(`Failed to push ${kind}: ${await response.text()}`);
|
|
898
|
+
}
|
|
899
|
+
const data = await parseJsonResponse(response, `${kind} push`);
|
|
900
|
+
if (hasJsonFlag()) {
|
|
901
|
+
printJson(data);
|
|
902
|
+
return;
|
|
903
|
+
}
|
|
904
|
+
const versionId = kind === 'runner' ? data.runner_version_id : data.optimizer_version_id;
|
|
905
|
+
printLine(`Pushed ${kind} ${sanitizeTerminalText(String(artifactName))} (${sanitizeTerminalText(String(versionId || 'unknown version'))})`);
|
|
906
|
+
}
|
|
907
|
+
async function pushPromptArtifact(kind) {
|
|
908
|
+
const promptDir = getPositionalArg(2);
|
|
909
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
910
|
+
const runnerVersionArg = getArg('--runner-version');
|
|
911
|
+
const parentVersionId = getArg('--parent') || undefined;
|
|
912
|
+
if (!promptDir) {
|
|
913
|
+
throw new Error(`Usage: orizu ${kind === 'judge' ? 'judges' : 'prompts'} push <dir> --project <team/project> [--runner-version <id>] [--parent <version-id>] [--json]`);
|
|
914
|
+
}
|
|
915
|
+
const promptRoot = expandHomePath(promptDir);
|
|
916
|
+
const manifest = readJsonFile(join(promptRoot, 'orizu.prompt.json'));
|
|
917
|
+
const primaryText = readPromptPrimaryText(manifest, promptRoot);
|
|
918
|
+
const sidecars = readPromptSidecars(manifest, promptRoot);
|
|
919
|
+
const runnerFromManifest = stringFromRecord(manifest, 'runner_version_id') ||
|
|
920
|
+
(isRecord(manifest.runner) ? stringFromRecord(manifest.runner, 'version_id') : undefined);
|
|
921
|
+
const runnerVersionId = runnerVersionArg || runnerFromManifest;
|
|
922
|
+
if (!runnerVersionId) {
|
|
923
|
+
throw new Error(`Usage: orizu ${kind === 'judge' ? 'judges' : 'prompts'} push <dir> --project <team/project> [--runner-version <id>] [--parent <version-id>] [--json]`);
|
|
924
|
+
}
|
|
925
|
+
const baseBundle = isRecord(manifest.bundle) ? manifest.bundle : {};
|
|
926
|
+
const endpoint = kind === 'judge' ? 'judges' : 'prompts';
|
|
927
|
+
const response = await authedFetch(`/api/cli/${endpoint}?project=${encodeURIComponent(project)}`, {
|
|
928
|
+
method: 'POST',
|
|
929
|
+
headers: { 'Content-Type': 'application/json' },
|
|
930
|
+
body: JSON.stringify({
|
|
931
|
+
name: manifest.name,
|
|
932
|
+
role: kind === 'judge' && manifest.role === undefined ? 'judge_per_row' : manifest.role,
|
|
933
|
+
description: manifest.description,
|
|
934
|
+
body: primaryText.body,
|
|
935
|
+
bodyKind: primaryText.bodyKind,
|
|
936
|
+
providerSettings: manifest.provider_settings || {},
|
|
937
|
+
bundle: {
|
|
938
|
+
...baseBundle,
|
|
939
|
+
tags: manifest.tags || [],
|
|
940
|
+
provenance: manifest.provenance || {},
|
|
941
|
+
primaryText: {
|
|
942
|
+
path: primaryText.path,
|
|
943
|
+
kind: primaryText.bodyKind,
|
|
944
|
+
},
|
|
945
|
+
sidecars,
|
|
946
|
+
},
|
|
947
|
+
runnerVersionId,
|
|
948
|
+
parentVersionId,
|
|
949
|
+
versionLabel: manifest.version_label,
|
|
950
|
+
createdBy: manifest.provenance || { kind: 'human-edit' },
|
|
951
|
+
}),
|
|
952
|
+
});
|
|
953
|
+
if (!response.ok) {
|
|
954
|
+
throw new Error(`Failed to push ${kind}: ${await response.text()}`);
|
|
955
|
+
}
|
|
956
|
+
const data = await parseJsonResponse(response, `${kind} push`);
|
|
957
|
+
if (hasJsonFlag()) {
|
|
958
|
+
printJson(data);
|
|
959
|
+
return;
|
|
960
|
+
}
|
|
961
|
+
printLine(`Pushed ${kind} ${sanitizeTerminalText(String(manifest.name || promptDir))} (${sanitizeTerminalText(String(data.prompt_version_id || 'unknown version'))})`);
|
|
962
|
+
}
|
|
963
|
+
async function pullPromptArtifact(kind) {
|
|
964
|
+
const promptRef = getPositionalArg(2);
|
|
965
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
966
|
+
const outDir = getArg('--out');
|
|
967
|
+
const label = getArg('--label');
|
|
968
|
+
const version = getArg('--version');
|
|
969
|
+
if (!promptRef || !outDir) {
|
|
970
|
+
throw new Error(`Usage: orizu ${kind === 'judge' ? 'judges' : 'prompts'} pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <version-id>] [--json]`);
|
|
971
|
+
}
|
|
972
|
+
if (label && version) {
|
|
973
|
+
throw new Error('Use either --label or --version, not both');
|
|
974
|
+
}
|
|
975
|
+
const params = new URLSearchParams({ project });
|
|
976
|
+
if (label)
|
|
977
|
+
params.set('label', label);
|
|
978
|
+
if (version)
|
|
979
|
+
params.set('version', version);
|
|
980
|
+
const endpoint = kind === 'judge' ? 'judges' : 'prompts';
|
|
981
|
+
const response = await authedFetch(`/api/cli/${endpoint}/${encodeURIComponent(promptRef)}?${params.toString()}`);
|
|
982
|
+
if (!response.ok) {
|
|
983
|
+
throw new Error(`Failed to pull ${kind}: ${await response.text()}`);
|
|
984
|
+
}
|
|
985
|
+
const data = await parseJsonResponse(response, `${kind} pull`);
|
|
986
|
+
const targetDir = expandHomePath(outDir);
|
|
987
|
+
mkdirSync(targetDir, { recursive: true });
|
|
988
|
+
const bundle = isRecord(data.version.bundle) ? data.version.bundle : {};
|
|
989
|
+
const bundlePrimaryText = isRecord(bundle.primaryText)
|
|
990
|
+
? bundle.primaryText
|
|
991
|
+
: isRecord(bundle.primary_text)
|
|
992
|
+
? bundle.primary_text
|
|
993
|
+
: null;
|
|
994
|
+
const primaryPath = safeRelativePath(stringFromRecord(bundlePrimaryText || {}, 'path') || 'prompt.md', 'prompt.md');
|
|
995
|
+
const bodyKind = stringFromRecord(bundlePrimaryText || {}, 'kind') || data.version.bodyKind || 'text';
|
|
996
|
+
writeTextFileEnsuringDir(join(targetDir, primaryPath), data.version.body || '');
|
|
997
|
+
const exportedSidecars = [];
|
|
998
|
+
const sidecars = Array.isArray(bundle.sidecars) ? bundle.sidecars : [];
|
|
999
|
+
for (const rawSidecar of sidecars) {
|
|
1000
|
+
if (!isRecord(rawSidecar))
|
|
1001
|
+
continue;
|
|
1002
|
+
const sidecarPathValue = stringFromRecord(rawSidecar, 'path');
|
|
1003
|
+
if (!sidecarPathValue)
|
|
1004
|
+
continue;
|
|
1005
|
+
const sidecarPath = safeRelativePath(sidecarPathValue, sidecarPathValue);
|
|
1006
|
+
const content = typeof rawSidecar.content === 'string' ? rawSidecar.content : null;
|
|
1007
|
+
const contentSha256 = stringFromRecord(rawSidecar, 'contentSha256') ||
|
|
1008
|
+
stringFromRecord(rawSidecar, 'content_sha256') ||
|
|
1009
|
+
(content !== null ? sha256Hex(content) : undefined);
|
|
1010
|
+
if (content !== null) {
|
|
1011
|
+
writeTextFileEnsuringDir(join(targetDir, sidecarPath), content);
|
|
1012
|
+
}
|
|
1013
|
+
const { content: _ignoredContent, contentSha256: _ignoredCamel, content_sha256: _ignoredSnake, ...metadata } = rawSidecar;
|
|
1014
|
+
exportedSidecars.push({
|
|
1015
|
+
...metadata,
|
|
1016
|
+
type: stringFromRecord(rawSidecar, 'type') || 'file',
|
|
1017
|
+
path: sidecarPath,
|
|
1018
|
+
...(contentSha256 ? { content_sha256: contentSha256 } : {}),
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
1021
|
+
const labels = (data.labels || [])
|
|
1022
|
+
.filter(item => item.promptVersionId === data.version.id)
|
|
1023
|
+
.map(item => item.label);
|
|
1024
|
+
const manifest = {
|
|
1025
|
+
schema_version: 'orizu.prompt.v1',
|
|
1026
|
+
name: data.prompt.name,
|
|
1027
|
+
role: data.prompt.role,
|
|
1028
|
+
description: data.prompt.description || undefined,
|
|
1029
|
+
primary_text: {
|
|
1030
|
+
path: primaryPath,
|
|
1031
|
+
kind: bodyKind,
|
|
1032
|
+
},
|
|
1033
|
+
provider_settings: data.version.providerSettings || {},
|
|
1034
|
+
runner_version_id: data.version.runnerVersionId || undefined,
|
|
1035
|
+
version_id: data.version.id,
|
|
1036
|
+
version_number: data.version.versionNumber,
|
|
1037
|
+
version_label: data.version.versionLabel || undefined,
|
|
1038
|
+
labels,
|
|
1039
|
+
sidecars: exportedSidecars,
|
|
1040
|
+
};
|
|
1041
|
+
writeTextFileEnsuringDir(join(targetDir, 'orizu.prompt.json'), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
1042
|
+
if (hasJsonFlag()) {
|
|
1043
|
+
printJson({
|
|
1044
|
+
prompt_id: data.prompt.id,
|
|
1045
|
+
prompt_version_id: data.version.id,
|
|
1046
|
+
path: targetDir,
|
|
1047
|
+
});
|
|
1048
|
+
return;
|
|
1049
|
+
}
|
|
1050
|
+
printLine(`Pulled ${kind} ${sanitizeTerminalText(data.prompt.name)} to ${sanitizeTerminalText(targetDir)}`);
|
|
1051
|
+
}
|
|
1052
|
+
async function setPromptLabel() {
|
|
1053
|
+
const promptName = getPositionalArg(3);
|
|
1054
|
+
const label = getPositionalArg(4);
|
|
1055
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
1056
|
+
const promptVersionId = getArg('--version');
|
|
1057
|
+
if (!promptName || !label || !promptVersionId) {
|
|
1058
|
+
throw new Error('Usage: orizu prompts labels set <prompt-name> <label> --version <prompt-version-id> [--project <team/project>] [--json]');
|
|
1059
|
+
}
|
|
1060
|
+
const response = await authedFetch(`/api/cli/prompts/labels?project=${encodeURIComponent(project)}`, {
|
|
1061
|
+
method: 'POST',
|
|
1062
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1063
|
+
body: JSON.stringify({
|
|
1064
|
+
promptName,
|
|
1065
|
+
label,
|
|
1066
|
+
promptVersionId,
|
|
1067
|
+
}),
|
|
1068
|
+
});
|
|
1069
|
+
if (!response.ok) {
|
|
1070
|
+
throw new Error(`Failed to set prompt label: ${await response.text()}`);
|
|
1071
|
+
}
|
|
1072
|
+
const data = await parseJsonResponse(response, 'Prompt label set');
|
|
1073
|
+
if (hasJsonFlag()) {
|
|
1074
|
+
printJson(data);
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
printLine(`Moved ${sanitizeTerminalText(label)} to ${sanitizeTerminalText(promptVersionId)}`);
|
|
1078
|
+
}
|
|
1079
|
+
async function bindPromptScorer(role) {
|
|
1080
|
+
const promptId = getPositionalArg(3);
|
|
1081
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
1082
|
+
const scorerVersionId = getArg('--scorer-version');
|
|
1083
|
+
if (!promptId || !scorerVersionId) {
|
|
1084
|
+
throw new Error(`Usage: orizu prompts scorers ${role === 'headline' ? 'set-headline' : 'add'} <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]`);
|
|
1085
|
+
}
|
|
1086
|
+
const response = await authedFetch(`/api/cli/prompts/scorers?project=${encodeURIComponent(project)}`, {
|
|
1087
|
+
method: 'POST',
|
|
1088
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1089
|
+
body: JSON.stringify({
|
|
1090
|
+
promptId,
|
|
1091
|
+
scorerVersionId,
|
|
1092
|
+
role,
|
|
1093
|
+
datasetVersionId: getArg('--dataset-version') || undefined,
|
|
1094
|
+
splitSetId: getArg('--split-set') || undefined,
|
|
1095
|
+
splitName: getArg('--split') || undefined,
|
|
1096
|
+
}),
|
|
1097
|
+
});
|
|
1098
|
+
if (!response.ok) {
|
|
1099
|
+
throw new Error(`Failed to bind prompt scorer: ${await response.text()}`);
|
|
1100
|
+
}
|
|
1101
|
+
const data = await parseJsonResponse(response, 'Prompt scorer bind');
|
|
1102
|
+
if (hasJsonFlag()) {
|
|
1103
|
+
printJson(data);
|
|
1104
|
+
return;
|
|
1105
|
+
}
|
|
1106
|
+
printLine(`${role === 'headline' ? 'Set headline' : 'Added'} scorer ${sanitizeTerminalText(scorerVersionId)} for ${sanitizeTerminalText(promptId)}`);
|
|
1107
|
+
}
|
|
1108
|
+
function getPositionalArg(index) {
|
|
1109
|
+
const value = cliArgs[index];
|
|
1110
|
+
return value && !value.startsWith('--') ? value : null;
|
|
1111
|
+
}
|
|
1112
|
+
function parseRatioFlag(name, fallback) {
|
|
1113
|
+
const value = getArg(name);
|
|
1114
|
+
if (!value) {
|
|
1115
|
+
return fallback;
|
|
1116
|
+
}
|
|
1117
|
+
const parsed = Number(value);
|
|
1118
|
+
if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
|
|
1119
|
+
throw new Error(`${name} must be a number between 0 and 1`);
|
|
1120
|
+
}
|
|
1121
|
+
return parsed;
|
|
1122
|
+
}
|
|
1123
|
+
function parsePositiveIntegerFlag(name, fallback) {
|
|
1124
|
+
const value = getArg(name);
|
|
1125
|
+
if (!value) {
|
|
1126
|
+
return fallback;
|
|
1127
|
+
}
|
|
1128
|
+
const parsed = Number(value);
|
|
1129
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
1130
|
+
throw new Error(`${name} must be a positive integer`);
|
|
1131
|
+
}
|
|
1132
|
+
return parsed;
|
|
1133
|
+
}
|
|
1134
|
+
function formatPercent(value) {
|
|
1135
|
+
if (value === null || !Number.isFinite(value)) {
|
|
1136
|
+
return 'unscored';
|
|
1137
|
+
}
|
|
1138
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
1139
|
+
}
|
|
1140
|
+
function looksLikeUuid(value) {
|
|
1141
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value);
|
|
1142
|
+
}
|
|
1143
|
+
async function resolveDatasetIdForProject(datasetRef, projectArg) {
|
|
1144
|
+
if (looksLikeUuid(datasetRef)) {
|
|
1145
|
+
return datasetRef;
|
|
1146
|
+
}
|
|
1147
|
+
projectArg = projectArg || process.env.ORIZU_PROJECT || null;
|
|
1148
|
+
if (!projectArg) {
|
|
1149
|
+
return datasetRef;
|
|
1150
|
+
}
|
|
1151
|
+
const project = projectArg || await resolveProjectSlug(null);
|
|
1152
|
+
const datasets = await fetchDatasets(project);
|
|
1153
|
+
const matches = datasets.filter(dataset => dataset.name === datasetRef || dataset.id === datasetRef);
|
|
1154
|
+
if (matches.length === 0) {
|
|
1155
|
+
throw new Error(`Dataset '${sanitizeTerminalText(datasetRef)}' not found in ${sanitizeTerminalText(project)}`);
|
|
1156
|
+
}
|
|
1157
|
+
return matches[0].id;
|
|
1158
|
+
}
|
|
1159
|
+
async function createDatasetVersion() {
|
|
1160
|
+
const datasetId = getPositionalArg(3) || getArg('--dataset');
|
|
1161
|
+
const versionLabel = getArg('--label') || getArg('--version-label') || null;
|
|
1162
|
+
const project = getArg('--project');
|
|
1163
|
+
if (!datasetId) {
|
|
1164
|
+
throw new Error('Usage: orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--json]');
|
|
1165
|
+
}
|
|
1166
|
+
const resolvedDatasetId = await resolveDatasetIdForProject(datasetId, project);
|
|
1167
|
+
const response = await authedFetch(`/api/cli/datasets/${encodeURIComponent(resolvedDatasetId)}/versions`, {
|
|
1168
|
+
method: 'POST',
|
|
1169
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1170
|
+
body: JSON.stringify({ versionLabel }),
|
|
1171
|
+
});
|
|
1172
|
+
if (!response.ok) {
|
|
1173
|
+
throw new Error(`Failed to create dataset version: ${await response.text()}`);
|
|
1174
|
+
}
|
|
1175
|
+
const data = await parseJsonResponse(response, 'Dataset version create');
|
|
1176
|
+
if (hasJsonFlag()) {
|
|
1177
|
+
printJson({
|
|
1178
|
+
dataset_version_id: data.datasetVersion.id,
|
|
1179
|
+
row_count: data.datasetVersion.rowCount,
|
|
1180
|
+
});
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
const details = [
|
|
1184
|
+
`${data.datasetVersion.rowCount} rows`,
|
|
1185
|
+
data.datasetVersion.artifactFormat,
|
|
1186
|
+
].filter(Boolean).join(', ');
|
|
1187
|
+
printLine(`Created dataset version ${sanitizeTerminalText(data.datasetVersion.id)}` +
|
|
1188
|
+
(details ? ` (${sanitizeTerminalText(details)})` : ''));
|
|
1189
|
+
}
|
|
1190
|
+
async function createDatasetSplitSet() {
|
|
1191
|
+
const datasetVersionId = getPositionalArg(3) || getArg('--dataset-version');
|
|
1192
|
+
const splitFile = getArg('--from-file');
|
|
1193
|
+
const splitSpec = splitFile ? readJsonFile(splitFile) : null;
|
|
1194
|
+
const name = getArg('--name') || (typeof splitSpec?.name === 'string' ? splitSpec.name : 'default');
|
|
1195
|
+
const strategy = getArg('--strategy') || (typeof splitSpec?.strategy === 'string' ? splitSpec.strategy : 'random');
|
|
1196
|
+
const seed = splitSpec && (splitSpec.seed === null || typeof splitSpec.seed === 'number')
|
|
1197
|
+
? splitSpec.seed
|
|
1198
|
+
: parsePositiveIntegerFlag('--seed', 1);
|
|
1199
|
+
const train = parseRatioFlag('--train', 0.7);
|
|
1200
|
+
const validation = parseRatioFlag('--validation', 0.2);
|
|
1201
|
+
const test = parseRatioFlag('--test', 0.1);
|
|
1202
|
+
if (!datasetVersionId) {
|
|
1203
|
+
throw new Error('Usage: orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--name <name>] [--seed <n>] [--train <ratio>] [--validation <ratio>] [--test <ratio>] [--json]');
|
|
1204
|
+
}
|
|
1205
|
+
const partitions = splitSpec && Array.isArray(splitSpec.partitions) ? splitSpec.partitions : undefined;
|
|
1206
|
+
const metadata = splitSpec && typeof splitSpec.metadata === 'object' && splitSpec.metadata !== null
|
|
1207
|
+
? splitSpec.metadata
|
|
1208
|
+
: undefined;
|
|
1209
|
+
const response = await authedFetch(`/api/cli/dataset-versions/${encodeURIComponent(datasetVersionId)}/split-sets`, {
|
|
1210
|
+
method: 'POST',
|
|
1211
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1212
|
+
body: JSON.stringify({
|
|
1213
|
+
name,
|
|
1214
|
+
strategy,
|
|
1215
|
+
seed,
|
|
1216
|
+
train,
|
|
1217
|
+
validation,
|
|
1218
|
+
test,
|
|
1219
|
+
partitions,
|
|
1220
|
+
metadata,
|
|
1221
|
+
}),
|
|
1222
|
+
});
|
|
1223
|
+
if (!response.ok) {
|
|
1224
|
+
throw new Error(`Failed to create dataset split set: ${await response.text()}`);
|
|
1225
|
+
}
|
|
1226
|
+
const data = await parseJsonResponse(response, 'Dataset split set create');
|
|
1227
|
+
if (hasJsonFlag()) {
|
|
1228
|
+
printJson({ split_set_id: data.splitSet.id });
|
|
1229
|
+
return;
|
|
1230
|
+
}
|
|
1231
|
+
printLine(`Created split set ${sanitizeTerminalText(data.splitSet.id)}`);
|
|
1232
|
+
}
|
|
1233
|
+
async function submitRunResults() {
|
|
1234
|
+
const resultsPath = getPositionalArg(2) || getArg('--file') || getArg('--results');
|
|
1235
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
1236
|
+
const promptVersionId = getArg('--prompt-version');
|
|
1237
|
+
const runnerVersionId = getArg('--runner-version');
|
|
1238
|
+
const datasetVersionId = getArg('--dataset-version');
|
|
1239
|
+
const splitSetId = getArg('--split-set');
|
|
1240
|
+
const splitName = getArg('--split');
|
|
1241
|
+
if (!resultsPath ||
|
|
1242
|
+
!promptVersionId ||
|
|
1243
|
+
!runnerVersionId ||
|
|
1244
|
+
!datasetVersionId ||
|
|
1245
|
+
!splitSetId ||
|
|
1246
|
+
!splitName) {
|
|
1247
|
+
throw new Error('Usage: orizu runs submit <results.jsonl> --project <team/project> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--judge-version <id>] [--judge-runner-version <id>]');
|
|
1248
|
+
}
|
|
1249
|
+
const resultBytes = readSourceBytes(resultsPath);
|
|
1250
|
+
const resultsJsonl = resultsPath.endsWith('.gz')
|
|
1251
|
+
? gunzipSync(resultBytes).toString('utf8')
|
|
1252
|
+
: resultBytes.toString('utf8');
|
|
1253
|
+
const response = await authedFetch(`/api/cli/runs/submit?project=${encodeURIComponent(project)}`, {
|
|
1254
|
+
method: 'POST',
|
|
1255
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1256
|
+
body: JSON.stringify({
|
|
1257
|
+
promptVersionId,
|
|
1258
|
+
runnerVersionId,
|
|
1259
|
+
datasetVersionId,
|
|
1260
|
+
splitSetId,
|
|
1261
|
+
splitName,
|
|
1262
|
+
judgeVersionId: getArg('--judge-version') || undefined,
|
|
1263
|
+
judgeRunnerVersionId: getArg('--judge-runner-version') || undefined,
|
|
1264
|
+
resultsJsonl,
|
|
1265
|
+
}),
|
|
1266
|
+
});
|
|
1267
|
+
if (!response.ok) {
|
|
1268
|
+
throw new Error(`Failed to submit run: ${await response.text()}`);
|
|
1269
|
+
}
|
|
1270
|
+
const data = await parseJsonResponse(response, 'Run submit');
|
|
1271
|
+
printLine(`Submitted run ${sanitizeTerminalText(data.run.id)} ` +
|
|
1272
|
+
`(${formatPercent(data.run.aggregateScore)}) -> ` +
|
|
1273
|
+
sanitizeTerminalText(data.run.perRowResultsStoragePath));
|
|
1274
|
+
}
|
|
1275
|
+
async function submitScoreResults() {
|
|
1276
|
+
const resultsPath = getPositionalArg(2) || getArg('--file') || getArg('--results');
|
|
1277
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
1278
|
+
const scorerVersionId = getArg('--scorer-version');
|
|
1279
|
+
const subjectPromptVersionId = getArg('--subject-version') || getArg('--prompt-version');
|
|
1280
|
+
const optimizationRunId = getArg('--optimization-run');
|
|
1281
|
+
const candidateId = getArg('--candidate');
|
|
1282
|
+
if (!resultsPath || !scorerVersionId || (!subjectPromptVersionId && (!optimizationRunId || !candidateId))) {
|
|
1283
|
+
throw new Error('Usage: orizu scores submit <results.jsonl|results.json> --project <team/project> --scorer-version <id> (--subject-version <prompt-version-id> | --optimization-run <id> --candidate <id>) [--dataset-version <id> --split-set <id> --split <name>] [--json]');
|
|
1284
|
+
}
|
|
1285
|
+
const resultBytes = readSourceBytes(resultsPath);
|
|
1286
|
+
const raw = resultsPath.endsWith('.gz')
|
|
1287
|
+
? gunzipSync(resultBytes).toString('utf8')
|
|
1288
|
+
: resultBytes.toString('utf8');
|
|
1289
|
+
const resultsJsonl = normalizeScoreResultsInput(resultsPath, raw);
|
|
1290
|
+
const response = await authedFetch(`/api/cli/scores/submit?project=${encodeURIComponent(project)}`, {
|
|
1291
|
+
method: 'POST',
|
|
1292
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1293
|
+
body: JSON.stringify({
|
|
1294
|
+
scorerVersionId,
|
|
1295
|
+
subjectPromptVersionId: subjectPromptVersionId || undefined,
|
|
1296
|
+
datasetVersionId: getArg('--dataset-version') || undefined,
|
|
1297
|
+
splitSetId: getArg('--split-set') || undefined,
|
|
1298
|
+
splitName: getArg('--split') || undefined,
|
|
1299
|
+
optimizationRunId: optimizationRunId || undefined,
|
|
1300
|
+
candidateId: candidateId || undefined,
|
|
1301
|
+
resultsJsonl,
|
|
1302
|
+
}),
|
|
1303
|
+
});
|
|
1304
|
+
if (!response.ok) {
|
|
1305
|
+
throw new Error(`Failed to submit score: ${await response.text()}`);
|
|
1306
|
+
}
|
|
1307
|
+
const data = await parseJsonResponse(response, 'Score submit');
|
|
1308
|
+
if (hasJsonFlag()) {
|
|
1309
|
+
printJson(data);
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
printLine(`Submitted score ${sanitizeTerminalText(data.scoreRun.id)} ` +
|
|
1313
|
+
`(${formatPercent(data.scoreRun.scoreValue)})`);
|
|
1314
|
+
}
|
|
1315
|
+
function normalizeScoreResultsInput(sourcePath, raw) {
|
|
1316
|
+
const logicalPath = sourcePath.endsWith('.gz') ? sourcePath.slice(0, -3) : sourcePath;
|
|
1317
|
+
if (!logicalPath.endsWith('.json'))
|
|
1318
|
+
return raw;
|
|
1319
|
+
try {
|
|
1320
|
+
const parsed = JSON.parse(raw);
|
|
1321
|
+
const rows = Array.isArray(parsed) ? parsed : [parsed];
|
|
1322
|
+
if (rows.some(row => !isRecord(row))) {
|
|
1323
|
+
throw new Error('JSON score results must be an object or an array of objects');
|
|
1324
|
+
}
|
|
1325
|
+
return rows.map(row => JSON.stringify(row)).join('\n');
|
|
1326
|
+
}
|
|
1327
|
+
catch (error) {
|
|
1328
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1329
|
+
throw new Error(`Invalid score results JSON '${sourcePath}': ${message}`);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
function parseOptionalNumberFlag(name) {
|
|
1333
|
+
const value = getArg(name);
|
|
1334
|
+
if (!value) {
|
|
1335
|
+
return undefined;
|
|
1336
|
+
}
|
|
1337
|
+
const parsed = Number(value);
|
|
1338
|
+
if (!Number.isFinite(parsed)) {
|
|
1339
|
+
throw new Error(`${name} must be a number`);
|
|
1340
|
+
}
|
|
1341
|
+
return parsed;
|
|
1342
|
+
}
|
|
1343
|
+
function hasObjectKeys(value) {
|
|
1344
|
+
return Object.keys(value).length > 0;
|
|
1345
|
+
}
|
|
1346
|
+
async function startOptimizationRun() {
|
|
1347
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
1348
|
+
const optimizerVersionId = getArg('--optimizer-version');
|
|
1349
|
+
const promptVersionIds = parseCommaSeparated(getArg('--prompt-version') || getArg('--prompt-versions'));
|
|
1350
|
+
const judgeVersionIds = parseCommaSeparated(getArg('--judge-version') || getArg('--judge-versions'));
|
|
1351
|
+
const selectionScorer = getArg('--selection-scorer') || getArg('--scorer-version');
|
|
1352
|
+
const reflectionScorer = getArg('--reflection-scorer');
|
|
1353
|
+
const paretoScorers = parseCommaSeparated(getArg('--pareto-scorer') || getArg('--pareto-scorers'));
|
|
1354
|
+
const bestScorers = parseCommaSeparated(getArg('--best-scorer') || getArg('--best-scorers'));
|
|
1355
|
+
const datasetVersionId = getArg('--dataset-version');
|
|
1356
|
+
const splitSetId = getArg('--split-set');
|
|
1357
|
+
const metadata = readJsonObjectArg(getArg('--metadata'), 'optimization metadata');
|
|
1358
|
+
if (!optimizerVersionId ||
|
|
1359
|
+
promptVersionIds.length === 0 ||
|
|
1360
|
+
!selectionScorer ||
|
|
1361
|
+
!datasetVersionId ||
|
|
1362
|
+
!splitSetId) {
|
|
1363
|
+
throw new Error('Usage: orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]');
|
|
1364
|
+
}
|
|
1365
|
+
const scorers = [
|
|
1366
|
+
{ scorerVersionId: selectionScorer, role: 'selection' },
|
|
1367
|
+
...(reflectionScorer ? [{ scorerVersionId: reflectionScorer, role: 'reflection' }] : []),
|
|
1368
|
+
...paretoScorers.map(scorerVersionId => ({
|
|
1369
|
+
scorerVersionId,
|
|
1370
|
+
role: 'tracked',
|
|
1371
|
+
trackedScope: 'pareto_candidates',
|
|
1372
|
+
})),
|
|
1373
|
+
...bestScorers.map(scorerVersionId => ({
|
|
1374
|
+
scorerVersionId,
|
|
1375
|
+
role: 'tracked',
|
|
1376
|
+
trackedScope: 'best_candidate',
|
|
1377
|
+
})),
|
|
1378
|
+
];
|
|
1379
|
+
const response = await authedFetch(`/api/cli/optimization-runs?project=${encodeURIComponent(project)}`, {
|
|
1380
|
+
method: 'POST',
|
|
1381
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1382
|
+
body: JSON.stringify({
|
|
1383
|
+
optimizerVersionId,
|
|
1384
|
+
promptVersionIds,
|
|
1385
|
+
judgeVersionIds,
|
|
1386
|
+
scorers,
|
|
1387
|
+
datasetVersionId,
|
|
1388
|
+
splitSetId,
|
|
1389
|
+
trainSplitName: getArg('--train-split') || undefined,
|
|
1390
|
+
validationSplitName: getArg('--validation-split') || undefined,
|
|
1391
|
+
metadata,
|
|
1392
|
+
}),
|
|
1393
|
+
});
|
|
1394
|
+
if (!response.ok) {
|
|
1395
|
+
throw new Error(`Failed to start optimization run: ${await response.text()}`);
|
|
1396
|
+
}
|
|
1397
|
+
const data = await parseJsonResponse(response, 'Optimization run start');
|
|
1398
|
+
if (hasJsonFlag()) {
|
|
1399
|
+
printJson(data);
|
|
1400
|
+
return;
|
|
1401
|
+
}
|
|
1402
|
+
printLine(`Started optimization run ${sanitizeTerminalText(data.optimization_run_id)} ` +
|
|
1403
|
+
`(${sanitizeTerminalText(data.optimizationRun.status)})`);
|
|
1404
|
+
}
|
|
1405
|
+
async function updateOptimizationRunLifecycle(action) {
|
|
1406
|
+
const runId = getPositionalArg(2) || getArg('--run-id');
|
|
1407
|
+
if (!runId) {
|
|
1408
|
+
throw new Error(`Usage: orizu optimizations ${action} <run-id>`);
|
|
1409
|
+
}
|
|
1410
|
+
const statusByAction = {
|
|
1411
|
+
pause: 'paused',
|
|
1412
|
+
resume: 'running',
|
|
1413
|
+
finish: 'succeeded',
|
|
1414
|
+
fail: 'failed',
|
|
1415
|
+
cancel: 'cancelled',
|
|
1416
|
+
};
|
|
1417
|
+
const body = {
|
|
1418
|
+
status: statusByAction[action],
|
|
1419
|
+
};
|
|
1420
|
+
const metadata = readJsonObjectArg(getArg('--metadata'), 'optimization metadata');
|
|
1421
|
+
const reason = getArg('--reason');
|
|
1422
|
+
if ((action === 'pause' || action === 'cancel') && reason) {
|
|
1423
|
+
metadata.reason = reason;
|
|
1424
|
+
}
|
|
1425
|
+
if (action === 'fail' && reason) {
|
|
1426
|
+
metadata.failure_reason = reason;
|
|
1427
|
+
body.failureReason = reason;
|
|
1428
|
+
}
|
|
1429
|
+
if (hasObjectKeys(metadata)) {
|
|
1430
|
+
body.metadata = metadata;
|
|
1431
|
+
}
|
|
1432
|
+
if (action === 'finish') {
|
|
1433
|
+
const bestScore = parseOptionalNumberFlag('--best-score');
|
|
1434
|
+
if (bestScore !== undefined) {
|
|
1435
|
+
body.bestScore = bestScore;
|
|
1436
|
+
}
|
|
1437
|
+
const bestCandidateId = getArg('--best-candidate');
|
|
1438
|
+
if (bestCandidateId) {
|
|
1439
|
+
body.bestCandidateId = bestCandidateId;
|
|
1440
|
+
}
|
|
1441
|
+
const resultPromptVersionId = getArg('--result-prompt-version');
|
|
1442
|
+
if (resultPromptVersionId) {
|
|
1443
|
+
body.resultPromptVersionId = resultPromptVersionId;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}`, {
|
|
1447
|
+
method: 'PATCH',
|
|
1448
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1449
|
+
body: JSON.stringify(body),
|
|
1450
|
+
});
|
|
1451
|
+
if (!response.ok) {
|
|
1452
|
+
throw new Error(`Failed to update optimization run: ${await response.text()}`);
|
|
1453
|
+
}
|
|
1454
|
+
const data = await parseJsonResponse(response, 'Optimization run update');
|
|
1455
|
+
if (hasJsonFlag()) {
|
|
1456
|
+
printJson(data);
|
|
1457
|
+
return;
|
|
1458
|
+
}
|
|
1459
|
+
const id = sanitizeTerminalText(data.optimizationRun.id);
|
|
1460
|
+
if (action === 'pause') {
|
|
1461
|
+
printLine(`Paused optimization run ${id}`);
|
|
1462
|
+
}
|
|
1463
|
+
else if (action === 'resume') {
|
|
1464
|
+
printLine(`Resumed optimization run ${id}`);
|
|
1465
|
+
}
|
|
1466
|
+
else if (action === 'finish') {
|
|
1467
|
+
printLine(`Finished optimization run ${id}`);
|
|
1468
|
+
}
|
|
1469
|
+
else if (action === 'fail') {
|
|
1470
|
+
printLine(`Marked optimization run ${id} failed`);
|
|
1471
|
+
}
|
|
1472
|
+
else {
|
|
1473
|
+
printLine(`Cancelled optimization run ${id}`);
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
function removeFlagWithValue(args, flag) {
|
|
1477
|
+
const filtered = [];
|
|
1478
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
1479
|
+
if (args[index] === flag) {
|
|
1480
|
+
index += 1;
|
|
1481
|
+
continue;
|
|
1482
|
+
}
|
|
1483
|
+
filtered.push(args[index]);
|
|
1484
|
+
}
|
|
1485
|
+
return filtered;
|
|
1486
|
+
}
|
|
1487
|
+
function bundledOrizuGepaPythonPath() {
|
|
1488
|
+
const candidate = fileURLToPath(new URL('../../orizu-gepa-python/src', import.meta.url));
|
|
1489
|
+
return existsSync(candidate) ? candidate : null;
|
|
1490
|
+
}
|
|
1491
|
+
async function runGepaOptimization() {
|
|
1492
|
+
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
1493
|
+
const baseUrl = getBaseUrl();
|
|
1494
|
+
const token = getStoredAuthTokenForBaseUrl(baseUrl);
|
|
1495
|
+
const python = getArg('--python') || process.env.PYTHON || 'python3';
|
|
1496
|
+
const bundledPythonPath = bundledOrizuGepaPythonPath();
|
|
1497
|
+
let forwardedArgs = removeFlagWithValue(cliArgs.slice(2), '--python');
|
|
1498
|
+
if (!forwardedArgs.includes('--project')) {
|
|
1499
|
+
forwardedArgs = ['--project', project, ...forwardedArgs];
|
|
1500
|
+
}
|
|
1501
|
+
const pythonPathEntries = [
|
|
1502
|
+
bundledPythonPath,
|
|
1503
|
+
process.env.PYTHONPATH,
|
|
1504
|
+
].filter((entry) => Boolean(entry));
|
|
1505
|
+
const result = spawnSync(python, ['-m', 'orizu_gepa.cli', ...forwardedArgs], {
|
|
1506
|
+
stdio: 'inherit',
|
|
1507
|
+
env: {
|
|
1508
|
+
...process.env,
|
|
1509
|
+
ORIZU_API_URL: baseUrl,
|
|
1510
|
+
ORIZU_TOKEN: token,
|
|
1511
|
+
ORIZU_PROJECT: project,
|
|
1512
|
+
PYTHONPATH: pythonPathEntries.join(delimiter),
|
|
1513
|
+
PYTHONUNBUFFERED: process.env.PYTHONUNBUFFERED || '1',
|
|
1514
|
+
},
|
|
1515
|
+
});
|
|
1516
|
+
if (result.error) {
|
|
1517
|
+
throw result.error;
|
|
1518
|
+
}
|
|
1519
|
+
if (result.status !== 0) {
|
|
1520
|
+
throw new Error(`orizu-gepa failed with exit code ${result.status}`);
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
641
1523
|
async function listTeams() {
|
|
642
1524
|
printTeams(await fetchTeams());
|
|
643
1525
|
}
|
|
@@ -720,6 +1602,24 @@ function readSourceFile(pathArg) {
|
|
|
720
1602
|
throw new Error(`Failed to read file '${expandedPath}': ${error?.message || String(error)}`);
|
|
721
1603
|
}
|
|
722
1604
|
}
|
|
1605
|
+
function readSourceBytes(pathArg) {
|
|
1606
|
+
const expandedPath = expandHomePath(pathArg);
|
|
1607
|
+
try {
|
|
1608
|
+
return readFileSync(expandedPath);
|
|
1609
|
+
}
|
|
1610
|
+
catch (error) {
|
|
1611
|
+
if (error?.code === 'ENOENT') {
|
|
1612
|
+
throw new Error(`File not found: ${expandedPath}`);
|
|
1613
|
+
}
|
|
1614
|
+
throw new Error(`Failed to read file '${expandedPath}': ${error?.message || String(error)}`);
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
function hasJsonFlag() {
|
|
1618
|
+
return hasArg('--json');
|
|
1619
|
+
}
|
|
1620
|
+
function printJson(value) {
|
|
1621
|
+
printLine(JSON.stringify(value));
|
|
1622
|
+
}
|
|
723
1623
|
function readJsonFile(pathArg) {
|
|
724
1624
|
const raw = readSourceFile(pathArg);
|
|
725
1625
|
try {
|
|
@@ -733,6 +1633,190 @@ function readJsonFile(pathArg) {
|
|
|
733
1633
|
throw new Error(`Invalid JSON file '${pathArg}': ${error?.message || String(error)}`);
|
|
734
1634
|
}
|
|
735
1635
|
}
|
|
1636
|
+
function isRecord(value) {
|
|
1637
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
1638
|
+
}
|
|
1639
|
+
function stringFromRecord(record, key) {
|
|
1640
|
+
const value = record[key];
|
|
1641
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
1642
|
+
}
|
|
1643
|
+
function safeRelativePath(value, fallback) {
|
|
1644
|
+
const raw = value && value.trim() ? value.trim() : fallback;
|
|
1645
|
+
if (raw.includes('\0')) {
|
|
1646
|
+
throw new Error(`Unsafe relative path: ${raw}`);
|
|
1647
|
+
}
|
|
1648
|
+
const normalized = normalize(raw);
|
|
1649
|
+
if (isAbsolute(raw) || normalized === '..' || normalized.startsWith('../') || normalized.startsWith('..\\')) {
|
|
1650
|
+
throw new Error(`Unsafe relative path: ${raw}`);
|
|
1651
|
+
}
|
|
1652
|
+
return normalized === '.' ? fallback : normalized;
|
|
1653
|
+
}
|
|
1654
|
+
function sha256Hex(value) {
|
|
1655
|
+
return createHash('sha256').update(value).digest('hex');
|
|
1656
|
+
}
|
|
1657
|
+
function readPromptPrimaryText(manifest, promptDir) {
|
|
1658
|
+
const primaryText = isRecord(manifest.primary_text) ? manifest.primary_text : null;
|
|
1659
|
+
const pathValue = primaryText
|
|
1660
|
+
? stringFromRecord(primaryText, 'path')
|
|
1661
|
+
: stringFromRecord(manifest, 'body_file');
|
|
1662
|
+
const bodyPath = safeRelativePath(pathValue || 'prompt.md', 'prompt.md');
|
|
1663
|
+
const bodyKind = primaryText
|
|
1664
|
+
? stringFromRecord(primaryText, 'kind') || stringFromRecord(manifest, 'body_kind') || 'text'
|
|
1665
|
+
: stringFromRecord(manifest, 'body_kind') || 'text';
|
|
1666
|
+
return {
|
|
1667
|
+
body: readSourceFile(join(promptDir, bodyPath)),
|
|
1668
|
+
bodyKind,
|
|
1669
|
+
path: bodyPath,
|
|
1670
|
+
};
|
|
1671
|
+
}
|
|
1672
|
+
function readPromptSidecars(manifest, promptDir) {
|
|
1673
|
+
const sidecars = Array.isArray(manifest.sidecars) ? manifest.sidecars : [];
|
|
1674
|
+
return sidecars
|
|
1675
|
+
.filter(isRecord)
|
|
1676
|
+
.map(sidecar => {
|
|
1677
|
+
const pathValue = stringFromRecord(sidecar, 'path');
|
|
1678
|
+
if (!pathValue) {
|
|
1679
|
+
throw new Error('Prompt sidecar entries require path');
|
|
1680
|
+
}
|
|
1681
|
+
const relativePath = safeRelativePath(pathValue, pathValue);
|
|
1682
|
+
const content = readSourceFile(join(promptDir, relativePath));
|
|
1683
|
+
const { content: _ignoredContent, content_sha256: _ignoredSnake, contentSha256: _ignoredCamel, ...metadata } = sidecar;
|
|
1684
|
+
return {
|
|
1685
|
+
...metadata,
|
|
1686
|
+
type: stringFromRecord(sidecar, 'type') || 'file',
|
|
1687
|
+
path: relativePath,
|
|
1688
|
+
content,
|
|
1689
|
+
contentSha256: sha256Hex(content),
|
|
1690
|
+
};
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
function writeTextFileEnsuringDir(pathArg, content) {
|
|
1694
|
+
mkdirSync(dirname(pathArg), { recursive: true });
|
|
1695
|
+
writeFileSync(pathArg, content);
|
|
1696
|
+
}
|
|
1697
|
+
function shouldExcludeArtifactPath(relativePath) {
|
|
1698
|
+
const parts = relativePath.split('/');
|
|
1699
|
+
return parts.some(part => part === '.git' ||
|
|
1700
|
+
part === '.DS_Store' ||
|
|
1701
|
+
part === '__pycache__' ||
|
|
1702
|
+
part === '.pytest_cache');
|
|
1703
|
+
}
|
|
1704
|
+
function collectArtifactFiles(sourceDir, relativeDir = '') {
|
|
1705
|
+
const absoluteDir = relativeDir ? join(sourceDir, relativeDir) : sourceDir;
|
|
1706
|
+
return readdirSync(absoluteDir, { withFileTypes: true })
|
|
1707
|
+
.flatMap(entry => {
|
|
1708
|
+
const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
|
|
1709
|
+
if (shouldExcludeArtifactPath(relativePath)) {
|
|
1710
|
+
return [];
|
|
1711
|
+
}
|
|
1712
|
+
if (entry.isDirectory()) {
|
|
1713
|
+
return collectArtifactFiles(sourceDir, relativePath);
|
|
1714
|
+
}
|
|
1715
|
+
return entry.isFile() ? [relativePath] : [];
|
|
1716
|
+
})
|
|
1717
|
+
.sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
|
|
1718
|
+
}
|
|
1719
|
+
const ZIP_GENERAL_PURPOSE_UTF8 = 0x0800;
|
|
1720
|
+
const ZIP_DOS_TIME_2000_01_01 = 0;
|
|
1721
|
+
const ZIP_DOS_DATE_2000_01_01 = ((2000 - 1980) << 9) | (1 << 5) | 1;
|
|
1722
|
+
const CRC32_TABLE = Array.from({ length: 256 }, (_, index) => {
|
|
1723
|
+
let value = index;
|
|
1724
|
+
for (let bit = 0; bit < 8; bit++) {
|
|
1725
|
+
value = (value & 1) ? (0xedb88320 ^ (value >>> 1)) : (value >>> 1);
|
|
1726
|
+
}
|
|
1727
|
+
return value >>> 0;
|
|
1728
|
+
});
|
|
1729
|
+
function crc32(bytes) {
|
|
1730
|
+
let crc = 0xffffffff;
|
|
1731
|
+
for (const byte of bytes) {
|
|
1732
|
+
crc = CRC32_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8);
|
|
1733
|
+
}
|
|
1734
|
+
return (crc ^ 0xffffffff) >>> 0;
|
|
1735
|
+
}
|
|
1736
|
+
function createStoredZip(entries) {
|
|
1737
|
+
const localParts = [];
|
|
1738
|
+
const centralParts = [];
|
|
1739
|
+
let offset = 0;
|
|
1740
|
+
for (const entry of entries) {
|
|
1741
|
+
const name = Buffer.from(entry.path, 'utf8');
|
|
1742
|
+
const crc = crc32(entry.data);
|
|
1743
|
+
const localHeader = Buffer.alloc(30);
|
|
1744
|
+
localHeader.writeUInt32LE(0x04034b50, 0);
|
|
1745
|
+
localHeader.writeUInt16LE(10, 4);
|
|
1746
|
+
localHeader.writeUInt16LE(ZIP_GENERAL_PURPOSE_UTF8, 6);
|
|
1747
|
+
localHeader.writeUInt16LE(0, 8);
|
|
1748
|
+
localHeader.writeUInt16LE(ZIP_DOS_TIME_2000_01_01, 10);
|
|
1749
|
+
localHeader.writeUInt16LE(ZIP_DOS_DATE_2000_01_01, 12);
|
|
1750
|
+
localHeader.writeUInt32LE(crc, 14);
|
|
1751
|
+
localHeader.writeUInt32LE(entry.data.length, 18);
|
|
1752
|
+
localHeader.writeUInt32LE(entry.data.length, 22);
|
|
1753
|
+
localHeader.writeUInt16LE(name.length, 26);
|
|
1754
|
+
localHeader.writeUInt16LE(0, 28);
|
|
1755
|
+
localParts.push(localHeader, name, entry.data);
|
|
1756
|
+
const centralHeader = Buffer.alloc(46);
|
|
1757
|
+
centralHeader.writeUInt32LE(0x02014b50, 0);
|
|
1758
|
+
centralHeader.writeUInt16LE(20, 4);
|
|
1759
|
+
centralHeader.writeUInt16LE(10, 6);
|
|
1760
|
+
centralHeader.writeUInt16LE(ZIP_GENERAL_PURPOSE_UTF8, 8);
|
|
1761
|
+
centralHeader.writeUInt16LE(0, 10);
|
|
1762
|
+
centralHeader.writeUInt16LE(ZIP_DOS_TIME_2000_01_01, 12);
|
|
1763
|
+
centralHeader.writeUInt16LE(ZIP_DOS_DATE_2000_01_01, 14);
|
|
1764
|
+
centralHeader.writeUInt32LE(crc, 16);
|
|
1765
|
+
centralHeader.writeUInt32LE(entry.data.length, 20);
|
|
1766
|
+
centralHeader.writeUInt32LE(entry.data.length, 24);
|
|
1767
|
+
centralHeader.writeUInt16LE(name.length, 28);
|
|
1768
|
+
centralHeader.writeUInt16LE(0, 30);
|
|
1769
|
+
centralHeader.writeUInt16LE(0, 32);
|
|
1770
|
+
centralHeader.writeUInt16LE(0, 34);
|
|
1771
|
+
centralHeader.writeUInt16LE(0, 36);
|
|
1772
|
+
centralHeader.writeUInt32LE(0, 38);
|
|
1773
|
+
centralHeader.writeUInt32LE(offset, 42);
|
|
1774
|
+
centralParts.push(centralHeader, name);
|
|
1775
|
+
offset += localHeader.length + name.length + entry.data.length;
|
|
1776
|
+
}
|
|
1777
|
+
const centralDirectoryOffset = offset;
|
|
1778
|
+
const centralDirectory = Buffer.concat(centralParts);
|
|
1779
|
+
const endOfCentralDirectory = Buffer.alloc(22);
|
|
1780
|
+
endOfCentralDirectory.writeUInt32LE(0x06054b50, 0);
|
|
1781
|
+
endOfCentralDirectory.writeUInt16LE(0, 4);
|
|
1782
|
+
endOfCentralDirectory.writeUInt16LE(0, 6);
|
|
1783
|
+
endOfCentralDirectory.writeUInt16LE(entries.length, 8);
|
|
1784
|
+
endOfCentralDirectory.writeUInt16LE(entries.length, 10);
|
|
1785
|
+
endOfCentralDirectory.writeUInt32LE(centralDirectory.length, 12);
|
|
1786
|
+
endOfCentralDirectory.writeUInt32LE(centralDirectoryOffset, 16);
|
|
1787
|
+
endOfCentralDirectory.writeUInt16LE(0, 20);
|
|
1788
|
+
return Buffer.concat([...localParts, centralDirectory, endOfCentralDirectory]);
|
|
1789
|
+
}
|
|
1790
|
+
function zipDirectoryToBase64(dirArg) {
|
|
1791
|
+
const sourceDir = expandHomePath(dirArg);
|
|
1792
|
+
try {
|
|
1793
|
+
const stats = statSync(sourceDir);
|
|
1794
|
+
if (!stats.isDirectory()) {
|
|
1795
|
+
throw new Error(`${sourceDir} is not a directory`);
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
catch (error) {
|
|
1799
|
+
if (error?.code === 'ENOENT') {
|
|
1800
|
+
throw new Error(`Directory not found: ${sourceDir}`);
|
|
1801
|
+
}
|
|
1802
|
+
throw error;
|
|
1803
|
+
}
|
|
1804
|
+
const files = collectArtifactFiles(sourceDir);
|
|
1805
|
+
if (files.length === 0) {
|
|
1806
|
+
throw new Error(`Directory contains no artifact files: ${sourceDir}`);
|
|
1807
|
+
}
|
|
1808
|
+
const bytes = createStoredZip(files.map(relativePath => ({
|
|
1809
|
+
path: relativePath,
|
|
1810
|
+
data: readFileSync(join(sourceDir, relativePath)),
|
|
1811
|
+
})));
|
|
1812
|
+
return {
|
|
1813
|
+
zipBase64: bytes.toString('base64'),
|
|
1814
|
+
contentSha256: createHash('sha256').update(bytes).digest('hex'),
|
|
1815
|
+
};
|
|
1816
|
+
}
|
|
1817
|
+
function readManifestFile(dirArg) {
|
|
1818
|
+
return readJsonFile(join(expandHomePath(dirArg), 'manifest.json'));
|
|
1819
|
+
}
|
|
736
1820
|
async function createAppFromFile() {
|
|
737
1821
|
const project = getArg('--project');
|
|
738
1822
|
const name = getArg('--name');
|
|
@@ -1194,6 +2278,31 @@ async function uploadDataset() {
|
|
|
1194
2278
|
printLine(`View dataset: ${formatTerminalLink(data.dataset.url)}`);
|
|
1195
2279
|
}
|
|
1196
2280
|
}
|
|
2281
|
+
async function pushDataset() {
|
|
2282
|
+
const projectArg = getArg('--project');
|
|
2283
|
+
const fileArg = getPositionalArg(2) || getArg('--file');
|
|
2284
|
+
const name = getArg('--name');
|
|
2285
|
+
if (!fileArg) {
|
|
2286
|
+
throw new Error('Usage: orizu datasets push <rows.csv|rows.json|rows.jsonl> [--project <team/project>] [--name <name>] [--json]');
|
|
2287
|
+
}
|
|
2288
|
+
const file = expandHomePath(fileArg);
|
|
2289
|
+
const project = await resolveProjectSlug(projectArg);
|
|
2290
|
+
const datasetName = name || basename(file);
|
|
2291
|
+
const { rows, sourceType } = parseDatasetFile(file);
|
|
2292
|
+
const data = await createDatasetFromRows(project, datasetName, sourceType, rows);
|
|
2293
|
+
if (hasJsonFlag()) {
|
|
2294
|
+
printJson({
|
|
2295
|
+
dataset_id: data.dataset.id,
|
|
2296
|
+
name: data.dataset.name,
|
|
2297
|
+
row_count: data.dataset.rowCount,
|
|
2298
|
+
});
|
|
2299
|
+
return;
|
|
2300
|
+
}
|
|
2301
|
+
printLine(`Uploaded dataset ${sanitizeTerminalText(data.dataset.name)} (${sanitizeTerminalText(data.dataset.id)}) with ${data.dataset.rowCount} rows.`);
|
|
2302
|
+
if (data.dataset.url) {
|
|
2303
|
+
printLine(`View dataset: ${formatTerminalLink(data.dataset.url)}`);
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
1197
2306
|
function getDatasetReferenceInput() {
|
|
1198
2307
|
const fromFlag = getArg('--dataset');
|
|
1199
2308
|
if (fromFlag) {
|
|
@@ -1537,6 +2646,178 @@ async function downloadAnnotations() {
|
|
|
1537
2646
|
writeFileSync(filename, bytes);
|
|
1538
2647
|
printLine(`Saved ${format.toUpperCase()} export to ${sanitizeTerminalText(filename)}`);
|
|
1539
2648
|
}
|
|
2649
|
+
function readRunnerManifest(runnerDir) {
|
|
2650
|
+
const manifestPath = join(runnerDir, 'manifest.json');
|
|
2651
|
+
const raw = readSourceFile(manifestPath);
|
|
2652
|
+
const manifest = JSON.parse(raw);
|
|
2653
|
+
if (!Array.isArray(manifest.command) || !manifest.command.every(item => typeof item === 'string')) {
|
|
2654
|
+
throw new Error(`Runner manifest at ${manifestPath} must include command: string[]`);
|
|
2655
|
+
}
|
|
2656
|
+
const supportedBodyKinds = manifest.supports_body_kinds ?? manifest.supports_body_kind;
|
|
2657
|
+
if (supportedBodyKinds !== undefined &&
|
|
2658
|
+
(!Array.isArray(supportedBodyKinds) ||
|
|
2659
|
+
!supportedBodyKinds.every(item => typeof item === 'string'))) {
|
|
2660
|
+
throw new Error(`Runner manifest at ${manifestPath} has invalid supports_body_kinds`);
|
|
2661
|
+
}
|
|
2662
|
+
return {
|
|
2663
|
+
command: manifest.command,
|
|
2664
|
+
supports_body_kind: supportedBodyKinds,
|
|
2665
|
+
};
|
|
2666
|
+
}
|
|
2667
|
+
async function materializeRunnerVersion(runnerVersionId) {
|
|
2668
|
+
const response = await authedFetch(`/api/cli/runner-versions/${encodeURIComponent(runnerVersionId)}/download`);
|
|
2669
|
+
if (!response.ok) {
|
|
2670
|
+
throw new Error(`Failed to download runner version: ${await response.text()}`);
|
|
2671
|
+
}
|
|
2672
|
+
const tempDir = mkdtempSync(join(tmpdir(), 'orizu-runner-version-'));
|
|
2673
|
+
const zipPath = join(tempDir, 'runner.zip');
|
|
2674
|
+
const runnerDir = join(tempDir, 'runner');
|
|
2675
|
+
writeFileSync(zipPath, new Uint8Array(await response.arrayBuffer()));
|
|
2676
|
+
const result = spawnSync('unzip', ['-q', zipPath, '-d', runnerDir], {
|
|
2677
|
+
encoding: 'utf8',
|
|
2678
|
+
});
|
|
2679
|
+
if (result.error) {
|
|
2680
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
2681
|
+
throw result.error;
|
|
2682
|
+
}
|
|
2683
|
+
if (result.status !== 0) {
|
|
2684
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
2685
|
+
throw new Error(`unzip failed: ${sanitizeTerminalText(result.stderr || result.stdout || '')}`);
|
|
2686
|
+
}
|
|
2687
|
+
return {
|
|
2688
|
+
runnerDir,
|
|
2689
|
+
cleanup: () => rmSync(tempDir, { recursive: true, force: true }),
|
|
2690
|
+
};
|
|
2691
|
+
}
|
|
2692
|
+
async function runnersExec() {
|
|
2693
|
+
const prompt = getArg('--prompt');
|
|
2694
|
+
const promptVersion = getArg('--prompt-version');
|
|
2695
|
+
const runnerVersion = getArg('--runner-version');
|
|
2696
|
+
const scorerVersion = getArg('--scorer-version');
|
|
2697
|
+
const datasetVersion = getArg('--dataset-version');
|
|
2698
|
+
const splitSet = getArg('--split-set');
|
|
2699
|
+
const split = getArg('--split');
|
|
2700
|
+
const runnerDirArg = getArg('--runner-dir');
|
|
2701
|
+
const outArg = getArg('--out');
|
|
2702
|
+
if ((!prompt && !promptVersion && !scorerVersion) ||
|
|
2703
|
+
(scorerVersion && (prompt || promptVersion)) ||
|
|
2704
|
+
(promptVersion && !runnerVersion) ||
|
|
2705
|
+
!datasetVersion ||
|
|
2706
|
+
!splitSet ||
|
|
2707
|
+
!split ||
|
|
2708
|
+
!outArg) {
|
|
2709
|
+
throw new Error('Usage: orizu runners exec (--prompt <prompt> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>');
|
|
2710
|
+
}
|
|
2711
|
+
const query = new URLSearchParams();
|
|
2712
|
+
if (scorerVersion) {
|
|
2713
|
+
query.set('scorerVersion', scorerVersion);
|
|
2714
|
+
}
|
|
2715
|
+
else if (promptVersion) {
|
|
2716
|
+
query.set('promptVersion', promptVersion);
|
|
2717
|
+
}
|
|
2718
|
+
else if (prompt) {
|
|
2719
|
+
query.set('prompt', prompt);
|
|
2720
|
+
}
|
|
2721
|
+
if (runnerVersion) {
|
|
2722
|
+
query.set('runnerVersion', runnerVersion);
|
|
2723
|
+
}
|
|
2724
|
+
query.set('datasetVersion', datasetVersion);
|
|
2725
|
+
query.set('splitSet', splitSet);
|
|
2726
|
+
query.set('split', split);
|
|
2727
|
+
const contextResponse = await authedFetch(`/api/cli/runners/exec-context?${query.toString()}`);
|
|
2728
|
+
if (!contextResponse.ok) {
|
|
2729
|
+
throw new Error(`Failed to fetch runner execution context: ${await contextResponse.text()}`);
|
|
2730
|
+
}
|
|
2731
|
+
const context = await parseJsonResponse(contextResponse, 'Runner exec context');
|
|
2732
|
+
const materializedRunner = runnerDirArg
|
|
2733
|
+
? { runnerDir: expandHomePath(runnerDirArg), cleanup: () => { } }
|
|
2734
|
+
: await materializeRunnerVersion(runnerVersion || context.prompt.runnerVersionId);
|
|
2735
|
+
const runnerDir = materializedRunner.runnerDir;
|
|
2736
|
+
const manifest = readRunnerManifest(runnerDir);
|
|
2737
|
+
try {
|
|
2738
|
+
if (manifest.supports_body_kind && !manifest.supports_body_kind.includes(context.prompt.bodyKind)) {
|
|
2739
|
+
throw new Error(`Runner does not support prompt body kind '${context.prompt.bodyKind}'. Supported kinds: ${manifest.supports_body_kind.join(', ')}`);
|
|
2740
|
+
}
|
|
2741
|
+
const resultLines = [];
|
|
2742
|
+
for (const row of context.rows) {
|
|
2743
|
+
const tempDir = mkdtempSync(join(tmpdir(), 'orizu-runner-'));
|
|
2744
|
+
const inputPath = join(tempDir, 'input.json');
|
|
2745
|
+
const outputPath = join(tempDir, 'output.json');
|
|
2746
|
+
try {
|
|
2747
|
+
const modelOutput = row.row.model_output ?? row.row.modelOutput ?? row.row.output ?? null;
|
|
2748
|
+
writeFileSync(inputPath, JSON.stringify({
|
|
2749
|
+
row: row.row,
|
|
2750
|
+
prompt: {
|
|
2751
|
+
body: context.prompt.body,
|
|
2752
|
+
body_kind: context.prompt.bodyKind,
|
|
2753
|
+
provider_settings: context.prompt.providerSettings,
|
|
2754
|
+
},
|
|
2755
|
+
subject: scorerVersion
|
|
2756
|
+
? {
|
|
2757
|
+
type: 'scorer_row',
|
|
2758
|
+
row_id: row.id,
|
|
2759
|
+
scorer_version_id: context.scorer?.versionId || scorerVersion,
|
|
2760
|
+
prompt_version_id: context.prompt.promptVersionId,
|
|
2761
|
+
}
|
|
2762
|
+
: {
|
|
2763
|
+
type: 'prompt_version',
|
|
2764
|
+
row_id: row.id,
|
|
2765
|
+
prompt_version_id: context.prompt.promptVersionId,
|
|
2766
|
+
},
|
|
2767
|
+
scorer: scorerVersion
|
|
2768
|
+
? {
|
|
2769
|
+
version_id: context.scorer?.versionId || scorerVersion,
|
|
2770
|
+
metric_key: context.scorer?.metricKey || 'score',
|
|
2771
|
+
higher_is_better: context.scorer?.higherIsBetter ?? true,
|
|
2772
|
+
}
|
|
2773
|
+
: null,
|
|
2774
|
+
model_output: modelOutput,
|
|
2775
|
+
prompt_version_id: context.prompt.promptVersionId,
|
|
2776
|
+
runner_version_id: context.prompt.runnerVersionId,
|
|
2777
|
+
run_id: null,
|
|
2778
|
+
}));
|
|
2779
|
+
const result = spawnSync(manifest.command[0], manifest.command.slice(1), {
|
|
2780
|
+
cwd: runnerDir,
|
|
2781
|
+
env: {
|
|
2782
|
+
...process.env,
|
|
2783
|
+
ORIZU_RUNNER_INPUT_PATH: inputPath,
|
|
2784
|
+
ORIZU_RUNNER_OUTPUT_PATH: outputPath,
|
|
2785
|
+
},
|
|
2786
|
+
encoding: 'utf8',
|
|
2787
|
+
});
|
|
2788
|
+
if (result.error) {
|
|
2789
|
+
throw result.error;
|
|
2790
|
+
}
|
|
2791
|
+
if (result.status !== 0) {
|
|
2792
|
+
throw new Error(`Runner failed for row ${row.id} with exit code ${result.status}: ${sanitizeTerminalText(result.stderr || result.stdout || '')}`);
|
|
2793
|
+
}
|
|
2794
|
+
const runnerOutput = JSON.parse(readFileSync(outputPath, 'utf8'));
|
|
2795
|
+
resultLines.push(JSON.stringify({
|
|
2796
|
+
row_id: row.id,
|
|
2797
|
+
prompt_version_id: context.prompt.promptVersionId,
|
|
2798
|
+
runner_version_id: context.prompt.runnerVersionId,
|
|
2799
|
+
...(scorerVersion ? { scorer_version_id: context.scorer?.versionId || scorerVersion } : {}),
|
|
2800
|
+
...runnerOutput,
|
|
2801
|
+
}));
|
|
2802
|
+
}
|
|
2803
|
+
finally {
|
|
2804
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
2805
|
+
}
|
|
2806
|
+
}
|
|
2807
|
+
const outPath = expandHomePath(outArg);
|
|
2808
|
+
const resultJsonl = `${resultLines.join('\n')}${resultLines.length > 0 ? '\n' : ''}`;
|
|
2809
|
+
if (outPath.endsWith('.gz')) {
|
|
2810
|
+
writeFileSync(outPath, gzipSync(Buffer.from(resultJsonl, 'utf8')));
|
|
2811
|
+
}
|
|
2812
|
+
else {
|
|
2813
|
+
writeFileSync(outPath, resultJsonl);
|
|
2814
|
+
}
|
|
2815
|
+
printLine(`Wrote ${context.rows.length} runner results to ${sanitizeTerminalText(outPath)}`);
|
|
2816
|
+
}
|
|
2817
|
+
finally {
|
|
2818
|
+
materializedRunner.cleanup();
|
|
2819
|
+
}
|
|
2820
|
+
}
|
|
1540
2821
|
export async function main(rawArgs = process.argv.slice(2)) {
|
|
1541
2822
|
const parsed = parseGlobalFlags(rawArgs);
|
|
1542
2823
|
setGlobalFlags(parsed.flags);
|
|
@@ -1549,6 +2830,7 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
1549
2830
|
}
|
|
1550
2831
|
if (!command) {
|
|
1551
2832
|
printUsage();
|
|
2833
|
+
printOptimizationUsage();
|
|
1552
2834
|
process.exit(1);
|
|
1553
2835
|
}
|
|
1554
2836
|
if (command === 'login') {
|
|
@@ -1563,6 +2845,14 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
1563
2845
|
await whoami();
|
|
1564
2846
|
return;
|
|
1565
2847
|
}
|
|
2848
|
+
if (command === 'env') {
|
|
2849
|
+
await printEnv();
|
|
2850
|
+
return;
|
|
2851
|
+
}
|
|
2852
|
+
if (command === 'log') {
|
|
2853
|
+
await logOptimizationEvent();
|
|
2854
|
+
return;
|
|
2855
|
+
}
|
|
1566
2856
|
if (command === 'teams' && subcommand === 'list') {
|
|
1567
2857
|
await listTeams();
|
|
1568
2858
|
return;
|
|
@@ -1596,6 +2886,95 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
1596
2886
|
await createProject();
|
|
1597
2887
|
return;
|
|
1598
2888
|
}
|
|
2889
|
+
if (command === 'prompts' && subcommand === 'list') {
|
|
2890
|
+
await listPrompts();
|
|
2891
|
+
return;
|
|
2892
|
+
}
|
|
2893
|
+
if (command === 'prompts' && subcommand === 'pull') {
|
|
2894
|
+
await pullPromptArtifact('prompt');
|
|
2895
|
+
return;
|
|
2896
|
+
}
|
|
2897
|
+
if (command === 'prompts' && subcommand === 'push') {
|
|
2898
|
+
await pushPromptArtifact('prompt');
|
|
2899
|
+
return;
|
|
2900
|
+
}
|
|
2901
|
+
if (command === 'prompts' && subcommand === 'labels' && cliArgs[2] === 'set') {
|
|
2902
|
+
await setPromptLabel();
|
|
2903
|
+
return;
|
|
2904
|
+
}
|
|
2905
|
+
if (command === 'prompts' && subcommand === 'scorers' && cliArgs[2] === 'set-headline') {
|
|
2906
|
+
await bindPromptScorer('headline');
|
|
2907
|
+
return;
|
|
2908
|
+
}
|
|
2909
|
+
if (command === 'prompts' && subcommand === 'scorers' && cliArgs[2] === 'add') {
|
|
2910
|
+
await bindPromptScorer('tracked');
|
|
2911
|
+
return;
|
|
2912
|
+
}
|
|
2913
|
+
if (command === 'judges' && subcommand === 'list') {
|
|
2914
|
+
await listJudges();
|
|
2915
|
+
return;
|
|
2916
|
+
}
|
|
2917
|
+
if (command === 'judges' && subcommand === 'pull') {
|
|
2918
|
+
await pullPromptArtifact('judge');
|
|
2919
|
+
return;
|
|
2920
|
+
}
|
|
2921
|
+
if (command === 'judges' && subcommand === 'push') {
|
|
2922
|
+
await pushPromptArtifact('judge');
|
|
2923
|
+
return;
|
|
2924
|
+
}
|
|
2925
|
+
if (command === 'scorers' && subcommand === 'list') {
|
|
2926
|
+
await listScorers();
|
|
2927
|
+
return;
|
|
2928
|
+
}
|
|
2929
|
+
if (command === 'scorers' && subcommand === 'register') {
|
|
2930
|
+
await registerScorer();
|
|
2931
|
+
return;
|
|
2932
|
+
}
|
|
2933
|
+
if (command === 'scorers' && subcommand === 'detail') {
|
|
2934
|
+
await showScorerDetail();
|
|
2935
|
+
return;
|
|
2936
|
+
}
|
|
2937
|
+
if (command === 'scorers' && subcommand === 'labels' && cliArgs[2] === 'set') {
|
|
2938
|
+
await setScorerLabel();
|
|
2939
|
+
return;
|
|
2940
|
+
}
|
|
2941
|
+
if (command === 'runners' && subcommand === 'push') {
|
|
2942
|
+
await pushRunnerArtifact('runner');
|
|
2943
|
+
return;
|
|
2944
|
+
}
|
|
2945
|
+
if (command === 'runners' && subcommand === 'exec') {
|
|
2946
|
+
await runnersExec();
|
|
2947
|
+
return;
|
|
2948
|
+
}
|
|
2949
|
+
if (command === 'optimizers' && subcommand === 'push') {
|
|
2950
|
+
await pushRunnerArtifact('optimizer');
|
|
2951
|
+
return;
|
|
2952
|
+
}
|
|
2953
|
+
if (command === 'runs' && subcommand === 'submit') {
|
|
2954
|
+
await submitRunResults();
|
|
2955
|
+
return;
|
|
2956
|
+
}
|
|
2957
|
+
if (command === 'scores' && subcommand === 'submit') {
|
|
2958
|
+
await submitScoreResults();
|
|
2959
|
+
return;
|
|
2960
|
+
}
|
|
2961
|
+
if (command === 'optimizations' && subcommand === 'start') {
|
|
2962
|
+
await startOptimizationRun();
|
|
2963
|
+
return;
|
|
2964
|
+
}
|
|
2965
|
+
if (command === 'optimizations' && subcommand === 'run-gepa') {
|
|
2966
|
+
await runGepaOptimization();
|
|
2967
|
+
return;
|
|
2968
|
+
}
|
|
2969
|
+
if (command === 'optimizations' &&
|
|
2970
|
+
(subcommand === 'pause' ||
|
|
2971
|
+
subcommand === 'resume' ||
|
|
2972
|
+
subcommand === 'finish' ||
|
|
2973
|
+
subcommand === 'fail' ||
|
|
2974
|
+
subcommand === 'cancel')) {
|
|
2975
|
+
await updateOptimizationRunLifecycle(subcommand);
|
|
2976
|
+
return;
|
|
2977
|
+
}
|
|
1599
2978
|
if (command === 'apps' && subcommand === 'list') {
|
|
1600
2979
|
await listApps();
|
|
1601
2980
|
return;
|
|
@@ -1644,6 +3023,10 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
1644
3023
|
await uploadDataset();
|
|
1645
3024
|
return;
|
|
1646
3025
|
}
|
|
3026
|
+
if (command === 'datasets' && subcommand === 'push') {
|
|
3027
|
+
await pushDataset();
|
|
3028
|
+
return;
|
|
3029
|
+
}
|
|
1647
3030
|
if (command === 'datasets' && subcommand === 'download') {
|
|
1648
3031
|
await downloadDataset();
|
|
1649
3032
|
return;
|
|
@@ -1652,6 +3035,15 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
1652
3035
|
await appendDatasetRows();
|
|
1653
3036
|
return;
|
|
1654
3037
|
}
|
|
3038
|
+
const datasetsAction = cliArgs[2];
|
|
3039
|
+
if (command === 'datasets' && subcommand === 'versions' && datasetsAction === 'create') {
|
|
3040
|
+
await createDatasetVersion();
|
|
3041
|
+
return;
|
|
3042
|
+
}
|
|
3043
|
+
if (command === 'datasets' && subcommand === 'splits' && datasetsAction === 'create') {
|
|
3044
|
+
await createDatasetSplitSet();
|
|
3045
|
+
return;
|
|
3046
|
+
}
|
|
1655
3047
|
if (command === 'datasets' && subcommand === 'edit-rows') {
|
|
1656
3048
|
await editDatasetRows();
|
|
1657
3049
|
return;
|
|
@@ -1677,9 +3069,22 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
1677
3069
|
return;
|
|
1678
3070
|
}
|
|
1679
3071
|
printUsage();
|
|
3072
|
+
printOptimizationUsage();
|
|
1680
3073
|
process.exit(1);
|
|
1681
3074
|
}
|
|
1682
|
-
|
|
3075
|
+
function isCliEntrypoint() {
|
|
3076
|
+
const entry = process.argv[1];
|
|
3077
|
+
if (!entry) {
|
|
3078
|
+
return false;
|
|
3079
|
+
}
|
|
3080
|
+
try {
|
|
3081
|
+
return realpathSync(entry) === realpathSync(fileURLToPath(import.meta.url));
|
|
3082
|
+
}
|
|
3083
|
+
catch {
|
|
3084
|
+
return false;
|
|
3085
|
+
}
|
|
3086
|
+
}
|
|
3087
|
+
if (isCliEntrypoint()) {
|
|
1683
3088
|
main().catch(error => {
|
|
1684
3089
|
console.error(sanitizeTerminalText(error instanceof Error ? error.message : 'Unknown error'));
|
|
1685
3090
|
process.exit(1);
|