orizu 0.5.17 → 0.5.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/archive-artifacts-cli.js +1 -3
- package/dist/artifact-pull.js +1 -3
- package/dist/diff-comments-cli.js +68 -32
- package/dist/error-response.js +17 -17
- package/dist/github-setup.js +3 -11
- package/dist/help.js +22 -2
- package/dist/index.js +40 -84
- package/dist/json-response.js +21 -0
- package/dist/markdown-report-input.js +67 -0
- package/dist/optimization-export-cli.js +30 -0
- package/dist/optimizations-list-cli.js +1 -3
- package/dist/prompt-draft-push.js +7 -5
- package/dist/prompt-report-cli.js +86 -0
- package/dist/scorer-draft-push.js +1 -22
- package/dist/scores-accept-cli.js +1 -3
- package/dist/zip-draft-push.js +1 -24
- package/package.json +1 -1
- package/src/archive-artifacts-cli.ts +1 -7
- package/src/artifact-pull.ts +1 -4
- package/src/diff-comments-cli.ts +86 -37
- package/src/error-response.ts +23 -21
- package/src/github-setup.ts +3 -12
- package/src/help.ts +22 -2
- package/src/index.ts +43 -106
- package/src/json-response.ts +37 -0
- package/src/markdown-report-input.ts +80 -0
- package/src/optimization-export-cli.ts +72 -0
- package/src/optimizations-list-cli.ts +1 -4
- package/src/prompt-draft-push.ts +8 -6
- package/src/prompt-report-cli.ts +135 -0
- package/src/scorer-draft-push.ts +1 -6
- package/src/scores-accept-cli.ts +1 -4
- package/src/zip-draft-push.ts +1 -6
- package/vendor/skills/orizu-cli/SKILL.md +1 -0
- package/vendor/skills/orizu-cli/references/cli-reference.md +5 -1
package/README.md
CHANGED
|
@@ -173,6 +173,14 @@ orizu logout
|
|
|
173
173
|
<td>Archive or restore a prompt</td>
|
|
174
174
|
<td><code>orizu prompts archive <promptIdOrName> --project ops-eval/support-qa</code><br><code>orizu prompts restore <promptIdOrName> --project ops-eval/support-qa</code></td>
|
|
175
175
|
</tr>
|
|
176
|
+
<tr>
|
|
177
|
+
<td>Push a prompt with a report</td>
|
|
178
|
+
<td><code>orizu prompts push ./prompts/support --project ops-eval/support-qa --report-file ./prompt-report.md</code></td>
|
|
179
|
+
</tr>
|
|
180
|
+
<tr>
|
|
181
|
+
<td>Upload a report for an existing prompt version</td>
|
|
182
|
+
<td><code>orizu prompts report set --prompt <promptIdOrName> --project ops-eval/support-qa --version 3 --report-file ./prompt-report.md</code><br><code>prompts report upload</code> is an exact alias; select exactly one of <code>--version</code> or <code>--version-id</code>.</td>
|
|
183
|
+
</tr>
|
|
176
184
|
<tr>
|
|
177
185
|
<td colspan="2"><strong>Report comments</strong></td>
|
|
178
186
|
</tr>
|
|
@@ -184,6 +192,13 @@ orizu logout
|
|
|
184
192
|
<td>Add or update comments</td>
|
|
185
193
|
<td><code>orizu comments add --run <runId> --body @comment.md --anchor "Summary" --lines 4:6</code><br><code>orizu comments reply <commentId> --body "Fixed"</code><br><code>orizu comments resolve <commentId></code></td>
|
|
186
194
|
</tr>
|
|
195
|
+
<tr>
|
|
196
|
+
<td colspan="2"><strong>Diff comments</strong></td>
|
|
197
|
+
</tr>
|
|
198
|
+
<tr>
|
|
199
|
+
<td>Export diff comments</td>
|
|
200
|
+
<td><code>orizu comments diff --run <optimizationRunId> [--from <candidateId> --to <candidateId>] [--detail hunk|diff|full] [--json]</code><br><code>orizu comments diff --prompt <promptId> [--from <version> --to <version>] [--detail hunk|diff|full] [--json]</code></td>
|
|
201
|
+
</tr>
|
|
187
202
|
<tr>
|
|
188
203
|
<td colspan="2"><strong>Datasets</strong></td>
|
|
189
204
|
</tr>
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { authedFetch } from './http.js';
|
|
9
9
|
import { extractErrorMessage } from './error-response.js';
|
|
10
|
+
import { sanitizeTerminalText } from './json-response.js';
|
|
10
11
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
11
12
|
const ARTIFACT_TYPES = {
|
|
12
13
|
app: 'app',
|
|
@@ -31,9 +32,6 @@ function argValue(args, flag) {
|
|
|
31
32
|
}
|
|
32
33
|
return args[index + 1];
|
|
33
34
|
}
|
|
34
|
-
function sanitizeTerminalText(value) {
|
|
35
|
-
return String(value).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, '');
|
|
36
|
-
}
|
|
37
35
|
function columnWidths(headers, rows) {
|
|
38
36
|
return headers.map((header, index) => Math.max(header.length, ...rows.map(row => row[index]?.length || 0)));
|
|
39
37
|
}
|
package/dist/artifact-pull.js
CHANGED
|
@@ -16,11 +16,9 @@ import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'fs';
|
|
|
16
16
|
import { tmpdir } from 'os';
|
|
17
17
|
import { dirname, join } from 'path';
|
|
18
18
|
import { authedFetch } from './http.js';
|
|
19
|
+
import { sanitizeTerminalText } from './json-response.js';
|
|
19
20
|
// Shared with the runners exec download path in index.ts.
|
|
20
21
|
export const ARTIFACT_MAX_BYTES = 25 * 1024 * 1024;
|
|
21
|
-
function sanitizeTerminalText(value) {
|
|
22
|
-
return String(value).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, '');
|
|
23
|
-
}
|
|
24
22
|
function expandHomePath(path) {
|
|
25
23
|
if (path.startsWith('~/')) {
|
|
26
24
|
const home = process.env.HOME || '';
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import { extractErrorMessage, isJsonErrorResponsePayload, } from './error-response.js';
|
|
2
|
+
import { sanitizeHumanInlineText } from './json-response.js';
|
|
1
3
|
const DETAIL_LEVELS = ['hunk', 'diff', 'full'];
|
|
4
|
+
const CONTEXT_UNAVAILABLE_WITHOUT_REASON = ' Context unavailable (no reason supplied)';
|
|
2
5
|
function isCandidatePair(pair) {
|
|
3
6
|
return 'candidateId' in pair.from;
|
|
4
7
|
}
|
|
@@ -8,11 +11,41 @@ function usage() {
|
|
|
8
11
|
function sanitizeHumanText(ctx, value) {
|
|
9
12
|
return ctx.sanitizeTerminalText(value).replaceAll('\r', '');
|
|
10
13
|
}
|
|
11
|
-
|
|
12
|
-
|
|
14
|
+
// Quote untrusted multiline blocks so none of their rows can impersonate a
|
|
15
|
+
// genuine hunk row, whose marker begins immediately after the four-space indent.
|
|
16
|
+
function printHumanTextBlock(ctx, value) {
|
|
17
|
+
for (const line of sanitizeHumanText(ctx, value).split('\n')) {
|
|
18
|
+
ctx.printLine(` │ ${line}`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function hasExplicitlyEmptyJsonError(response, rawBody) {
|
|
22
|
+
try {
|
|
23
|
+
const payload = JSON.parse(rawBody);
|
|
24
|
+
return isJsonErrorResponsePayload(response, payload) && payload.error.length === 0;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
async function extractDiffCommentsErrorMessage(response) {
|
|
31
|
+
const statusFallback = `HTTP ${response.status}`;
|
|
32
|
+
try {
|
|
33
|
+
const rawBody = await response.text();
|
|
34
|
+
if (hasExplicitlyEmptyJsonError(response, rawBody)) {
|
|
35
|
+
return statusFallback;
|
|
36
|
+
}
|
|
37
|
+
const copiedResponse = new Response(rawBody, { headers: response.headers });
|
|
38
|
+
const extractedMessage = await extractErrorMessage(copiedResponse);
|
|
39
|
+
return extractedMessage.trim() ? extractedMessage : statusFallback;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return statusFallback;
|
|
43
|
+
}
|
|
13
44
|
}
|
|
14
45
|
function humanLineNumber(ctx, line) {
|
|
15
|
-
return line === null
|
|
46
|
+
return line === null
|
|
47
|
+
? '-'
|
|
48
|
+
: sanitizeHumanInlineText(ctx.sanitizeTerminalText, String(line + 1));
|
|
16
49
|
}
|
|
17
50
|
function buildParams(ctx) {
|
|
18
51
|
const runId = ctx.getArg('--run');
|
|
@@ -29,9 +62,15 @@ function buildParams(ctx) {
|
|
|
29
62
|
]) {
|
|
30
63
|
ctx.rejectDashPrefixedOptionValue(name, value);
|
|
31
64
|
}
|
|
65
|
+
if (runId === '' || promptId === '') {
|
|
66
|
+
throw new Error(usage());
|
|
67
|
+
}
|
|
32
68
|
if (Number(Boolean(runId)) + Number(Boolean(promptId)) !== 1) {
|
|
33
69
|
throw new Error(usage());
|
|
34
70
|
}
|
|
71
|
+
if (from === '' || to === '') {
|
|
72
|
+
throw new Error('--from and --to must not be empty');
|
|
73
|
+
}
|
|
35
74
|
if (Boolean(from) !== Boolean(to)) {
|
|
36
75
|
throw new Error('--from and --to must be provided together');
|
|
37
76
|
}
|
|
@@ -65,15 +104,14 @@ function printDiffComments(ctx, payload) {
|
|
|
65
104
|
const to = hasCandidateIds ? pair.to.candidateId : String(pair.to.version);
|
|
66
105
|
ctx.printLine('');
|
|
67
106
|
ctx.printLine(`${hasCandidateIds ? 'Candidates' : 'Versions'} ` +
|
|
68
|
-
`${sanitizeHumanInlineText(ctx, from)} →
|
|
107
|
+
`${sanitizeHumanInlineText(ctx.sanitizeTerminalText, from)} → ` +
|
|
108
|
+
`${sanitizeHumanInlineText(ctx.sanitizeTerminalText, to)}`);
|
|
69
109
|
if (pair.degraded) {
|
|
70
|
-
ctx.printLine(` Context unavailable: ${
|
|
110
|
+
ctx.printLine(` Context unavailable: ${sanitizeHumanInlineText(ctx.sanitizeTerminalText, pair.degraded.reason)}`);
|
|
71
111
|
}
|
|
72
112
|
if (pair.diff !== null) {
|
|
73
113
|
ctx.printLine(' Diff:');
|
|
74
|
-
|
|
75
|
-
ctx.printLine(` ${line}`);
|
|
76
|
-
}
|
|
114
|
+
printHumanTextBlock(ctx, pair.diff);
|
|
77
115
|
}
|
|
78
116
|
if (pair.bodies !== null) {
|
|
79
117
|
for (const [label, body] of [
|
|
@@ -81,24 +119,31 @@ function printDiffComments(ctx, payload) {
|
|
|
81
119
|
['To body', pair.bodies.to],
|
|
82
120
|
]) {
|
|
83
121
|
ctx.printLine(` ${label}:`);
|
|
84
|
-
|
|
85
|
-
ctx.printLine(` ${line}`);
|
|
86
|
-
}
|
|
122
|
+
printHumanTextBlock(ctx, body);
|
|
87
123
|
}
|
|
88
124
|
}
|
|
89
125
|
for (const comment of pair.comments) {
|
|
90
126
|
const authorName = comment.author?.person?.name || comment.author?.name || 'Unknown';
|
|
91
|
-
ctx.printLine(` ${sanitizeHumanInlineText(ctx, comment.id)} · ` +
|
|
92
|
-
`${sanitizeHumanInlineText(ctx, authorName)} · ` +
|
|
93
|
-
`${sanitizeHumanInlineText(ctx, comment.createdAt)} · ` +
|
|
94
|
-
`${sanitizeHumanInlineText(ctx, comment.anchor.side)} line ` +
|
|
95
|
-
`${sanitizeHumanInlineText(ctx, String(comment.anchor.line + 1))}`);
|
|
96
|
-
|
|
97
|
-
|
|
127
|
+
ctx.printLine(` ${sanitizeHumanInlineText(ctx.sanitizeTerminalText, comment.id)} · ` +
|
|
128
|
+
`${sanitizeHumanInlineText(ctx.sanitizeTerminalText, authorName)} · ` +
|
|
129
|
+
`${sanitizeHumanInlineText(ctx.sanitizeTerminalText, comment.createdAt)} · ` +
|
|
130
|
+
`${sanitizeHumanInlineText(ctx.sanitizeTerminalText, comment.anchor.side)} line ` +
|
|
131
|
+
`${sanitizeHumanInlineText(ctx.sanitizeTerminalText, String(comment.anchor.line + 1))}`);
|
|
132
|
+
printHumanTextBlock(ctx, comment.body);
|
|
133
|
+
if (comment.contextUnavailableReason) {
|
|
134
|
+
ctx.printLine(` Context unavailable: ${sanitizeHumanInlineText(ctx.sanitizeTerminalText, comment.contextUnavailableReason)}`);
|
|
135
|
+
if (comment.context) {
|
|
136
|
+
ctx.printLine(` Anchored line: ${sanitizeHumanInlineText(ctx.sanitizeTerminalText, comment.context.lineText)}`);
|
|
137
|
+
}
|
|
138
|
+
continue;
|
|
98
139
|
}
|
|
99
140
|
if (!comment.context) {
|
|
100
|
-
ctx.printLine(
|
|
101
|
-
|
|
141
|
+
ctx.printLine(CONTEXT_UNAVAILABLE_WITHOUT_REASON);
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
if (!comment.context.hunk) {
|
|
145
|
+
ctx.printLine(CONTEXT_UNAVAILABLE_WITHOUT_REASON);
|
|
146
|
+
ctx.printLine(` Anchored line: ${sanitizeHumanInlineText(ctx.sanitizeTerminalText, comment.context.lineText)}`);
|
|
102
147
|
continue;
|
|
103
148
|
}
|
|
104
149
|
for (const line of comment.context.hunk.lines) {
|
|
@@ -109,7 +154,7 @@ function printDiffComments(ctx, payload) {
|
|
|
109
154
|
const marker = isAnchored ? '>' : ' ';
|
|
110
155
|
ctx.printLine(` ${marker} old:${humanLineNumber(ctx, line.oldLine)} ` +
|
|
111
156
|
`new:${humanLineNumber(ctx, line.newLine)} | ` +
|
|
112
|
-
`${prefix}${
|
|
157
|
+
`${prefix}${sanitizeHumanInlineText(ctx.sanitizeTerminalText, line.text)}`);
|
|
113
158
|
}
|
|
114
159
|
}
|
|
115
160
|
}
|
|
@@ -118,17 +163,8 @@ export async function diffCommentsCommand(ctx) {
|
|
|
118
163
|
const params = buildParams(ctx);
|
|
119
164
|
const response = await ctx.authedFetch(`/api/cli/diff-comments?${params.toString()}`);
|
|
120
165
|
if (!response.ok) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
const payload = await response.clone().json();
|
|
124
|
-
if (typeof payload.error === 'string' && payload.error.trim()) {
|
|
125
|
-
message = sanitizeHumanInlineText(ctx, payload.error);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
catch {
|
|
129
|
-
// Keep the status-only error for non-JSON responses.
|
|
130
|
-
}
|
|
131
|
-
throw new Error(`Failed to fetch diff comments: ${message}`);
|
|
166
|
+
const message = await extractDiffCommentsErrorMessage(response);
|
|
167
|
+
throw new Error(`Failed to fetch diff comments: ${sanitizeHumanInlineText(ctx.sanitizeTerminalText, message)}`);
|
|
132
168
|
}
|
|
133
169
|
const payload = await ctx.parseJsonResponse(response, 'Diff comments');
|
|
134
170
|
if (ctx.hasJsonFlag()) {
|
package/dist/error-response.js
CHANGED
|
@@ -1,26 +1,26 @@
|
|
|
1
|
+
import { sanitizeTerminalText } from './json-response.js';
|
|
1
2
|
const ERROR_BODY_PREVIEW_LIMIT = 180;
|
|
3
|
+
export function isJsonErrorResponsePayload(response, payload) {
|
|
4
|
+
const contentType = response.headers.get('content-type') || '';
|
|
5
|
+
return (contentType.includes('application/json') &&
|
|
6
|
+
payload !== null &&
|
|
7
|
+
typeof payload === 'object' &&
|
|
8
|
+
!Array.isArray(payload) &&
|
|
9
|
+
typeof payload.error === 'string');
|
|
10
|
+
}
|
|
2
11
|
function boundedTerminalPreview(value) {
|
|
3
|
-
return
|
|
4
|
-
.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, '')
|
|
5
|
-
.slice(0, ERROR_BODY_PREVIEW_LIMIT);
|
|
12
|
+
return sanitizeTerminalText(value).slice(0, ERROR_BODY_PREVIEW_LIMIT);
|
|
6
13
|
}
|
|
7
14
|
export async function extractErrorMessage(response) {
|
|
8
|
-
const contentType = response.headers.get('content-type') || '';
|
|
9
15
|
const rawBody = await response.text();
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
typeof payload === 'object' &&
|
|
15
|
-
!Array.isArray(payload) &&
|
|
16
|
-
typeof payload.error === 'string' &&
|
|
17
|
-
payload.error.length > 0) {
|
|
18
|
-
return boundedTerminalPreview(payload.error);
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
catch {
|
|
22
|
-
// Fall through to raw body
|
|
16
|
+
try {
|
|
17
|
+
const payload = JSON.parse(rawBody);
|
|
18
|
+
if (isJsonErrorResponsePayload(response, payload) && payload.error.length > 0) {
|
|
19
|
+
return boundedTerminalPreview(payload.error);
|
|
23
20
|
}
|
|
24
21
|
}
|
|
22
|
+
catch {
|
|
23
|
+
// Fall through to raw body
|
|
24
|
+
}
|
|
25
25
|
return boundedTerminalPreview(rawBody);
|
|
26
26
|
}
|
package/dist/github-setup.js
CHANGED
|
@@ -17,16 +17,8 @@ import { spawnSync } from 'child_process';
|
|
|
17
17
|
import { existsSync, mkdirSync } from 'fs';
|
|
18
18
|
import { join } from 'path';
|
|
19
19
|
import { authedFetch } from './http.js';
|
|
20
|
+
import { sanitizeTerminalText } from './json-response.js';
|
|
20
21
|
import { assertWorkspaceDirUsable, workspaceExists } from './workspace.js';
|
|
21
|
-
/**
|
|
22
|
-
* Strip terminal control characters from server-supplied strings before they
|
|
23
|
-
* are printed (mirrors index.ts `sanitizeTerminalText`), so a hostile or
|
|
24
|
-
* mangled provisioning `warnings[]`/`defaultBranch` value can't inject escape
|
|
25
|
-
* sequences into the user's terminal.
|
|
26
|
-
*/
|
|
27
|
-
function sanitizeServerText(value) {
|
|
28
|
-
return String(value).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, '');
|
|
29
|
-
}
|
|
30
22
|
const DEFAULT_POLL_INTERVAL_MS = 2000;
|
|
31
23
|
// ALI-1141: long enough for a slow org-picker + repo-selection walk, but the
|
|
32
24
|
// wait MUST be finite — when the App is already installed in the target org,
|
|
@@ -231,10 +223,10 @@ export async function runHostedAttach(options, io) {
|
|
|
231
223
|
repoFullName = data.repo;
|
|
232
224
|
provisioned = true;
|
|
233
225
|
if (data.rulesetApplied === true) {
|
|
234
|
-
io.print(` Default-branch protection applied (ruleset "orizu-default-branch-protection"${data.defaultBranch ? ` on ${
|
|
226
|
+
io.print(` Default-branch protection applied (ruleset "orizu-default-branch-protection"${data.defaultBranch ? ` on ${sanitizeTerminalText(data.defaultBranch)}` : ''}).`);
|
|
235
227
|
}
|
|
236
228
|
for (const warning of data.warnings ?? []) {
|
|
237
|
-
io.print(` ⚠ ${
|
|
229
|
+
io.print(` ⚠ ${sanitizeTerminalText(warning)}`);
|
|
238
230
|
}
|
|
239
231
|
}
|
|
240
232
|
const cleanUrl = `https://github.com/${repoFullName}.git`;
|
package/dist/help.js
CHANGED
|
@@ -337,10 +337,30 @@ export const COMMAND_DOCS = [
|
|
|
337
337
|
},
|
|
338
338
|
{
|
|
339
339
|
path: ['prompts', 'push'],
|
|
340
|
-
usage: 'orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--session <session-id>] [--json]',
|
|
340
|
+
usage: 'orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--session <session-id>] [--report-file <path>] [--json]',
|
|
341
341
|
summary: 'Push a local prompt artifact directory as a new version (with --session: a commit-first git draft).',
|
|
342
342
|
group: 'Prompts and judges',
|
|
343
343
|
},
|
|
344
|
+
{
|
|
345
|
+
path: ['prompts', 'report', 'set'],
|
|
346
|
+
usage: 'orizu prompts report set --prompt <id-or-name> --project <team/project> (--version <n> | --version-id <id>) (--report <markdown|@file> | --report-file <path>) [--json]',
|
|
347
|
+
summary: 'Upload or replace the Markdown report for one explicit prompt version.',
|
|
348
|
+
group: 'Prompts and judges',
|
|
349
|
+
aliases: [['prompts', 'report', 'upload']],
|
|
350
|
+
options: [
|
|
351
|
+
{ name: '--prompt <id-or-name>', help: 'Prompt ID or name.', required: true },
|
|
352
|
+
{ name: '--project <team/project>', help: 'Project slug containing the prompt.', required: true },
|
|
353
|
+
{ name: '--version <n>', help: 'Positive prompt version number.' },
|
|
354
|
+
{ name: '--version-id <id>', help: 'Explicit prompt version ID.' },
|
|
355
|
+
{ name: '--report <markdown|@file>', help: 'Inline Markdown or an @file reference.' },
|
|
356
|
+
{ name: '--report-file <path>', help: 'Markdown file to upload.' },
|
|
357
|
+
{ name: '--json', help: 'Emit the updated prompt report payload.' },
|
|
358
|
+
],
|
|
359
|
+
examples: [
|
|
360
|
+
'orizu prompts report set --prompt Generator --project core/evals --version 3 --report-file ./report.md',
|
|
361
|
+
'orizu prompts report upload --prompt Generator --project core/evals --version-id <id> --report @./report.md',
|
|
362
|
+
],
|
|
363
|
+
},
|
|
344
364
|
{
|
|
345
365
|
path: ['prompts', 'labels', 'set'],
|
|
346
366
|
usage: 'orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]',
|
|
@@ -373,7 +393,7 @@ export const COMMAND_DOCS = [
|
|
|
373
393
|
},
|
|
374
394
|
{
|
|
375
395
|
path: ['judges', 'push'],
|
|
376
|
-
usage: 'orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--session <session-id>] [--json]',
|
|
396
|
+
usage: 'orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--session <session-id>] [--report-file <path>] [--json]',
|
|
377
397
|
summary: 'Push a local judge artifact directory as a new version (with --session: a commit-first git draft).',
|
|
378
398
|
group: 'Prompts and judges',
|
|
379
399
|
},
|
package/dist/index.js
CHANGED
|
@@ -41,12 +41,17 @@ import { hostedCommand } from './hosted-session-cli.js';
|
|
|
41
41
|
import { workspaceSyncCommand } from './workspace-sync.js';
|
|
42
42
|
import { runGitCredentialInvocation } from './git-credential.js';
|
|
43
43
|
import { pushPromptDraft } from './prompt-draft-push.js';
|
|
44
|
+
import { readMarkdownReportInput } from './markdown-report-input.js';
|
|
45
|
+
import { promptPushErrorMessage, promptReportCommand } from './prompt-report-cli.js';
|
|
44
46
|
import { assertSnapshotManifestConfined, verifyGepaRunnerDirsFromArgs, verifyRunnerDirRegistered } from './runner-dir-verify.js';
|
|
45
47
|
import { runScorersRegister } from './scorer-draft-push.js';
|
|
46
48
|
import { runZipArtifactPush } from './zip-draft-push.js';
|
|
47
49
|
import { runGithubLink, runInteractiveHostedSetup } from './github-setup.js';
|
|
50
|
+
import { parseJsonResponse, sanitizeTerminalText } from './json-response.js';
|
|
48
51
|
import { reportCommentsCommand, throwDeprecatedPromptCommentsCommand, } from './report-comments-cli.js';
|
|
49
52
|
import { diffCommentsCommand } from './diff-comments-cli.js';
|
|
53
|
+
import { exportOptimizationRunCommand } from './optimization-export-cli.js';
|
|
54
|
+
export { parseJsonResponse, sanitizeTerminalText } from './json-response.js';
|
|
50
55
|
function getErrorMessage(error) {
|
|
51
56
|
return error instanceof Error ? error.message : String(error);
|
|
52
57
|
}
|
|
@@ -176,9 +181,6 @@ function createCodeVerifier() {
|
|
|
176
181
|
export function createCodeChallenge(verifier) {
|
|
177
182
|
return createHash('sha256').update(verifier).digest('base64url');
|
|
178
183
|
}
|
|
179
|
-
export function sanitizeTerminalText(value) {
|
|
180
|
-
return String(value).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, '');
|
|
181
|
-
}
|
|
182
184
|
function printError(message) {
|
|
183
185
|
console.error(sanitizeTerminalText(message));
|
|
184
186
|
}
|
|
@@ -241,21 +243,6 @@ export function formatTerminalLink(url) {
|
|
|
241
243
|
}
|
|
242
244
|
return `\u001B]8;;${safeUrl}\u0007${safeUrl}\u001B]8;;\u0007`;
|
|
243
245
|
}
|
|
244
|
-
export async function parseJsonResponse(response, context) {
|
|
245
|
-
const contentType = response.headers.get('content-type') || '';
|
|
246
|
-
const rawBody = await response.text();
|
|
247
|
-
if (!contentType.includes('application/json')) {
|
|
248
|
-
throw new Error(`${context} returned non-JSON response (status ${response.status}). ` +
|
|
249
|
-
`Body preview: ${sanitizeTerminalText(rawBody.slice(0, 180))}`);
|
|
250
|
-
}
|
|
251
|
-
try {
|
|
252
|
-
return JSON.parse(rawBody);
|
|
253
|
-
}
|
|
254
|
-
catch {
|
|
255
|
-
throw new Error(`${context} returned invalid JSON (status ${response.status}). ` +
|
|
256
|
-
`Body preview: ${sanitizeTerminalText(rawBody.slice(0, 180))}`);
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
246
|
function shellQuote(value) {
|
|
260
247
|
return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
|
261
248
|
}
|
|
@@ -1014,14 +1001,17 @@ async function pushRunnerArtifact(kind) {
|
|
|
1014
1001
|
}
|
|
1015
1002
|
async function pushPromptArtifact(kind) {
|
|
1016
1003
|
const promptDir = getPositionalArg(2);
|
|
1017
|
-
const
|
|
1004
|
+
const projectArg = getArg('--project');
|
|
1018
1005
|
const runnerVersionArg = getArg('--runner-version');
|
|
1019
1006
|
const parentVersionId = getArg('--parent') || undefined;
|
|
1020
1007
|
const sessionId = getArg('--session') || undefined;
|
|
1021
|
-
const usage = `Usage: orizu ${kind === 'judge' ? 'judges' : 'prompts'} push <dir> --project <team/project> [--runner-version <id>] [--parent <version-id>] [--session <session-id>] [--json]`;
|
|
1022
|
-
if (!promptDir)
|
|
1008
|
+
const usage = `Usage: orizu ${kind === 'judge' ? 'judges' : 'prompts'} push <dir> --project <team/project> [--runner-version <id>] [--parent <version-id>] [--session <session-id>] [--report-file <path>] [--json]`;
|
|
1009
|
+
if (!promptDir)
|
|
1023
1010
|
throw new Error(usage);
|
|
1024
|
-
|
|
1011
|
+
if (cliArgs.some(arg => arg === '--report' || arg.startsWith('--report=')))
|
|
1012
|
+
throw new Error(`${kind === 'judge' ? 'judges' : 'prompts'} push accepts report input only through --report-file`);
|
|
1013
|
+
const report = readMarkdownReportInput(cliArgs, 'Prompt');
|
|
1014
|
+
const project = projectArg || await resolveProjectSlug(null);
|
|
1025
1015
|
const promptRoot = expandHomePath(promptDir);
|
|
1026
1016
|
const manifest = readJsonFile(join(promptRoot, 'orizu.prompt.json'));
|
|
1027
1017
|
const primaryText = readPromptPrimaryText(manifest, promptRoot);
|
|
@@ -1040,6 +1030,7 @@ async function pushPromptArtifact(kind) {
|
|
|
1040
1030
|
kind, project, sessionId, manifest, sidecars, runnerVersionId, parentVersionId,
|
|
1041
1031
|
displayName: String(manifest.name || promptDir),
|
|
1042
1032
|
bodyKind: primaryText.bodyKind, primaryBody: primaryText.body,
|
|
1033
|
+
report,
|
|
1043
1034
|
});
|
|
1044
1035
|
if (hasJsonFlag())
|
|
1045
1036
|
printJson(data);
|
|
@@ -1074,10 +1065,16 @@ async function pushPromptArtifact(kind) {
|
|
|
1074
1065
|
parentVersionId,
|
|
1075
1066
|
versionLabel: manifest.version_label,
|
|
1076
1067
|
createdBy: manifest.provenance || { kind: 'human-edit' },
|
|
1068
|
+
...(report
|
|
1069
|
+
? {
|
|
1070
|
+
reportMarkdown: report.markdown,
|
|
1071
|
+
reportSourceName: report.sourceName,
|
|
1072
|
+
}
|
|
1073
|
+
: {}),
|
|
1077
1074
|
}),
|
|
1078
1075
|
});
|
|
1079
1076
|
if (!response.ok) {
|
|
1080
|
-
throw new Error(
|
|
1077
|
+
throw new Error(await promptPushErrorMessage(response, kind, Boolean(report)));
|
|
1081
1078
|
}
|
|
1082
1079
|
const data = await parseJsonResponse(response, `${kind} push`);
|
|
1083
1080
|
if (hasJsonFlag()) {
|
|
@@ -1658,7 +1655,6 @@ async function execScorer() {
|
|
|
1658
1655
|
printLine(`Wrote scorer result to ${sanitizeTerminalText(outPath)}`);
|
|
1659
1656
|
}
|
|
1660
1657
|
}
|
|
1661
|
-
const MARKDOWN_REPORT_MAX_BYTES = 2 * 1024 * 1024;
|
|
1662
1658
|
function parseOptionalNumberFlag(name) {
|
|
1663
1659
|
const value = getArg(name);
|
|
1664
1660
|
if (!value) {
|
|
@@ -1673,47 +1669,11 @@ function parseOptionalNumberFlag(name) {
|
|
|
1673
1669
|
function hasObjectKeys(value) {
|
|
1674
1670
|
return Object.keys(value).length > 0;
|
|
1675
1671
|
}
|
|
1676
|
-
function readMarkdownReportInput(reportLabel) {
|
|
1677
|
-
const report = getArg('--report');
|
|
1678
|
-
const reportFile = getArg('--report-file');
|
|
1679
|
-
rejectDashPrefixedOptionValue('--report', report);
|
|
1680
|
-
rejectDashPrefixedOptionValue('--report-file', reportFile);
|
|
1681
|
-
if (report && reportFile) {
|
|
1682
|
-
throw new Error('Use either --report or --report-file, not both');
|
|
1683
|
-
}
|
|
1684
|
-
if (!report && !reportFile) {
|
|
1685
|
-
return null;
|
|
1686
|
-
}
|
|
1687
|
-
let markdown;
|
|
1688
|
-
let sourceName;
|
|
1689
|
-
if (reportFile) {
|
|
1690
|
-
const expandedPath = expandHomePath(reportFile);
|
|
1691
|
-
markdown = readSourceFile(reportFile);
|
|
1692
|
-
sourceName = basename(expandedPath);
|
|
1693
|
-
}
|
|
1694
|
-
else if (report?.startsWith('@')) {
|
|
1695
|
-
const path = report.slice(1);
|
|
1696
|
-
const expandedPath = expandHomePath(path);
|
|
1697
|
-
markdown = readSourceFile(path);
|
|
1698
|
-
sourceName = basename(expandedPath);
|
|
1699
|
-
}
|
|
1700
|
-
else {
|
|
1701
|
-
markdown = report || '';
|
|
1702
|
-
sourceName = 'inline';
|
|
1703
|
-
}
|
|
1704
|
-
if (!markdown.trim()) {
|
|
1705
|
-
throw new Error(`${reportLabel} report markdown must not be blank`);
|
|
1706
|
-
}
|
|
1707
|
-
if (Buffer.byteLength(markdown, 'utf8') > MARKDOWN_REPORT_MAX_BYTES) {
|
|
1708
|
-
throw new Error(`${reportLabel} report exceeds ${MARKDOWN_REPORT_MAX_BYTES} bytes`);
|
|
1709
|
-
}
|
|
1710
|
-
return { markdown, sourceName };
|
|
1711
|
-
}
|
|
1712
1672
|
function readOptimizationReportInput() {
|
|
1713
|
-
return readMarkdownReportInput('Optimization');
|
|
1673
|
+
return readMarkdownReportInput(cliArgs, 'Optimization');
|
|
1714
1674
|
}
|
|
1715
1675
|
function readTaskReportInput() {
|
|
1716
|
-
return readMarkdownReportInput('Task');
|
|
1676
|
+
return readMarkdownReportInput(cliArgs, 'Task');
|
|
1717
1677
|
}
|
|
1718
1678
|
async function startOptimizationRun() {
|
|
1719
1679
|
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
@@ -1853,27 +1813,6 @@ async function updateOptimizationRunLifecycle(action) {
|
|
|
1853
1813
|
printLine(`Cancelled optimization run ${id}`);
|
|
1854
1814
|
}
|
|
1855
1815
|
}
|
|
1856
|
-
async function exportOptimizationRun() {
|
|
1857
|
-
const runId = getPositionalArg(2) || getArg('--run-id');
|
|
1858
|
-
const outPathArg = getArg('--out');
|
|
1859
|
-
if (!runId) {
|
|
1860
|
-
throw new Error('Usage: orizu optimizations export <run-id> [--out <path>] [--json]');
|
|
1861
|
-
}
|
|
1862
|
-
const response = await authedFetch(`/api/cli/optimization-runs/${encodeURIComponent(runId)}/export`);
|
|
1863
|
-
if (!response.ok) {
|
|
1864
|
-
throw new Error(`Failed to export optimization run: ${await response.text()}`);
|
|
1865
|
-
}
|
|
1866
|
-
const data = await parseJsonResponse(response, 'Optimization export');
|
|
1867
|
-
if (hasJsonFlag() && !outPathArg) {
|
|
1868
|
-
printJson(data);
|
|
1869
|
-
return;
|
|
1870
|
-
}
|
|
1871
|
-
const filename = outPathArg
|
|
1872
|
-
? expandHomePath(outPathArg)
|
|
1873
|
-
: `${runId}.optimization.json`;
|
|
1874
|
-
writeTextFileEnsuringDir(filename, `${JSON.stringify(data, null, 2)}\n`);
|
|
1875
|
-
printLine(`Saved optimization export to ${sanitizeTerminalText(filename)}`);
|
|
1876
|
-
}
|
|
1877
1816
|
// ALI-1073 list moved to optimizations-list-cli.ts (line ratchet, ALI-976);
|
|
1878
1817
|
// ALI-1175 labels its best scores as agent-reported evidence there.
|
|
1879
1818
|
function removeFlagWithValue(args, flag) {
|
|
@@ -4929,6 +4868,13 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
4929
4868
|
await pushPromptArtifact('prompt');
|
|
4930
4869
|
return;
|
|
4931
4870
|
}
|
|
4871
|
+
if (command === 'prompts' && subcommand === 'report' &&
|
|
4872
|
+
(cliArgs[2] === 'set' || cliArgs[2] === 'upload')) {
|
|
4873
|
+
await promptReportCommand(cliArgs, {
|
|
4874
|
+
resolveProjectSlug, json: hasJsonFlag(), printJson, printLine,
|
|
4875
|
+
});
|
|
4876
|
+
return;
|
|
4877
|
+
}
|
|
4932
4878
|
if (command === 'prompts' && subcommand === 'archive') {
|
|
4933
4879
|
await setPromptArchivedState(true);
|
|
4934
4880
|
return;
|
|
@@ -5032,7 +4978,17 @@ export async function main(rawArgs = process.argv.slice(2)) {
|
|
|
5032
4978
|
return;
|
|
5033
4979
|
}
|
|
5034
4980
|
if (command === 'optimizations' && subcommand === 'export') {
|
|
5035
|
-
await
|
|
4981
|
+
await exportOptimizationRunCommand({
|
|
4982
|
+
getArg,
|
|
4983
|
+
getPositionalArg,
|
|
4984
|
+
authedFetch,
|
|
4985
|
+
hasJsonFlag,
|
|
4986
|
+
printJson,
|
|
4987
|
+
printLine,
|
|
4988
|
+
expandHomePath,
|
|
4989
|
+
writeTextFileEnsuringDir,
|
|
4990
|
+
sanitizeTerminalText,
|
|
4991
|
+
});
|
|
5036
4992
|
return;
|
|
5037
4993
|
}
|
|
5038
4994
|
if (command === 'optimizations' && subcommand === 'list') {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export function sanitizeTerminalText(value) {
|
|
2
|
+
return String(value).replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, '');
|
|
3
|
+
}
|
|
4
|
+
export function sanitizeHumanInlineText(sanitizer, value) {
|
|
5
|
+
return sanitizer(value).replaceAll('\r', '').replaceAll('\n', '\\n');
|
|
6
|
+
}
|
|
7
|
+
export async function parseJsonResponse(response, context) {
|
|
8
|
+
const contentType = response.headers.get('content-type') || '';
|
|
9
|
+
const rawBody = await response.text();
|
|
10
|
+
if (!contentType.includes('application/json')) {
|
|
11
|
+
throw new Error(`${context} returned non-JSON response (status ${response.status}). ` +
|
|
12
|
+
`Body preview: ${sanitizeTerminalText(rawBody.slice(0, 180))}`);
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
return JSON.parse(rawBody);
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
throw new Error(`${context} returned invalid JSON (status ${response.status}). ` +
|
|
19
|
+
`Body preview: ${sanitizeTerminalText(rawBody.slice(0, 180))}`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { readFileSync } from 'fs';
|
|
2
|
+
import { basename } from 'path';
|
|
3
|
+
export const MARKDOWN_REPORT_MAX_BYTES = 2 * 1024 * 1024;
|
|
4
|
+
function getArg(args, name) {
|
|
5
|
+
const index = args.indexOf(name);
|
|
6
|
+
if (index === -1 || index + 1 >= args.length)
|
|
7
|
+
return null;
|
|
8
|
+
return args[index + 1];
|
|
9
|
+
}
|
|
10
|
+
function expandHomePath(path) {
|
|
11
|
+
if (!path.startsWith('~/'))
|
|
12
|
+
return path;
|
|
13
|
+
return `${process.env.HOME || ''}/${path.slice(2)}`;
|
|
14
|
+
}
|
|
15
|
+
function rejectDashPrefixedOptionValue(name, value) {
|
|
16
|
+
if (value?.startsWith('-')) {
|
|
17
|
+
throw new Error(`Invalid value for ${name}: option values cannot start with a dash`);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function readSourceFile(pathArg) {
|
|
21
|
+
const expandedPath = expandHomePath(pathArg);
|
|
22
|
+
try {
|
|
23
|
+
return readFileSync(expandedPath, 'utf8');
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
|
|
27
|
+
throw new Error(`File not found: ${expandedPath}`);
|
|
28
|
+
}
|
|
29
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
30
|
+
throw new Error(`Failed to read file '${expandedPath}': ${message}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
export function readMarkdownReportInput(args, reportLabel) {
|
|
34
|
+
const report = getArg(args, '--report');
|
|
35
|
+
const reportFile = getArg(args, '--report-file');
|
|
36
|
+
rejectDashPrefixedOptionValue('--report', report);
|
|
37
|
+
rejectDashPrefixedOptionValue('--report-file', reportFile);
|
|
38
|
+
if (report && reportFile) {
|
|
39
|
+
throw new Error('Use either --report or --report-file, not both');
|
|
40
|
+
}
|
|
41
|
+
if (!report && !reportFile)
|
|
42
|
+
return null;
|
|
43
|
+
let markdown;
|
|
44
|
+
let sourceName;
|
|
45
|
+
if (reportFile) {
|
|
46
|
+
const expandedPath = expandHomePath(reportFile);
|
|
47
|
+
markdown = readSourceFile(reportFile);
|
|
48
|
+
sourceName = basename(expandedPath);
|
|
49
|
+
}
|
|
50
|
+
else if (report?.startsWith('@')) {
|
|
51
|
+
const path = report.slice(1);
|
|
52
|
+
const expandedPath = expandHomePath(path);
|
|
53
|
+
markdown = readSourceFile(path);
|
|
54
|
+
sourceName = basename(expandedPath);
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
markdown = report || '';
|
|
58
|
+
sourceName = 'inline';
|
|
59
|
+
}
|
|
60
|
+
if (!markdown.trim()) {
|
|
61
|
+
throw new Error(`${reportLabel} report markdown must not be blank`);
|
|
62
|
+
}
|
|
63
|
+
if (Buffer.byteLength(markdown, 'utf8') > MARKDOWN_REPORT_MAX_BYTES) {
|
|
64
|
+
throw new Error(`${reportLabel} report exceeds ${MARKDOWN_REPORT_MAX_BYTES} bytes`);
|
|
65
|
+
}
|
|
66
|
+
return { markdown, sourceName };
|
|
67
|
+
}
|