orizu 0.5.21 → 0.5.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -15
- package/dist/cleanup.d.ts +6 -0
- package/dist/cleanup.d.ts.map +1 -0
- package/dist/cleanup.js +27 -0
- package/dist/gepa-engine-dispatch.d.ts.map +1 -1
- package/dist/gepa-engine-dispatch.js +20 -1
- package/dist/gepa-python-paths.d.ts.map +1 -1
- package/dist/gepa-python-paths.js +25 -6
- package/dist/help.d.ts.map +1 -1
- package/dist/help.js +37 -15
- package/dist/hosted-commands.d.ts +13 -0
- package/dist/hosted-commands.d.ts.map +1 -0
- package/dist/hosted-commands.js +10 -0
- package/dist/hosted-gepa-cli.d.ts +11 -0
- package/dist/hosted-gepa-cli.d.ts.map +1 -0
- package/dist/hosted-gepa-cli.js +110 -0
- package/dist/hosted-gepa-numeric-values.d.ts +3 -0
- package/dist/hosted-gepa-numeric-values.d.ts.map +1 -0
- package/dist/hosted-gepa-numeric-values.js +49 -0
- package/dist/hosted-provider-settings.d.ts +3 -0
- package/dist/hosted-provider-settings.d.ts.map +1 -0
- package/dist/hosted-provider-settings.js +69 -0
- package/dist/hosted-session-cli.d.ts.map +1 -1
- package/dist/hosted-session-cli.js +6 -1
- package/dist/http.d.ts +1 -0
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +72 -74
- package/dist/instruction-set-loader.d.ts.map +1 -1
- package/dist/instruction-set-loader.js +57 -5
- package/dist/instruction-sets-cli.d.ts +5 -2
- package/dist/instruction-sets-cli.d.ts.map +1 -1
- package/dist/instruction-sets-cli.js +134 -24
- package/dist/optimizations-promote-cli.d.ts +3 -0
- package/dist/optimizations-promote-cli.d.ts.map +1 -1
- package/dist/optimizations-promote-cli.js +75 -7
- package/dist/runner-dir-verify.d.ts.map +1 -1
- package/dist/runner-dir-verify.js +4 -7
- package/dist/skilled-proposer-bake-report.d.ts +2 -0
- package/dist/skilled-proposer-bake-report.d.ts.map +1 -0
- package/dist/skilled-proposer-bake-report.js +1 -0
- package/dist/skilled-proposer-config.d.ts +2 -0
- package/dist/skilled-proposer-config.d.ts.map +1 -0
- package/dist/skilled-proposer-config.js +212 -0
- package/dist/skilled-proposer-launch.d.ts +26 -0
- package/dist/skilled-proposer-launch.d.ts.map +1 -1
- package/dist/skilled-proposer-launch.js +103 -6
- package/package.json +1 -1
- package/scripts/vendor-gepa-python.mjs +81 -12
- package/src/cleanup.ts +37 -0
- package/src/gepa-engine-dispatch.ts +21 -1
- package/src/gepa-python-paths.ts +24 -6
- package/src/help.ts +37 -15
- package/src/hosted-commands.ts +4 -0
- package/src/hosted-gepa-cli.ts +107 -0
- package/src/hosted-gepa-numeric-values.ts +60 -0
- package/src/hosted-provider-settings.ts +69 -0
- package/src/hosted-session-cli.ts +4 -1
- package/src/http.ts +8 -0
- package/src/index.ts +112 -89
- package/src/instruction-set-loader.ts +55 -5
- package/src/instruction-sets-cli.ts +173 -19
- package/src/optimizations-promote-cli.ts +85 -7
- package/src/runner-dir-verify.ts +11 -6
- package/src/skilled-proposer-bake-report.ts +1 -0
- package/src/skilled-proposer-config.ts +191 -0
- package/src/skilled-proposer-launch.ts +96 -7
- package/src/skilled-proposer-venv-manager.mjs +44 -27
- package/vendor/orizu-gepa/src/orizu_gepa_connector/engine.py +36 -3
- package/vendor/orizu-gepa/src/orizu_gepa_connector/instruction_set_loader.py +31 -3
- package/vendor/orizu-gepa/src/orizu_gepa_connector/runtime.py +7 -2
- package/vendor/orizu-gepa/src/orizu_gepa_connector/skilled_proposer_bridge.py +354 -5
- package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py +1 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/client.py +15 -0
- package/vendor/skills/orizu-cli/SKILL.md +94 -168
- package/vendor/skills/orizu-cli/references/assess-and-plan.md +107 -0
- package/vendor/skills/orizu-cli/references/building-judges.md +442 -180
- package/vendor/skills/orizu-cli/references/cli-reference.md +69 -39
- package/vendor/skills/orizu-cli/references/dataset-design.md +100 -0
- package/vendor/skills/orizu-cli/references/eval-strategy.md +64 -0
- package/vendor/skills/orizu-cli/references/instructions-after-prompts.md +115 -0
- package/vendor/skills/orizu-cli/references/migrate-existing-gepa-setup.md +150 -82
- package/vendor/skills/orizu-cli/references/optimization-reports.md +196 -104
- package/vendor/skills/orizu-cli/references/optimization-with-gepa.md +57 -37
- package/vendor/skills/orizu-cli/references/primer.md +43 -18
- package/vendor/skills/orizu-cli/references/prompt-control-plane.md +199 -131
|
@@ -43,13 +43,30 @@ const SAFE_SEGMENT = /^[A-Za-z0-9._-]+$/u
|
|
|
43
43
|
export interface SyncComponent { body?: string; repoPath?: string; contentSha?: string; commitSha?: string }
|
|
44
44
|
export interface SyncMaterial { profileVersionId: string; versionNumber: number; modelConfigIdentity: string; resolvedFrom: string; components: Record<string, SyncComponent> }
|
|
45
45
|
export interface SyncProfile { modelConfigIdentity: string; resolvedFrom: string; production: SyncMaterial | null }
|
|
46
|
-
export interface SyncSet {
|
|
46
|
+
export interface SyncSet {
|
|
47
|
+
projectId?: string
|
|
48
|
+
instructionSetId?: string
|
|
49
|
+
name: string
|
|
50
|
+
slug?: string
|
|
51
|
+
description?: string | null
|
|
52
|
+
shape: string[]
|
|
53
|
+
default: SyncMaterial
|
|
54
|
+
profiles: SyncProfile[]
|
|
55
|
+
filteredTo?: string[]
|
|
56
|
+
}
|
|
47
57
|
function safeSegment(value: string) { if (!SAFE_SEGMENT.test(value) || value === '.' || value === '..' || value.startsWith('.')) throw new Error('instruction_set_path_unsafe') }
|
|
58
|
+
function isSafeSegment(value: string) {
|
|
59
|
+
try { safeSegment(value); return true } catch { return false }
|
|
60
|
+
}
|
|
48
61
|
function slug(identity: string) { return identity.replaceAll('/', '__').replace(/[^A-Za-z0-9._-]/gu, '_') }
|
|
49
62
|
function stable(value: unknown): string { return `${JSON.stringify(value, null, 2)}\n` }
|
|
50
63
|
function sha256(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex') }
|
|
51
64
|
|
|
52
|
-
export interface SyncFileOps {
|
|
65
|
+
export interface SyncFileOps {
|
|
66
|
+
writeFileSync: typeof writeFileSync
|
|
67
|
+
renameSync?: typeof renameSync
|
|
68
|
+
rmSync?: typeof rmSync
|
|
69
|
+
}
|
|
53
70
|
|
|
54
71
|
function writeMaterial(root: string, material: SyncMaterial, shape: string[], manifestRoot: string, fileOps: SyncFileOps = { writeFileSync }) {
|
|
55
72
|
const files: Record<string, string> = {}
|
|
@@ -77,16 +94,76 @@ function removeOwnScratch(paths: Array<string | null>) {
|
|
|
77
94
|
for (const path of paths) if (path && existsSync(path)) rmSync(path, { recursive: true, force: true })
|
|
78
95
|
}
|
|
79
96
|
|
|
80
|
-
function existingDestinationIsSynced(
|
|
97
|
+
function existingDestinationIsSynced(
|
|
98
|
+
destination: string,
|
|
99
|
+
set: Pick<SyncSet, 'projectId' | 'instructionSetId' | 'name' | 'slug'>
|
|
100
|
+
) {
|
|
101
|
+
if (
|
|
102
|
+
set.slug &&
|
|
103
|
+
set.projectId === undefined &&
|
|
104
|
+
set.instructionSetId === undefined
|
|
105
|
+
) {
|
|
106
|
+
return existingDestinationIsLegacySynced(destination, set.name, set.slug)
|
|
107
|
+
}
|
|
108
|
+
try {
|
|
109
|
+
if (!statSync(destination).isDirectory()) return false
|
|
110
|
+
const manifest = JSON.parse(readFileSync(join(destination, 'manifest.json'), 'utf8')) as {
|
|
111
|
+
manifestVersion?: number
|
|
112
|
+
projectId?: string
|
|
113
|
+
instructionSetId?: string
|
|
114
|
+
name?: string
|
|
115
|
+
slug?: string
|
|
116
|
+
}
|
|
117
|
+
return manifest.manifestVersion === 1 && (set.slug
|
|
118
|
+
? manifest.slug === set.slug &&
|
|
119
|
+
manifest.projectId === set.projectId &&
|
|
120
|
+
manifest.instructionSetId === set.instructionSetId
|
|
121
|
+
: manifest.name === set.name)
|
|
122
|
+
} catch { return false }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function existingDestinationIsLegacySynced(
|
|
126
|
+
destination: string,
|
|
127
|
+
name: string,
|
|
128
|
+
slug?: string
|
|
129
|
+
) {
|
|
81
130
|
try {
|
|
82
131
|
if (!statSync(destination).isDirectory()) return false
|
|
83
|
-
const manifest = JSON.parse(readFileSync(join(destination, 'manifest.json'), 'utf8')) as {
|
|
84
|
-
|
|
132
|
+
const manifest = JSON.parse(readFileSync(join(destination, 'manifest.json'), 'utf8')) as {
|
|
133
|
+
manifestVersion?: number
|
|
134
|
+
name?: string
|
|
135
|
+
slug?: string
|
|
136
|
+
}
|
|
137
|
+
return manifest.manifestVersion === 1 &&
|
|
138
|
+
manifest.name === name &&
|
|
139
|
+
manifest.slug === slug
|
|
85
140
|
} catch { return false }
|
|
86
141
|
}
|
|
87
142
|
|
|
143
|
+
function removePublishedBackups(
|
|
144
|
+
paths: string[],
|
|
145
|
+
remove: typeof rmSync
|
|
146
|
+
) {
|
|
147
|
+
let firstError: unknown = null
|
|
148
|
+
for (const path of paths) {
|
|
149
|
+
if (!existsSync(path)) continue
|
|
150
|
+
try {
|
|
151
|
+
remove(path, { recursive: true, force: true })
|
|
152
|
+
} catch (error) {
|
|
153
|
+
firstError ??= error
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (firstError) throw new Error('instruction_set_sync_cleanup_failed')
|
|
157
|
+
}
|
|
158
|
+
|
|
88
159
|
export function syncToDisk(out: string, set: SyncSet, fileOps: SyncFileOps = { writeFileSync }) {
|
|
89
|
-
|
|
160
|
+
const directoryName = set.slug || set.name
|
|
161
|
+
safeSegment(directoryName)
|
|
162
|
+
const hasProjectId = set.projectId !== undefined
|
|
163
|
+
const hasInstructionSetId = set.instructionSetId !== undefined
|
|
164
|
+
if (set.slug && hasProjectId !== hasInstructionSetId) {
|
|
165
|
+
throw new Error('instruction_set_sync_identity_missing')
|
|
166
|
+
}
|
|
90
167
|
for (const key of set.shape) safeSegment(key)
|
|
91
168
|
const profileSlugs = new Set<string>()
|
|
92
169
|
for (const profile of set.profiles) {
|
|
@@ -96,11 +173,28 @@ export function syncToDisk(out: string, set: SyncSet, fileOps: SyncFileOps = { w
|
|
|
96
173
|
profileSlugs.add(value)
|
|
97
174
|
}
|
|
98
175
|
mkdirSync(out, { recursive: true })
|
|
99
|
-
const destination = join(out,
|
|
100
|
-
|
|
101
|
-
|
|
176
|
+
const destination = join(out, directoryName)
|
|
177
|
+
const legacyDestination = set.slug && set.name !== set.slug && isSafeSegment(set.name)
|
|
178
|
+
? join(out, set.name)
|
|
179
|
+
: null
|
|
180
|
+
if (existsSync(destination)) {
|
|
181
|
+
if (set.slug && existingDestinationIsLegacySynced(destination, set.name)) {
|
|
182
|
+
throw new Error('instruction_set_sync_legacy_identity_required')
|
|
183
|
+
}
|
|
184
|
+
if (!existingDestinationIsSynced(destination, set)) {
|
|
185
|
+
throw new Error('instruction_set_sync_destination_not_synced')
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (legacyDestination && existsSync(legacyDestination)) {
|
|
189
|
+
if (existingDestinationIsLegacySynced(legacyDestination, set.name)) {
|
|
190
|
+
throw new Error('instruction_set_sync_legacy_identity_required')
|
|
191
|
+
}
|
|
192
|
+
throw new Error('instruction_set_sync_destination_not_synced')
|
|
193
|
+
}
|
|
194
|
+
const priorDestinations = [destination]
|
|
195
|
+
const temp = join(out, `.${directoryName}.${randomUUID()}.tmp`)
|
|
102
196
|
const move = fileOps.renameSync || renameSync
|
|
103
|
-
|
|
197
|
+
const backups: Array<{ original: string; backup: string }> = []
|
|
104
198
|
mkdirSync(temp)
|
|
105
199
|
try {
|
|
106
200
|
const defaultMaterial = writeMaterial(temp, set.default, set.shape, 'default', fileOps)
|
|
@@ -114,18 +208,40 @@ export function syncToDisk(out: string, set: SyncSet, fileOps: SyncFileOps = { w
|
|
|
114
208
|
? Object.entries(profile.production.files).map(([key, path]) => ({ key, modelConfig: profile.modelConfigIdentity, path, syncedContentSha256: profile.production!.syncedContentSha256[key] }))
|
|
115
209
|
: []),
|
|
116
210
|
]
|
|
117
|
-
const manifest = {
|
|
211
|
+
const manifest = {
|
|
212
|
+
manifestVersion: 1,
|
|
213
|
+
...(set.projectId ? { projectId: set.projectId } : {}),
|
|
214
|
+
...(set.instructionSetId ? { instructionSetId: set.instructionSetId } : {}),
|
|
215
|
+
name: set.name,
|
|
216
|
+
...(set.slug ? { slug: set.slug } : {}),
|
|
217
|
+
...(Object.hasOwn(set, 'description') ? { description: set.description ?? null } : {}),
|
|
218
|
+
shape: set.shape,
|
|
219
|
+
components,
|
|
220
|
+
default: defaultMaterial,
|
|
221
|
+
profiles,
|
|
222
|
+
...(set.filteredTo ? { filteredTo: [...set.filteredTo].sort() } : {}),
|
|
223
|
+
}
|
|
118
224
|
fileOps.writeFileSync(join(temp, 'manifest.json'), stable(manifest))
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
225
|
+
for (const prior of priorDestinations) {
|
|
226
|
+
if (!existsSync(prior)) continue
|
|
227
|
+
const backup = join(out, `.${directoryName}.${randomUUID()}.bak`)
|
|
228
|
+
move(prior, backup)
|
|
229
|
+
backups.push({ original: prior, backup })
|
|
230
|
+
}
|
|
231
|
+
move(temp, destination)
|
|
123
232
|
} catch (error) {
|
|
124
233
|
removeOwnScratch([temp])
|
|
125
|
-
|
|
126
|
-
|
|
234
|
+
for (const { original, backup } of [...backups].reverse()) {
|
|
235
|
+
if (existsSync(backup) && !existsSync(original)) move(backup, original)
|
|
236
|
+
}
|
|
237
|
+
removeOwnScratch(backups.map(entry => entry.backup))
|
|
127
238
|
throw error
|
|
128
239
|
}
|
|
240
|
+
removePublishedBackups(
|
|
241
|
+
backups.map(entry => entry.backup),
|
|
242
|
+
fileOps.rmSync || rmSync
|
|
243
|
+
)
|
|
244
|
+
return destination
|
|
129
245
|
}
|
|
130
246
|
|
|
131
247
|
export async function instructionSetsCommand(args: string[], io: InstructionSetsCommandIo): Promise<number> {
|
|
@@ -143,6 +259,44 @@ export async function instructionSetsCommand(args: string[], io: InstructionSets
|
|
|
143
259
|
writeManifest = loadInstructionSetManifest(reference)
|
|
144
260
|
}
|
|
145
261
|
const project = await io.resolveProjectSlug(argValue(args, '--project'))
|
|
262
|
+
if (subcommand === 'scorers') {
|
|
263
|
+
const action = reference
|
|
264
|
+
const set = positionals[2]
|
|
265
|
+
const componentKey = argValue(args, '--key')
|
|
266
|
+
const scorerVersionId = argValue(args, '--scorer-version')
|
|
267
|
+
if (
|
|
268
|
+
(action !== 'set-headline' && action !== 'add') ||
|
|
269
|
+
!set ||
|
|
270
|
+
!componentKey ||
|
|
271
|
+
!scorerVersionId
|
|
272
|
+
) {
|
|
273
|
+
throw new Error(
|
|
274
|
+
'Usage: orizu instructions scorers set-headline|add <set> --key <component-key> --scorer-version <id> --project <team/project> [--dataset-version <id> --split-set <id> --split <name>] [--json]'
|
|
275
|
+
)
|
|
276
|
+
}
|
|
277
|
+
const payload = await responsePayload(await authedFetch(
|
|
278
|
+
`/api/cli/instruction-sets/${encodeURIComponent(set)}/scorers?project=${encodeURIComponent(project)}`,
|
|
279
|
+
{
|
|
280
|
+
method: 'POST',
|
|
281
|
+
headers: { 'content-type': 'application/json' },
|
|
282
|
+
body: JSON.stringify({
|
|
283
|
+
componentKey,
|
|
284
|
+
scorerVersionId,
|
|
285
|
+
role: action === 'set-headline' ? 'headline' : 'tracked',
|
|
286
|
+
datasetVersionId: argValue(args, '--dataset-version') || undefined,
|
|
287
|
+
splitSetId: argValue(args, '--split-set') || undefined,
|
|
288
|
+
splitName: argValue(args, '--split') || undefined,
|
|
289
|
+
}),
|
|
290
|
+
}
|
|
291
|
+
), `Instruction sets scorers ${action}`)
|
|
292
|
+
if (io.json) io.print(JSON.stringify(payload))
|
|
293
|
+
else {
|
|
294
|
+
io.print(
|
|
295
|
+
`${action === 'set-headline' ? 'Set headline' : 'Added'} scorer ${sanitizeTerminalText(scorerVersionId)} for ${sanitizeTerminalText(set)} / ${sanitizeTerminalText(componentKey)}`
|
|
296
|
+
)
|
|
297
|
+
}
|
|
298
|
+
return 0
|
|
299
|
+
}
|
|
146
300
|
if (subcommand === 'default') {
|
|
147
301
|
const operation = reference; const set = positionals[2]; const identity = argValue(args, '--model-config'); const version = argValue(args, '--version')
|
|
148
302
|
if ((operation !== 'show' && operation !== 'move') || !set) throw new Error('Usage: orizu instructions default show|move <set> --project <team/project> [--model-config <identity> --version <n>] [--json]')
|
|
@@ -270,8 +424,8 @@ export async function instructionSetsCommand(args: string[], io: InstructionSets
|
|
|
270
424
|
if (!set || (set.slug !== reference && set.name !== reference) || (modelConfig && (!Array.isArray(set.filteredTo) || set.filteredTo.length !== 1 || set.filteredTo[0] !== modelConfig))) {
|
|
271
425
|
throw new Error('instruction_set_sync_response_mismatch')
|
|
272
426
|
}
|
|
273
|
-
syncToDisk(output, set)
|
|
274
|
-
if (io.json) io.print(JSON.stringify(payload)); else io.print(`Synced ${set.name} to ${
|
|
427
|
+
const destination = syncToDisk(output, set)
|
|
428
|
+
if (io.json) io.print(JSON.stringify(payload)); else io.print(`Synced ${set.name} to ${destination}`)
|
|
275
429
|
return 0
|
|
276
430
|
}
|
|
277
431
|
if (subcommand === 'show' && (!reference || reference.startsWith('--'))) throw new Error('Instruction set name is required')
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { extractErrorMessage } from './error-response.js'
|
|
2
|
-
import { sanitizeTerminalText } from './json-response.js'
|
|
2
|
+
import { sanitizeHumanInlineText, sanitizeTerminalText } from './json-response.js'
|
|
3
|
+
import type { GlobalFlags } from './global-flags.js'
|
|
3
4
|
|
|
4
5
|
export interface OptimizationsPromoteIo {
|
|
5
6
|
json: boolean
|
|
7
|
+
origin: GlobalFlags
|
|
6
8
|
print: (line: string) => void
|
|
9
|
+
printErr: (line: string) => void
|
|
7
10
|
resolveProjectSlug: (projectArg: string | null) => Promise<string>
|
|
8
11
|
fetcher: (path: string, init?: RequestInit) => Promise<Response>
|
|
9
12
|
}
|
|
@@ -13,6 +16,57 @@ function option(args: readonly string[], name: string): string | null {
|
|
|
13
16
|
return index === -1 || !args[index + 1] || args[index + 1].startsWith('--') ? null : args[index + 1]
|
|
14
17
|
}
|
|
15
18
|
|
|
19
|
+
function shellArgument(value: string): string {
|
|
20
|
+
if (/^[A-Za-z0-9._/:@-]+$/.test(value)) return value
|
|
21
|
+
if (/[\u0000-\u001F\u007F-\u009F\u2028\u2029]/.test(value)) {
|
|
22
|
+
let escaped = ''
|
|
23
|
+
for (const character of value) {
|
|
24
|
+
if (character === '\\') escaped += '\\\\'
|
|
25
|
+
else if (character === "'") escaped += "\\'"
|
|
26
|
+
else if (character === '\n') escaped += '\\n'
|
|
27
|
+
else if (character === '\r') escaped += '\\r'
|
|
28
|
+
else if (character === '\t') escaped += '\\t'
|
|
29
|
+
else {
|
|
30
|
+
const codePoint = character.codePointAt(0)!
|
|
31
|
+
if (codePoint <= 0x1f || codePoint === 0x7f) {
|
|
32
|
+
escaped += `\\x${codePoint.toString(16).padStart(2, '0')}`
|
|
33
|
+
} else if ((codePoint >= 0x80 && codePoint <= 0x9f) || codePoint === 0x2028 || codePoint === 0x2029) {
|
|
34
|
+
for (const byte of new TextEncoder().encode(character)) {
|
|
35
|
+
escaped += `\\x${byte.toString(16).padStart(2, '0')}`
|
|
36
|
+
}
|
|
37
|
+
} else {
|
|
38
|
+
escaped += character
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return `$'${escaped}'`
|
|
43
|
+
}
|
|
44
|
+
const safeValue = sanitizeTerminalText(value)
|
|
45
|
+
return `'${safeValue.replace(/'/g, `'\\''`)}'`
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function inlineTerminalText(value: string): string {
|
|
49
|
+
return sanitizeHumanInlineText(sanitizeTerminalText, value)
|
|
50
|
+
.replaceAll('\u2028', '\\u2028')
|
|
51
|
+
.replaceAll('\u2029', '\\u2029')
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const APPLIED_LABEL_VERSION_SKEW_NOTE = 'Server response predates the appliedLabel discriminator; printing the production-promotion follow-up as a fail-safe.'
|
|
55
|
+
|
|
56
|
+
function productionPromotionReplay(
|
|
57
|
+
runId: string,
|
|
58
|
+
candidateId: string,
|
|
59
|
+
project: string,
|
|
60
|
+
origin: GlobalFlags
|
|
61
|
+
): string {
|
|
62
|
+
const originArguments = origin.local
|
|
63
|
+
? ' --local'
|
|
64
|
+
: origin.server
|
|
65
|
+
? ` --server ${shellArgument(origin.server)}`
|
|
66
|
+
: ''
|
|
67
|
+
return `Promote this profile with: orizu${originArguments} optimizations promote ${shellArgument(runId)} --candidate ${shellArgument(candidateId)} --label production --project ${shellArgument(project)}`
|
|
68
|
+
}
|
|
69
|
+
|
|
16
70
|
export async function promoteOptimizationCommand(args: string[], io: OptimizationsPromoteIo): Promise<void> {
|
|
17
71
|
const runId = args.find(arg => !arg.startsWith('--') && arg !== option(args, '--candidate') && arg !== option(args, '--label') && arg !== option(args, '--project'))
|
|
18
72
|
const candidateId = option(args, '--candidate')
|
|
@@ -25,17 +79,41 @@ export async function promoteOptimizationCommand(args: string[], io: Optimizatio
|
|
|
25
79
|
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ candidateId, label }),
|
|
26
80
|
})
|
|
27
81
|
if (!response.ok) throw new Error(`Failed to promote optimization candidate: ${await extractErrorMessage(response)}`)
|
|
28
|
-
const data = await response.json() as
|
|
82
|
+
const data = await response.json() as
|
|
83
|
+
| { promptVersionId: string }
|
|
84
|
+
| {
|
|
85
|
+
profileVersionId: string
|
|
86
|
+
versionNumber: number
|
|
87
|
+
appliedLabel?: string | null
|
|
88
|
+
components: Array<{ key: string; status: string }>
|
|
89
|
+
}
|
|
90
|
+
const isProfilePromotion = 'profileVersionId' in data
|
|
91
|
+
const replay = isProfilePromotion && data.appliedLabel !== 'production'
|
|
92
|
+
? productionPromotionReplay(runId, candidateId, project, io.origin)
|
|
93
|
+
: null
|
|
29
94
|
if (io.json) {
|
|
30
|
-
io.print(JSON.stringify(
|
|
31
|
-
? {
|
|
95
|
+
io.print(JSON.stringify(isProfilePromotion
|
|
96
|
+
? {
|
|
97
|
+
profileVersionId: data.profileVersionId,
|
|
98
|
+
versionNumber: data.versionNumber,
|
|
99
|
+
appliedLabel: data.appliedLabel,
|
|
100
|
+
components: data.components,
|
|
101
|
+
}
|
|
32
102
|
: { promptVersionId: data.promptVersionId }))
|
|
103
|
+
if (isProfilePromotion && data.appliedLabel === undefined) {
|
|
104
|
+
io.printErr(APPLIED_LABEL_VERSION_SKEW_NOTE)
|
|
105
|
+
io.printErr(replay!)
|
|
106
|
+
}
|
|
33
107
|
return
|
|
34
108
|
}
|
|
35
|
-
if (
|
|
109
|
+
if (isProfilePromotion) {
|
|
36
110
|
io.print(`Promoted profile version ${sanitizeTerminalText(data.profileVersionId)}`)
|
|
37
|
-
|
|
111
|
+
if (data.appliedLabel === undefined) {
|
|
112
|
+
io.printErr(APPLIED_LABEL_VERSION_SKEW_NOTE)
|
|
113
|
+
}
|
|
114
|
+
if (replay) io.print(replay)
|
|
115
|
+
for (const component of data.components) io.print(`${inlineTerminalText(component.key)}: ${inlineTerminalText(component.status)}`)
|
|
38
116
|
return
|
|
39
117
|
}
|
|
40
|
-
io.print(`Promoted prompt version ${sanitizeTerminalText(data.promptVersionId
|
|
118
|
+
io.print(`Promoted prompt version ${sanitizeTerminalText(data.promptVersionId)}`)
|
|
41
119
|
}
|
package/src/runner-dir-verify.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
shouldExcludeArtifactPath,
|
|
10
10
|
} from './artifact-archive.js'
|
|
11
11
|
import { authedFetch } from './http.js'
|
|
12
|
+
import { cleanupAll, throwWithCleanup } from './cleanup.js'
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* ALI-1159 (ADR-007): ad-hoc `--runner-dir` bytes may only execute when they
|
|
@@ -159,7 +160,7 @@ export async function verifyRunnerDirRegistered(input: VerifyRunnerDirInput): Pr
|
|
|
159
160
|
|
|
160
161
|
if (registeredSha.toLowerCase() !== localSha.toLowerCase()) {
|
|
161
162
|
throw new Error(
|
|
162
|
-
`Local runner bytes at ${dir} are not registered runner version ${input.runnerVersionId} ` +
|
|
163
|
+
`runner_version_bytes_mismatch: Local runner bytes at ${dir} are not registered runner version ${input.runnerVersionId} ` +
|
|
163
164
|
`(content sha ${localSha} != ${registeredSha}). Runs may only execute registered runner bytes ` +
|
|
164
165
|
'(ADR-007): register the directory first with `orizu runners push` ' +
|
|
165
166
|
'(use --session <session-id> inside a workspace session) and re-run against the new version, ' +
|
|
@@ -979,9 +980,10 @@ export async function verifyGepaRunnerDirsFromArgs(
|
|
|
979
980
|
let rewritten = [...args]
|
|
980
981
|
const verifiedDirs: string[] = []
|
|
981
982
|
const cleanups: Array<() => void> = []
|
|
982
|
-
const cleanup = () =>
|
|
983
|
-
|
|
984
|
-
|
|
983
|
+
const cleanup = () => cleanupAll(
|
|
984
|
+
cleanups,
|
|
985
|
+
'Failed to clean verified runner snapshots'
|
|
986
|
+
)
|
|
985
987
|
|
|
986
988
|
try {
|
|
987
989
|
for (const pair of GEPA_RUNNER_FLAG_PAIRS) {
|
|
@@ -1021,8 +1023,11 @@ export async function verifyGepaRunnerDirsFromArgs(
|
|
|
1021
1023
|
rewritten = rewriteFlagValue(rewritten, pair.dirFlag, verified.snapshotDir)
|
|
1022
1024
|
}
|
|
1023
1025
|
} catch (error) {
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
+
throwWithCleanup(
|
|
1027
|
+
error,
|
|
1028
|
+
cleanups,
|
|
1029
|
+
'Runner verification failed and snapshot cleanup also failed'
|
|
1030
|
+
)
|
|
1026
1031
|
}
|
|
1027
1032
|
|
|
1028
1033
|
return { args: rewritten, verifiedDirs, cleanup }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const SKILLED_PROPOSER_BAKE_REPORT_FIELDS = ['venv', 'publishKey', 'publishedVenv', 'waitedForPublishLock', 'pythonPathEntries', 'executedPipArgv', 'sslCertFile', 'warnings', 'marker', 'python', 'packageRoot', 'manager', 'lock', 'vendoredGepaPath', 'launcherVenv', 'launcherSslCertFile'] as const
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { readFileSync, statSync } from 'node:fs'
|
|
3
|
+
import { basename, dirname, extname, relative, resolve, sep } from 'node:path'
|
|
4
|
+
import { fileURLToPath } from 'node:url'
|
|
5
|
+
|
|
6
|
+
interface NormalizedTextSource { source: string, content: string, sha256: string, byteLength: number }
|
|
7
|
+
interface NormalizedSkill extends NormalizedTextSource { name: string, description: string | null }
|
|
8
|
+
|
|
9
|
+
const CONFIG_KEYS = new Set(['schemaVersion', 'skills', 'additionalInstructionsFile', 'baseInstructionsFile',
|
|
10
|
+
'maxWords', 'maxTokens', 'maxExamples', 'onError',
|
|
11
|
+
])
|
|
12
|
+
const PATH_SKILL_KEYS = new Set(['path', 'name', 'description'])
|
|
13
|
+
const INLINE_SKILL_KEYS = new Set(['name', 'description', 'inline'])
|
|
14
|
+
|
|
15
|
+
function configError(field: string, detail: string): never { throw new Error(`--candidate-proposer-config ${field} ${detail}`) }
|
|
16
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
17
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
18
|
+
}
|
|
19
|
+
function rejectUnknownKeys(value: Record<string, unknown>, allowed: Set<string>, field = ''): void {
|
|
20
|
+
const unknown = Object.keys(value).find(key => !allowed.has(key))
|
|
21
|
+
if (unknown) configError(field ? `${field}.${unknown}` : unknown, 'is not supported')
|
|
22
|
+
}
|
|
23
|
+
function sha256(bytes: Uint8Array): string { return createHash('sha256').update(bytes).digest('hex') }
|
|
24
|
+
function decodeUtf8(bytes: Uint8Array, field: string): string {
|
|
25
|
+
try {
|
|
26
|
+
return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes)
|
|
27
|
+
} catch {
|
|
28
|
+
configError(field, 'must contain valid UTF-8 text')
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function readRegularFile(path: string, field: string): Buffer {
|
|
32
|
+
try {
|
|
33
|
+
if (!statSync(path).isFile()) configError(field, 'must resolve to a file')
|
|
34
|
+
return readFileSync(path)
|
|
35
|
+
} catch (error) {
|
|
36
|
+
if (error instanceof Error && error.message.startsWith('--candidate-proposer-config')) throw error
|
|
37
|
+
configError(field, `is missing or unreadable: ${error instanceof Error ? error.message : String(error)}`)
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function logicalSource(configDirectory: string, path: string): string {
|
|
41
|
+
return relative(configDirectory, path).split(sep).join('/') || basename(path)
|
|
42
|
+
}
|
|
43
|
+
function normalizedText(bytes: Buffer, source: string, field: string): NormalizedTextSource {
|
|
44
|
+
const content = decodeUtf8(bytes, field)
|
|
45
|
+
if (!content.trim()) configError(field, 'must not be empty')
|
|
46
|
+
return { source, content, sha256: sha256(bytes), byteLength: bytes.byteLength }
|
|
47
|
+
}
|
|
48
|
+
function parseFrontmatter(content: string, field: string): { name?: string, description?: string, body: string } {
|
|
49
|
+
const lines = content.split(/\r?\n/)
|
|
50
|
+
if (lines[0]?.replace(/^\uFEFF/, '').trim() !== '---') return { body: content }
|
|
51
|
+
const end = lines.slice(1).findIndex(line => line.trim() === '---')
|
|
52
|
+
if (end < 0) configError(field, 'has malformed frontmatter without a closing ---')
|
|
53
|
+
const metadata: Record<string, string> = {}
|
|
54
|
+
for (const raw of lines.slice(1, end + 1)) {
|
|
55
|
+
if (/^[ \t]/.test(raw) || !raw.includes(':')) continue
|
|
56
|
+
const colon = raw.indexOf(':')
|
|
57
|
+
metadata[raw.slice(0, colon).trim()] = raw.slice(colon + 1).trim().replace(/^(['"])(.*)\1$/, '$2')
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
name: metadata.name || undefined,
|
|
61
|
+
description: metadata.description || undefined,
|
|
62
|
+
body: lines.slice(end + 2).join('\n'),
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
function wellFormedString(value: string, field: string): string {
|
|
66
|
+
if (!(value as string & { isWellFormed(): boolean }).isWellFormed()) {
|
|
67
|
+
configError(field, 'must contain well-formed Unicode text')
|
|
68
|
+
}
|
|
69
|
+
return value
|
|
70
|
+
}
|
|
71
|
+
function nonEmptyString(value: unknown, field: string): string {
|
|
72
|
+
if (typeof value !== 'string' || !value.trim()) configError(field, 'must be a non-empty string')
|
|
73
|
+
return wellFormedString(value, field)
|
|
74
|
+
}
|
|
75
|
+
function nullableDescription(value: unknown, field: string): string | null {
|
|
76
|
+
if (value === null) return null
|
|
77
|
+
if (typeof value !== 'string') configError(field, 'must be a string or null')
|
|
78
|
+
return wellFormedString(value, field) || null
|
|
79
|
+
}
|
|
80
|
+
function normalizePathSkill(entry: Record<string, unknown>, index: number, configDirectory: string): NormalizedSkill {
|
|
81
|
+
const field = `skills[${index}]`
|
|
82
|
+
rejectUnknownKeys(entry, PATH_SKILL_KEYS, field)
|
|
83
|
+
const configuredPath = nonEmptyString(entry.path, `${field}.path`)
|
|
84
|
+
let resolvedPath = resolve(configDirectory, configuredPath)
|
|
85
|
+
try {
|
|
86
|
+
if (statSync(resolvedPath).isDirectory()) {
|
|
87
|
+
resolvedPath = resolve(resolvedPath, 'SKILL.md')
|
|
88
|
+
try {
|
|
89
|
+
if (!statSync(resolvedPath).isFile()) configError(`${field}.path`, 'directory must contain SKILL.md')
|
|
90
|
+
} catch (error) {
|
|
91
|
+
if (error instanceof Error && error.message.startsWith('--candidate-proposer-config')) throw error
|
|
92
|
+
configError(`${field}.path`, 'directory must contain SKILL.md')
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
} catch (error) {
|
|
96
|
+
if (error instanceof Error && error.message.startsWith('--candidate-proposer-config')) throw error
|
|
97
|
+
configError(`${field}.path`, 'is missing or unreadable')
|
|
98
|
+
}
|
|
99
|
+
const text = normalizedText(readRegularFile(resolvedPath, `${field}.path`),
|
|
100
|
+
logicalSource(configDirectory, resolvedPath), `${field}.path`)
|
|
101
|
+
const frontmatter = parseFrontmatter(text.content, `${field}.path`)
|
|
102
|
+
if (!frontmatter.body.trim()) configError(`${field}.path`, 'must contain non-empty skill content')
|
|
103
|
+
const fallbackName = extname(resolvedPath)
|
|
104
|
+
? basename(resolvedPath, extname(resolvedPath))
|
|
105
|
+
: basename(resolvedPath)
|
|
106
|
+
const name = entry.name === undefined
|
|
107
|
+
? frontmatter.name ?? (basename(resolvedPath) === 'SKILL.md' ? basename(dirname(resolvedPath)) : fallbackName)
|
|
108
|
+
: nonEmptyString(entry.name, `${field}.name`)
|
|
109
|
+
const description = entry.description === undefined
|
|
110
|
+
? frontmatter.description ?? null
|
|
111
|
+
: nullableDescription(entry.description, `${field}.description`)
|
|
112
|
+
return { name, description, ...text }
|
|
113
|
+
}
|
|
114
|
+
function normalizeInlineSkill(entry: Record<string, unknown>, index: number): NormalizedSkill {
|
|
115
|
+
const field = `skills[${index}]`
|
|
116
|
+
rejectUnknownKeys(entry, INLINE_SKILL_KEYS, field)
|
|
117
|
+
const name = nonEmptyString(entry.name, `${field}.name`)
|
|
118
|
+
const description = nonEmptyString(entry.description, `${field}.description`)
|
|
119
|
+
const content = nonEmptyString(entry.inline, `${field}.inline`)
|
|
120
|
+
const bytes = Buffer.from(content, 'utf8')
|
|
121
|
+
return { name, description, source: `inline:${name}`, content,
|
|
122
|
+
sha256: sha256(bytes), byteLength: bytes.byteLength }
|
|
123
|
+
}
|
|
124
|
+
function normalizeSkill(value: unknown, index: number, configDirectory: string): NormalizedSkill {
|
|
125
|
+
const field = `skills[${index}]`
|
|
126
|
+
if (!isRecord(value)) configError(field, 'must be an object')
|
|
127
|
+
const hasPath = Object.hasOwn(value, 'path')
|
|
128
|
+
const hasInline = Object.hasOwn(value, 'inline')
|
|
129
|
+
if (hasPath === hasInline) configError(field, 'must contain exactly one of path or inline')
|
|
130
|
+
return hasPath ? normalizePathSkill(value, index, configDirectory) : normalizeInlineSkill(value, index)
|
|
131
|
+
}
|
|
132
|
+
function optionalPositiveInteger(value: unknown, field: string): number | null {
|
|
133
|
+
if (value === undefined || value === null) return null
|
|
134
|
+
if (!Number.isSafeInteger(value) || (value as number) <= 0) configError(field, 'must be a positive integer or null')
|
|
135
|
+
return value as number
|
|
136
|
+
}
|
|
137
|
+
function normalizeInstructionFile(value: unknown, field: string, configDirectory: string): NormalizedTextSource | null {
|
|
138
|
+
if (value === undefined || value === null) return null
|
|
139
|
+
const configuredPath = nonEmptyString(value, field)
|
|
140
|
+
const resolvedPath = resolve(configDirectory, configuredPath)
|
|
141
|
+
return normalizedText(readRegularFile(resolvedPath, field), logicalSource(configDirectory, resolvedPath), field)
|
|
142
|
+
}
|
|
143
|
+
function lockedPackageVersion(): string {
|
|
144
|
+
const lockPath = fileURLToPath(new URL('../requirements/skilled-proposer.lock', import.meta.url))
|
|
145
|
+
const match = /^skilled-proposer==([^\s\\]+)/m.exec(readFileSync(lockPath, 'utf8'))
|
|
146
|
+
if (!match) throw new Error('ALI_1505_SKILLED_PROPOSER_LOCK_VERSION_MISSING')
|
|
147
|
+
return match[1]
|
|
148
|
+
}
|
|
149
|
+
export function normalizedSkilledProposerConfig(value: string): string {
|
|
150
|
+
const suppliedPath = nonEmptyString(value.startsWith('@') ? value.slice(1) : value, 'path')
|
|
151
|
+
const configPath = resolve(suppliedPath)
|
|
152
|
+
const configBytes = readRegularFile(configPath, 'path')
|
|
153
|
+
const raw = decodeUtf8(configBytes, 'path').replace(/^\uFEFF/, '')
|
|
154
|
+
let parsed: unknown
|
|
155
|
+
try {
|
|
156
|
+
parsed = JSON.parse(raw)
|
|
157
|
+
} catch (error) {
|
|
158
|
+
configError('path', `must contain valid JSON: ${error instanceof Error ? error.message : String(error)}`)
|
|
159
|
+
}
|
|
160
|
+
if (!isRecord(parsed)) configError('path', 'must contain a JSON object')
|
|
161
|
+
rejectUnknownKeys(parsed, CONFIG_KEYS)
|
|
162
|
+
if (parsed.schemaVersion !== 1) configError('schemaVersion', 'must be 1')
|
|
163
|
+
if (!Array.isArray(parsed.skills)) configError('skills', 'must be an array')
|
|
164
|
+
|
|
165
|
+
const configDirectory = dirname(configPath)
|
|
166
|
+
const skills = parsed.skills.map((skill, index) => normalizeSkill(skill, index, configDirectory))
|
|
167
|
+
const names = new Set<string>()
|
|
168
|
+
for (const skill of skills) {
|
|
169
|
+
if (names.has(skill.name)) configError('skills', `contains duplicate resolved name ${skill.name}`)
|
|
170
|
+
names.add(skill.name)
|
|
171
|
+
}
|
|
172
|
+
const onError = parsed.onError ?? 'keep'
|
|
173
|
+
if (onError !== 'keep' && onError !== 'raise') configError('onError', 'must be keep or raise')
|
|
174
|
+
|
|
175
|
+
const normalized = {
|
|
176
|
+
schemaVersion: 1,
|
|
177
|
+
implementation: 'cmpnd-ai/skilled-proposer',
|
|
178
|
+
packageVersion: lockedPackageVersion(),
|
|
179
|
+
skills,
|
|
180
|
+
additionalInstructions: normalizeInstructionFile(parsed.additionalInstructionsFile,
|
|
181
|
+
'additionalInstructionsFile', configDirectory),
|
|
182
|
+
baseInstructions: normalizeInstructionFile(parsed.baseInstructionsFile, 'baseInstructionsFile', configDirectory),
|
|
183
|
+
maxWords: optionalPositiveInteger(parsed.maxWords, 'maxWords'),
|
|
184
|
+
maxTokens: optionalPositiveInteger(parsed.maxTokens, 'maxTokens'),
|
|
185
|
+
maxExamples: optionalPositiveInteger(parsed.maxExamples, 'maxExamples'),
|
|
186
|
+
onError,
|
|
187
|
+
}
|
|
188
|
+
const configFileSha256 = sha256(configBytes)
|
|
189
|
+
const configSha256 = sha256(Buffer.from(JSON.stringify({ configFileSha256, ...normalized }), 'utf8'))
|
|
190
|
+
return JSON.stringify({ ...normalized, configFileSha256, configSha256 })
|
|
191
|
+
}
|