orizu 0.5.21 → 0.5.22

package/src/help.ts

@@ -89,6 +89,40 @@ export const COMMAND_DOCS: CliCommandDoc[] = [
   },
   { path: ['instructions', 'archive'], aliases: [['instruction-sets', 'archive']], usage: 'orizu instructions archive <slug-or-exact-name> --project <team/project> [--json]', summary: 'Archive an instruction set without changing resolution or sync.', group: 'Instruction sets', options: [{ name: '--project <team/project>', help: 'Project slug.', required: true }, { name: '--json', help: 'Print JSON.' }], examples: ['orizu instructions archive planner --project core/evals'] },
   { path: ['instructions', 'restore'], aliases: [['instruction-sets', 'restore']], usage: 'orizu instructions restore <slug-or-exact-name> --project <team/project> [--json]', summary: 'Restore an archived instruction set.', group: 'Instruction sets', options: [{ name: '--project <team/project>', help: 'Project slug.', required: true }, { name: '--json', help: 'Print JSON.' }], examples: ['orizu instructions restore planner --project core/evals'] },
+  {
+    path: ['instructions', 'scorers', 'set-headline'],
+    aliases: [['instruction-sets', 'scorers', 'set-headline']],
+    usage: 'orizu instructions scorers set-headline <set> --key <component-key> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
+    summary: 'Bind the headline scorer for one instruction-set component.',
+    group: 'Instruction sets',
+    options: [
+      { name: '--key <component-key>', help: 'Instruction-set component key.', required: true },
+      { name: '--scorer-version <id>', help: 'Scorer version to bind.', required: true },
+      { name: '--dataset-version <id>', help: 'Default dataset version.' },
+      { name: '--split-set <id>', help: 'Default dataset split set.' },
+      { name: '--split <name>', help: 'Default split name.' },
+      { name: '--project <team/project>', help: 'Project slug.' },
+      { name: '--json', help: 'Print JSON.' },
+    ],
+    examples: ['orizu instructions scorers set-headline planner --key system --scorer-version <id> --project core/evals'],
+  },
+  {
+    path: ['instructions', 'scorers', 'add'],
+    aliases: [['instruction-sets', 'scorers', 'add']],
+    usage: 'orizu instructions scorers add <set> --key <component-key> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
+    summary: 'Track an additional scorer for one instruction-set component.',
+    group: 'Instruction sets',
+    options: [
+      { name: '--key <component-key>', help: 'Instruction-set component key.', required: true },
+      { name: '--scorer-version <id>', help: 'Scorer version to bind.', required: true },
+      { name: '--dataset-version <id>', help: 'Default dataset version.' },
+      { name: '--split-set <id>', help: 'Default dataset split set.' },
+      { name: '--split <name>', help: 'Default split name.' },
+      { name: '--project <team/project>', help: 'Project slug.' },
+      { name: '--json', help: 'Print JSON.' },
+    ],
+    examples: ['orizu instructions scorers add planner --key system --scorer-version <id> --project core/evals'],
+  },
   {
     path: ['login'],
     usage: 'orizu login [--no-prompt-if-logged-in]',
@@ -413,18 +447,6 @@ export const COMMAND_DOCS: CliCommandDoc[] = [
     summary: 'Move or set a non-production prompt label; use instructions profiles promote for production.',
     group: 'Prompts and judges',
   },
-  {
-    path: ['prompts', 'scorers', 'set-headline'],
-    usage: 'orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
-    summary: 'Bind the headline scorer for a prompt.',
-    group: 'Scorers and runners',
-  },
-  {
-    path: ['prompts', 'scorers', 'add'],
-    usage: 'orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]',
-    summary: 'Track an additional scorer for a prompt.',
-    group: 'Scorers and runners',
-  },
   {
     path: ['judges', 'list'],
     usage: 'orizu judges list --project <team/project> [--status active|archived|all]',
@@ -578,8 +600,8 @@ export const COMMAND_DOCS: CliCommandDoc[] = [
   },
   {
     path: ['optimizations', 'run-gepa'],
-    usage: 'orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> (--candidate-version-id <id> | --instruction-set <name> --model-config <identity> [--component-selector round-robin|all]) --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> [--engine official|legacy] [--scorer-input-contract gepa|flat_row] [--scorer-candidate-field <row-field>] [--allow-degenerate-seed] --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--budget auto|light|medium|heavy | --max-metric-calls <n> | --max-full-evals <n> | --max-iterations <n> | --max-candidate-proposals <n>] [--num-threads auto|N] [--reflection-retry-attempts N] [--reflection-http-timeout-seconds N] [--log-dir logs]',
-    summary: 'Run official GEPA locally and stream events; pass --engine legacy only for the frozen safety hatch. --instruction-set requires --model-config and cannot be combined with --candidate-version-id; round-robin updates one component-map entry per round and all updates every component. --max-candidate-proposals is official-engine-only. Validates the scorer contract on the seed before iterating; judge runners built for flat-row score runs need --scorer-input-contract flat_row (ALI-1158).',
+    usage: 'orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> (--candidate-version-id <id> | --instruction-set <name> --model-config <identity> [--component-selector round-robin|all]) --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> [--engine official|legacy] [--candidate-proposer skilled-proposer [--candidate-proposer-config <@file|path>] [--proposal-max-calls <n>] [--proposal-max-tokens <n>]] [--scorer-input-contract gepa|flat_row] [--scorer-candidate-field <row-field>] [--allow-degenerate-seed] --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--budget auto|light|medium|heavy | --max-metric-calls <n> | --max-full-evals <n> | --max-iterations <n> | --max-candidate-proposals <n>] [--num-threads auto|N] [--reflection-max-tokens N] [--reflection-retry-attempts N] [--reflection-http-timeout-seconds N] [--log-dir logs]',
+    summary: 'Run official GEPA locally and stream events; pass --engine legacy only for the frozen safety hatch. Skilled-proposer is official-only, its config requires that selection, and the selection cannot be combined with --reflection-prompt-template. --instruction-set requires --model-config and cannot be combined with --candidate-version-id; round-robin updates one component-map entry per round and all updates every component. --max-candidate-proposals is official-engine-only. Validates the scorer contract on the seed before iterating; judge runners built for flat-row score runs need --scorer-input-contract flat_row (ALI-1158).',
     group: 'Optimizations',
   },
   {

package/src/index.ts

@@ -117,7 +117,7 @@ import { dispatchGepaEngine } from './gepa-engine-dispatch.js'
 import { resolveGepaInstructionSetProfileVersion } from './instruction-set-gepa-launch.js'
 import { getGepaPythonCommand } from './gepa-python-command.js'
 import { getGepaPythonPathEntries } from './gepa-python-paths.js'
-import { prepareSkilledProposerLaunch } from './skilled-proposer-launch.js'
+import { prepareSkilledProposerLaunch, spawnSkilledProposerChild } from './skilled-proposer-launch.js'
 
 export { parseJsonResponse, sanitizeTerminalText } from './json-response.js'
 
@@ -1767,53 +1767,6 @@ function refusePromptMutation(replacement: string): never {
   throw new Error(`Use: orizu instructions ${replacement}`)
 }
 
-async function bindPromptScorer(role: 'headline' | 'tracked') {
-  const promptId = getPositionalArg(3)
-  const project = getArg('--project') || await resolveProjectSlug(null)
-  const scorerVersionId = getArg('--scorer-version')
-
-  if (!promptId || !scorerVersionId) {
-    throw new Error(`Usage: orizu prompts scorers ${role === 'headline' ? 'set-headline' : 'add'} <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]`)
-  }
-
-  const response = await authedFetch(`/api/cli/prompts/scorers?project=${encodeURIComponent(project)}`, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      promptId,
-      scorerVersionId,
-      role,
-      datasetVersionId: getArg('--dataset-version') || undefined,
-      splitSetId: getArg('--split-set') || undefined,
-      splitName: getArg('--split') || undefined,
-    }),
-  })
-
-  if (!response.ok) {
-    throw new Error(`Failed to bind prompt scorer: ${await response.text()}`)
-  }
-
-  const data = await parseJsonResponse<{
-    promptScorer: Record<string, unknown>
-    owner?: {
-      instructionSetSlug: string
-      componentKey: string
-    }
-  }>(response, 'Prompt scorer bind')
-  if (hasJsonFlag()) {
-    printJson(data)
-    return
-  }
-
-  printLine(`${role === 'headline' ? 'Set headline' : 'Added'} scorer ${sanitizeTerminalText(scorerVersionId)} for ${sanitizeTerminalText(promptId)}`)
-  if (data.owner) {
-    printLine(
-      `Owner: ${sanitizeTerminalText(promptId)} -> ` +
-      `${sanitizeTerminalText(data.owner.instructionSetSlug)} / ${sanitizeTerminalText(data.owner.componentKey)}`
-    )
-  }
-}
-
 function getPositionalArg(index: number): string | null {
   const value = cliArgs[index]
   return value && !value.startsWith('--') ? value : null
@@ -2539,7 +2492,9 @@ async function runGepaOptimization() {
     if (instructionSetProfileVersionId) dispatch.environment.ORIZU_INSTRUCTION_SET_PROFILE_VERSION_ID = instructionSetProfileVersionId
     selectedEngine = dispatch.engine
     const launch = prepareSkilledProposerLaunch(python, dispatch.engine, dispatch.environment)
-    result = spawnSync(launch.python, ['-m', dispatch.module, ...dispatch.args], { stdio: 'inherit', env: launch.environment })
+    result = await spawnSkilledProposerChild(
+      launch.python, ['-m', dispatch.module, ...dispatch.args], dispatch.engine, launch.environment,
+    )
   } finally {
     verified.cleanup()
   }
@@ -6236,13 +6191,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
   }
 
   if (command === 'prompts' && subcommand === 'scorers' && cliArgs[2] === 'set-headline') {
-    await bindPromptScorer('headline')
-    return
+    refusePromptMutation('scorers set-headline <set> --key <component-key> --scorer-version <id> --project <team/project>')
   }
 
   if (command === 'prompts' && subcommand === 'scorers' && cliArgs[2] === 'add') {
-    await bindPromptScorer('tracked')
-    return
+    refusePromptMutation('scorers add <set> --key <component-key> --scorer-version <id> --project <team/project>')
   }
 
   if (command === 'judges' && subcommand === 'list') {

package/src/instruction-set-loader.ts

@@ -1,4 +1,4 @@
-import { existsSync, realpathSync, readFileSync, statSync } from 'node:fs'
+import { existsSync, realpathSync, readFileSync, readdirSync, statSync } from 'node:fs'
 import { resolve, sep } from 'node:path'
 
 export class InstructionSetLoaderError extends Error {
@@ -6,14 +6,64 @@ export class InstructionSetLoaderError extends Error {
 }
 
 interface Material { files: Record<string, string>; pinnedComponents?: Record<string, unknown> }
-interface Manifest { manifestVersion: number; name: string; shape: string[]; default: Material; profiles: Array<{ modelConfigIdentity: string; production: Material | null }>; filteredTo?: string[] }
+interface Manifest { manifestVersion: number; projectId?: string; instructionSetId?: string; name: string; slug?: string; shape: string[]; default: Material; profiles: Array<{ modelConfigIdentity: string; production: Material | null }>; filteredTo?: string[] }
 
 function inside(root: string, candidate: string) { return candidate === root || candidate.startsWith(`${root}${sep}`) }
 
+function manifestMatchesReference(candidate: string, reference: string): boolean {
+  try {
+    const manifest = JSON.parse(
+      readFileSync(resolve(candidate, 'manifest.json'), 'utf8')
+    ) as Pick<Manifest, 'name' | 'slug'>
+    return manifest.name === reference || manifest.slug === reference
+  } catch {
+    return false
+  }
+}
+
+function resolveSetRoot(directory: string, reference: string): string {
+  let directoryRoot: string
+  try { directoryRoot = realpathSync(directory) } catch { throw new InstructionSetLoaderError('instruction_set_not_synced') }
+  const direct = resolve(directoryRoot, reference)
+  try {
+    if (inside(directoryRoot, direct) && existsSync(direct) && statSync(direct).isDirectory()) {
+      const realDirect = realpathSync(direct)
+      if (inside(directoryRoot, realDirect)) {
+        try {
+          const manifest = JSON.parse(
+            readFileSync(resolve(realDirect, 'manifest.json'), 'utf8')
+          ) as Pick<Manifest, 'name' | 'slug'>
+          if (manifest.name === reference || manifest.slug === reference) {
+            return realDirect
+          }
+        } catch {
+          // Preserve the exact-path loader's specific missing/invalid manifest error.
+          return realDirect
+        }
+      }
+    }
+  } catch {
+    // A stale or unreadable direct-path candidate must not shadow a valid set.
+  }
+  const matches = readdirSync(directoryRoot, { withFileTypes: true }).flatMap(entry => {
+    try {
+      if (entry.name.startsWith('.') || !entry.isDirectory()) return []
+      const candidate = realpathSync(resolve(directoryRoot, entry.name))
+      return inside(directoryRoot, candidate) && manifestMatchesReference(candidate, reference)
+        ? [candidate]
+        : []
+    } catch {
+      return []
+    }
+  })
+  if (matches.length !== 1) throw new InstructionSetLoaderError(
+    matches.length > 1 ? 'instruction_set_reference_ambiguous' : 'instruction_set_not_synced'
+  )
+  return matches[0]!
+}
+
 export function loadInstructionSet(directory: string, name: string, modelConfigIdentity: string): Record<string, string> {
-  const unresolvedRoot = resolve(directory, name)
-  if (!existsSync(unresolvedRoot) || !statSync(unresolvedRoot).isDirectory()) throw new InstructionSetLoaderError('instruction_set_not_synced')
-  const setRoot = realpathSync(unresolvedRoot)
+  const setRoot = resolveSetRoot(directory, name)
   const manifestPath = resolve(setRoot, 'manifest.json')
   if (!existsSync(manifestPath)) throw new InstructionSetLoaderError('instruction_set_manifest_missing')
   let manifest: Manifest

package/src/instruction-sets-cli.ts

@@ -43,13 +43,30 @@ const SAFE_SEGMENT = /^[A-Za-z0-9._-]+$/u
 export interface SyncComponent { body?: string; repoPath?: string; contentSha?: string; commitSha?: string }
 export interface SyncMaterial { profileVersionId: string; versionNumber: number; modelConfigIdentity: string; resolvedFrom: string; components: Record<string, SyncComponent> }
 export interface SyncProfile { modelConfigIdentity: string; resolvedFrom: string; production: SyncMaterial | null }
-export interface SyncSet { name: string; slug?: string; description?: string | null; shape: string[]; default: SyncMaterial; profiles: SyncProfile[]; filteredTo?: string[] }
+export interface SyncSet {
+  projectId?: string
+  instructionSetId?: string
+  name: string
+  slug?: string
+  description?: string | null
+  shape: string[]
+  default: SyncMaterial
+  profiles: SyncProfile[]
+  filteredTo?: string[]
+}
 function safeSegment(value: string) { if (!SAFE_SEGMENT.test(value) || value === '.' || value === '..' || value.startsWith('.')) throw new Error('instruction_set_path_unsafe') }
+function isSafeSegment(value: string) {
+  try { safeSegment(value); return true } catch { return false }
+}
 function slug(identity: string) { return identity.replaceAll('/', '__').replace(/[^A-Za-z0-9._-]/gu, '_') }
 function stable(value: unknown): string { return `${JSON.stringify(value, null, 2)}\n` }
 function sha256(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex') }
 
-export interface SyncFileOps { writeFileSync: typeof writeFileSync; renameSync?: typeof renameSync }
+export interface SyncFileOps {
+  writeFileSync: typeof writeFileSync
+  renameSync?: typeof renameSync
+  rmSync?: typeof rmSync
+}
 
 function writeMaterial(root: string, material: SyncMaterial, shape: string[], manifestRoot: string, fileOps: SyncFileOps = { writeFileSync }) {
   const files: Record<string, string> = {}
@@ -77,16 +94,76 @@ function removeOwnScratch(paths: Array<string | null>) {
   for (const path of paths) if (path && existsSync(path)) rmSync(path, { recursive: true, force: true })
 }
 
-function existingDestinationIsSynced(destination: string, name: string) {
+function existingDestinationIsSynced(
+  destination: string,
+  set: Pick<SyncSet, 'projectId' | 'instructionSetId' | 'name' | 'slug'>
+) {
+  if (
+    set.slug &&
+    set.projectId === undefined &&
+    set.instructionSetId === undefined
+  ) {
+    return existingDestinationIsLegacySynced(destination, set.name, set.slug)
+  }
+  try {
+    if (!statSync(destination).isDirectory()) return false
+    const manifest = JSON.parse(readFileSync(join(destination, 'manifest.json'), 'utf8')) as {
+      manifestVersion?: number
+      projectId?: string
+      instructionSetId?: string
+      name?: string
+      slug?: string
+    }
+    return manifest.manifestVersion === 1 && (set.slug
+      ? manifest.slug === set.slug &&
+        manifest.projectId === set.projectId &&
+        manifest.instructionSetId === set.instructionSetId
+      : manifest.name === set.name)
+  } catch { return false }
+}
+
+function existingDestinationIsLegacySynced(
+  destination: string,
+  name: string,
+  slug?: string
+) {
   try {
     if (!statSync(destination).isDirectory()) return false
-    const manifest = JSON.parse(readFileSync(join(destination, 'manifest.json'), 'utf8')) as { manifestVersion?: number; name?: string }
-    return manifest.manifestVersion === 1 && manifest.name === name
+    const manifest = JSON.parse(readFileSync(join(destination, 'manifest.json'), 'utf8')) as {
+      manifestVersion?: number
+      name?: string
+      slug?: string
+    }
+    return manifest.manifestVersion === 1 &&
+      manifest.name === name &&
+      manifest.slug === slug
   } catch { return false }
 }
 
+function removePublishedBackups(
+  paths: string[],
+  remove: typeof rmSync
+) {
+  let firstError: unknown = null
+  for (const path of paths) {
+    if (!existsSync(path)) continue
+    try {
+      remove(path, { recursive: true, force: true })
+    } catch (error) {
+      firstError ??= error
+    }
+  }
+  if (firstError) throw new Error('instruction_set_sync_cleanup_failed')
+}
+
 export function syncToDisk(out: string, set: SyncSet, fileOps: SyncFileOps = { writeFileSync }) {
-  safeSegment(set.name)
+  const directoryName = set.slug || set.name
+  safeSegment(directoryName)
+  const hasProjectId = set.projectId !== undefined
+  const hasInstructionSetId = set.instructionSetId !== undefined
+  if (set.slug && hasProjectId !== hasInstructionSetId) {
+    throw new Error('instruction_set_sync_identity_missing')
+  }
   for (const key of set.shape) safeSegment(key)
   const profileSlugs = new Set<string>()
   for (const profile of set.profiles) {
@@ -96,11 +173,28 @@ export function syncToDisk(out: string, set: SyncSet, fileOps: SyncFileOps = { w
     profileSlugs.add(value)
   }
   mkdirSync(out, { recursive: true })
-  const destination = join(out, set.name)
-  if (existsSync(destination) && !existingDestinationIsSynced(destination, set.name)) throw new Error('instruction_set_sync_destination_not_synced')
-  const temp = join(out, `.${set.name}.${randomUUID()}.tmp`)
+  const destination = join(out, directoryName)
+  const legacyDestination = set.slug && set.name !== set.slug && isSafeSegment(set.name)
+    ? join(out, set.name)
+    : null
+  if (existsSync(destination)) {
+    if (set.slug && existingDestinationIsLegacySynced(destination, set.name)) {
+      throw new Error('instruction_set_sync_legacy_identity_required')
+    }
+    if (!existingDestinationIsSynced(destination, set)) {
+      throw new Error('instruction_set_sync_destination_not_synced')
+    }
+  }
+  if (legacyDestination && existsSync(legacyDestination)) {
+    if (existingDestinationIsLegacySynced(legacyDestination, set.name)) {
+      throw new Error('instruction_set_sync_legacy_identity_required')
+    }
+    throw new Error('instruction_set_sync_destination_not_synced')
+  }
+  const priorDestinations = [destination]
+  const temp = join(out, `.${directoryName}.${randomUUID()}.tmp`)
   const move = fileOps.renameSync || renameSync
-  let backup: string | null = null
+  const backups: Array<{ original: string; backup: string }> = []
   mkdirSync(temp)
   try {
     const defaultMaterial = writeMaterial(temp, set.default, set.shape, 'default', fileOps)
@@ -114,18 +208,40 @@ export function syncToDisk(out: string, set: SyncSet, fileOps: SyncFileOps = { w
         ? Object.entries(profile.production.files).map(([key, path]) => ({ key, modelConfig: profile.modelConfigIdentity, path, syncedContentSha256: profile.production!.syncedContentSha256[key] }))
         : []),
     ]
-    const manifest = { manifestVersion: 1, name: set.name, ...(Object.hasOwn(set, 'description') ? { description: set.description ?? null } : {}), shape: set.shape, components, default: defaultMaterial, profiles, ...(set.filteredTo ? { filteredTo: [...set.filteredTo].sort() } : {}) }
+    const manifest = {
+      manifestVersion: 1,
+      ...(set.projectId ? { projectId: set.projectId } : {}),
+      ...(set.instructionSetId ? { instructionSetId: set.instructionSetId } : {}),
+      name: set.name,
+      ...(set.slug ? { slug: set.slug } : {}),
+      ...(Object.hasOwn(set, 'description') ? { description: set.description ?? null } : {}),
+      shape: set.shape,
+      components,
+      default: defaultMaterial,
+      profiles,
+      ...(set.filteredTo ? { filteredTo: [...set.filteredTo].sort() } : {}),
+    }
     fileOps.writeFileSync(join(temp, 'manifest.json'), stable(manifest))
-    backup = join(out, `.${set.name}.${randomUUID()}.bak`)
-    if (existsSync(destination)) move(destination, backup)
-    try { move(temp, destination) } catch (error) { if (existsSync(backup)) move(backup, destination); throw error }
-    removeOwnScratch([backup])
+    for (const prior of priorDestinations) {
+      if (!existsSync(prior)) continue
+      const backup = join(out, `.${directoryName}.${randomUUID()}.bak`)
+      move(prior, backup)
+      backups.push({ original: prior, backup })
+    }
+    move(temp, destination)
   } catch (error) {
     removeOwnScratch([temp])
-    if (backup && existsSync(backup) && !existsSync(destination)) move(backup, destination)
-    removeOwnScratch([backup])
+    for (const { original, backup } of [...backups].reverse()) {
+      if (existsSync(backup) && !existsSync(original)) move(backup, original)
+    }
+    removeOwnScratch(backups.map(entry => entry.backup))
     throw error
   }
+  removePublishedBackups(
+    backups.map(entry => entry.backup),
+    fileOps.rmSync || rmSync
+  )
+  return destination
 }
 
 export async function instructionSetsCommand(args: string[], io: InstructionSetsCommandIo): Promise<number> {
@@ -143,6 +259,44 @@ export async function instructionSetsCommand(args: string[], io: InstructionSets
     writeManifest = loadInstructionSetManifest(reference)
   }
   const project = await io.resolveProjectSlug(argValue(args, '--project'))
+  if (subcommand === 'scorers') {
+    const action = reference
+    const set = positionals[2]
+    const componentKey = argValue(args, '--key')
+    const scorerVersionId = argValue(args, '--scorer-version')
+    if (
+      (action !== 'set-headline' && action !== 'add') ||
+      !set ||
+      !componentKey ||
+      !scorerVersionId
+    ) {
+      throw new Error(
+        'Usage: orizu instructions scorers set-headline|add <set> --key <component-key> --scorer-version <id> --project <team/project> [--dataset-version <id> --split-set <id> --split <name>] [--json]'
+      )
+    }
+    const payload = await responsePayload(await authedFetch(
+      `/api/cli/instruction-sets/${encodeURIComponent(set)}/scorers?project=${encodeURIComponent(project)}`,
+      {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+          componentKey,
+          scorerVersionId,
+          role: action === 'set-headline' ? 'headline' : 'tracked',
+          datasetVersionId: argValue(args, '--dataset-version') || undefined,
+          splitSetId: argValue(args, '--split-set') || undefined,
+          splitName: argValue(args, '--split') || undefined,
+        }),
+      }
+    ), `Instruction sets scorers ${action}`)
+    if (io.json) io.print(JSON.stringify(payload))
+    else {
+      io.print(
+        `${action === 'set-headline' ? 'Set headline' : 'Added'} scorer ${sanitizeTerminalText(scorerVersionId)} for ${sanitizeTerminalText(set)} / ${sanitizeTerminalText(componentKey)}`
+      )
+    }
+    return 0
+  }
   if (subcommand === 'default') {
     const operation = reference; const set = positionals[2]; const identity = argValue(args, '--model-config'); const version = argValue(args, '--version')
     if ((operation !== 'show' && operation !== 'move') || !set) throw new Error('Usage: orizu instructions default show|move <set> --project <team/project> [--model-config <identity> --version <n>] [--json]')
@@ -270,8 +424,8 @@ export async function instructionSetsCommand(args: string[], io: InstructionSets
     if (!set || (set.slug !== reference && set.name !== reference) || (modelConfig && (!Array.isArray(set.filteredTo) || set.filteredTo.length !== 1 || set.filteredTo[0] !== modelConfig))) {
       throw new Error('instruction_set_sync_response_mismatch')
     }
-    syncToDisk(output, set)
-    if (io.json) io.print(JSON.stringify(payload)); else io.print(`Synced ${set.name} to ${join(output, set.name)}`)
+    const destination = syncToDisk(output, set)
+    if (io.json) io.print(JSON.stringify(payload)); else io.print(`Synced ${set.name} to ${destination}`)
     return 0
   }
   if (subcommand === 'show' && (!reference || reference.startsWith('--'))) throw new Error('Instruction set name is required')