orizu 0.3.4 → 0.3.6

package/src/index.ts

@@ -23,7 +23,10 @@ import { clearServerCredentials, getServerCredentials, saveServerCredentials } f
 import { parseDatasetFile } from './file-parser.js'
 import { streamJsonlRowChunks } from './jsonl-stream.js'
 import { parseDatasetReference } from './dataset-download.js'
+import { extractErrorMessage } from './error-response.js'
 import { parseGlobalFlags } from './global-flags.js'
+import { getCapabilities, renderHelpForArgs, renderRootHelp } from './help.js'
+import { runLocalAppPreview } from './preview-runtime.js'
 import {
   assertSecureTokenTransport,
   authedFetch,
@@ -31,6 +34,14 @@ import {
   resolveLoginBaseUrl,
   setGlobalFlags,
 } from './http.js'
+import {
+  getSkillInstallPath,
+  installSkillTarget,
+  isSkillInstallTarget,
+  SKILL_INSTALL_TARGETS,
+  SkillInstallTarget,
+  targetNeedsOverwrite,
+} from './skill-installer.js'
 import { formatTaskCreateError } from './task-create-error.js'
 import { LoginResponse } from './types.js'
 
@@ -163,6 +174,12 @@ interface ScorerSummary {
   implementationKind: string
 }
 
+interface ScoreSubmitInput {
+  resultsJsonl?: string
+  aggregate?: Record<string, unknown>
+  aggregateWarning?: string
+}
+
 interface RunnerExecContext {
   prompt: {
     body: string | null
@@ -263,14 +280,19 @@ function printVersion() {
 }
 
 function printUsage() {
-  printLine(`orizu global options:\n\n  --local                 Use http://localhost:3000\n  --server <url>          Use a specific server origin (for example: https://preview.example.com)\n  --version, -v           Print the orizu CLI version\n\norizu commands:\n\n  orizu login [--no-prompt-if-logged-in]\n  orizu logout\n  orizu whoami\n  orizu env [--project <team/project>] [--project-id <projectId>]\n  orizu log <event_type> --run-id <id> --sequence <n> --payload @event.json\n  orizu teams list\n  orizu teams create [--name <name>]\n  orizu teams members list [--team <teamSlug>]\n  orizu teams members add --email <email> [--team <teamSlug>]\n  orizu teams members remove --email <email> [--team <teamSlug>]\n  orizu teams members role --team <teamSlug> --email <email> --role <admin|member>\n  orizu projects list [--team <teamSlug>]\n  orizu projects create --name <name> [--team <teamSlug>]\n  orizu prompts list --project <team/project>\n  orizu prompts comments <prompt-id-or-name> --project <team/project> [--label <label> | --version <id>] [--json]\n  orizu prompts pull <prompt-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n  orizu prompts push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n  orizu prompts labels set <prompt-name> <label> --version <version-id> [--project <team/project>] [--json]\n  orizu prompts scorers set-headline <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n  orizu prompts scorers add <prompt-id> --scorer-version <id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n  orizu scorers list --project <team/project>\n  orizu scorers register --project <team/project> --name <name> --manifest <manifest.json> [--prompt-version <id>] [--runner-version <id>] [--label <label>] [--json]\n  orizu scorers detail <scorer-id-or-name> --project <team/project> [--json]\n  orizu scorers labels set <scorer-name> <label> --version <scorer-version-id> [--project <team/project>] [--json]\n  orizu scores submit <results.jsonl|results.json> --scorer-version <id> --subject-version <prompt-version-id> [--dataset-version <id> --split-set <id> --split <name>] [--project <team/project>] [--json]\n  orizu judges list --project <team/project>\n  orizu judges pull <judge-id-or-name> --project <team/project> --out <dir> [--label <label> | --version <id>] [--json]\n  orizu judges push <dir> [--runner-version <id>] [--project <team/project>] [--parent <version-id>] [--json]\n  orizu runners push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n  orizu runners exec (--prompt <prompt-version-id> | --prompt-version <id> --runner-version <id> | --scorer-version <id>) --dataset-version <id> --split-set <id-or-name> --split <name> [--runner-dir <dir>] --out <results.jsonl|results.jsonl.gz>\n  orizu optimizers push <dir> [--project <team/project>] [--name <name>] [--label <label>] [--json]\n  orizu runs submit <results.jsonl|results.jsonl.gz> --prompt-version <id> --runner-version <id> --dataset-version <id> --split-set <id> --split <name> [--project <team/project>]\n  orizu apps list [--project <team/project>]\n  orizu apps create --project <team/project> --name <name> --dataset <datasetId> --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n  orizu apps update [--app <appId>] [--project <team/project>] --file <path> --input-schema <json-path> --output-schema <json-path> [--component <name>]\n  orizu apps link-dataset --dataset <datasetId> [--app <appId>] [--project <team/project>] [--version <n>]\n  orizu apps detail --app <appId> [--project <team/project>] [--json]\n  orizu apps export [--app <appId>] [--project <team/project>] [--version <n>] [--out <path>]\n  orizu tasks list [--project <team/project>]\n  orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]\n  orizu tasks assign --task <taskId> --assignees <userId1,userId2>\n  orizu tasks status --task <taskId> [--json]\n  orizu tasks pause --task <taskId>\n  orizu tasks unpause --task <taskId>\n  orizu datasets upload --file <path> [--project <team/project>] [--name <name>] [--readme-file <README.md> | --readme-text <markdown>]\n  orizu datasets push <path> [--project <team/project>] [--name <name>] [--readme-file <README.md> | --readme-text <markdown>] [--json]\n  orizu datasets readme set <datasetId|dataset-name> [--project <team/project>] (--readme-file <README.md> | --readme-text <markdown>) [--json]\n  orizu datasets versions create <datasetId|dataset-name> [--project <team/project>] [--label <label>] [--readme-file <README.md> | --readme-text <markdown>] [--json]\n  orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--json]\n  orizu datasets download [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--format <csv|json|jsonl>] [--out <path>]\n  orizu datasets append [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n  orizu datasets edit-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --file <path>\n  orizu datasets delete-rows [--dataset <datasetId|datasetUrl>] [--project <team/project>] --row-ids <id1,id2>\n  orizu datasets delete [--dataset <datasetId|datasetUrl>] [--project <team/project>]\n  orizu datasets lock [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--reason <text>]\n  orizu datasets clone [--dataset <datasetId|datasetUrl>] [--project <team/project>] [--name <name>]\n  orizu tasks export [--task <taskId>] [--format <csv|json|jsonl>] [--out <path>]`)
+  printLine(renderRootHelp())
+}
+
+function printPreviewUsage() {
+  printLine(`\nLocal app preview:\n\n  orizu apps preview --file <path> --input-schema <json-path> --output-schema <json-path> --sample-row <json-path> [--screenshot <png-path>] [--headed] [--keep-open] [--component <name>]`)
 }
 
 function printOptimizationUsage() {
-  printLine(`\nOptimization lifecycle commands:\n\n  orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n  orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs]\n  orizu optimizations export <run-id> [--out <path>] [--json]\n  orizu optimizations pause <run-id> [--reason <text>] [--json]\n  orizu optimizations resume <run-id> [--json]\n  orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations fail <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations cancel <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--json]`)
+  printLine(`\nOptimization lifecycle commands:\n\n  orizu optimizations start --project <team/project> --optimizer-version <id> --prompt-version <id[,id]> --selection-scorer <id> [--reflection-scorer <id>] [--pareto-scorer <id>] [--best-scorer <id>] --dataset-version <id> --split-set <id> [--train-split <name>] [--validation-split <name>] [--metadata <json|@file>] [--json]\n  orizu optimizations run-gepa --project <team/project> --optimizer-version-id <id> --candidate-version-id <id> --runner-version-id <id> --candidate-runner-dir <dir> --scorer-version-id <id> --scorer-runner-version-id <id> --scorer-runner-dir <dir> --dataset-version-id <id> --split-set-id <id> [--train-split train] [--val-split validation] [--log-dir logs] [--no-skip-perfect-parent-reflection]\n  orizu optimizations export <run-id> [--out <path>] [--json]\n  orizu optimizations pause <run-id> [--reason <text>] [--json]\n  orizu optimizations resume <run-id> [--json]\n  orizu optimizations finish <run-id> [--best-score <n>] [--best-candidate <id>] [--result-prompt-version <id>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations fail <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--metadata <json|@file>] [--json]\n  orizu optimizations cancel <run-id> [--reason <text>] [--report <markdown|@file> | --report-file <path>] [--json]`)
 }
 
 let cliArgs = process.argv.slice(2)
+let cliJsonOutput = false
 
 function getArg(name: string): string | null {
   const index = cliArgs.indexOf(name)
@@ -281,6 +303,31 @@ function getArg(name: string): string | null {
   return cliArgs[index + 1]
 }
 
+function getArgs(name: string): string[] {
+  const values: string[] = []
+  for (let index = 0; index < cliArgs.length; index += 1) {
+    if (cliArgs[index] !== name) {
+      continue
+    }
+
+    const value = cliArgs[index + 1]
+    if (!value || value.startsWith('-')) {
+      throw new Error(`Usage: ${name} <value>`)
+    }
+
+    values.push(value)
+    index += 1
+  }
+
+  return values
+}
+
+function rejectDashPrefixedOptionValue(name: string, value: string | null) {
+  if (value && value.startsWith('-')) {
+    throw new Error(`Invalid value for ${name}: option values cannot start with a dash`)
+  }
+}
+
 export function normalizeSlugInput(slug: string): string {
   return slug.trim().toLowerCase()
 }
@@ -2001,18 +2048,23 @@ async function submitScoreResults() {
   const candidateId = getArg('--candidate')
 
   if (!resultsPath || !scorerVersionId || (!subjectPromptVersionId && (!optimizationRunId || !candidateId))) {
-    throw new Error('Usage: orizu scores submit <results.jsonl|results.json> --project <team/project> --scorer-version <id> (--subject-version <prompt-version-id> | --optimization-run <id> --candidate <id>) [--dataset-version <id> --split-set <id> --split <name>] [--json]')
+    throw new Error('Usage: orizu scores submit <results.jsonl|results.json> --project <team/project> --scorer-version <id> (--subject-version <prompt-version-id> | --optimization-run <id> --candidate <id>) [--aggregate] [--dataset-version <id> --split-set <id> --split <name>] [--json]')
   }
 
   const resultBytes = readSourceBytes(resultsPath)
   const raw = resultsPath.endsWith('.gz')
     ? gunzipSync(resultBytes).toString('utf8')
     : resultBytes.toString('utf8')
-  const resultsJsonl = normalizeScoreResultsInput(resultsPath, raw)
+  const scoreInput = normalizeScoreResultsInput(resultsPath, raw, hasArg('--aggregate'))
+  if (scoreInput.aggregateWarning) {
+    console.warn(scoreInput.aggregateWarning)
+  }
+  const aggregatePayload = scoreInput.aggregate || {}
   const response = await authedFetch(`/api/cli/scores/submit?project=${encodeURIComponent(project)}`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({
+      ...aggregatePayload,
       scorerVersionId,
       subjectPromptVersionId: subjectPromptVersionId || undefined,
       datasetVersionId: getArg('--dataset-version') || undefined,
@@ -2020,7 +2072,7 @@ async function submitScoreResults() {
       splitName: getArg('--split') || undefined,
       optimizationRunId: optimizationRunId || undefined,
       candidateId: candidateId || undefined,
-      resultsJsonl,
+      resultsJsonl: scoreInput.resultsJsonl,
     }),
   })
 
@@ -2046,25 +2098,161 @@ async function submitScoreResults() {
   )
 }
 
-function normalizeScoreResultsInput(sourcePath: string, raw: string): string {
+function looksLikeAggregateScoreObject(value: Record<string, unknown>): boolean {
+  return (
+    value.scoreValue !== undefined ||
+    value.score_value !== undefined ||
+    value.diagnostics !== undefined ||
+    value.feedbackSummary !== undefined ||
+    value.feedback_summary !== undefined ||
+    value.rowEvidence !== undefined ||
+    value.row_evidence !== undefined ||
+    value.dependencyScoreRunIds !== undefined ||
+    value.dependency_score_run_ids !== undefined
+  )
+}
+
+function numberFromUnknown(value: unknown): number | null {
+  if (typeof value === 'number' && Number.isFinite(value)) return value
+  if (typeof value === 'string' && value.trim()) {
+    const parsed = Number(value)
+    return Number.isFinite(parsed) ? parsed : null
+  }
+  if (typeof value === 'boolean') return value ? 1 : 0
+  return null
+}
+
+function normalizeScoreResultsInput(sourcePath: string, raw: string, aggregateMode = false): ScoreSubmitInput {
   const logicalPath = sourcePath.endsWith('.gz') ? sourcePath.slice(0, -3) : sourcePath
-  if (!logicalPath.endsWith('.json')) return raw
+  if (!logicalPath.endsWith('.json')) return { resultsJsonl: raw }
 
   try {
     const parsed = JSON.parse(raw) as unknown
+    if (aggregateMode) {
+      if (!isRecord(parsed)) {
+        throw new Error('Aggregate score results JSON must be an object')
+      }
+      return { aggregate: parsed }
+    }
+    if (isRecord(parsed) && looksLikeAggregateScoreObject(parsed)) {
+      return {
+        aggregate: parsed,
+        aggregateWarning: 'Warning: aggregate score JSON detected. Pass --aggregate to make this explicit.',
+      }
+    }
     const rows = Array.isArray(parsed) ? parsed : [parsed]
     if (rows.some(row => !isRecord(row))) {
       throw new Error('JSON score results must be an object or an array of objects')
     }
-    return rows.map(row => JSON.stringify(row)).join('\n')
+    return { resultsJsonl: rows.map(row => JSON.stringify(row)).join('\n') }
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error)
     throw new Error(`Invalid score results JSON '${sourcePath}': ${message}`)
   }
 }
 
+function readOptionalJsonlFile(pathArg: string | null): string | undefined {
+  if (!pathArg) return undefined
+  const bytes = readSourceBytes(pathArg)
+  return pathArg.endsWith('.gz')
+    ? gunzipSync(bytes).toString('utf8')
+    : bytes.toString('utf8')
+}
+
+function parseAliasAssignments(value: string | null): Record<string, string> {
+  const result: Record<string, string> = {}
+  if (!value) return result
+  for (const item of parseCommaSeparated(value)) {
+    const equalsIndex = item.indexOf('=')
+    if (equalsIndex <= 0 || equalsIndex === item.length - 1) {
+      throw new Error(`Expected alias assignment in form alias=value, got '${item}'`)
+    }
+    result[item.slice(0, equalsIndex)] = item.slice(equalsIndex + 1)
+  }
+  return result
+}
+
+function readDependencyResultAssignments(value: string | null): Record<string, string> | undefined {
+  const assignments = parseAliasAssignments(value)
+  const result: Record<string, string> = {}
+  for (const [alias, path] of Object.entries(assignments)) {
+    result[alias] = readOptionalJsonlFile(path) || ''
+  }
+  return Object.keys(result).length > 0 ? result : undefined
+}
+
+async function execScorer() {
+  const project = getArg('--project') || await resolveProjectSlug(null)
+  const scorerVersionId = getArg('--scorer-version')
+  const subjectPromptVersionId = getArg('--subject-version') || getArg('--prompt-version')
+  const optimizationRunId = getArg('--optimization-run')
+  const candidateId = getArg('--candidate')
+  const datasetVersionId = getArg('--dataset-version')
+  const splitSetId = getArg('--split-set')
+  const splitName = getArg('--split')
+  const outPath = getArg('--out')
+
+  if (
+    !scorerVersionId ||
+    (!subjectPromptVersionId && (!optimizationRunId || !candidateId)) ||
+    !datasetVersionId ||
+    !splitSetId ||
+    !splitName
+  ) {
+    throw new Error('Usage: orizu scorers exec --scorer-version <id> (--subject-version <prompt-version-id> | --optimization-run <id> --candidate <id>) --dataset-version <id> --split-set <id> --split <name> [--subject-results <jsonl>] [--dependency-score-run <alias=id>] [--dependency-results <alias=path>] [--no-submit] [--out <score.json>] [--project <team/project>] [--json]')
+  }
+
+  const response = await authedFetch(`/api/cli/scorers/exec?project=${encodeURIComponent(project)}`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      scorerVersionId,
+      subjectPromptVersionId: subjectPromptVersionId || undefined,
+      optimizationRunId: optimizationRunId || undefined,
+      candidateId: candidateId || undefined,
+      datasetVersionId,
+      splitSetId,
+      splitName,
+      submit: !hasArg('--no-submit'),
+      reuseExisting: !hasArg('--no-reuse-existing'),
+      subjectResultsJsonl: readOptionalJsonlFile(getArg('--subject-results') || getArg('--outputs')),
+      dependencyScoreRunIds: parseAliasAssignments(getArg('--dependency-score-run') || getArg('--dependency-score-runs')),
+      dependencyResultsJsonl: readDependencyResultAssignments(getArg('--dependency-results')),
+    }),
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to execute scorer: ${await response.text()}`)
+  }
+
+  const data = await parseJsonResponse<Record<string, unknown>>(response, 'Scorer exec')
+  if (outPath) {
+    writeFileSync(expandHomePath(outPath), `${JSON.stringify(data.scoreResult || data, null, 2)}\n`)
+  }
+  if (hasJsonFlag()) {
+    printJson(data)
+    return
+  }
+
+  const scoreRun = isRecord(data.scoreRun) ? data.scoreRun : null
+  const scoreResult = isRecord(data.scoreResult) ? data.scoreResult : null
+  const scoreValue = scoreRun
+    ? scoreRun.scoreValue
+    : scoreResult
+      ? scoreResult.scoreValue
+      : null
+  if (scoreRun && typeof scoreRun.id === 'string') {
+    printLine(`Executed scorer ${sanitizeTerminalText(scorerVersionId)} -> score ${sanitizeTerminalText(scoreRun.id)} (${formatPercent(numberFromUnknown(scoreValue))})`)
+  } else {
+    printLine(`Executed scorer ${sanitizeTerminalText(scorerVersionId)} (${formatPercent(numberFromUnknown(scoreValue))})`)
+  }
+  if (outPath) {
+    printLine(`Wrote scorer result to ${sanitizeTerminalText(outPath)}`)
+  }
+}
+
 type OptimizationLifecycleAction = 'pause' | 'resume' | 'finish' | 'fail' | 'cancel'
-const OPTIMIZATION_REPORT_MAX_BYTES = 2 * 1024 * 1024
+const MARKDOWN_REPORT_MAX_BYTES = 2 * 1024 * 1024
 
 function parseOptionalNumberFlag(name: string): number | undefined {
   const value = getArg(name)
@@ -2084,10 +2272,15 @@ function hasObjectKeys(value: Record<string, unknown>): boolean {
   return Object.keys(value).length > 0
 }
 
-function readOptimizationReportInput(): { markdown: string; sourceName: string | null } | null {
+function readMarkdownReportInput(
+  reportLabel: 'Optimization' | 'Task'
+): { markdown: string; sourceName: string | null } | null {
   const report = getArg('--report')
   const reportFile = getArg('--report-file')
 
+  rejectDashPrefixedOptionValue('--report', report)
+  rejectDashPrefixedOptionValue('--report-file', reportFile)
+
   if (report && reportFile) {
     throw new Error('Use either --report or --report-file, not both')
   }
@@ -2114,16 +2307,24 @@ function readOptimizationReportInput(): { markdown: string; sourceName: string |
   }
 
   if (!markdown.trim()) {
-    throw new Error('Optimization report markdown must not be blank')
+    throw new Error(`${reportLabel} report markdown must not be blank`)
   }
 
-  if (Buffer.byteLength(markdown, 'utf8') > OPTIMIZATION_REPORT_MAX_BYTES) {
-    throw new Error(`Optimization report exceeds ${OPTIMIZATION_REPORT_MAX_BYTES} bytes`)
+  if (Buffer.byteLength(markdown, 'utf8') > MARKDOWN_REPORT_MAX_BYTES) {
+    throw new Error(`${reportLabel} report exceeds ${MARKDOWN_REPORT_MAX_BYTES} bytes`)
   }
 
   return { markdown, sourceName }
 }
 
+function readOptimizationReportInput(): { markdown: string; sourceName: string | null } | null {
+  return readMarkdownReportInput('Optimization')
+}
+
+function readTaskReportInput(): { markdown: string; sourceName: string | null } | null {
+  return readMarkdownReportInput('Task')
+}
+
 async function startOptimizationRun() {
   const project = getArg('--project') || await resolveProjectSlug(null)
   const optimizerVersionId = getArg('--optimizer-version')
@@ -2521,13 +2722,120 @@ function readSourceBytes(pathArg: string): Buffer {
 }
 
 function hasJsonFlag(): boolean {
-  return hasArg('--json')
+  return cliJsonOutput || hasArg('--json')
 }
 
 function printJson(value: Record<string, unknown>) {
   printLine(JSON.stringify(value))
 }
 
+function printCapabilities() {
+  const capabilities = getCapabilities(getCliVersion())
+  if (hasJsonFlag()) {
+    printLine(JSON.stringify(capabilities))
+    return
+  }
+
+  printLine('orizu capabilities')
+  printLine('')
+  printLine('Run `orizu capabilities --json` for a machine-readable command manifest.')
+  printLine('')
+  for (const command of capabilities.commands) {
+    printLine(`  ${command.name.padEnd(34)} ${command.help}`)
+  }
+}
+
+async function askYesNo(question: string, defaultYes: boolean): Promise<boolean> {
+  const rl = createInterface({ input, output })
+  try {
+    const suffix = defaultYes ? 'Y/n' : 'y/N'
+    const answer = (await rl.question(`${question} (${suffix}) `)).trim().toLowerCase()
+    if (!answer) {
+      return defaultYes
+    }
+    return answer === 'y' || answer === 'yes'
+  } finally {
+    rl.close()
+  }
+}
+
+async function promptSkillInstallTargets(): Promise<SkillInstallTarget[]> {
+  if (!isInteractiveTerminal()) {
+    throw new Error(
+      `Usage: orizu install-skill --target <${SKILL_INSTALL_TARGETS.join('|')}> [--yes] [--dry-run]`
+    )
+  }
+
+  printLine('Where should the Orizu CLI skill be installed?')
+  const targets: SkillInstallTarget[] = []
+  for (const target of SKILL_INSTALL_TARGETS) {
+    const defaultYes = target === 'agent-user'
+    if (await askYesNo(`  Install to ${target}?`, defaultYes)) {
+      targets.push(target)
+    }
+  }
+
+  return targets
+}
+
+function parseSkillInstallTargets(): SkillInstallTarget[] {
+  const rawTargets = getArgs('--target')
+  const targets: SkillInstallTarget[] = []
+  for (const target of rawTargets) {
+    if (!isSkillInstallTarget(target)) {
+      throw new Error(
+        `Unknown skill install target '${target}'. Available targets: ${SKILL_INSTALL_TARGETS.join(', ')}`
+      )
+    }
+    if (!targets.includes(target)) {
+      targets.push(target)
+    }
+  }
+  return targets
+}
+
+function formatSkillInstallAction(action: string): string {
+  if (action === 'created') return 'Wrote'
+  if (action === 'updated') return 'Updated'
+  if (action === 'would-create') return 'Would write'
+  return 'Would update'
+}
+
+async function installSkillCommand() {
+  const skipConfirm = hasArg('--yes')
+  const dryRun = hasArg('--dry-run')
+  let targets = parseSkillInstallTargets()
+  if (targets.length === 0) {
+    targets = await promptSkillInstallTargets()
+  }
+
+  if (targets.length === 0) {
+    printLine('No targets selected; nothing to do.')
+    return
+  }
+
+  for (const target of targets) {
+    let overwrite = skipConfirm
+    const installPath = getSkillInstallPath(target)
+    if (!dryRun && !overwrite && targetNeedsOverwrite(target)) {
+      if (!isInteractiveTerminal()) {
+        throw new Error(`${installPath} already exists. Pass --yes to replace it.`)
+      }
+      overwrite = await askYesNo(`Replace ${installPath}?`, true)
+      if (!overwrite) {
+        printLine(`Skipped ${installPath}`)
+        continue
+      }
+    }
+
+    const result = installSkillTarget(target, {
+      overwrite,
+      dryRun,
+    })
+    printLine(`${formatSkillInstallAction(result.action)} ${result.path}`)
+  }
+}
+
 function readJsonFile(pathArg: string): Record<string, unknown> {
   const raw = readSourceFile(pathArg)
   try {
@@ -2789,15 +3097,61 @@ async function createAppFromFile() {
   }
 
   const data = await parseJsonResponse<{
-    app: { id: string; name: string; versionNum: number; componentName?: string }
+    app: { id: string; name: string; versionNum: number; componentName?: string; url?: string }
     warnings?: string[]
   }>(response, 'App create')
   printLine(`Created app ${sanitizeTerminalText(data.app.name)} (${sanitizeTerminalText(data.app.id)}) v${data.app.versionNum}`)
+  if (data.app.url) {
+    printLine(`View app: ${formatTerminalLink(data.app.url)}`)
+  }
   if (data.warnings?.length) {
     printLine(`Warnings: ${sanitizeTerminalText(data.warnings.join('; '))}`)
   }
 }
 
+async function previewAppFromFile() {
+  const filePath = getArg('--file')
+  const inputSchemaPath = getArg('--input-schema')
+  const outputSchemaPath = getArg('--output-schema')
+  const sampleRowPath = getArg('--sample-row')
+  const screenshotPath = getArg('--screenshot')
+  const component = getArg('--component') || undefined
+  const headed = hasArg('--headed')
+  const keepOpen = hasArg('--keep-open')
+
+  if (!filePath || !inputSchemaPath || !outputSchemaPath || !sampleRowPath) {
+    throw new Error('Usage: orizu apps preview --file <path> --input-schema <json-path> --output-schema <json-path> --sample-row <json-path> [--screenshot <png-path>] [--headed] [--keep-open] [--component <name>]')
+  }
+
+  const expandedFilePath = realpathSync(expandHomePath(filePath))
+  const inputJsonSchema = readJsonFile(inputSchemaPath)
+  const outputJsonSchema = readJsonFile(outputSchemaPath)
+  const sampleRow = readJsonFile(sampleRowPath)
+  const expandedScreenshotPath = screenshotPath ? expandHomePath(screenshotPath) : undefined
+
+  const result = await runLocalAppPreview({
+    filePath: expandedFilePath,
+    inputSchema: inputJsonSchema,
+    outputSchema: outputJsonSchema,
+    sampleRow,
+    screenshotPath: expandedScreenshotPath,
+    componentName: component,
+    headed,
+    keepOpen,
+  })
+
+  printLine(`Preview rendered: ${formatTerminalLink(result.url)}`)
+  if (result.screenshotPath) {
+    printLine(`Screenshot: ${sanitizeTerminalText(result.screenshotPath)}`)
+  }
+  if (headed || keepOpen) {
+    printLine('Headed preview is running. Close Chromium or stop the command when you are done.')
+  }
+  if (result.warnings.length > 0) {
+    printLine(`Warnings: ${sanitizeTerminalText(result.warnings.join('; '))}`)
+  }
+}
+
 async function updateAppFromFile() {
   const filePath = getArg('--file')
   const inputSchemaPath = getArg('--input-schema')
@@ -2905,6 +3259,7 @@ async function createTask() {
   const appId = getArg('--app')
   const title = getArg('--title')
   const assignees = parseCommaSeparated(getArg('--assignees'))
+  const publish = hasArg('--publish')
   const versionArg = getArg('--version')
   const instructions = getArg('--instructions')
   const labelsPerItemArg = getArg('--labels-per-item')
@@ -2913,8 +3268,8 @@ async function createTask() {
   const versionNum =
     Number.isInteger(parsedVersionNum) && parsedVersionNum > 0 ? parsedVersionNum : null
 
-  if (!projectSlug || !datasetId || !appId || !title || assignees.length === 0) {
-    throw new Error('Usage: orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> --assignees <userIdOrEmail1,userIdOrEmail2> [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]')
+  if (!projectSlug || !datasetId || !appId || !title || (publish && assignees.length === 0)) {
+    throw new Error('Usage: orizu tasks create --project <team/project> --dataset <datasetId> --app <appId> --title <title> [--assignees <userIdOrEmail1,userIdOrEmail2>] [--publish] [--version <n>] [--instructions <text>] [--labels-per-item <n>] [--json]')
   }
 
   if (versionArg && versionNum === null) {
@@ -2931,6 +3286,7 @@ async function createTask() {
       versionNum,
       title,
       memberIds: assignees,
+      publish,
       instructions,
       requiredAssignmentsPerRow: labelsPerItem,
     }),
@@ -2973,6 +3329,13 @@ async function createTask() {
       status: data.task.status,
       requiredAssignmentsPerRow: data.task.requiredAssignmentsPerRow,
       assignmentsCreated: data.assignmentsCreated,
+      draft: data.task.status === 'draft',
+      ...(data.task.status === 'draft'
+        ? {
+            message: 'Draft created. Test it manually before assigning.',
+            publishCommand: `orizu tasks publish --task ${data.task.id} --assignees <userId1,userId2>`,
+          }
+        : {}),
       ...(data.assignmentShortfall !== undefined ? { assignmentShortfall: data.assignmentShortfall } : {}),
       ...(data.warning ? { warning: data.warning } : {}),
     }, null, 2))
@@ -2981,15 +3344,76 @@ async function createTask() {
 
   const baseUrl = getBaseUrl()
   const taskUrl = `${baseUrl}/d/${projectSlug}/tasks/${data.task.id}`
+  const isDraft = data.task.status === 'draft'
   printLine(
-    `Created task ${sanitizeTerminalText(data.task.title)} (${sanitizeTerminalText(data.task.id)}) [${sanitizeTerminalText(data.task.status)}]` +
+    `Created ${isDraft ? 'draft ' : ''}task ${sanitizeTerminalText(data.task.title)} (${sanitizeTerminalText(data.task.id)}) [${sanitizeTerminalText(data.task.status)}]` +
     `\n  Task ID:    ${sanitizeTerminalText(data.task.id)}` +
     `\n  Dataset ID: ${sanitizeTerminalText(datasetId)}` +
     `\n  Version:    v${data.task.versionNum} (${sanitizeTerminalText(data.task.versionId)})` +
     `\n  Labels/row: ${data.task.requiredAssignmentsPerRow}` +
     `\n  Assignments: ${data.assignmentsCreated}` +
     (data.warning ? `\n  Warning:    ${sanitizeTerminalText(data.warning)}` : '') +
-    `\n  URL:        ${sanitizeTerminalText(taskUrl)}`
+    `\n  URL:        ${sanitizeTerminalText(taskUrl)}` +
+    (isDraft
+      ? `\n\nThis task is a draft. Test it manually before assigning.\nPublish after approval: orizu tasks publish --task ${sanitizeTerminalText(data.task.id)} --assignees <userId1,userId2>`
+      : '')
+  )
+}
+
+async function publishTask() {
+  const taskId = getArg('--task')
+  const assignees = parseCommaSeparated(getArg('--assignees'))
+
+  if (!taskId || assignees.length === 0) {
+    throw new Error('Usage: orizu tasks publish --task <taskId> --assignees <userId1,userId2> [--json]')
+  }
+
+  const response = await authedFetch(`/api/cli/tasks/${encodeURIComponent(taskId)}/publish`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ memberIds: assignees }),
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to publish task: ${await extractErrorMessage(response)}`)
+  }
+
+  const data = await parseJsonResponse<{
+    task: { id: string; status: string }
+    assignmentsProcessed: number
+    assignmentsCreated: number
+    assignmentsRemoved?: number
+    assignmentShortfall?: number
+    warning?: string
+  }>(
+    response,
+    'Task publish'
+  )
+
+  if (hasArg('--json')) {
+    printLine(JSON.stringify({
+      taskId: data.task.id,
+      status: data.task.status,
+      assignmentsCreated: data.assignmentsCreated,
+      assignmentsProcessed: data.assignmentsProcessed,
+      ...(data.assignmentsRemoved !== undefined
+        ? { assignmentsRemoved: data.assignmentsRemoved }
+        : {}),
+      ...(data.assignmentShortfall !== undefined
+        ? { assignmentShortfall: data.assignmentShortfall }
+        : {}),
+      ...(data.warning ? { warning: data.warning } : {}),
+    }, null, 2))
+    return
+  }
+
+  printLine(
+    `Published task ${sanitizeTerminalText(data.task.id)} [${sanitizeTerminalText(data.task.status)}]` +
+    `\n  Assignments: ${data.assignmentsCreated}` +
+    (data.assignmentsRemoved !== undefined
+      ? `\n  Replaced:    ${data.assignmentsRemoved} old assignments removed`
+      : '') +
+    (data.warning ? `\n  Warning:     ${sanitizeTerminalText(data.warning)}` : '')
   )
 }
 
@@ -3082,6 +3506,58 @@ async function updateTaskStatus(targetStatus: 'paused' | 'active') {
   printLine(`${action} task ${sanitizeTerminalText(data.task.id)} [${sanitizeTerminalText(data.task.status)}]`)
 }
 
+async function setTaskReport() {
+  const taskId = getArg('--task')
+  rejectDashPrefixedOptionValue('--task', taskId)
+  if (!taskId) {
+    throw new Error(
+      'Usage: orizu tasks report set --task <taskId> (--report <markdown|@file> | --report-file <path>) [--json]'
+    )
+  }
+
+  const report = readTaskReportInput()
+  if (!report) {
+    throw new Error(
+      'Usage: orizu tasks report set --task <taskId> (--report <markdown|@file> | --report-file <path>) [--json]'
+    )
+  }
+
+  const response = await authedFetch(`/api/cli/tasks/${encodeURIComponent(taskId)}/report`, {
+    method: 'PATCH',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      reportMarkdown: report.markdown,
+      reportSourceName: report.sourceName,
+    }),
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to update task report: ${await response.text()}`)
+  }
+
+  const data = await parseJsonResponse<{
+    task: {
+      id: string
+      status: string
+      report: {
+        markdown: string
+        sourceName: string | null
+        createdAt: string | null
+      }
+    }
+  }>(response, 'Task report update')
+
+  if (hasJsonFlag()) {
+    printJson(data)
+    return
+  }
+
+  printLine(
+    `Uploaded task report for ${sanitizeTerminalText(data.task.id)} ` +
+    `[${sanitizeTerminalText(data.task.status)}]`
+  )
+}
+
 interface AppDetailPayload {
   id: string
   name: string
@@ -4236,11 +4712,45 @@ async function runnersExec() {
   }
 }
 
+function isHelpToken(value: string): boolean {
+  return value === '--help' || value === '-h'
+}
+
+function extractHelpTarget(args: string[]): string[] | null {
+  if (args[0] === 'help') {
+    return helpTargetFromArgs(args.slice(1))
+  }
+
+  if (!args.some(isHelpToken)) {
+    return null
+  }
+
+  return helpTargetFromArgs(args)
+}
+
+function helpTargetFromArgs(args: string[]): string[] {
+  const targetArgs = args.filter(arg => !isHelpToken(arg) && arg !== '--json')
+  const optionIndex = targetArgs.findIndex(arg => arg.startsWith('-'))
+  return optionIndex === -1 ? targetArgs : targetArgs.slice(0, optionIndex)
+}
+
 export async function main(rawArgs = process.argv.slice(2)) {
   const parsed = parseGlobalFlags(rawArgs)
   setGlobalFlags(parsed.flags)
   cliArgs = parsed.args
 
+  cliJsonOutput = false
+  if (cliArgs[0] === '--json') {
+    cliJsonOutput = true
+    cliArgs = cliArgs.slice(1)
+  }
+
+  const helpTarget = extractHelpTarget(cliArgs)
+  if (helpTarget !== null) {
+    printLine(renderHelpForArgs(helpTarget))
+    return
+  }
+
   const command = cliArgs[0]
   const subcommand = cliArgs[1]
 
@@ -4251,10 +4761,24 @@ export async function main(rawArgs = process.argv.slice(2)) {
 
   if (!command) {
     printUsage()
-    printOptimizationUsage()
     process.exit(1)
   }
 
+  if (command === 'capabilities') {
+    printCapabilities()
+    return
+  }
+
+  if (command === 'install-skill') {
+    await installSkillCommand()
+    return
+  }
+
+  if (command === 'skills' && subcommand === 'install') {
+    await installSkillCommand()
+    return
+  }
+
   if (command === 'login') {
     await login()
     return
@@ -4388,6 +4912,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
     return
   }
 
+  if (command === 'scorers' && subcommand === 'exec') {
+    await execScorer()
+    return
+  }
+
   if (command === 'runners' && subcommand === 'push') {
     await pushRunnerArtifact('runner')
     return
@@ -4452,6 +4981,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
     return
   }
 
+  if (command === 'apps' && subcommand === 'preview') {
+    await previewAppFromFile()
+    return
+  }
+
   if (command === 'apps' && subcommand === 'update') {
     await updateAppFromFile()
     return
@@ -4482,6 +5016,11 @@ export async function main(rawArgs = process.argv.slice(2)) {
     return
   }
 
+  if (command === 'tasks' && subcommand === 'publish') {
+    await publishTask()
+    return
+  }
+
   if (command === 'tasks' && subcommand === 'assign') {
     await assignTask()
     return
@@ -4492,6 +5031,15 @@ export async function main(rawArgs = process.argv.slice(2)) {
     return
   }
 
+  if (
+    command === 'tasks' &&
+    subcommand === 'report' &&
+    (cliArgs[2] === 'set' || cliArgs[2] === 'upload')
+  ) {
+    await setTaskReport()
+    return
+  }
+
   if (command === 'tasks' && subcommand === 'pause') {
     await updateTaskStatus('paused')
     return
@@ -4569,7 +5117,6 @@ export async function main(rawArgs = process.argv.slice(2)) {
   }
 
   printUsage()
-  printOptimizationUsage()
   process.exit(1)
 }