@wyattjoh/demur 0.5.0 → 0.7.0

package/src/cli.ts

@@ -1,8 +1,13 @@
 #!/usr/bin/env bun
-import { createInterface } from "node:readline/promises";
 import { Predicate } from "effect";
 import { loadCostTotals } from "../extensions/demur/cost-tracker.ts";
 import {
+  loadDemurSettings,
+  saveDemurSettings,
+  type DemurSettings,
+} from "../extensions/demur/settings.ts";
+import {
+  isTrainingCorrectionReason,
   loadTrainingRecords,
   loadTrainingReviews,
   recordTrainingReview,
@@ -11,26 +16,39 @@ import {
   type TrainingReviewInput,
 } from "../extensions/demur/training-store.ts";
 import { guard } from "./guard.ts";
+import { judgeRenderedState, type JudgeResult } from "./judge.ts";
 import {
   deleteApiKey,
   resolveApiKey,
   storeApiKey,
   type ResolvedApiKey,
 } from "./key.ts";
+import {
+  analyzeTrainingFeedback,
+  replayTrainingQuestions,
+} from "./training-evaluation.ts";
 import {
   buildTrainingReviewEntries,
+  createTrainingReviewInput,
+  filterTrainingReviewEntries,
+  getLatestTrainingReview,
   getTrainingReviewFilter,
+  type TrainingReviewEntry,
+  type TrainingReviewFilter,
   type TrainingReviewSnapshot,
 } from "./training-review-model.ts";
 import type { TrainingReviewTuiResult } from "./training-review-tui.tsx";
-import type { Verdict } from "./types.ts";
+import type { RenderedCommandState, Verdict } from "./types.ts";
 
 const USAGE = `Usage:
   demur
   demur auth login
   demur auth status
   demur auth logout
-  demur training review [--plain]
+  demur training list [--status=<all|unreviewed|allow|ask|deny>] [--cwd=<query>] [--json]
+  demur training evaluate [--replay] [--limit=<1-100>] [--json]
+  demur training review
+  demur training review <record-id> --decision=<allow|ask|deny> [--reason=<correction-reason>] [--note=<text>] [--json]
   demur judge "<command>" [--cwd=<path>]`;
 
 /**
@@ -38,6 +56,7 @@ const USAGE = `Usage:
  */
 export type CliDependencies = {
   judge(command: string, cwd: string): Promise<Verdict>;
+  judgeTrainingState(state: RenderedCommandState): Promise<JudgeResult>;
   resolveApiKey(): Promise<ResolvedApiKey | undefined>;
   storeApiKey(value: string): Promise<void>;
   deleteApiKey(): Promise<boolean>;
@@ -45,28 +64,25 @@ export type CliDependencies = {
   loadTrainingReviews(): Promise<ReadonlyArray<TrainingReview>>;
   loadGlobalEstimatedCostUsd(): Promise<number>;
   recordTrainingReview(input: TrainingReviewInput): Promise<TrainingReview>;
+  loadDemurSettings(): Promise<DemurSettings>;
+  saveDemurSettings(settings: DemurSettings): Promise<void>;
   runTrainingReviewTui(
     snapshot: TrainingReviewSnapshot,
+    settings: DemurSettings,
     reloadSnapshot: () => Promise<TrainingReviewSnapshot>,
     recordReview: (input: TrainingReviewInput) => Promise<TrainingReview>,
+    saveSettings: (settings: DemurSettings) => Promise<void>,
   ): Promise<TrainingReviewTuiResult>;
   isInteractive(): boolean;
   readSecret(prompt: string): Promise<string>;
-  readLine(prompt: string): Promise<string>;
   cwd(): string;
   stdout(message: string): void;
   stderr(message: string): void;
 };
 
-let lineInput:
-  | {
-    terminal: ReturnType<typeof createInterface>;
-    lines: AsyncIterator<string>;
-  }
-  | undefined;
-
 const defaultDependencies: CliDependencies = {
   judge: (command, cwd) => guard(command, cwd, "cli"),
+  judgeTrainingState: judgeRenderedState,
   resolveApiKey,
   storeApiKey,
   deleteApiKey,
@@ -75,15 +91,28 @@ const defaultDependencies: CliDependencies = {
   loadGlobalEstimatedCostUsd: async () =>
     (await loadCostTotals()).estimatedCostUsd,
   recordTrainingReview,
-  runTrainingReviewTui: async (snapshot, reloadSnapshot, recordReview) => {
+  loadDemurSettings,
+  saveDemurSettings,
+  runTrainingReviewTui: async (
+    snapshot,
+    settings,
+    reloadSnapshot,
+    recordReview,
+    saveSettings,
+  ) => {
     const { runTrainingReviewTui } = await import(
       "./training-review-tui.tsx"
     );
-    return runTrainingReviewTui(snapshot, reloadSnapshot, recordReview);
+    return runTrainingReviewTui(
+      snapshot,
+      settings,
+      reloadSnapshot,
+      recordReview,
+      saveSettings,
+    );
   },
   isInteractive: () => Boolean(process.stdin.isTTY && process.stdout.isTTY),
   readSecret,
-  readLine,
   cwd: () => process.cwd(),
   stdout: (message) => console.log(message),
   stderr: (message) => console.error(message),
@@ -104,7 +133,7 @@ export async function runCli(
     if (args.length === 0) {
       if (!dependencies.isInteractive()) {
         dependencies.stderr(
-          "demur: the interactive app requires a terminal; use `demur training review --plain` for line-oriented review.",
+          "demur: the interactive app requires a terminal; use `demur training list --json` or review a record by ID.",
         );
         return 2;
       }
@@ -127,10 +156,17 @@ export async function runCli(
     const judgeArgs = args[0] === "judge" ? args.slice(1) : args;
     return await runJudge(judgeArgs, dependencies);
   } catch (error: unknown) {
-    dependencies.stderr(`demur: ${errorDetail(error)}`);
+    if (isTrainingJsonRequest(args)) {
+      writeTrainingJsonError(
+        trainingOperation(args),
+        "unexpected_error",
+        errorDetail(error),
+        dependencies,
+      );
+    } else {
+      dependencies.stderr(`demur: ${errorDetail(error)}`);
+    }
     return 1;
-  } finally {
-    closeLineInput();
   }
 }
 
@@ -201,43 +237,364 @@ async function runAuth(
   return 2;
 }
 
+type ParsedArguments = {
+  positionals: ReadonlyArray<string>;
+  options: ReadonlyMap<string, string | true>;
+};
+
+type ParseArgumentsResult =
+  | { parsed: ParsedArguments; error: undefined }
+  | { parsed: undefined; error: string };
+
 async function runTraining(
   args: ReadonlyArray<string>,
   dependencies: CliDependencies,
 ): Promise<number> {
-  const plain = args.includes("--plain");
-  const operands = args.filter((arg) => arg !== "--plain");
-  if (
-    operands.length !== 1 ||
-    operands[0] !== "review" ||
-    args.filter((arg) => arg === "--plain").length > 1
-  ) {
-    dependencies.stderr(USAGE);
+  const command = args[0];
+  if (command === "list") {
+    return runTrainingList(args.slice(1), dependencies);
+  }
+
+  if (command === "evaluate") {
+    return runTrainingEvaluate(args.slice(1), dependencies);
+  }
+
+  if (command === "review") {
+    if (args.length === 1) {
+      return runInteractiveTrainingReview(dependencies);
+    }
+    return runTrainingReview(args.slice(1), dependencies);
+  }
+
+  return writeTrainingUsageError(
+    trainingOperation(["training", ...args]),
+    args.includes("--json"),
+    "expected `training list`, `training evaluate`, or `training review`",
+    dependencies,
+  );
+}
+
+async function runInteractiveTrainingReview(
+  dependencies: CliDependencies,
+): Promise<number> {
+  if (!dependencies.isInteractive()) {
+    dependencies.stderr(
+      "demur: `training review` requires a terminal; use `training list --json` and review a record by ID.",
+    );
     return 2;
   }
 
   const loadSnapshot = () => loadTrainingReviewSnapshot(dependencies);
   const snapshot = await loadSnapshot();
+  const settings = await dependencies.loadDemurSettings();
+  const result = await dependencies.runTrainingReviewTui(
+    snapshot,
+    settings,
+    loadSnapshot,
+    dependencies.recordTrainingReview,
+    dependencies.saveDemurSettings,
+  );
+  printReviewSummary(result, dependencies);
+  return 0;
+}
 
-  if (!plain && dependencies.isInteractive()) {
-    const result = await dependencies.runTrainingReviewTui(
-      snapshot,
-      loadSnapshot,
-      dependencies.recordTrainingReview,
+async function runTrainingList(
+  args: ReadonlyArray<string>,
+  dependencies: CliDependencies,
+): Promise<number> {
+  const operation = "training.list";
+  const parsedResult = parseArguments(
+    args,
+    new Set(["json"]),
+    new Set(["status", "cwd"]),
+  );
+  const json = args.includes("--json");
+  if (parsedResult.error !== undefined) {
+    return writeTrainingUsageError(
+      operation,
+      json,
+      parsedResult.error,
+      dependencies,
+    );
+  }
+
+  const parsed = parsedResult.parsed;
+  if (parsed.positionals.length !== 0) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`training list` does not accept positional arguments",
+      dependencies,
     );
-    printReviewSummary(result, dependencies);
+  }
+
+  const statusValue = getStringOption(parsed, "status") ?? "all";
+  if (!isTrainingReviewFilter(statusValue)) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      `unsupported status: ${statusValue}`,
+      dependencies,
+    );
+  }
+
+  const cwdQuery = getStringOption(parsed, "cwd") ?? "";
+  const entries = filterTrainingReviewEntries(
+    await loadTrainingEntries(dependencies),
+    statusValue,
+    cwdQuery,
+  );
+
+  if (hasOption(parsed, "json")) {
+    writeTrainingJsonSuccess(
+      operation,
+      {
+        filter: {
+          status: statusValue,
+          cwd: cwdQuery === "" ? null : cwdQuery,
+        },
+        records: entries.map(trainingListRecord),
+      },
+      dependencies,
+    );
+    return 0;
+  }
+
+  if (entries.length === 0) {
+    dependencies.stdout("No matching demur training records.");
     return 0;
   }
 
-  const pending = buildTrainingReviewEntries(snapshot)
-    .filter((entry) => getTrainingReviewFilter(entry) === "unreviewed")
-    .map((entry) => entry.record);
-  if (pending.length === 0) {
-    dependencies.stdout("No unreviewed demur training records.");
+  dependencies.stdout(
+    `${entries.length} matching training record${entries.length === 1 ? "" : "s"}:`,
+  );
+  for (const entry of entries) {
+    const record = entry.record;
+    dependencies.stdout(
+      [
+        record.id,
+        getTrainingReviewFilter(entry),
+        `model=${record.verdict.decision}`,
+        record.cwd,
+        summarizeTrainingCommand(record.command),
+      ].join("\t"),
+    );
+  }
+  return 0;
+}
+
+async function runTrainingEvaluate(
+  args: ReadonlyArray<string>,
+  dependencies: CliDependencies,
+): Promise<number> {
+  const operation = "training.evaluate";
+  const parsedResult = parseArguments(
+    args,
+    new Set(["json", "replay"]),
+    new Set(["limit"]),
+  );
+  const json = args.includes("--json");
+  if (parsedResult.error !== undefined) {
+    return writeTrainingUsageError(
+      operation,
+      json,
+      parsedResult.error,
+      dependencies,
+    );
+  }
+
+  const parsed = parsedResult.parsed;
+  if (parsed.positionals.length !== 0) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`training evaluate` does not accept positional arguments",
+      dependencies,
+    );
+  }
+
+  const limitValue = getStringOption(parsed, "limit");
+  if (limitValue !== undefined && !hasOption(parsed, "replay")) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`--limit` requires `--replay`",
+      dependencies,
+    );
+  }
+  const replayLimit = limitValue === undefined ? 20 : Number(limitValue);
+  if (
+    !Number.isInteger(replayLimit) ||
+    replayLimit < 1 ||
+    replayLimit > 100
+  ) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`--limit` must be an integer from 1 to 100",
+      dependencies,
+    );
+  }
+
+  const entries = await loadTrainingEntries(dependencies);
+  const report = analyzeTrainingFeedback(entries);
+  const replay = hasOption(parsed, "replay")
+    ? await replayTrainingQuestions(
+      entries,
+      dependencies.judgeTrainingState,
+      replayLimit,
+    )
+    : undefined;
+  if (hasOption(parsed, "json")) {
+    writeTrainingJsonSuccess(
+      operation,
+      replay === undefined ? report : { offline: report, replay },
+      dependencies,
+    );
     return 0;
   }
 
-  return runPlainTrainingReview(pending, dependencies);
+  dependencies.stdout(
+    `Reviewed ${report.reviewed}; evaluable ${report.evaluable}; unavailable ${report.unavailable}.`,
+  );
+  dependencies.stdout(
+    `Replay fidelity: ${report.completeReplayRecords} complete, ${report.policyOnlyRecords} policy-only legacy.`,
+  );
+  dependencies.stdout(
+    `Current policy: ${report.current.matches}/${report.current.evaluated} match; weighted loss ${report.current.weightedLoss}.`,
+  );
+
+  const reasons = Object.entries(report.correctionsByReason);
+  if (reasons.length > 0) {
+    dependencies.stdout(
+      `Correction reasons: ${reasons.map(([reason, count]) => `${reason}=${count}`).join(", ")}.`,
+    );
+  }
+
+  if (report.candidates.length === 0) {
+    dependencies.stdout("No single-threshold candidate improved observed weighted loss.");
+  } else {
+    dependencies.stdout("Exploratory single-threshold candidates:");
+    for (const candidate of report.candidates) {
+      dependencies.stdout(
+        `  ${candidate.field}=${candidate.value}: ${candidate.metrics.matches}/${candidate.metrics.evaluated} match; weighted loss ${candidate.metrics.weightedLoss}`,
+      );
+    }
+  }
+
+  if (report.warning !== undefined) dependencies.stdout(`Warning: ${report.warning}`);
+  dependencies.stdout(
+    "Candidates are not applied automatically; validate them on an independent holdout and the synthetic corpus.",
+  );
+  if (replay !== undefined) {
+    dependencies.stdout(
+      `Current questions replay: ${replay.metrics.matches}/${replay.metrics.evaluated} match; ${replay.improved} improved; ${replay.regressed} regressed; ${replay.unavailable} unavailable; ${replay.skipped} skipped by limit.`,
+    );
+    dependencies.stdout(
+      `Replay usage: ${replay.inputTokens} input / ${replay.outputTokens} output tokens. Stored commands remained data and were never executed.`,
+    );
+  }
+  return 0;
+}
+
+async function runTrainingReview(
+  args: ReadonlyArray<string>,
+  dependencies: CliDependencies,
+): Promise<number> {
+  const operation = "training.review";
+  const parsedResult = parseArguments(
+    args,
+    new Set(["json"]),
+    new Set(["decision", "reason", "note"]),
+  );
+  const json = args.includes("--json");
+  if (parsedResult.error !== undefined) {
+    return writeTrainingUsageError(
+      operation,
+      json,
+      parsedResult.error,
+      dependencies,
+    );
+  }
+
+  const parsed = parsedResult.parsed;
+  if (parsed.positionals.length !== 1) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`training review` requires exactly one record ID",
+      dependencies,
+    );
+  }
+
+  const decisionValue = getStringOption(parsed, "decision");
+  if (decisionValue === undefined || !isDecision(decisionValue)) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`--decision` must be allow, ask, or deny",
+      dependencies,
+    );
+  }
+
+  const recordId = parsed.positionals[0];
+  const entries = await loadTrainingEntries(dependencies);
+  const entry = entries.find((candidate) => candidate.record.id === recordId);
+  if (entry === undefined) {
+    return writeTrainingDomainError(
+      operation,
+      hasOption(parsed, "json"),
+      "record_not_found",
+      `training record not found: ${recordId}`,
+      dependencies,
+    );
+  }
+
+  const corrected = decisionValue !== entry.record.verdict.decision;
+  const reasonValue = getStringOption(parsed, "reason");
+  if (corrected && !isTrainingCorrectionReason(reasonValue)) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "corrected reviews require `--reason` with a supported correction reason",
+      dependencies,
+    );
+  }
+  if (!corrected && reasonValue !== undefined) {
+    return writeTrainingUsageError(
+      operation,
+      hasOption(parsed, "json"),
+      "`--reason` is only valid when correcting the model decision",
+      dependencies,
+    );
+  }
+
+  const correctionReason = corrected && isTrainingCorrectionReason(reasonValue)
+    ? reasonValue
+    : undefined;
+  const previousReview = getLatestTrainingReview(entry) ?? null;
+  const review = await dependencies.recordTrainingReview(
+    createTrainingReviewInput(
+      entry.record,
+      decisionValue,
+      correctionReason,
+      getStringOption(parsed, "note"),
+    ),
+  );
+
+  if (hasOption(parsed, "json")) {
+    writeTrainingJsonSuccess(
+      operation,
+      { review, corrected, previousReview },
+      dependencies,
+    );
+    return 0;
+  }
+
+  dependencies.stdout(
+    `Recorded ${decisionValue.toUpperCase()} review for ${recordId}${corrected ? ` (model: ${entry.record.verdict.decision.toUpperCase()})` : " (accepted model decision)"}.`,
+  );
+  return 0;
 }
 
 async function loadTrainingReviewSnapshot(
@@ -251,44 +608,18 @@ async function loadTrainingReviewSnapshot(
   return { records, reviews, globalEstimatedCostUsd };
 }
 
-async function runPlainTrainingReview(
-  pending: ReadonlyArray<TrainingRecord>,
+async function loadTrainingEntries(
   dependencies: CliDependencies,
-): Promise<number> {
-  let reviewed = 0;
-  let corrected = 0;
-  let skipped = 0;
-  for (const [index, record] of pending.entries()) {
-    printTrainingRecord(record, index + 1, pending.length, dependencies);
-    const expectedDecision = await readExpectedDecision(record, dependencies);
-    if (expectedDecision === "quit") break;
-    if (expectedDecision === "skip") {
-      skipped += 1;
-      continue;
-    }
-
-    const isCorrection = expectedDecision !== record.verdict.decision;
-    const note = isCorrection
-      ? (await dependencies.readLine("Correction note (optional): ")).trim() ||
-        undefined
-      : undefined;
-    await dependencies.recordTrainingReview({
-      recordId: record.id,
-      originalDecision: record.verdict.decision,
-      expectedDecision,
-      note,
-    });
-    reviewed += 1;
-    if (isCorrection) corrected += 1;
-    dependencies.stdout(
-      isCorrection
-        ? `Recorded correction: ${record.verdict.decision} → ${expectedDecision}.`
-        : `Accepted ${record.verdict.decision} decision.`,
-    );
-  }
-
-  printReviewSummary({ reviewed, corrected, skipped }, dependencies);
-  return 0;
+): Promise<ReadonlyArray<TrainingReviewEntry>> {
+  const [records, reviews] = await Promise.all([
+    dependencies.loadTrainingRecords(),
+    dependencies.loadTrainingReviews(),
+  ]);
+  return buildTrainingReviewEntries({
+    records,
+    reviews,
+    globalEstimatedCostUsd: 0,
+  });
 }
 
 function printReviewSummary(
@@ -303,54 +634,162 @@ function printReviewSummary(
   );
 }
 
-async function readExpectedDecision(
-  record: TrainingRecord,
-  dependencies: CliDependencies,
-): Promise<"allow" | "ask" | "deny" | "skip" | "quit"> {
-  while (true) {
-    const input = (
-      await dependencies.readLine(
-        `Expected decision [enter=${record.verdict.decision}, allow, ask, deny, skip, quit]: `,
-      )
-    ).trim().toLowerCase();
-
-    if (input === "") return record.verdict.decision;
-    if (
-      input === "allow" ||
-      input === "ask" ||
-      input === "deny" ||
-      input === "skip" ||
-      input === "quit"
-    ) {
-      return input;
+function parseArguments(
+  args: ReadonlyArray<string>,
+  booleanOptions: ReadonlySet<string>,
+  valueOptions: ReadonlySet<string>,
+): ParseArgumentsResult {
+  const positionals: Array<string> = [];
+  const options = new Map<string, string | true>();
+
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+    if (argument === undefined) continue;
+    if (!argument.startsWith("--")) {
+      positionals.push(argument);
+      continue;
+    }
+
+    const separator = argument.indexOf("=");
+    const name = argument.slice(2, separator < 0 ? undefined : separator);
+    const inlineValue = separator < 0 ? undefined : argument.slice(separator + 1);
+    if (options.has(name)) {
+      return { parsed: undefined, error: `duplicate option: --${name}` };
     }
-    dependencies.stderr(`Unknown review choice: ${input}`);
+
+    if (booleanOptions.has(name)) {
+      if (inlineValue !== undefined) {
+        return {
+          parsed: undefined,
+          error: `option --${name} does not accept a value`,
+        };
+      }
+      options.set(name, true);
+      continue;
+    }
+
+    if (!valueOptions.has(name)) {
+      return { parsed: undefined, error: `unknown option: --${name}` };
+    }
+
+    const followingValue = args[index + 1];
+    const value = inlineValue ??
+      (followingValue !== undefined && !followingValue.startsWith("--")
+        ? followingValue
+        : undefined);
+    if (value === undefined) {
+      return {
+        parsed: undefined,
+        error: `option --${name} requires a value`,
+      };
+    }
+    if (inlineValue === undefined) index += 1;
+    options.set(name, value);
   }
+
+  return { parsed: { positionals, options }, error: undefined };
 }
 
-function printTrainingRecord(
-  record: TrainingRecord,
-  index: number,
-  total: number,
+function getStringOption(
+  parsed: ParsedArguments,
+  name: string,
+): string | undefined {
+  const value = parsed.options.get(name);
+  return typeof value === "string" ? value : undefined;
+}
+
+function hasOption(parsed: ParsedArguments, name: string): boolean {
+  return parsed.options.has(name);
+}
+
+function isTrainingReviewFilter(value: string): value is TrainingReviewFilter {
+  return value === "all" ||
+    value === "unreviewed" ||
+    value === "allow" ||
+    value === "ask" ||
+    value === "deny";
+}
+
+function isDecision(value: string): value is "allow" | "ask" | "deny" {
+  return value === "allow" || value === "ask" || value === "deny";
+}
+
+function trainingListRecord(entry: TrainingReviewEntry): object {
+  return {
+    status: getTrainingReviewFilter(entry),
+    record: entry.record,
+    reviews: entry.reviews,
+    latestReview: getLatestTrainingReview(entry) ?? null,
+  };
+}
+
+function summarizeTrainingCommand(command: string): string {
+  return command.replaceAll(/\s+/g, " ").trim();
+}
+
+function writeTrainingUsageError(
+  operation: string,
+  json: boolean,
+  message: string,
   dependencies: CliDependencies,
-): void {
-  dependencies.stdout("");
-  dependencies.stdout(
-    `[${index}/${total}] ${record.recordedAt} · ${record.mode} · host ${record.hostAction}`,
-  );
-  dependencies.stdout(`cwd: ${record.cwd}`);
-  dependencies.stdout(`command: ${record.command}`);
-  dependencies.stdout(
-    `verdict: ${record.verdict.decision.toUpperCase()} — ${record.verdict.reason}`,
-  );
-  if (record.verdict.judgments !== undefined) {
-    dependencies.stdout(
-      `judgments: ${JSON.stringify(record.verdict.judgments)}`,
+): number {
+  if (json) {
+    writeTrainingJsonError(
+      operation,
+      "invalid_arguments",
+      message,
+      dependencies,
     );
+  } else {
+    dependencies.stderr(`demur: ${message}`);
+    dependencies.stderr(USAGE);
   }
-  if (record.verdict.failure !== undefined) {
-    dependencies.stdout(`failure: ${record.verdict.failure}`);
+  return 2;
+}
+
+function writeTrainingDomainError(
+  operation: string,
+  json: boolean,
+  code: string,
+  message: string,
+  dependencies: CliDependencies,
+): number {
+  if (json) {
+    writeTrainingJsonError(operation, code, message, dependencies);
+  } else {
+    dependencies.stderr(`demur: ${message}`);
   }
+  return 1;
+}
+
+function writeTrainingJsonSuccess(
+  operation: string,
+  result: unknown,
+  dependencies: CliDependencies,
+): void {
+  dependencies.stdout(JSON.stringify({ version: 1, ok: true, operation, result }));
+}
+
+function writeTrainingJsonError(
+  operation: string,
+  code: string,
+  message: string,
+  dependencies: CliDependencies,
+): void {
+  dependencies.stdout(JSON.stringify({
+    version: 1,
+    ok: false,
+    operation,
+    error: { code, message },
+  }));
+}
+
+function isTrainingJsonRequest(args: ReadonlyArray<string>): boolean {
+  return args[0] === "training" && args.includes("--json");
+}
+
+function trainingOperation(args: ReadonlyArray<string>): string {
+  return `training.${args[1] ?? "unknown"}`;
 }
 
 async function runJudge(
@@ -403,26 +842,6 @@ async function runJudge(
   return 0;
 }
 
-async function readLine(prompt: string): Promise<string> {
-  if (lineInput === undefined) {
-    const terminal = createInterface({ input: process.stdin });
-    lineInput = {
-      terminal,
-      lines: terminal[Symbol.asyncIterator](),
-    };
-  }
-
-  process.stderr.write(prompt);
-  const next = await lineInput.lines.next();
-  if (next.done) throw new Error("training review input ended");
-  return next.value;
-}
-
-function closeLineInput(): void {
-  lineInput?.terminal.close();
-  lineInput = undefined;
-}
-
 async function readSecret(prompt: string): Promise<string> {
   if (!process.stdin.isTTY || typeof process.stdin.setRawMode !== "function") {
     return (await Bun.stdin.text()).trim();