crewhaus 0.1.6 → 0.1.8

package/dist/index.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env bun
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { basename, dirname, join, relative, resolve } from "node:path";
 import { SpecParseError, compile, lower } from "@crewhaus/compiler";
 import { buildContextBundle, discoverRoots } from "@crewhaus/context-bundle";
@@ -9,6 +10,7 @@ import { parseGradersConfig } from "@crewhaus/eval-grader";
 import { optimizeSpec } from "@crewhaus/eval-optimizer-orchestrator";
 import { diffReports, loadRun, renderReport } from "@crewhaus/eval-report";
 import { runEval as runEvalLib } from "@crewhaus/eval-runner";
+import { openEventLog } from "@crewhaus/event-log";
 import { loadHooks } from "@crewhaus/hooks-engine";
 import { ArgParseError, parseArgs, } from "@crewhaus/infra-utils";
 import { createLogger } from "@crewhaus/logging";
@@ -33,6 +35,10 @@ import { buildCredentialChecks, extractSpecModel } from "./doctor-checks";
 // imports the optional semantic package only when "semantic" is selected, so
 // the default path pulls in no embedding dependency.
 import { DEFAULT_EGRESS_EMBEDDER_MODEL, InvalidEgressMatcherChoiceError, createEgressMatcher, resolveEgressMatcherChoice, } from "./egress-matcher";
+// Response-feedback core — pure, side-effect-free so it is unit-testable
+// (this entry file runs an argv switch on import). Powers `rate`/`feedback`
+// (capture) and `distill` (ratings → eval dataset + graders).
+import { FEEDBACK_EVENT_KIND, buildFeedbackRecord, deriveTurns, distill as distillFeedback, extractFeedbackRecords, gradersConfigToYaml, samplesToJsonl, } from "./feedback";
 // FR-004 — Pillar 3 intent-gate judge + durable audit-sink resolution, in a
 // side-effect-free module so it is unit-testable (this entry file runs an
 // argv switch on import).
@@ -134,9 +140,15 @@ const OPTIMIZE_SCHEMA = {
     flags: [
         { name: "dataset", takesValue: true },
         { name: "graders", takesValue: true },
+        // Inline-distill user ratings into the training set (Pillar 2 — close the
+        // loop from real feedback). Value: a session id (sess_<16 hex>) or "all".
+        // Synthesizes the dataset (and, when --graders is omitted, the graders too).
+        { name: "ratings", takesValue: true },
+        { name: "min-score", takesValue: true },
         { name: "mutator", takesValue: true },
         { name: "iterations", takesValue: true },
         { name: "seed", takesValue: true },
+        { name: "concurrency", takesValue: true },
         { name: "improvement-threshold", takesValue: true },
         // FR-003 — dollar ceiling for model-driven runs (composes with --iterations).
         { name: "budget-usd", takesValue: true },
@@ -170,6 +182,40 @@ const COST_SUMMARY_SCHEMA = {
         { name: "help", short: "h" },
     ],
 };
+const RATE_SCHEMA = {
+    flags: [
+        { name: "session", takesValue: true },
+        { name: "turn", takesValue: true },
+        { name: "thumbs", takesValue: true },
+        { name: "stars", takesValue: true },
+        { name: "score", takesValue: true },
+        { name: "comment", takesValue: true },
+        { name: "rater", takesValue: true },
+        { name: "help", short: "h" },
+    ],
+};
+const FEEDBACK_SCHEMA = {
+    flags: [
+        { name: "session", takesValue: true },
+        { name: "turn", takesValue: true },
+        { name: "text", takesValue: true },
+        { name: "correction", takesValue: true },
+        { name: "rater", takesValue: true },
+        { name: "help", short: "h" },
+    ],
+};
+const DISTILL_SCHEMA = {
+    flags: [
+        { name: "session", takesValue: true },
+        { name: "all-sessions", takesValue: false },
+        { name: "out", short: "o", takesValue: true },
+        { name: "graders-out", takesValue: true },
+        { name: "min-score", takesValue: true },
+        { name: "judge", takesValue: false },
+        { name: "judge-model", takesValue: true },
+        { name: "help", short: "h" },
+    ],
+};
 const SANDBOX_SCHEMA = {
     flags: [
         { name: "probe", takesValue: false },
@@ -268,6 +314,7 @@ function usageText() {
         "       [-o <out-dir>]",
         "  optimize <spec.yaml> --dataset <data> --graders <graders.yaml>",
         "       [--mutator rule-based|claude] [--iterations N] [--seed N]",
+        "       [--ratings <session>|all]            distill user ratings into the training set (Pillar 2)",
         "       [--budget-usd N]                     stop a model-driven run before it exceeds $N (FR-003)",
         "       [--write-back] [-o <out-dir>]        active eval-driven optimization (Pillar 2)",
         "  init [name]                          scaffold a new crewhaus.yaml",
@@ -275,6 +322,12 @@ function usageText() {
         "  context --bundle [-o <file>]         emit a single-markdown orientation manifest",
         "       [--factory-root <p>] [--docs-root <p>] [--demos-root <p>]",
         "  cost-summary --session <id>          summarize cost_accrual events for a session",
+        "  rate --session <id> [--turn N]       rate an assistant turn 👍/👎, ⭐, or 0–1",
+        "       (--thumbs up|down | --stars 1-5 | --score 0-1) [--comment <t>]",
+        "  feedback --session <id> --text <msg> attach a comment/correction to a turn",
+        "       [--turn N] [--correction <better answer>]",
+        "  distill --session <id> -o <ds.jsonl> turn ratings into an eval dataset + graders",
+        "       [--all-sessions] [--graders-out <g.yaml>] [--min-score F]",
         "  secrets doctor                       list known secrets via the configured backend",
         "  secrets rotate <name> [--value V]    rotate a named secret (file backend)",
         "  spec put|list|get|pin|alias ...      versioned spec storage (Section 28 spec-registry)",
@@ -1331,8 +1384,8 @@ async function runDoctorPhilosophyAlignment() {
  */
 async function runOptimize(args) {
     if (args.flags["help"]) {
-        process.stdout.write("usage: crewhaus optimize <spec.yaml> --dataset <data> --graders <graders.yaml> " +
-            "[--mutator rule-based|claude] [--iterations N] [--seed N] " +
+        process.stdout.write("usage: crewhaus optimize <spec.yaml> (--dataset <data> --graders <graders.yaml> | --ratings <session>) " +
+            "[--min-score F] [--mutator rule-based|claude] [--iterations N] [--seed N] [--concurrency N] " +
             "[--improvement-threshold F] [--budget-usd N] [--write-back] [-o <out-dir>]\n");
         return;
     }
@@ -1341,10 +1394,18 @@ async function runOptimize(args) {
         die("missing <spec.yaml>");
     const datasetPath = args.flags["dataset"];
     const gradersPath = args.flags["graders"];
-    if (typeof datasetPath !== "string")
-        die("missing --dataset <data>");
-    if (typeof gradersPath !== "string")
-        die("missing --graders <graders.yaml>");
+    const ratingsArg = args.flags["ratings"];
+    if (typeof datasetPath !== "string" && typeof ratingsArg !== "string") {
+        die("missing --dataset <data> (or --ratings <session> to distill one from feedback)");
+    }
+    if (typeof gradersPath !== "string" && typeof ratingsArg !== "string") {
+        die("missing --graders <graders.yaml> (or --ratings to synthesize one from feedback)");
+    }
+    const ratingsMinScore = floatFlag(args, "min-score") ?? 0.7;
+    if (ratingsMinScore < 0 || ratingsMinScore > 1) {
+        die(`invalid --min-score "${ratingsMinScore}" — must be in [0,1]`);
+    }
+    const ratingsDistill = typeof ratingsArg === "string" ? distillRatings(ratingsArg, ratingsMinScore) : undefined;
     const iterationsFlag = args.flags["iterations"];
     const seedFlag = args.flags["seed"];
     const thresholdFlag = args.flags["improvement-threshold"];
@@ -1354,6 +1415,15 @@ async function runOptimize(args) {
     if (Number.isNaN(iterations) || iterations < 1) {
         die(`invalid --iterations "${iterationsFlag}" — must be positive integer`);
     }
+    // Per-candidate eval concurrency. Each iteration runs a full eval pass on
+    // the dev set; on a low provider rate-limit tier a high fan-out trips 429s,
+    // so this is exposed (mirroring `crewhaus eval --concurrency`) and defaults
+    // to 4. The nightly flywheel sets `--concurrency 1` on constrained tiers.
+    const concurrencyFlag = args.flags["concurrency"];
+    const concurrency = typeof concurrencyFlag === "string" ? Number.parseInt(concurrencyFlag, 10) : 4;
+    if (Number.isNaN(concurrency) || concurrency < 1) {
+        die(`invalid --concurrency "${concurrencyFlag}" — must be a positive integer`);
+    }
     // FR-003 — optional dollar budget for model-driven runs. Omit → today's
     // behaviour (iterations cap only). On a rule-based run the gate is inert
     // (no model calls → $0), so passing it is harmless.
@@ -1373,25 +1443,40 @@ async function runOptimize(args) {
         ? resolve(outDirArg)
         : resolve(join(".crewhaus", "optimize", runId));
     let gradersYaml;
-    try {
-        gradersYaml = readFileSync(resolve(gradersPath), "utf-8");
+    if (typeof gradersPath === "string") {
+        try {
+            gradersYaml = readFileSync(resolve(gradersPath), "utf-8");
+        }
+        catch (err) {
+            die(`could not read ${gradersPath}: ${err.message}`);
+        }
     }
-    catch (err) {
-        die(`could not read ${gradersPath}: ${err.message}`);
+    else {
+        // No --graders: use the grader synthesized from the ratings (guaranteed
+        // present here because the missing-flags gate required --ratings).
+        gradersYaml = ratingsDistill.gradersYaml;
     }
     const { compiled } = parseGradersConfig(gradersYaml);
-    const dataset = await loadDataset(resolve(datasetPath));
-    // Materialize the dataset once — we'll re-iterate per fitness call.
+    // Materialize the training set once — we'll re-iterate per fitness call. It
+    // is the union of the file dataset (if any) and the distilled ratings (if
+    // any); at least one is present per the missing-flags gate above.
     const samples = [];
-    for await (const s of dataset.samples) {
-        samples.push({
-            id: s.id,
-            input: s.input,
-            ...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
-        });
+    let datasetName = "ratings";
+    if (typeof datasetPath === "string") {
+        const dataset = await loadDataset(resolve(datasetPath));
+        datasetName = dataset.name;
+        for await (const s of dataset.samples) {
+            samples.push({
+                id: s.id,
+                input: s.input,
+                ...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
+            });
+        }
     }
+    if (ratingsDistill !== undefined)
+        samples.push(...ratingsDistill.samples);
     if (samples.length === 0)
-        die(`dataset "${dataset.name}" yielded zero samples`);
+        die(`dataset "${datasetName}" yielded zero samples`);
     // Train/dev split: 70/30 deterministic split by sample id ordering.
     const splitIdx = Math.max(1, Math.floor(samples.length * 0.7));
     const trainSet = samples.slice(0, splitIdx);
@@ -1399,8 +1484,16 @@ async function runOptimize(args) {
     if (devSet.length === 0) {
         die(`dataset has ${samples.length} samples — need at least 2 (70/30 split needs a dev split)`);
     }
+    // Index the dev set by id so the fitness fn can join each graded
+    // sample-result back to the input + reference it was scored against.
+    const devById = new Map(devSet.map((s) => [s.id, s]));
     // Fitness fn: patch the spec with the candidate prompt, lower to IR,
-    // run eval-runner, return passRate. Each call is one full eval pass.
+    // run eval-runner, and return the pass-rate PLUS per-sample grades. The
+    // aggregate `passRate` still drives the search (unchanged scoring); the
+    // grades are additive — they carry each sample's overall score and the
+    // grader's rationale to the mutator (via OptimizerState.bestGrades) so
+    // a model-driven rewrite can target the samples the prompt actually
+    // fails and the reason it fails them. Each call is one full eval pass.
     const fitness = async (prompt) => {
         const yamlText = readFileSync(absSpec, "utf-8");
         // Re-parse to capture spec.target without depending on the
@@ -1422,7 +1515,7 @@ async function runOptimize(args) {
         catch (err) {
             if (err instanceof SpecParseError) {
                 process.stderr.write("[optimize] candidate compiled invalid spec, skipping\n");
-                return 0;
+                return { score: 0 };
             }
             throw err;
         }
@@ -1431,15 +1524,24 @@ async function runOptimize(args) {
         }
         const summary = await runEvalLib({
             ir,
-            dataset: { name: dataset.name, samples: makeAsyncIterable(devSet) },
+            dataset: { name: datasetName, samples: makeAsyncIterable(devSet) },
             compiledGraders: compiled,
             opts: {
                 outDir: join(outDir, "evals", `${prompt.length}_${ir.agent.instructions.length}`),
-                concurrency: 4,
+                concurrency,
                 seed,
             },
         });
-        return summary.aggregates.passRate;
+        const grades = summary.samples.map((r) => {
+            const dev = devById.get(r.sampleId);
+            return {
+                input: dev?.input ?? r.sampleId,
+                score: r.grades.overall.score,
+                ...(dev?.expected_output !== undefined ? { expected: dev.expected_output } : {}),
+                rationale: r.grades.overall.rationale,
+            };
+        });
+        return { score: summary.aggregates.passRate, grades };
     };
     const mutator = args.flags["mutator"];
     let mutatorImpl;
@@ -1462,7 +1564,7 @@ async function runOptimize(args) {
     else if (mutator !== undefined && mutator !== "rule-based") {
         die(`unknown --mutator "${mutator}" — supported: rule-based, claude`);
     }
-    process.stdout.write(`[optimize] runId=${runId} spec=${specPath} dataset=${dataset.name} ` +
+    process.stdout.write(`[optimize] runId=${runId} spec=${specPath} dataset=${datasetName} ` +
         `(${trainSet.length} train / ${devSet.length} dev) iterations=${iterations} ` +
         `mutator=${mutator ?? "rule-based"}\n`);
     // FR-003 — thread a real trace bus into the optimize run so the per-call
@@ -1709,6 +1811,272 @@ async function runCostSummary(args) {
         }
     }
 }
+// -------- response feedback: rate / feedback / distill --------
+const SESSIONS_SUBDIR = join(".crewhaus", "sessions");
+const FEEDBACK_SUBDIR = join(".crewhaus", "feedback");
+function sessionJsonlPath(session) {
+    return join(process.cwd(), SESSIONS_SUBDIR, `${session}.jsonl`);
+}
+/** Parse a JSONL blob into objects, skipping blank/malformed lines. */
+function parseJsonlObjects(text) {
+    const out = [];
+    for (const line of text.split("\n")) {
+        if (line.trim() === "")
+            continue;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            // A single malformed line must not abort a read.
+        }
+    }
+    return out;
+}
+function readSessionEvents(session) {
+    const file = sessionJsonlPath(session);
+    if (!existsSync(file))
+        die(`session log not found at ${file}`);
+    return parseJsonlObjects(readFileSync(file, "utf-8"));
+}
+function strFlag(args, name) {
+    const v = args.flags[name];
+    return typeof v === "string" ? v : undefined;
+}
+function intFlag(args, name) {
+    const v = args.flags[name];
+    if (typeof v !== "string")
+        return undefined;
+    const n = Number.parseInt(v, 10);
+    if (Number.isNaN(n))
+        die(`invalid --${name} "${v}" — must be an integer`);
+    return n;
+}
+function floatFlag(args, name) {
+    const v = args.flags[name];
+    if (typeof v !== "string")
+        return undefined;
+    const n = Number.parseFloat(v);
+    if (Number.isNaN(n))
+        die(`invalid --${name} "${v}" — must be a number`);
+    return n;
+}
+/** Resolve --turn against the transcript-derived turns; default to the last. */
+function resolveTurn(args, turns) {
+    const flag = args.flags["turn"];
+    const last = turns[turns.length - 1];
+    if (typeof flag !== "string")
+        return last.turnNumber;
+    const n = Number.parseInt(flag, 10);
+    if (Number.isNaN(n))
+        die(`invalid --turn "${flag}" — must be an integer`);
+    if (!turns.some((t) => t.turnNumber === n)) {
+        die(`turn ${n} not found — session has turns 1..${turns.length}`);
+    }
+    return n;
+}
+/** Shared capture path for `rate` and `feedback`: validate the session, derive
+ *  turns, build a FeedbackRecord, and append it as a `user_feedback` event. */
+async function captureFeedback(args, source, fields) {
+    const session = args.flags["session"];
+    if (typeof session !== "string")
+        die("missing --session <id>");
+    if (!SESSION_ID_REGEX.test(session))
+        die(`invalid --session "${session}" — expected sess_<16 hex>`);
+    const turns = deriveTurns(readSessionEvents(session));
+    if (turns.length === 0)
+        die(`session ${session} has no user turns to rate`);
+    const turnNumber = resolveTurn(args, turns);
+    let record;
+    try {
+        record = buildFeedbackRecord({
+            id: `fb_${randomBytes(6).toString("hex")}`,
+            sessionId: session,
+            turnNumber,
+            ts: new Date().toISOString(),
+            source,
+            ...fields,
+        });
+    }
+    catch (err) {
+        die(err instanceof Error ? err.message : String(err));
+    }
+    const log = await openEventLog(session, { rootDir: join(process.cwd(), SESSIONS_SUBDIR) });
+    await log.append({ kind: FEEDBACK_EVENT_KIND, payload: record });
+    process.stdout.write(`recorded ${record.modality} feedback on ${session} turn ${turnNumber}\n`);
+}
+async function runRate(args) {
+    if (args.flags["help"]) {
+        process.stdout.write("usage: crewhaus rate --session <id> [--turn N] " +
+            "(--thumbs up|down | --stars 1-5 | --score 0-1) [--comment <text>] [--rater <who>]\n");
+        return;
+    }
+    const thumbsFlag = args.flags["thumbs"];
+    let thumbs;
+    if (typeof thumbsFlag === "string") {
+        if (thumbsFlag !== "up" && thumbsFlag !== "down")
+            die(`--thumbs must be "up" or "down" (got "${thumbsFlag}")`);
+        thumbs = thumbsFlag;
+    }
+    const stars = intFlag(args, "stars");
+    const score = floatFlag(args, "score");
+    if (thumbs === undefined && stars === undefined && score === undefined) {
+        die("give one of --thumbs up|down, --stars 1-5, or --score 0-1");
+    }
+    await captureFeedback(args, "cli", {
+        ...(thumbs !== undefined ? { thumbs } : {}),
+        ...(stars !== undefined ? { stars } : {}),
+        ...(score !== undefined ? { score } : {}),
+        ...(strFlag(args, "comment") !== undefined ? { comment: strFlag(args, "comment") } : {}),
+        ...(strFlag(args, "rater") !== undefined ? { rater: strFlag(args, "rater") } : {}),
+    });
+}
+async function runFeedbackCmd(args) {
+    if (args.flags["help"]) {
+        process.stdout.write("usage: crewhaus feedback --session <id> [--turn N] --text <msg> [--correction <better answer>] [--rater <who>]\n");
+        return;
+    }
+    const text = strFlag(args, "text");
+    const correction = strFlag(args, "correction");
+    if (text === undefined && correction === undefined) {
+        die("give --text <msg> and/or --correction <better answer>");
+    }
+    await captureFeedback(args, "cli", {
+        ...(text !== undefined ? { comment: text } : {}),
+        ...(correction !== undefined ? { correction } : {}),
+        ...(strFlag(args, "rater") !== undefined ? { rater: strFlag(args, "rater") } : {}),
+    });
+}
+/** List `sess_*` ids that have a transcript under `.crewhaus/sessions`. */
+function listSessionIds(sessionsDir) {
+    if (!existsSync(sessionsDir))
+        return [];
+    return readdirSync(sessionsDir)
+        .filter((f) => f.endsWith(".jsonl"))
+        .map((f) => f.slice(0, -".jsonl".length))
+        .filter((id) => SESSION_ID_REGEX.test(id));
+}
+/** Read bare FeedbackRecords from `.crewhaus/feedback/*.jsonl` (the web-UI host
+ *  sink, which has no event-log handle). */
+function readFeedbackDir(feedbackDir) {
+    if (!existsSync(feedbackDir))
+        return [];
+    const objects = [];
+    for (const f of readdirSync(feedbackDir)) {
+        if (!f.endsWith(".jsonl"))
+            continue;
+        objects.push(...parseJsonlObjects(readFileSync(join(feedbackDir, f), "utf-8")));
+    }
+    return extractFeedbackRecords(objects);
+}
+async function runDistill(args) {
+    if (args.flags["help"]) {
+        process.stdout.write("usage: crewhaus distill (--session <id> | --all-sessions) -o <dataset.jsonl> " +
+            "[--graders-out <graders.yaml>] [--min-score F] [--judge] [--judge-model <model>]\n");
+        return;
+    }
+    const outPath = args.flags["out"];
+    if (typeof outPath !== "string")
+        die("missing -o <dataset.jsonl>");
+    const allSessions = args.flags["all-sessions"] === true;
+    const session = args.flags["session"];
+    if (!allSessions && typeof session !== "string")
+        die("missing --session <id> (or --all-sessions)");
+    if (typeof session === "string" && !SESSION_ID_REGEX.test(session)) {
+        die(`invalid --session "${session}" — expected sess_<16 hex>`);
+    }
+    const minScore = floatFlag(args, "min-score") ?? 0.7;
+    if (minScore < 0 || minScore > 1)
+        die(`invalid --min-score "${minScore}" — must be in [0,1]`);
+    const useJudge = args.flags["judge"] === true;
+    const judgeModel = args.flags["judge-model"];
+    const sessionsDir = join(process.cwd(), SESSIONS_SUBDIR);
+    const sessionIds = allSessions ? listSessionIds(sessionsDir) : [session];
+    if (sessionIds.length === 0)
+        die(`no sessions found under ${sessionsDir}`);
+    // Derive turns per session (tagged with the sessionId join key) and gather
+    // feedback from both the in-transcript events and the web-UI feedback dir.
+    const turns = [];
+    const feedback = [];
+    for (const id of sessionIds) {
+        const events = readSessionEvents(id);
+        for (const t of deriveTurns(events))
+            turns.push({ ...t, sessionId: id });
+        feedback.push(...extractFeedbackRecords(events));
+    }
+    feedback.push(...readFeedbackDir(join(process.cwd(), FEEDBACK_SUBDIR)));
+    if (feedback.length === 0) {
+        die("no feedback found — record some with `crewhaus rate` / `crewhaus feedback` first");
+    }
+    const result = distillFeedback(turns, feedback, {
+        minScore,
+        ...(useJudge ? { judge: true } : {}),
+        ...(typeof judgeModel === "string" ? { judgeModel } : {}),
+    });
+    for (const w of result.warnings)
+        process.stderr.write(`[distill] warning: ${w}\n`);
+    if (result.samples.length === 0)
+        die("no rated turns could be matched to the transcript(s)");
+    const absOut = resolve(outPath);
+    mkdirSync(dirname(absOut), { recursive: true });
+    writeFileSync(absOut, samplesToJsonl(result.samples), { mode: 0o600 });
+    const gradersOut = args.flags["graders-out"];
+    if (typeof gradersOut === "string") {
+        const absGraders = resolve(gradersOut);
+        mkdirSync(dirname(absGraders), { recursive: true });
+        writeFileSync(absGraders, gradersConfigToYaml(result.graders), { mode: 0o600 });
+    }
+    const { stats } = result;
+    process.stdout.write(`[distill] ${stats.matchedTurns} rated turn(s) → ${result.samples.length} sample(s) ` +
+        `(${stats.positives} positive, ${stats.negatives} low-rated) → ${absOut}\n`);
+    if (typeof gradersOut === "string") {
+        const g = result.graders.graders[0];
+        process.stdout.write(`[distill] grader: ${g?.name} (${g?.type}) → ${resolve(gradersOut)}\n`);
+    }
+    if (stats.unmatchedFeedback > 0) {
+        process.stdout.write(`[distill] ${stats.unmatchedFeedback} rating(s) had no matching turn (skipped)\n`);
+    }
+}
+/**
+ * Inline-distill ratings for `crewhaus optimize --ratings`. `ratingsArg` is a
+ * session id or "all". Returns optimizer-shaped samples plus a synthesized
+ * graders.yaml (used only when the user did not pass their own --graders).
+ */
+function distillRatings(ratingsArg, minScore) {
+    const sessionsDir = join(process.cwd(), SESSIONS_SUBDIR);
+    let sessionIds;
+    if (ratingsArg === "all") {
+        sessionIds = listSessionIds(sessionsDir);
+    }
+    else {
+        if (!SESSION_ID_REGEX.test(ratingsArg)) {
+            die(`invalid --ratings "${ratingsArg}" — expected a session id (sess_<16 hex>) or "all"`);
+        }
+        sessionIds = [ratingsArg];
+    }
+    if (sessionIds.length === 0)
+        die(`no sessions found under ${sessionsDir}`);
+    const turns = [];
+    const feedback = [];
+    for (const id of sessionIds) {
+        const events = readSessionEvents(id);
+        for (const t of deriveTurns(events))
+            turns.push({ ...t, sessionId: id });
+        feedback.push(...extractFeedbackRecords(events));
+    }
+    feedback.push(...readFeedbackDir(join(process.cwd(), FEEDBACK_SUBDIR)));
+    if (feedback.length === 0) {
+        die("no feedback found for --ratings — record some with `crewhaus rate` / `crewhaus feedback` first");
+    }
+    const result = distillFeedback(turns, feedback, { minScore });
+    for (const w of result.warnings)
+        process.stderr.write(`[optimize] ratings warning: ${w}\n`);
+    const samples = result.samples.map((s) => ({
+        id: s.id,
+        input: s.input,
+        ...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
+    }));
+    return { samples, gradersYaml: gradersConfigToYaml(result.graders) };
+}
 /**
  * Section 27 — `crewhaus secrets <action> <name> [opts]`. Two actions:
  *   doctor                 list configured secrets and report missing
@@ -2283,6 +2651,15 @@ switch (subcommand) {
     case "cost-summary":
         await runCostSummary(parseFor(rest, COST_SUMMARY_SCHEMA));
         break;
+    case "rate":
+        await runRate(parseFor(rest, RATE_SCHEMA));
+        break;
+    case "feedback":
+        await runFeedbackCmd(parseFor(rest, FEEDBACK_SCHEMA));
+        break;
+    case "distill":
+        await runDistill(parseFor(rest, DISTILL_SCHEMA));
+        break;
     case "secrets": {
         const action = rest[0] ?? "";
         if (action !== "doctor" && action !== "rotate") {