crewhaus 0.1.7 → 0.2.0

package/dist/feedback.js

@@ -0,0 +1,612 @@
+/**
+ * Response-feedback core — the pure, side-effect-free half of the
+ * `crewhaus rate` / `crewhaus feedback` / `crewhaus distill` subcommands.
+ *
+ * A `FeedbackRecord` is one human rating placed on one assistant turn (a
+ * thumbs up/down, a 1–5 star vote, an arbitrary scale vote, and/or a
+ * free-text comment or correction). The capture surfaces write it durably as
+ * a `user_feedback` event-log line (payload = FeedbackRecord); the web-UI host
+ * writes bare records to `.crewhaus/feedback/*.jsonl`. `distill` reads both,
+ * pairs each rating to a conversation turn, and emits the two artifacts the
+ * R-eval stack already consumes: a `Sample[]` dataset (`@crewhaus/eval-dataset`)
+ * and a `graders.yaml` (`@crewhaus/eval-grader`).
+ *
+ * Everything here is pure so it is unit-testable; all filesystem access lives
+ * in `apps/cli/src/index.ts` (mirrors `doctor-checks.ts` / `scope-audit.ts`).
+ *
+ * turnNumber convention: the 1-based ordinal of USER-TEXT turns in a session.
+ * A `user_message` whose content is a string (or an array containing a `text`
+ * block and no `tool_result` block) starts a new turn; a `user_message` that
+ * carries only `tool_result` blocks is a tool-result echo, and one marked
+ * `synthetic: true` (a runtime-injected loop/continue/tombstone nudge) is NOT a
+ * turn. This is the same count the CLI `--turn` flag, the web-UI (prompts
+ * sent), and the runtime's runContext.turnNumber all compute, so a rating keys
+ * back to the exact exchange it rated even after a recovery fires mid-session.
+ */
+/** The event-log EventKind under which a FeedbackRecord is persisted. */
+export const FEEDBACK_EVENT_KIND = "user_feedback";
+export const FEEDBACK_SCHEMA_VERSION = 1;
+const SESSION_ID_REGEX = /^sess_[0-9a-f]{16}$/;
+/** Max stored length for a free-text comment/correction. Untrusted input (a
+ *  hand-edited or web-UI-supplied `.crewhaus/feedback` line) flows into
+ *  expected_output/metadata and the optimizer meta-prompt, so it is bounded at
+ *  ingestion and stripped of control chars (newline/tab kept). */
+export const MAX_FEEDBACK_TEXT = 8192;
+export function clipFeedbackText(s) {
+    // Drop C0/C1 control chars and DEL (keeping tab, newline, carriage return),
+    // then bound the length. Built by code point so the source carries no
+    // literal control characters.
+    let out = "";
+    for (const ch of s) {
+        const c = ch.codePointAt(0) ?? 0;
+        const printable = c === 9 || c === 10 || c === 13 || (c >= 0x20 && c !== 0x7f && !(c >= 0x80 && c <= 0x9f));
+        if (printable)
+            out += ch;
+        if (out.length >= MAX_FEEDBACK_TEXT)
+            break;
+    }
+    return out.slice(0, MAX_FEEDBACK_TEXT);
+}
+function isValidRating(rating) {
+    const { thumbs, stars, scale } = rating;
+    if (thumbs !== undefined && thumbs !== "up" && thumbs !== "down")
+        return false;
+    if (stars !== undefined) {
+        if (typeof stars !== "number" || !Number.isInteger(stars) || stars < 1 || stars > 5)
+            return false;
+    }
+    if (scale !== undefined) {
+        if (typeof scale !== "object" || scale === null)
+            return false;
+        const s = scale;
+        if (!["value", "min", "max"].every((k) => typeof s[k] === "number" && Number.isFinite(s[k]))) {
+            return false;
+        }
+    }
+    return true;
+}
+/** Narrow an arbitrary parsed JSON value to a FeedbackRecord. Tolerant of the
+ *  optional fields; rejects anything missing the required core OR carrying a
+ *  malformed rating (so a corrupt/crafted line can't coerce a NaN/spurious
+ *  score downstream). */
+export function isFeedbackRecord(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const v = value;
+    if (v["schemaVersion"] !== 1)
+        return false;
+    if (typeof v["id"] !== "string")
+        return false;
+    if (typeof v["sessionId"] !== "string" || !SESSION_ID_REGEX.test(v["sessionId"])) {
+        return false;
+    }
+    if (typeof v["turnNumber"] !== "number" || !Number.isFinite(v["turnNumber"]))
+        return false;
+    const modality = v["modality"];
+    if (modality !== "binary" &&
+        modality !== "stars" &&
+        modality !== "scale" &&
+        modality !== "comment") {
+        return false;
+    }
+    if (typeof v["rating"] !== "object" || v["rating"] === null)
+        return false;
+    if (!isValidRating(v["rating"]))
+        return false;
+    if (typeof v["source"] !== "string")
+        return false;
+    if (typeof v["ts"] !== "string")
+        return false;
+    return true;
+}
+/**
+ * Assemble + validate a FeedbackRecord from capture-surface inputs. The
+ * modality is inferred from which rating fields are present (thumbs → binary,
+ * stars → stars, score → scale, else → comment). Throws a plain Error (the CLI
+ * routes it through `die`) when no rating signal at all is supplied.
+ */
+export function buildFeedbackRecord(input) {
+    const rating = {};
+    let modality;
+    if (input.thumbs !== undefined) {
+        rating.thumbs = input.thumbs;
+        modality = "binary";
+    }
+    if (input.stars !== undefined) {
+        if (!Number.isInteger(input.stars) || input.stars < 1 || input.stars > 5) {
+            throw new Error(`--stars must be an integer 1–5 (got ${input.stars})`);
+        }
+        rating.stars = input.stars;
+        modality = "stars";
+    }
+    if (input.score !== undefined) {
+        if (!Number.isFinite(input.score) || input.score < 0 || input.score > 1) {
+            throw new Error(`--score must be a number in [0,1] (got ${input.score})`);
+        }
+        rating.scale = { value: input.score, min: 0, max: 1 };
+        modality = "scale";
+    }
+    if (modality === undefined) {
+        if (input.comment === undefined && input.correction === undefined) {
+            throw new Error("no rating supplied — give --thumbs, --stars, --score, --text, or --correction");
+        }
+        modality = "comment";
+    }
+    return {
+        schemaVersion: FEEDBACK_SCHEMA_VERSION,
+        id: input.id,
+        sessionId: input.sessionId,
+        ...(input.runId !== undefined ? { runId: input.runId } : {}),
+        turnNumber: input.turnNumber,
+        ...(input.targetSpanId !== undefined ? { targetSpanId: input.targetSpanId } : {}),
+        modality,
+        rating,
+        ...(input.comment !== undefined ? { comment: clipFeedbackText(input.comment) } : {}),
+        ...(input.correction !== undefined ? { correction: clipFeedbackText(input.correction) } : {}),
+        source: input.source,
+        ...(input.rater !== undefined ? { rater: input.rater } : {}),
+        ts: input.ts,
+    };
+}
+/**
+ * Normalize a rating to [0,1] — the same convention `GradeResult.score` uses
+ * ((n-1)/4 for a 1–5 judge score). Returns `undefined` for a comment-only
+ * record (no numeric signal). Thumbs: up→1, down→0. Stars n→(n-1)/4. Scale
+ * value→(value-min)/(max-min), clamped.
+ */
+export function normalizeRating(r) {
+    const { thumbs, stars, scale } = r.rating;
+    if (thumbs !== undefined)
+        return thumbs === "up" ? 1 : 0;
+    if (stars !== undefined)
+        return clamp01((stars - 1) / 4);
+    if (scale !== undefined) {
+        const span = scale.max - scale.min;
+        if (span <= 0)
+            return undefined;
+        return clamp01((scale.value - scale.min) / span);
+    }
+    return undefined;
+}
+function clamp01(n) {
+    return n < 0 ? 0 : n > 1 ? 1 : n;
+}
+function turnKey(sessionId, turnNumber) {
+    return `${sessionId}#${turnNumber}`;
+}
+function contentBlocks(payload) {
+    const content = payload?.content;
+    if (typeof content === "string")
+        return { blocks: [], text: content };
+    if (Array.isArray(content))
+        return { blocks: content };
+    return { blocks: [] };
+}
+/** The text when a `user_message` is a real user-text turn (string content, or
+ *  an array with ≥1 text block and no tool_result block); undefined otherwise.
+ *  Runtime-injected nudges (loop warnings, "continue"/tombstone recovery) carry
+ *  `synthetic: true` and are NOT turns — skipping them keeps deriveTurns's turn
+ *  ordinal aligned with the runtime's runContext.turnNumber and the web UI. */
+function userTurnText(payload) {
+    if (payload !== null &&
+        typeof payload === "object" &&
+        payload.synthetic === true) {
+        return undefined;
+    }
+    const { blocks, text } = contentBlocks(payload);
+    if (text !== undefined)
+        return text;
+    const hasToolResult = blocks.some((b) => b.type === "tool_result");
+    if (hasToolResult)
+        return undefined;
+    const texts = blocks
+        .filter((b) => b.type === "text" && typeof b.text === "string")
+        .map((b) => b.text);
+    return texts.length > 0 ? texts.join("\n") : undefined;
+}
+function assistantText(payload) {
+    const { blocks, text } = contentBlocks(payload);
+    if (text !== undefined)
+        return text;
+    return blocks
+        .filter((b) => b.type === "text" && typeof b.text === "string")
+        .map((b) => b.text)
+        .join("\n");
+}
+function assistantToolNames(payload) {
+    const { blocks } = contentBlocks(payload);
+    return blocks
+        .filter((b) => b.type === "tool_use" && typeof b.name === "string")
+        .map((b) => b.name);
+}
+/**
+ * Reconstruct conversation turns from a session transcript. Each user-text
+ * message opens a turn; the assistant's messages until the next user-text
+ * message form its response. `output` is the LAST assistant text (the answer);
+ * `toolNames` are the unique tool names the assistant called, in first-seen
+ * order.
+ */
+export function deriveTurns(events) {
+    const turns = [];
+    let current;
+    let counter = 0;
+    const finalize = () => {
+        if (!current)
+            return;
+        turns.push({
+            turnNumber: current.turnNumber,
+            input: current.input,
+            output: current.texts.length > 0 ? current.texts[current.texts.length - 1] : "",
+            toolNames: current.tools,
+        });
+    };
+    for (const ev of events) {
+        if (ev.kind === "user_message") {
+            const text = userTurnText(ev.payload);
+            if (text !== undefined) {
+                finalize();
+                counter += 1;
+                current = { turnNumber: counter, input: text, texts: [], tools: [], toolSeen: new Set() };
+            }
+        }
+        else if (ev.kind === "assistant_message" && current) {
+            const t = assistantText(ev.payload);
+            if (t !== "")
+                current.texts.push(t);
+            for (const name of assistantToolNames(ev.payload)) {
+                if (!current.toolSeen.has(name)) {
+                    current.toolSeen.add(name);
+                    current.tools.push(name);
+                }
+            }
+        }
+    }
+    finalize();
+    return turns;
+}
+/** Extract FeedbackRecords from parsed JSON objects. Accepts both event-log
+ *  envelopes (`{ kind: "user_feedback", payload }`) and bare records (the
+ *  web-UI host's `.crewhaus/feedback/*.jsonl`). */
+export function extractFeedbackRecords(objects) {
+    const out = [];
+    for (const obj of objects) {
+        if (obj && typeof obj === "object" && obj.kind === FEEDBACK_EVENT_KIND) {
+            const payload = obj.payload;
+            if (isFeedbackRecord(payload))
+                out.push(payload);
+        }
+        else if (isFeedbackRecord(obj)) {
+            out.push(obj);
+        }
+    }
+    return out;
+}
+/**
+ * Collapse records to one per (sessionId, turnNumber). Rather than pure
+ * last-write-wins — which would let a later comment-only `feedback` erase an
+ * earlier `rate` on the same turn — fields are merged chronologically: the
+ * newest value of each field wins, but a later comment-only record keeps the
+ * earlier record's rating. So `rate --thumbs up` then `feedback --text "…"`
+ * yields one record carrying BOTH.
+ */
+export function mergeFeedback(records) {
+    const sorted = [...records].sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0));
+    const byKey = new Map();
+    for (const r of sorted) {
+        const key = turnKey(r.sessionId, r.turnNumber);
+        const existing = byKey.get(key);
+        byKey.set(key, existing === undefined ? r : foldRecords(existing, r));
+    }
+    return [...byKey.values()];
+}
+function hasRating(r) {
+    return (r.rating.thumbs !== undefined || r.rating.stars !== undefined || r.rating.scale !== undefined);
+}
+/** Merge an earlier + later record for the same turn (later ts wins per field;
+ *  a comment-only later record does not erase the earlier rating). */
+function foldRecords(earlier, later) {
+    const ratingRecord = hasRating(later) ? later : hasRating(earlier) ? earlier : later;
+    const comment = later.comment ?? earlier.comment;
+    const correction = later.correction ?? earlier.correction;
+    const rater = later.rater ?? earlier.rater;
+    const runId = later.runId ?? earlier.runId;
+    const targetSpanId = later.targetSpanId ?? earlier.targetSpanId;
+    return {
+        schemaVersion: FEEDBACK_SCHEMA_VERSION,
+        id: later.id,
+        sessionId: later.sessionId,
+        turnNumber: later.turnNumber,
+        modality: hasRating(ratingRecord) ? ratingRecord.modality : "comment",
+        rating: ratingRecord.rating,
+        source: later.source,
+        ts: later.ts,
+        ...(runId !== undefined ? { runId } : {}),
+        ...(targetSpanId !== undefined ? { targetSpanId } : {}),
+        ...(comment !== undefined ? { comment } : {}),
+        ...(correction !== undefined ? { correction } : {}),
+        ...(rater !== undefined ? { rater } : {}),
+    };
+}
+/**
+ * Tag-all distillation. Every rated turn becomes a Sample; positively-rated
+ * turns (normalized ≥ minScore) — and any turn with a `correction` — become
+ * GOLD (expected_output set), the rest carry a low `metadata.user_rating` and
+ * no expected_output so they feed the optimizer's failure channel. A single
+ * deterministic grader is synthesized from the positive turns' behavior
+ * (exactly one, to avoid the hard-AND collapse in `all(...)`).
+ */
+export function distill(turns, feedback, opts) {
+    const turnByKey = new Map();
+    for (const t of turns)
+        turnByKey.set(turnKey(t.sessionId, t.turnNumber), t);
+    const merged = mergeFeedback(feedback);
+    const samples = [];
+    const positiveTurns = [];
+    const positiveComments = [];
+    const negativeComments = [];
+    const warnings = [];
+    let matched = 0;
+    let unmatched = 0;
+    let positives = 0;
+    let negatives = 0;
+    for (const fb of merged) {
+        const turn = turnByKey.get(turnKey(fb.sessionId, fb.turnNumber));
+        if (turn === undefined) {
+            unmatched += 1;
+            warnings.push(`rating for ${fb.sessionId} turn ${fb.turnNumber} has no matching turn in the transcript — skipped`);
+            continue;
+        }
+        matched += 1;
+        const score = normalizeRating(fb);
+        const isPositive = (score !== undefined && score >= opts.minScore) || fb.correction !== undefined;
+        // Gold = the user's correction, else the assistant's answer for a positive
+        // turn. Omit an empty answer (e.g. a turn that errored before replying) so
+        // we never emit a meaningless empty-string gold.
+        const goldCandidate = fb.correction ?? (isPositive ? turn.output : undefined);
+        const expectedOutput = goldCandidate !== undefined && goldCandidate.trim() !== "" ? goldCandidate : undefined;
+        const metadata = {
+            sessionId: fb.sessionId,
+            turnNumber: fb.turnNumber,
+            source: fb.source,
+            raw_rating: fb.rating,
+        };
+        if (score !== undefined)
+            metadata["user_rating"] = score;
+        if (fb.comment !== undefined)
+            metadata["comment"] = fb.comment;
+        if (fb.correction !== undefined)
+            metadata["correction"] = fb.correction;
+        if (fb.rater !== undefined)
+            metadata["rater"] = fb.rater;
+        const sample = { id: `${fb.sessionId}_t${fb.turnNumber}`, input: turn.input, metadata };
+        if (expectedOutput !== undefined)
+            sample.expected_output = expectedOutput;
+        if (isPositive && turn.toolNames.length > 0)
+            sample.expected_tools = turn.toolNames;
+        samples.push(sample);
+        if (isPositive) {
+            positives += 1;
+            positiveTurns.push(turn);
+            if (fb.comment !== undefined)
+                positiveComments.push(fb.comment);
+        }
+        else {
+            negatives += 1;
+            if (fb.comment !== undefined)
+                negativeComments.push(fb.comment);
+        }
+    }
+    return {
+        samples,
+        graders: opts.judge === true
+            ? { graders: [buildJudgeRubricGrader(positiveComments, negativeComments, opts.judgeModel)] }
+            : synthesizeGraders(positiveTurns, warnings),
+        stats: {
+            totalFeedback: merged.length,
+            matchedTurns: matched,
+            unmatchedFeedback: unmatched,
+            positives,
+            negatives,
+        },
+        warnings,
+    };
+}
+const STOPWORDS = new Set([
+    "the",
+    "and",
+    "that",
+    "this",
+    "with",
+    "from",
+    "your",
+    "have",
+    "will",
+    "here",
+    "there",
+    "what",
+    "when",
+    "which",
+    "would",
+    "could",
+    "should",
+    "about",
+    "into",
+    "then",
+    "than",
+    "them",
+    "they",
+    "you",
+    "for",
+    "are",
+    "was",
+    "were",
+    "its",
+]);
+/**
+ * Pick the single deterministic grader that best captures the up-rated
+ * behavior. Prefers a `tool_call_sequence` over the tools every tool-using
+ * positive turn shared (falling back to the most common tool); if the positive
+ * turns used no tools, a `contains` over a distinctive token common to their
+ * answers; failing that, a non-empty-answer floor (with a warning).
+ */
+export function synthesizeGraders(positiveTurns, warnings = []) {
+    const toolCounts = new Map();
+    let turnsWithTools = 0;
+    for (const t of positiveTurns) {
+        const uniq = [...new Set(t.toolNames)];
+        if (uniq.length > 0)
+            turnsWithTools += 1;
+        for (const name of uniq)
+            toolCounts.set(name, (toolCounts.get(name) ?? 0) + 1);
+    }
+    if (turnsWithTools > 0 && toolCounts.size > 0) {
+        const inEvery = [...toolCounts.entries()]
+            .filter(([, c]) => c === turnsWithTools)
+            .map(([n]) => n);
+        let expected = inEvery;
+        if (expected.length === 0) {
+            const top = [...toolCounts.entries()].sort((a, b) => b[1] - a[1])[0];
+            expected = top ? [top[0]] : [];
+        }
+        if (expected.length > 0) {
+            return {
+                graders: [{ name: "preferred_tools", type: "tool_call_sequence", expected, mode: "set" }],
+            };
+        }
+    }
+    const token = commonToken(positiveTurns.map((t) => t.output));
+    if (token !== undefined) {
+        return {
+            graders: [
+                { name: "preferred_phrase", type: "contains", substring: token, case_insensitive: true },
+            ],
+        };
+    }
+    warnings.push("no consistent tool or phrase signal in the up-rated turns — emitted a non-empty-answer floor grader; add ratings or edit graders.yaml for a sharper eval");
+    return { graders: [{ name: "non_empty_answer", type: "regex", pattern: "\\S" }] };
+}
+/**
+ * Build a single `llm_judge` grader seeded from the aggregated comment themes.
+ * One criterion ("user_preference") with fixed 1–5 anchors and a description
+ * folding a short, quoted, clipped summary of what users praised vs criticized.
+ * Emitted as the SOLE grader so it is never hard-ANDed with a deterministic one
+ * (the `all(...)` min-collapse). The untrusted comment text is bounded and
+ * quoted-as-data; the rubric is trusted context sent verbatim to the judge.
+ */
+export function buildJudgeRubricGrader(positiveComments, negativeComments, model) {
+    const praised = summarizeComments(positiveComments);
+    const criticized = summarizeComments(negativeComments);
+    const praisedNote = praised !== undefined ? ` Users praised responses like: '${praised}'.` : "";
+    const criticizedNote = criticized !== undefined ? ` Users criticized responses like: '${criticized}'.` : "";
+    const description = `Judge how well the response matches the qualities real users prefer, learned from their ratings.${praisedNote}${criticizedNote}`;
+    return {
+        name: "user_preference",
+        type: "llm_judge",
+        rubric: {
+            criteria: [
+                {
+                    name: "user_preference",
+                    description,
+                    anchors: {
+                        "1": "Strongly conflicts with what users liked; clearly exhibits the criticized traits.",
+                        "2": "Mostly misses the qualities users preferred.",
+                        "3": "Partially matches user preferences; a mixed response.",
+                        "4": "Largely matches the qualities users praised.",
+                        "5": "Strongly matches the user-preferred style and avoids the criticized traits.",
+                    },
+                },
+            ],
+            passing_score: 3,
+        },
+        ...(model !== undefined ? { model } : {}),
+    };
+}
+/** Dedupe, join, and clip a batch of comments into one short quoted summary for
+ *  a rubric description (~400 chars). Undefined when there are no comments. */
+function summarizeComments(comments) {
+    const cleaned = [...new Set(comments.map((c) => c.trim()).filter((c) => c !== ""))];
+    if (cleaned.length === 0)
+        return undefined;
+    const joined = clipFeedbackText(cleaned.slice(0, 8).join("; ")).replace(/'/g, "");
+    return joined.length > 400 ? `${joined.slice(0, 400)}…` : joined;
+}
+/** The most frequent distinctive token (alnum, length ≥ 4, non-stopword)
+ *  appearing in at least half of the answers. Undefined when none qualifies. */
+function commonToken(answers) {
+    const nonEmpty = answers.filter((a) => a.trim() !== "");
+    if (nonEmpty.length === 0)
+        return undefined;
+    const docFreq = new Map();
+    for (const answer of nonEmpty) {
+        const seen = new Set();
+        for (const raw of answer.toLowerCase().split(/[^a-z0-9]+/)) {
+            if (raw.length < 4 || STOPWORDS.has(raw) || seen.has(raw))
+                continue;
+            seen.add(raw);
+            docFreq.set(raw, (docFreq.get(raw) ?? 0) + 1);
+        }
+    }
+    const threshold = Math.ceil(nonEmpty.length / 2);
+    const ranked = [...docFreq.entries()]
+        .filter(([, c]) => c >= threshold)
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+    return ranked.length > 0 ? ranked[0]?.[0] : undefined;
+}
+// -------- serialization --------
+/** One SampleSchema-valid JSON object per line (round-trips through loadJsonl). */
+export function samplesToJsonl(samples) {
+    return `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`;
+}
+/** The default graders.yaml header — distill's provenance note. Callers with
+ *  a different provenance (scaffold-evals item 13, graders suggest item 4)
+ *  pass their own header lines instead. */
+const DISTILL_GRADERS_HEADER = [
+    "# Synthesized by `crewhaus distill` from user ratings.",
+    "# Exactly one grader — stacking graders hard-ANDs their scores (see eval-grader `all`).",
+];
+/** Emit a GradersConfigSchema-valid graders.yaml. Scalars are JSON-encoded so
+ *  arbitrary substrings/tool names stay YAML-safe (JSON is a YAML subset).
+ *  `headerLines` replaces the distill provenance header (each line must be a
+ *  full `# …` YAML comment). */
+export function gradersConfigToYaml(config, headerLines = DISTILL_GRADERS_HEADER) {
+    const lines = [...headerLines, "graders:"];
+    for (const g of config.graders) {
+        lines.push(`  - name: ${JSON.stringify(g.name)}`);
+        lines.push(`    type: ${g.type}`);
+        if (g.type === "tool_call_sequence") {
+            lines.push(`    expected: ${JSON.stringify(g.expected)}`);
+            lines.push(`    mode: ${g.mode}`);
+        }
+        else if (g.type === "contains") {
+            lines.push(`    substring: ${JSON.stringify(g.substring)}`);
+            if (g.case_insensitive !== undefined)
+                lines.push(`    case_insensitive: ${g.case_insensitive}`);
+        }
+        else if (g.type === "json_path") {
+            lines.push(`    path: ${JSON.stringify(g.path)}`);
+            if (g.expected !== undefined)
+                lines.push(`    expected: ${JSON.stringify(g.expected)}`);
+        }
+        else if (g.type === "llm_judge") {
+            lines.push("    rubric:");
+            lines.push("      criteria:");
+            for (const c of g.rubric.criteria) {
+                lines.push(`        - name: ${JSON.stringify(c.name)}`);
+                lines.push(`          description: ${JSON.stringify(c.description)}`);
+                lines.push("          anchors:");
+                for (const k of ["1", "2", "3", "4", "5"]) {
+                    lines.push(`            ${JSON.stringify(k)}: ${JSON.stringify(c.anchors[k])}`);
+                }
+            }
+            if (g.rubric.passing_score !== undefined) {
+                lines.push(`      passing_score: ${g.rubric.passing_score}`);
+            }
+            if (g.model !== undefined)
+                lines.push(`    model: ${JSON.stringify(g.model)}`);
+        }
+        else {
+            lines.push(`    pattern: ${JSON.stringify(g.pattern)}`);
+        }
+    }
+    return `${lines.join("\n")}\n`;
+}

package/dist/fewshot.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Item #54 — harvest up-rated turns into a golden few-shot pool.
+ *
+ * `crewhaus fewshot harvest` mines POSITIVELY-rated conversation turns (the
+ * same `user_feedback` ratings `distill` consumes) into a curated pool of
+ * input→good-output few-shot examples. The pool is a versioned, provenance-
+ * tagged JSONL under `.crewhaus/fewshot/<spec>.jsonl`; `crewhaus optimize
+ * --few-shot <file>` injects the top-K examples into the agent prompt as
+ * in-context demonstrations, and `agent.instructions` can carry them at
+ * compile time via `formatFewShotForPrompt`.
+ *
+ * Everything here is pure + deterministic (stable ordering, id-based dedupe)
+ * so it is unit-testable; all filesystem access + the model/redactor wiring
+ * lives in `apps/cli/src/index.ts` (mirrors `feedback.ts` / `dataset-mine.ts`).
+ * The harvested OUTPUT text is the rated assistant answer (or the user's
+ * correction), so it is PII/secret-redacted before it lands in the pool: the
+ * caller threads a redactor built from `SYNTHESIZE_PII_DETECTORS` (the shared
+ * secret/API-key + DEFAULT_PII detector set) so a pasted credential never
+ * survives into the pool or the optimizer meta-prompt.
+ */
+import type { FeedbackRecord, SessionTurn } from "./feedback";
+/** Schema version of a persisted few-shot pool line. */
+export declare const FEWSHOT_SCHEMA_VERSION: 1;
+/** One curated few-shot example: an input paired with a known-good output. */
+export type FewShotExample = {
+    readonly schemaVersion: 1;
+    /** Stable id: `${sessionId}_t${turnNumber}` — the join key, dedupes reruns. */
+    readonly id: string;
+    readonly input: string;
+    readonly output: string;
+    /** [0,1] normalized rating that qualified this turn (or 1 for a correction). */
+    readonly score: number;
+    /** Provenance for audit + optimizer trust. */
+    readonly provenance: {
+        readonly sessionId: string;
+        readonly turnNumber: number;
+        /** "rating" (up-rated answer) | "correction" (user-supplied better answer). */
+        readonly source: "rating" | "correction";
+        readonly rater?: string;
+    };
+};
+export type HarvestOptions = {
+    /** Normalized score at/above which a rated turn qualifies. Default 0.7. */
+    readonly minScore?: number;
+    /** Async redactor applied to the harvested OUTPUT (and correction). The CLI
+     *  passes a `SYNTHESIZE_PII_DETECTORS`-backed redactor; tests pass a stub. */
+    readonly redact?: (text: string) => Promise<string>;
+};
+export type HarvestStats = {
+    readonly totalFeedback: number;
+    readonly qualified: number;
+    readonly skippedUnmatched: number;
+    readonly skippedEmpty: number;
+};
+export type HarvestResult = {
+    readonly examples: FewShotExample[];
+    readonly stats: HarvestStats;
+};
+/**
+ * Harvest positively-rated turns into few-shot examples. A turn qualifies when
+ * its merged rating normalizes to ≥ `minScore`, OR it carries a `correction`
+ * (the user's own better answer — always gold). The example OUTPUT is the
+ * correction when present, else the assistant's answer; empty answers are
+ * skipped. Deterministic: examples are sorted by (score desc, id asc) and
+ * deduped by id, so re-harvesting the same feedback is idempotent.
+ */
+export declare function harvestFewShot(turns: ReadonlyArray<SessionTurn>, feedback: ReadonlyArray<FeedbackRecord>, opts?: HarvestOptions): Promise<HarvestResult>;
+/** Narrow an arbitrary parsed JSON value to a persisted FewShotExample. */
+export declare function isFewShotExample(value: unknown): value is FewShotExample;
+/**
+ * Merge a freshly-harvested pool into an existing one, deduping by id (the
+ * newer example wins so re-runs refresh outputs without duplicating). Returns
+ * the deterministically-ordered union.
+ */
+export declare function mergePools(existing: ReadonlyArray<FewShotExample>, fresh: ReadonlyArray<FewShotExample>): FewShotExample[];
+/** Serialize a pool to JSONL (one example per line). */
+export declare function poolToJsonl(examples: ReadonlyArray<FewShotExample>): string;
+/**
+ * Render the top-K examples as an in-context few-shot block for the system
+ * prompt / `agent.instructions`. Empty input → "" so callers append
+ * unconditionally. `k` caps how many examples are injected (default 5).
+ */
+export declare function formatFewShotForPrompt(examples: ReadonlyArray<FewShotExample>, k?: number): string;