crewhaus 0.1.8 → 0.2.0

package/dist/intents.js

@@ -0,0 +1,292 @@
+/**
+ * Item 67 — `crewhaus intents`: end-user intent analytics. What are people
+ * actually asking the harness, and how well is it serving them?
+ *
+ * Clusters `user_message` inputs across sessions with the SAME deterministic
+ * greedy-Jaccard token clustering the `faq` / `graders suggest` / coverage
+ * features use (no model call), then ranks each intent cluster by:
+ *
+ *   - FREQUENCY:     total occurrences + distinct sessions.
+ *   - SATISFACTION:  mean normalized rating of the cluster's rated turns.
+ *   - FAILURE RATE:  share of the cluster's turns that hit a struggle signal
+ *                    (error / tool-error / loop / retry — mined by the same
+ *                    `dataset mine` signals), i.e. an UNMET intent.
+ *   - RECENCY:       first vs second half of the session window → a "rising"
+ *                    intent asked much more in the recent half.
+ *
+ * From those it surfaces four views: top intents, rising intents, low-
+ * satisfaction intents (down-rated / high-error), and unmet intents (the agent
+ * couldn't answer — errors/loops/retries). This is the product-analytics feed
+ * for `dataset mine` / `faq distill` / few-shot harvest.
+ *
+ * Everything here is pure + deterministic (the CLI entry file reads sessions +
+ * feedback and PII/secret-redacts the rendered examples via a passed-in
+ * callback). No credential/secret text enters a rendered example un-redacted.
+ */
+import { mergeFeedback, normalizeRating } from "./feedback";
+import { normalizeEvidenceTokens } from "./graders-suggest";
+/** Thrown on malformed inputs / bad flags. The CLI routes it through `die()`. */
+export class IntentsError extends Error {
+    name = "IntentsError";
+}
+function jaccard(a, b) {
+    let intersection = 0;
+    for (const t of a)
+        if (b.has(t))
+            intersection += 1;
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+/** session#turn composite key — `#` never collides (session ids are hex,
+ *  turnNumber an int), so no control/NUL byte enters a map key. */
+function turnKey(sessionId, turnNumber) {
+    return `${sessionId}#${turnNumber}`;
+}
+/**
+ * Cluster ordered user turns into ranked intents. Deterministic: turns are
+ * clustered greedily by normalized-token Jaccard overlap in `order` sequence;
+ * each cluster's representative is the highest-token-frequency question (ties →
+ * shortest, then lexicographic). The recency split is the median `order`.
+ */
+export function clusterIntents(turns, feedback, failedTurnKeys, opts = {}) {
+    const threshold = opts.threshold ?? 0.5;
+    const lowBelow = opts.lowSatisfactionBelow ?? 0.5;
+    const unmetAtLeast = opts.unmetFailureAtLeast ?? 0.34;
+    const topN = opts.topN ?? 5;
+    const examplesPer = opts.examplesPerIntent ?? 3;
+    const ratingByKey = new Map();
+    for (const fb of mergeFeedback(feedback)) {
+        const r = normalizeRating(fb);
+        if (r !== undefined)
+            ratingByKey.set(turnKey(fb.sessionId, fb.turnNumber), r);
+    }
+    const failedKeys = new Set(failedTurnKeys.map((s) => turnKey(s.sessionId, s.turnNumber)));
+    const records = turns
+        .filter((t) => t.input.trim() !== "")
+        .map((t) => ({
+        sessionId: t.sessionId,
+        turnNumber: t.turnNumber,
+        question: t.input.trim(),
+        order: t.order,
+        rating: ratingByKey.get(turnKey(t.sessionId, t.turnNumber)),
+        failed: failedKeys.has(turnKey(t.sessionId, t.turnNumber)),
+    }))
+        // Stable clustering order: chronological, tie-broken deterministically.
+        .sort((a, b) => a.order - b.order || a.sessionId.localeCompare(b.sessionId) || a.turnNumber - b.turnNumber);
+    const totalSessions = new Set(records.map((r) => r.sessionId)).size;
+    // Median order for the rising split (older half < median ≤ recent half).
+    const orders = records.map((r) => r.order).sort((a, b) => a - b);
+    const medianOrder = orders.length === 0 ? 0 : orders[Math.floor(orders.length / 2)];
+    const clusters = [];
+    for (const rec of records) {
+        const tokens = new Set(normalizeEvidenceTokens(rec.question));
+        if (tokens.size === 0)
+            continue;
+        let bestIdx = -1;
+        let bestOverlap = 0;
+        for (let i = 0; i < clusters.length; i += 1) {
+            const overlap = jaccard(tokens, clusters[i].seed);
+            if (overlap > bestOverlap) {
+                bestOverlap = overlap;
+                bestIdx = i;
+            }
+        }
+        if (bestIdx >= 0 && bestOverlap >= threshold) {
+            const c = clusters[bestIdx];
+            c.items.push(rec);
+            for (const t of tokens)
+                c.tokenFreq.set(t, (c.tokenFreq.get(t) ?? 0) + 1);
+        }
+        else {
+            const tokenFreq = new Map();
+            for (const t of tokens)
+                tokenFreq.set(t, 1);
+            clusters.push({ seed: tokens, items: [rec], tokenFreq });
+        }
+    }
+    const intents = clusters.map((c) => {
+        const occurrences = c.items.length;
+        const sessionCount = new Set(c.items.map((r) => r.sessionId)).size;
+        const rated = c.items.filter((r) => r.rating !== undefined);
+        const meanRating = rated.length === 0
+            ? undefined
+            : rated.reduce((acc, r) => acc + r.rating, 0) / rated.length;
+        const failedTurns = c.items.filter((r) => r.failed).length;
+        const earlyCount = c.items.filter((r) => r.order < medianOrder).length;
+        const recentCount = occurrences - earlyCount;
+        // Representative: score each question by summed token frequency; higher is
+        // more central. Ties → shorter question, then lexicographic (deterministic).
+        const scored = c.items
+            .map((r) => {
+            const toks = normalizeEvidenceTokens(r.question);
+            const freq = toks.reduce((acc, t) => acc + (c.tokenFreq.get(t) ?? 0), 0);
+            return { question: r.question, freq };
+        })
+            .sort((a, b) => b.freq - a.freq ||
+            a.question.length - b.question.length ||
+            a.question.localeCompare(b.question));
+        const representative = scored[0]?.question ?? "";
+        // Distinct example questions, most-central first.
+        const examples = [];
+        for (const s of scored) {
+            if (!examples.includes(s.question))
+                examples.push(s.question);
+            if (examples.length >= examplesPer)
+                break;
+        }
+        return {
+            representative,
+            occurrences,
+            sessionCount,
+            meanRating,
+            failedTurns,
+            failureRate: occurrences === 0 ? 0 : failedTurns / occurrences,
+            earlyCount,
+            recentCount,
+            examples,
+        };
+    });
+    // Rank: frequency-first for the master list.
+    const byFrequency = [...intents].sort((a, b) => b.occurrences - a.occurrences ||
+        b.sessionCount - a.sessionCount ||
+        a.representative.localeCompare(b.representative));
+    const topIntents = byFrequency.slice(0, topN);
+    // Rising: recentCount > earlyCount, ranked by the delta (then frequency).
+    const risingIntents = [...intents]
+        .filter((i) => i.recentCount > i.earlyCount)
+        .sort((a, b) => b.recentCount - b.earlyCount - (a.recentCount - a.earlyCount) ||
+        b.occurrences - a.occurrences ||
+        a.representative.localeCompare(b.representative))
+        .slice(0, topN);
+    // Low satisfaction: rated AND below the threshold, worst first (then frequency).
+    const lowSatisfactionIntents = [...intents]
+        .filter((i) => i.meanRating !== undefined && i.meanRating < lowBelow)
+        .sort((a, b) => a.meanRating - b.meanRating ||
+        b.occurrences - a.occurrences ||
+        a.representative.localeCompare(b.representative))
+        .slice(0, topN);
+    // Unmet: failure rate at/above the threshold, highest first (then frequency).
+    const unmetIntents = [...intents]
+        .filter((i) => i.failureRate >= unmetAtLeast && i.failedTurns > 0)
+        .sort((a, b) => b.failureRate - a.failureRate ||
+        b.occurrences - a.occurrences ||
+        a.representative.localeCompare(b.representative))
+        .slice(0, topN);
+    return {
+        totalTurns: records.length,
+        totalSessions,
+        intents: byFrequency,
+        topIntents,
+        risingIntents,
+        lowSatisfactionIntents,
+        unmetIntents,
+    };
+}
+/** Build ordered turns from per-session events, oldest session first. The
+ *  caller passes sessions already in chronological order (oldest→newest). */
+export function orderedTurnsFromSessions(perSession, deriveTurns) {
+    const out = [];
+    let order = 0;
+    for (const { sessionId, events } of perSession) {
+        for (const t of deriveTurns(events)) {
+            out.push({ ...t, sessionId, order });
+            order += 1;
+        }
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+// Rendering — text / json / html. `redact` is applied to every example string.
+// ---------------------------------------------------------------------------
+function redactExamples(intent, redact) {
+    return {
+        ...intent,
+        representative: redact(intent.representative),
+        examples: intent.examples.map(redact),
+    };
+}
+/** Apply the redactor to every rendered string in the digest. */
+export function redactDigest(digest, redact) {
+    const map = (list) => list.map((i) => redactExamples(i, redact));
+    return {
+        ...digest,
+        intents: map(digest.intents),
+        topIntents: map(digest.topIntents),
+        risingIntents: map(digest.risingIntents),
+        lowSatisfactionIntents: map(digest.lowSatisfactionIntents),
+        unmetIntents: map(digest.unmetIntents),
+    };
+}
+function fmtRating(r) {
+    return r === undefined ? "n/a" : `${(r * 100).toFixed(0)}%`;
+}
+/** Render the digest as plain text. */
+export function renderIntentsText(digest) {
+    const lines = [];
+    lines.push(`intent analytics: ${digest.totalTurns} user turns across ${digest.totalSessions} sessions, ${digest.intents.length} intent(s)`);
+    const section = (title, list, detail) => {
+        lines.push("");
+        lines.push(title);
+        if (list.length === 0) {
+            lines.push("  (none)");
+            return;
+        }
+        for (const i of list)
+            lines.push(`  - ${i.representative} — ${detail(i)}`);
+    };
+    section("TOP INTENTS", digest.topIntents, (i) => `${i.occurrences}× in ${i.sessionCount} session(s), rating ${fmtRating(i.meanRating)}`);
+    section("RISING INTENTS", digest.risingIntents, (i) => `${i.earlyCount} → ${i.recentCount} (older → recent)`);
+    section("LOW SATISFACTION", digest.lowSatisfactionIntents, (i) => `rating ${fmtRating(i.meanRating)} over ${i.occurrences} turn(s)`);
+    section("UNMET (errors/loops/retries)", digest.unmetIntents, (i) => `${(i.failureRate * 100).toFixed(0)}% failed (${i.failedTurns}/${i.occurrences})`);
+    return `${lines.join("\n")}\n`;
+}
+/** Render the digest as pretty JSON. */
+export function renderIntentsJson(digest) {
+    return `${JSON.stringify(digest, null, 2)}\n`;
+}
+function esc(s) {
+    return s
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;");
+}
+/** Render the digest as a self-contained HTML report. */
+export function renderIntentsHtml(digest) {
+    const rows = (list, cols) => list.length === 0
+        ? '<tr><td class="empty">(none)</td></tr>'
+        : list
+            .map((i) => `<tr><td>${esc(i.representative)}</td>${cols(i)
+            .map((c) => `<td>${esc(c)}</td>`)
+            .join("")}</tr>`)
+            .join("");
+    return `<!doctype html>
+<html><head><meta charset="utf-8"><title>Intent analytics</title>
+<style>
+ body{font:14px system-ui,sans-serif;margin:2rem;color:#111}
+ h1{font-size:1.4rem} h2{font-size:1.05rem;margin-top:1.5rem}
+ table{border-collapse:collapse;width:100%;margin-top:.5rem}
+ th,td{border:1px solid #ddd;padding:.4rem .6rem;text-align:left;vertical-align:top}
+ th{background:#f5f5f5} td.empty{color:#999;font-style:italic}
+</style></head><body>
+<h1>Intent analytics</h1>
+<p>${digest.totalTurns} user turns across ${digest.totalSessions} sessions, ${digest.intents.length} intent(s).</p>
+<h2>Top intents</h2>
+<table><tr><th>Intent</th><th>Occurrences</th><th>Sessions</th><th>Rating</th></tr>
+${rows(digest.topIntents, (i) => [String(i.occurrences), String(i.sessionCount), fmtRating(i.meanRating)])}
+</table>
+<h2>Rising intents</h2>
+<table><tr><th>Intent</th><th>Older</th><th>Recent</th></tr>
+${rows(digest.risingIntents, (i) => [String(i.earlyCount), String(i.recentCount)])}
+</table>
+<h2>Low satisfaction</h2>
+<table><tr><th>Intent</th><th>Rating</th><th>Turns</th></tr>
+${rows(digest.lowSatisfactionIntents, (i) => [fmtRating(i.meanRating), String(i.occurrences)])}
+</table>
+<h2>Unmet (errors / loops / retries)</h2>
+<table><tr><th>Intent</th><th>Failure rate</th><th>Failed / total</th></tr>
+${rows(digest.unmetIntents, (i) => [`${(i.failureRate * 100).toFixed(0)}%`, `${i.failedTurns}/${i.occurrences}`])}
+</table>
+</body></html>
+`;
+}

package/dist/judge-calibrate.d.ts

@@ -0,0 +1,137 @@
+/**
+ * Item 8 — `crewhaus judge calibrate`: measure how well an llm_judge grader
+ * agrees with real human ratings, and recommend the score cut that best
+ * separates the answers users liked from the ones they didn't.
+ *
+ * The command pairs, for each turn that carries BOTH a human `user_feedback`
+ * rating AND can be judged, the (human rating, llm_judge score). The judge
+ * call itself is a model-dependent seam that lives in the CLI entry file
+ * (re-running the judge over the rated transcript turn); this module owns the
+ * PURE statistics over the resulting pairs so they are unit-testable without a
+ * provider:
+ *   - agreement: Pearson correlation + a confusion matrix at the current cut.
+ *   - systematic bias: judge mean − human mean (both normalized to [0,1]).
+ *   - the ROC-optimal cut: the judge score threshold maximizing Youden's J
+ *     (TPR − FPR) at separating up- from down-rated turns.
+ *   - per-rubric disagreement flags: rubrics whose judge-vs-human disagreement
+ *     exceeds a threshold, naming the specific exemplars to re-anchor.
+ *
+ * `--apply` writes the per-spec calibrated `--min-score` default to
+ * `.crewhaus/judge-calibration.json` (a small documented file distill/optimize
+ * could later consult). Kept side-effect-free (the CLI entry file runs an argv
+ * switch on import) mirroring `feedback.ts` / `graders-suggest.ts`; all
+ * filesystem access + the judge model call live in `apps/cli/src/index.ts`.
+ */
+/** Thrown on malformed flags / unusable inputs. The CLI entry file routes it
+ *  through `die()`; tests assert on `.message`. */
+export declare class JudgeCalibrateError extends Error {
+    readonly name = "JudgeCalibrateError";
+}
+/** The [0,1] score at/above which a turn is treated as "up-rated" (positive)
+ *  for confusion + ROC — the same 0.5 midpoint normalizeRating maps a thumbs
+ *  up/down onto. */
+export declare const POSITIVE_THRESHOLD = 0.5;
+/** Default judge cut (normalized [0,1]) the confusion matrix reports against,
+ *  matching distill/eval's default --min-score 0.7 → passing_score 3/5. */
+export declare const DEFAULT_JUDGE_CUT = 0.5;
+/** One paired observation: a human rating and the llm_judge's score for the
+ *  same turn. Both are recorded in native units; helpers normalize. */
+export type CalibrationPair = {
+    readonly sessionId: string;
+    readonly turnNumber: number;
+    /** Human rating normalized to [0,1] (thumbs up=1/down=0, stars (n-1)/4, …). */
+    readonly human: number;
+    /** Raw 1–5 llm_judge score. */
+    readonly judge: number;
+    /** Per-criterion 1–5 judge scores, when the rubric had named criteria. */
+    readonly criterionScores?: Readonly<Record<string, number>>;
+};
+/** Normalize a 1–5 judge score to [0,1] — the same (n-1)/4 map GradeResult uses. */
+export declare function normalizeJudge(score: number): number;
+export type ConfusionMatrix = {
+    /** judge-pass & human-up. */
+    readonly tp: number;
+    /** judge-pass & human-down. */
+    readonly fp: number;
+    /** judge-fail & human-down. */
+    readonly tn: number;
+    /** judge-fail & human-up. */
+    readonly fn: number;
+};
+/** Confusion of judge (normalized ≥ `judgeCut`) vs human (≥ POSITIVE_THRESHOLD). */
+export declare function confusionAt(pairs: ReadonlyArray<CalibrationPair>, judgeCut: number): ConfusionMatrix;
+/** Accuracy of the judge's pass/fail vs the human up/down at a cut. */
+export declare function accuracyAt(pairs: ReadonlyArray<CalibrationPair>, judgeCut: number): number;
+/**
+ * Pearson correlation between the human ratings and the normalized judge
+ * scores. Returns 0 when there is no variance in either series (a degenerate
+ * "all the same" set correlates with nothing).
+ */
+export declare function pearson(pairs: ReadonlyArray<CalibrationPair>): number;
+/** Systematic bias: mean(normalized judge) − mean(human). Positive ⇒ the judge
+ *  is more generous than users; negative ⇒ harsher. */
+export declare function judgeBias(pairs: ReadonlyArray<CalibrationPair>): number;
+export type RocPoint = {
+    readonly cut: number;
+    readonly tpr: number;
+    readonly fpr: number;
+    readonly youdenJ: number;
+};
+/**
+ * Sweep candidate judge cut points (the 5 normalized anchors 0, .25, .5, .75, 1
+ * plus midpoints between observed scores) and return the one that maximizes
+ * Youden's J = TPR − FPR at separating up-rated (human ≥ 0.5) from down-rated
+ * turns. Ties break toward the LOWER cut (more inclusive), then the midpoint
+ * default. Returns undefined when the set has no positives or no negatives (a
+ * ROC point is undefined without both classes).
+ */
+export declare function rocOptimalCut(pairs: ReadonlyArray<CalibrationPair>): RocPoint | undefined;
+export type RubricDisagreement = {
+    readonly criterion: string;
+    /** Mean |normalized judge criterion − human| for this criterion. */
+    readonly meanAbsError: number;
+    /** The sessionId#turn exemplars where they disagreed most (up to 3). */
+    readonly exemplars: ReadonlyArray<string>;
+};
+/**
+ * Flag rubric criteria whose judge-vs-human disagreement exceeds `threshold`
+ * (mean absolute error on [0,1]). For each flagged criterion the worst
+ * exemplars are named so the user knows exactly which transcripts to look at
+ * when re-anchoring the rubric. Criteria absent from the pairs are skipped.
+ */
+export declare function flagDisagreements(pairs: ReadonlyArray<CalibrationPair>, threshold?: number): RubricDisagreement[];
+export type CalibrationCard = {
+    readonly specName?: string;
+    readonly model?: string;
+    readonly pairCount: number;
+    readonly correlation: number;
+    readonly bias: number;
+    readonly accuracyAtDefault: number;
+    readonly confusionAtDefault: ConfusionMatrix;
+    readonly recommendedCut?: RocPoint;
+    readonly disagreements: ReadonlyArray<RubricDisagreement>;
+};
+/** Compute the full calibration card from the paired data. */
+export declare function buildCalibrationCard(pairs: ReadonlyArray<CalibrationPair>, opts?: {
+    specName?: string;
+    model?: string;
+}): CalibrationCard;
+/** The on-disk `.crewhaus/judge-calibration.json` schema `--apply` writes. */
+export type JudgeCalibrationFile = {
+    readonly version: 1;
+    /** Per spec name → the calibrated recommended --min-score cut ([0,1]). */
+    readonly calibrations: Record<string, {
+        readonly minScore: number;
+        readonly model?: string;
+        readonly correlation: number;
+        readonly bias: number;
+        readonly pairCount: number;
+        readonly updatedAt: string;
+    }>;
+};
+/** Build/merge the persisted calibration entry for a spec. Existing entries for
+ *  OTHER specs are preserved. Returns the file object to write. The recommended
+ *  cut is the ROC-optimal cut when available, else the default. */
+export declare function buildCalibrationFile(existing: JudgeCalibrationFile | undefined, card: CalibrationCard, now: string): JudgeCalibrationFile;
+/** Render the calibration card as a terminal report. */
+export declare function renderCalibrationCard(card: CalibrationCard): string;

package/dist/judge-calibrate.js

@@ -0,0 +1,247 @@
+/**
+ * Item 8 — `crewhaus judge calibrate`: measure how well an llm_judge grader
+ * agrees with real human ratings, and recommend the score cut that best
+ * separates the answers users liked from the ones they didn't.
+ *
+ * The command pairs, for each turn that carries BOTH a human `user_feedback`
+ * rating AND can be judged, the (human rating, llm_judge score). The judge
+ * call itself is a model-dependent seam that lives in the CLI entry file
+ * (re-running the judge over the rated transcript turn); this module owns the
+ * PURE statistics over the resulting pairs so they are unit-testable without a
+ * provider:
+ *   - agreement: Pearson correlation + a confusion matrix at the current cut.
+ *   - systematic bias: judge mean − human mean (both normalized to [0,1]).
+ *   - the ROC-optimal cut: the judge score threshold maximizing Youden's J
+ *     (TPR − FPR) at separating up- from down-rated turns.
+ *   - per-rubric disagreement flags: rubrics whose judge-vs-human disagreement
+ *     exceeds a threshold, naming the specific exemplars to re-anchor.
+ *
+ * `--apply` writes the per-spec calibrated `--min-score` default to
+ * `.crewhaus/judge-calibration.json` (a small documented file distill/optimize
+ * could later consult). Kept side-effect-free (the CLI entry file runs an argv
+ * switch on import) mirroring `feedback.ts` / `graders-suggest.ts`; all
+ * filesystem access + the judge model call live in `apps/cli/src/index.ts`.
+ */
+/** Thrown on malformed flags / unusable inputs. The CLI entry file routes it
+ *  through `die()`; tests assert on `.message`. */
+export class JudgeCalibrateError extends Error {
+    name = "JudgeCalibrateError";
+}
+/** The [0,1] score at/above which a turn is treated as "up-rated" (positive)
+ *  for confusion + ROC — the same 0.5 midpoint normalizeRating maps a thumbs
+ *  up/down onto. */
+export const POSITIVE_THRESHOLD = 0.5;
+/** Default judge cut (normalized [0,1]) the confusion matrix reports against,
+ *  matching distill/eval's default --min-score 0.7 → passing_score 3/5. */
+export const DEFAULT_JUDGE_CUT = 0.5;
+/** Normalize a 1–5 judge score to [0,1] — the same (n-1)/4 map GradeResult uses. */
+export function normalizeJudge(score) {
+    return clamp01((score - 1) / 4);
+}
+function clamp01(n) {
+    return n < 0 ? 0 : n > 1 ? 1 : n;
+}
+/** Confusion of judge (normalized ≥ `judgeCut`) vs human (≥ POSITIVE_THRESHOLD). */
+export function confusionAt(pairs, judgeCut) {
+    let tp = 0;
+    let fp = 0;
+    let tn = 0;
+    let fn = 0;
+    for (const p of pairs) {
+        const judgePass = normalizeJudge(p.judge) >= judgeCut;
+        const humanUp = p.human >= POSITIVE_THRESHOLD;
+        if (judgePass && humanUp)
+            tp += 1;
+        else if (judgePass && !humanUp)
+            fp += 1;
+        else if (!judgePass && !humanUp)
+            tn += 1;
+        else
+            fn += 1;
+    }
+    return { tp, fp, tn, fn };
+}
+/** Accuracy of the judge's pass/fail vs the human up/down at a cut. */
+export function accuracyAt(pairs, judgeCut) {
+    if (pairs.length === 0)
+        return 0;
+    const { tp, fp, tn, fn } = confusionAt(pairs, judgeCut);
+    return (tp + tn) / (tp + fp + tn + fn);
+}
+/**
+ * Pearson correlation between the human ratings and the normalized judge
+ * scores. Returns 0 when there is no variance in either series (a degenerate
+ * "all the same" set correlates with nothing).
+ */
+export function pearson(pairs) {
+    const n = pairs.length;
+    if (n < 2)
+        return 0;
+    const xs = pairs.map((p) => p.human);
+    const ys = pairs.map((p) => normalizeJudge(p.judge));
+    const mx = mean(xs);
+    const my = mean(ys);
+    let cov = 0;
+    let vx = 0;
+    let vy = 0;
+    for (let i = 0; i < n; i += 1) {
+        const dx = xs[i] - mx;
+        const dy = ys[i] - my;
+        cov += dx * dy;
+        vx += dx * dx;
+        vy += dy * dy;
+    }
+    if (vx === 0 || vy === 0)
+        return 0;
+    return cov / Math.sqrt(vx * vy);
+}
+/** Systematic bias: mean(normalized judge) − mean(human). Positive ⇒ the judge
+ *  is more generous than users; negative ⇒ harsher. */
+export function judgeBias(pairs) {
+    if (pairs.length === 0)
+        return 0;
+    return mean(pairs.map((p) => normalizeJudge(p.judge))) - mean(pairs.map((p) => p.human));
+}
+function mean(xs) {
+    return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+/**
+ * Sweep candidate judge cut points (the 5 normalized anchors 0, .25, .5, .75, 1
+ * plus midpoints between observed scores) and return the one that maximizes
+ * Youden's J = TPR − FPR at separating up-rated (human ≥ 0.5) from down-rated
+ * turns. Ties break toward the LOWER cut (more inclusive), then the midpoint
+ * default. Returns undefined when the set has no positives or no negatives (a
+ * ROC point is undefined without both classes).
+ */
+export function rocOptimalCut(pairs) {
+    const positives = pairs.filter((p) => p.human >= POSITIVE_THRESHOLD);
+    const negatives = pairs.filter((p) => p.human < POSITIVE_THRESHOLD);
+    if (positives.length === 0 || negatives.length === 0)
+        return undefined;
+    const scores = [...new Set(pairs.map((p) => normalizeJudge(p.judge)))].sort((a, b) => a - b);
+    const candidateCuts = new Set([0, 0.25, 0.5, 0.75, 1]);
+    for (let i = 0; i < scores.length; i += 1) {
+        const s = scores[i];
+        candidateCuts.add(s);
+        const next = scores[i + 1];
+        if (next !== undefined)
+            candidateCuts.add((s + next) / 2);
+    }
+    let best;
+    for (const cut of [...candidateCuts].sort((a, b) => a - b)) {
+        let tp = 0;
+        let fp = 0;
+        for (const p of positives)
+            if (normalizeJudge(p.judge) >= cut)
+                tp += 1;
+        for (const p of negatives)
+            if (normalizeJudge(p.judge) >= cut)
+                fp += 1;
+        const tpr = tp / positives.length;
+        const fpr = fp / negatives.length;
+        const youdenJ = tpr - fpr;
+        if (best === undefined || youdenJ > best.youdenJ) {
+            best = { cut, tpr, fpr, youdenJ };
+        }
+    }
+    return best;
+}
+/**
+ * Flag rubric criteria whose judge-vs-human disagreement exceeds `threshold`
+ * (mean absolute error on [0,1]). For each flagged criterion the worst
+ * exemplars are named so the user knows exactly which transcripts to look at
+ * when re-anchoring the rubric. Criteria absent from the pairs are skipped.
+ */
+export function flagDisagreements(pairs, threshold = 0.34) {
+    const byCriterion = new Map();
+    for (const p of pairs) {
+        if (p.criterionScores === undefined)
+            continue;
+        for (const [name, raw] of Object.entries(p.criterionScores)) {
+            const err = Math.abs(normalizeJudge(raw) - p.human);
+            const bucket = byCriterion.get(name) ?? [];
+            bucket.push({ ref: `${p.sessionId}#${p.turnNumber}`, err });
+            byCriterion.set(name, bucket);
+        }
+    }
+    const out = [];
+    for (const [criterion, errs] of byCriterion) {
+        const meanAbsError = mean(errs.map((e) => e.err));
+        if (meanAbsError < threshold)
+            continue;
+        const exemplars = [...errs]
+            .sort((a, b) => b.err - a.err || a.ref.localeCompare(b.ref))
+            .slice(0, 3)
+            .map((e) => e.ref);
+        out.push({ criterion, meanAbsError, exemplars });
+    }
+    return out.sort((a, b) => b.meanAbsError - a.meanAbsError || a.criterion.localeCompare(b.criterion));
+}
+/** Compute the full calibration card from the paired data. */
+export function buildCalibrationCard(pairs, opts = {}) {
+    const roc = rocOptimalCut(pairs);
+    return {
+        ...(opts.specName !== undefined ? { specName: opts.specName } : {}),
+        ...(opts.model !== undefined ? { model: opts.model } : {}),
+        pairCount: pairs.length,
+        correlation: round(pearson(pairs)),
+        bias: round(judgeBias(pairs)),
+        accuracyAtDefault: round(accuracyAt(pairs, DEFAULT_JUDGE_CUT)),
+        confusionAtDefault: confusionAt(pairs, DEFAULT_JUDGE_CUT),
+        ...(roc !== undefined ? { recommendedCut: roc } : {}),
+        disagreements: flagDisagreements(pairs),
+    };
+}
+function round(n) {
+    return Math.round(n * 1000) / 1000;
+}
+/** Build/merge the persisted calibration entry for a spec. Existing entries for
+ *  OTHER specs are preserved. Returns the file object to write. The recommended
+ *  cut is the ROC-optimal cut when available, else the default. */
+export function buildCalibrationFile(existing, card, now) {
+    const specKey = card.specName ?? "default";
+    const minScore = card.recommendedCut?.cut ?? DEFAULT_JUDGE_CUT;
+    return {
+        version: 1,
+        calibrations: {
+            ...(existing?.calibrations ?? {}),
+            [specKey]: {
+                minScore: round(minScore),
+                ...(card.model !== undefined ? { model: card.model } : {}),
+                correlation: card.correlation,
+                bias: card.bias,
+                pairCount: card.pairCount,
+                updatedAt: now,
+            },
+        },
+    };
+}
+// -------- rendering --------
+/** Render the calibration card as a terminal report. */
+export function renderCalibrationCard(card) {
+    const lines = [];
+    const forSpec = card.specName !== undefined ? ` for "${card.specName}"` : "";
+    const withModel = card.model !== undefined ? ` (judge: ${card.model})` : "";
+    lines.push(`judge calibration${forSpec}${withModel}`);
+    lines.push(`  paired ratings:   ${card.pairCount}`);
+    lines.push(`  correlation:      ${card.correlation.toFixed(3)} (human vs judge)`);
+    lines.push(`  systematic bias:  ${card.bias >= 0 ? "+" : ""}${card.bias.toFixed(3)} (${card.bias > 0.05 ? "judge too GENEROUS" : card.bias < -0.05 ? "judge too HARSH" : "well-centered"})`);
+    const c = card.confusionAtDefault;
+    lines.push(`  at cut ${DEFAULT_JUDGE_CUT}:  accuracy ${(card.accuracyAtDefault * 100).toFixed(0)}%  ` +
+        `(tp ${c.tp} / fp ${c.fp} / tn ${c.tn} / fn ${c.fn})`);
+    if (card.recommendedCut !== undefined) {
+        const r = card.recommendedCut;
+        lines.push(`  ROC-optimal cut:  ${r.cut.toFixed(3)}  (TPR ${(r.tpr * 100).toFixed(0)}%, FPR ${(r.fpr * 100).toFixed(0)}%, Youden J ${r.youdenJ.toFixed(3)})`);
+    }
+    else {
+        lines.push("  ROC-optimal cut:  n/a (need both up- and down-rated turns)");
+    }
+    if (card.disagreements.length > 0) {
+        lines.push("");
+        lines.push(`  ${card.disagreements.length} rubric criterion/criteria disagree with users — re-anchor:`);
+        for (const d of card.disagreements) {
+            lines.push(`    - "${d.criterion}" (mean abs error ${d.meanAbsError.toFixed(3)}) e.g. ${d.exemplars.join(", ")}`);
+        }
+    }
+    return `${lines.join("\n")}\n`;
+}