@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/dist/decisions.d.ts

@@ -0,0 +1,84 @@
+import type { CheckMeasures } from "./stats.js";
+import type { CheckResult } from "./types.js";
+/**
+ * Scoring for decision models: models that answer typed questions about a
+ * state (pick one option, say true or false, rate on a scale) with
+ * probabilities, like the AI SDK's `experimental_decide`. A decision is
+ * graded twice: did it pick right, and how good were its probabilities?
+ * Two models that pick right equally often can differ a lot in the second:
+ * one says 90% and is right 90% of the time, the other says 99% and is
+ * right 80%. Pure.
+ */
+/** A decision model's answer to one question, as the AI SDK returns it. */
+export type DecisionAnswerLike = {
+    type: "choice";
+    choice: string;
+    probabilities?: Record<string, number> | undefined;
+} | {
+    type: "boolean";
+    probability: number;
+} | {
+    type: "score";
+    score: number;
+    probabilities?: Record<string, number> | undefined;
+} | {
+    type: "refusal";
+};
+/** The right answer to one question: an option, true or false, or a level. */
+export type DecisionExpected = string | boolean | number;
+/** How one decision did. Probability measures are null when the model gave none. */
+export type DecisionScore = {
+    /** Picked the right option, side, or level. */
+    correct: boolean;
+    /** The model declined to answer. */
+    refused: boolean;
+    /** The probability the model gave the right answer. */
+    probabilityOfTruth: number | null;
+    /** The probability the model gave its own pick: how sure it was. */
+    confidence: number | null;
+    /**
+     * Brier score, from 0 (all probability on the right answer) to 1 (all on
+     * a wrong one): squared error, scaled to 0–1 for any number of options.
+     */
+    brier: number | null;
+    /** Log loss of the right answer, capped so one sure miss can't dominate. */
+    logLoss: number | null;
+};
+/** Probabilities below this count as this, so log loss stays finite. */
+export declare const MIN_PROBABILITY = 0.0001;
+/** Scores one decision against its right answer. */
+export declare function scoreDecision(answer: DecisionAnswerLike, expected: DecisionExpected, opts?: {
+    /** A score question's number of levels, for the answer's level scale. */
+    levels?: number;
+    /** A boolean answer counts as true at or above this. Default 0.5. */
+    threshold?: number;
+}): DecisionScore;
+/**
+ * One question's check: passes when the model picked right, and carries
+ * its probability measures (brier, logLoss, confidence, correct), which a
+ * test averages and turns into a calibration error. Doesn't apply (null)
+ * when there's no right answer to compare with.
+ */
+export declare function decisionCheck(key: string, answer: DecisionAnswerLike | undefined, expected: DecisionExpected | undefined, opts?: {
+    levels?: number;
+    threshold?: number;
+    describe?: (v: DecisionExpected) => string;
+}): CheckResult;
+/**
+ * Expected calibration error: answers grouped by how sure the model was
+ * (10 equal bins), and the average gap between how sure it was and how
+ * often it was right, weighted by bin size. 0 is perfectly calibrated.
+ * Null with no answers that carry a confidence.
+ */
+export declare function calibrationError(answers: {
+    confidence: number;
+    correct: boolean;
+}[], bins?: number): number | null;
+/**
+ * A test's probability quality: 1 minus the mean Brier score over its
+ * checks that measured one. 1 is perfect; null when no check had
+ * probabilities (a prompted model's choices, for instance).
+ */
+export declare function probabilityQuality(measures: Record<string, CheckMeasures>): number | null;
+/** A test's calibration error: the mean over its checks that had one. */
+export declare function meanCalibrationError(measures: Record<string, CheckMeasures>): number | null;

package/dist/decisions.js

@@ -0,0 +1,146 @@
+/** Probabilities below this count as this, so log loss stays finite. */
+export const MIN_PROBABILITY = 1e-4;
+const logLossOf = (p) => -Math.log(Math.max(p, MIN_PROBABILITY));
+/** The option with the most probability. */
+function topOf(probabilities) {
+    let best = null;
+    let max = -Infinity;
+    for (const [k, p] of Object.entries(probabilities))
+        if (p > max) {
+            max = p;
+            best = k;
+        }
+    return best;
+}
+/** Scores one decision against its right answer. */
+export function scoreDecision(answer, expected, opts = {}) {
+    const none = {
+        probabilityOfTruth: null,
+        confidence: null,
+        brier: null,
+        logLoss: null,
+    };
+    if (answer.type === "refusal")
+        return { correct: false, refused: true, ...none };
+    if (answer.type === "boolean") {
+        const p = answer.probability;
+        const truth = expected === true;
+        const pTruth = truth ? p : 1 - p;
+        return {
+            correct: p >= (opts.threshold ?? 0.5) === truth,
+            refused: false,
+            probabilityOfTruth: pTruth,
+            confidence: Math.max(p, 1 - p),
+            brier: (p - (truth ? 1 : 0)) ** 2,
+            logLoss: logLossOf(pTruth),
+        };
+    }
+    const right = String(expected);
+    const picked = answer.type === "choice" ? answer.choice : String(Math.round(answer.score));
+    const correct = picked === right;
+    const probs = answer.probabilities;
+    if (!probs || !Object.keys(probs).length)
+        return { correct, refused: false, ...none };
+    // Brier over every option, scaled to 0–1 (its most is 2).
+    const keys = new Set([...Object.keys(probs), right]);
+    let squared = 0;
+    for (const k of keys)
+        squared += ((probs[k] ?? 0) - (k === right ? 1 : 0)) ** 2;
+    const pTruth = probs[right] ?? 0;
+    const top = topOf(probs);
+    return {
+        correct,
+        refused: false,
+        probabilityOfTruth: pTruth,
+        confidence: top === null ? null : (probs[top] ?? null),
+        brier: squared / 2,
+        logLoss: logLossOf(pTruth),
+    };
+}
+/**
+ * One question's check: passes when the model picked right, and carries
+ * its probability measures (brier, logLoss, confidence, correct), which a
+ * test averages and turns into a calibration error. Doesn't apply (null)
+ * when there's no right answer to compare with.
+ */
+export function decisionCheck(key, answer, expected, opts = {}) {
+    if (expected === undefined)
+        return { key, pass: null };
+    const say = opts.describe ?? ((v) => String(v));
+    if (!answer)
+        return {
+            key,
+            pass: false,
+            note: "No answer to this question",
+            measures: { correct: 0 },
+        };
+    const s = scoreDecision(answer, expected, opts);
+    const measures = { correct: s.correct ? 1 : 0 };
+    if (s.brier !== null)
+        measures.brier = s.brier;
+    if (s.logLoss !== null)
+        measures.logLoss = s.logLoss;
+    if (s.confidence !== null)
+        measures.confidence = s.confidence;
+    const given = answer.type === "refusal"
+        ? "refused"
+        : answer.type === "boolean"
+            ? `${Math.round(answer.probability * 100)}% true`
+            : answer.type === "choice"
+                ? say(answer.choice)
+                : `level ${answer.score.toFixed(1)}`;
+    return {
+        key,
+        pass: s.correct,
+        ...(s.correct ? {} : { note: `Expected ${say(expected)}, got ${given}` }),
+        measures,
+    };
+}
+/**
+ * Expected calibration error: answers grouped by how sure the model was
+ * (10 equal bins), and the average gap between how sure it was and how
+ * often it was right, weighted by bin size. 0 is perfectly calibrated.
+ * Null with no answers that carry a confidence.
+ */
+export function calibrationError(answers, bins = 10) {
+    if (!answers.length)
+        return null;
+    const groups = Array.from({ length: bins }, () => ({
+        n: 0,
+        sure: 0,
+        right: 0,
+    }));
+    for (const a of answers) {
+        const i = Math.min(bins - 1, Math.max(0, Math.floor(a.confidence * bins)));
+        const g = groups[i];
+        if (!g)
+            continue;
+        g.n++;
+        g.sure += a.confidence;
+        g.right += a.correct ? 1 : 0;
+    }
+    let gap = 0;
+    for (const g of groups)
+        if (g.n)
+            gap += (g.n / answers.length) * Math.abs(g.sure / g.n - g.right / g.n);
+    return gap;
+}
+/**
+ * A test's probability quality: 1 minus the mean Brier score over its
+ * checks that measured one. 1 is perfect; null when no check had
+ * probabilities (a prompted model's choices, for instance).
+ */
+export function probabilityQuality(measures) {
+    const briers = Object.values(measures).flatMap((m) => m.mean.brier === undefined ? [] : [m.mean.brier]);
+    return briers.length
+        ? 1 - briers.reduce((t, b) => t + b, 0) / briers.length
+        : null;
+}
+/** A test's calibration error: the mean over its checks that had one. */
+export function meanCalibrationError(measures) {
+    const errors = Object.values(measures).flatMap((m) => m.calibrationError === null ? [] : [m.calibrationError]);
+    return errors.length
+        ? errors.reduce((t, e) => t + e, 0) / errors.length
+        : null;
+}
+//# sourceMappingURL=decisions.js.map

package/dist/decisions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decisions.js","sourceRoot":"","sources":["../src/decisions.ts"],"names":[],"mappings":"AAkDA,wEAAwE;AACxE,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAEpC,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;AAEzE,4CAA4C;AAC5C,SAAS,KAAK,CAAC,aAAqC;IAClD,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IACpB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC;QAChD,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;YACZ,GAAG,GAAG,CAAC,CAAC;YACR,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,aAAa,CAC3B,MAA0B,EAC1B,QAA0B,EAC1B,OAKI,EAAE;IAEN,MAAM,IAAI,GAAG;QACX,kBAAkB,EAAE,IAAI;QACxB,UAAU,EAAE,IAAI;QAChB,KAAK,EAAE,IAAI;QACX,OAAO,EAAE,IAAI;KACd,CAAC;IACF,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS;QAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;IAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ,KAAK,IAAI,CAAC;QAChC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjC,OAAO;YACL,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,KAAK,KAAK;YAC/C,OAAO,EAAE,KAAK;YACd,kBAAkB,EAAE,MAAM;YAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;YAC9B,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACjC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC;SAC3B,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,MAAM,GACV,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,MAAM,KAAK,KAAK,CAAC;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC;IACnC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM;QACtC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IAE9C,0DAA0D;IAC1D,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,IAAI;QAClB,OAAO,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACzB,OAAO;QACL,OAAO;QACP,OAAO,EAAE,KAAK;QACd,kBAAkB,EAAE,MAAM;QAC1B,UAAU,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QACtD,KAAK,EAAE,OAAO,GAAG,CAAC;QAClB,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC;KAC3B,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,GAAW,EACX,MAAsC,EACtC,QAAsC,EACtC,OAII,EAAE;IAEN,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,MAAM;QACT,OAAO;YACL,GAAG;YACH,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,4BAA4B;YAClC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;SACzB,CAAC;IACJ,MAAM,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAChD,MAAM,QAAQ,GAA2B,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI;QAAE,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAC/C,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI;QAAE,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;IACrD,IAAI,CAAC,CAAC,UAAU,KAAK,IAAI;QAAE,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;IAC9D,MAAM,KAAK,GACT,MAAM,CAAC,IAAI,KAAK,SAAS;QACvB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS;YACzB,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,QAAQ;YACjD,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ;gBACxB,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;gBACpB,CAAC,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7C,OAAO;QACL,GAAG;QACH,IAAI,EAAE,CAAC,CAAC,OAAO;QACf,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,CAAC,QAAQ,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;QACzE,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAmD,EACnD,IAAI,GAAG,EAAE;IAET,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACjD,CAAC,EAAE,CAAC;QACJ,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,CAAC;KACT,CAAC,CAAC,CAAC;IACJ,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,CAAC,CAAC,CAAC,EAAE,CAAC;QACN,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC;QACvB,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,MAAM;QACpB,IAAI,CAAC,CAAC,CAAC;YACL,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAAuC;IAEvC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACnD,CAAC,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CACjD,CAAC;IACF,OAAO,MAAM,CAAC,MAAM;QAClB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACvD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,oBAAoB,CAClC,QAAuC;IAEvC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACnD,CAAC,CAAC,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CACxD,CAAC;IACF,OAAO,MAAM,CAAC,MAAM;QAClB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACnD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC"}

package/dist/feedback.d.ts

@@ -0,0 +1,114 @@
+import type { NewCase } from "./store.js";
+import type { AnySuite } from "./types.js";
+/**
+ * Thumbs up and down on AI answers in your product, and turning them into
+ * draft cases. A rating says how a real answer landed; a case made from it
+ * tests the next model on the same input. Pure, except the store.
+ */
+/** A thumbs up or a thumbs down. */
+export type Rating = "UP" | "DOWN";
+/** OPEN: waiting for a look. CASED: made into a case. DISMISSED: not useful. */
+export type FeedbackStatus = "OPEN" | "CASED" | "DISMISSED";
+/** A person's rating of one AI answer in your product, as stored. */
+export type FeedbackRecord = {
+    id: string;
+    /** The AI feature that wrote the answer. */
+    operation: string;
+    /** What was rated, in your app: a message id, a summary id, … */
+    targetId: string;
+    /** Who rated it, in your app; one rating per person per answer. */
+    raterId: string;
+    rating: Rating;
+    /** What was wrong, in their words (a thumbs down). */
+    reason: string | null;
+    /** The text they rated, when your app doesn't keep it elsewhere. */
+    excerpt: string | null;
+    /**
+     * The suite input that produced the answer, de-identified, kept only
+     * with the person's consent. Without it a rating can't become a case.
+     */
+    input: unknown;
+    /** The answer as the operation returned it. */
+    output: unknown;
+    status: FeedbackStatus;
+    /** The case made from it. */
+    caseId: string | null;
+    createdAt: Date;
+    updatedAt: Date;
+};
+/** A rating to save; `cleanRating` trims and bounds it. */
+export type NewRating = {
+    operation: string;
+    targetId: string;
+    raterId: string;
+    rating: Rating;
+    reason?: string | null;
+    excerpt?: string | null;
+    input?: unknown;
+    output?: unknown;
+};
+/** Where ratings live. `@crowandraven/evalkeeper-prisma` and `memoryStore` implement it. */
+export interface FeedbackStore {
+    /**
+     * Saves a person's rating of an answer, replacing their earlier one. A
+     * changed rating reopens it.
+     */
+    rate(rating: NewRating): Promise<FeedbackRecord>;
+    /** Takes a person's rating back. */
+    clearRating(raterId: string, operation: string, targetId: string): Promise<void>;
+    /** Newest first. */
+    listFeedback(opts?: {
+        operation?: string;
+        rating?: Rating;
+        status?: FeedbackStatus;
+        since?: Date;
+        limit?: number;
+    }): Promise<FeedbackRecord[]>;
+    setFeedbackStatus(id: string, status: FeedbackStatus, caseId?: string | null): Promise<void>;
+}
+/** Longest reason and excerpt kept, so one rating can't fill a table. */
+export declare const REASON_MAX = 1000;
+/** The longest rated excerpt kept. */
+export declare const EXCERPT_MAX = 2000;
+/** A rating as it should be stored: trimmed, bounded, a reason only on a thumbs down. */
+export declare function cleanRating(r: NewRating): Required<NewRating>;
+/**
+ * Why a rating can't become a case for this suite, or null when it can. A
+ * case needs the input that produced the answer, in the suite's shape.
+ */
+export declare function whyNotACase(f: FeedbackRecord, suite: AnySuite | undefined): string | null;
+/**
+ * A draft case from a rating, for a person to finish. A thumbs down has no
+ * answer yet: the person writes what a good one is. A thumbs up proposes
+ * the model's own answer, pending until a person confirms it. Never
+ * confirmed on anyone's behalf. Null when it can't become a case.
+ *
+ * @example
+ * ```ts
+ * const suite = SUITES.find((s) => s.operation === rating.operation);
+ * const draft = suite ? caseFromFeedback(rating, suite) : null;
+ * if (draft) await store.createCase(draft);
+ * ```
+ */
+export declare function caseFromFeedback(f: FeedbackRecord, suite: AnySuite): NewCase | null;
+/**
+ * One feature's ratings over a period: up, down, the share down, whether it needs a look,
+ * and how many thumbs down nobody has looked at.
+ */
+export type ThumbsSummary = {
+    operation: string;
+    up: number;
+    down: number;
+    /** Share of ratings that were thumbs down; null with none. */
+    downShare: number | null;
+    /** Enough ratings, and enough of them down, to look at. */
+    alert: boolean;
+    /** Thumbs down nobody has looked at yet. */
+    open: number;
+};
+/**
+ * Ratings per feature since a date, most thumbs down first. A feature is
+ * flagged when it has at least THUMBS_MIN ratings and at least
+ * THUMBS_DOWN_SHARE of them are down.
+ */
+export declare function thumbsSummary(feedback: FeedbackRecord[], since: Date): ThumbsSummary[];

package/dist/feedback.js

@@ -0,0 +1,116 @@
+import { thumbsAlert } from "./upkeepRules.js";
+/** Longest reason and excerpt kept, so one rating can't fill a table. */
+export const REASON_MAX = 1_000;
+/** The longest rated excerpt kept. */
+export const EXCERPT_MAX = 2_000;
+/** A rating as it should be stored: trimmed, bounded, a reason only on a thumbs down. */
+export function cleanRating(r) {
+    const trim = (s, max) => {
+        const t = s?.trim().slice(0, max);
+        return t ? t : null;
+    };
+    return {
+        operation: r.operation,
+        targetId: r.targetId.slice(0, 200),
+        raterId: r.raterId,
+        rating: r.rating,
+        reason: r.rating === "DOWN" ? trim(r.reason, REASON_MAX) : null,
+        excerpt: trim(r.excerpt, EXCERPT_MAX),
+        input: r.input ?? null,
+        output: r.output ?? null,
+    };
+}
+/**
+ * Why a rating can't become a case for this suite, or null when it can. A
+ * case needs the input that produced the answer, in the suite's shape.
+ */
+export function whyNotACase(f, suite) {
+    if (!suite)
+        return "No suite tests this feature yet.";
+    if (f.input === null || f.input === undefined)
+        return "The input behind this answer wasn't kept, so there's nothing to test.";
+    if (!suite.input.safeParse(f.input).success)
+        return "The kept input doesn't match the suite's input any more.";
+    return null;
+}
+/**
+ * A draft case from a rating, for a person to finish. A thumbs down has no
+ * answer yet: the person writes what a good one is. A thumbs up proposes
+ * the model's own answer, pending until a person confirms it. Never
+ * confirmed on anyone's behalf. Null when it can't become a case.
+ *
+ * @example
+ * ```ts
+ * const suite = SUITES.find((s) => s.operation === rating.operation);
+ * const draft = suite ? caseFromFeedback(rating, suite) : null;
+ * if (draft) await store.createCase(draft);
+ * ```
+ */
+export function caseFromFeedback(f, suite) {
+    if (whyNotACase(f, suite))
+        return null;
+    const proposed = f.rating === "UP" && suite.labelFromOutput && f.output !== null
+        ? suite.labelFromOutput(f.output)
+        : null;
+    const note = f.rating === "DOWN"
+        ? `From a thumbs down${f.reason ? `: "${f.reason}"` : "."} Write what a good answer is.`
+        : "From a thumbs up: the model's answer is proposed. Confirm it if it's right.";
+    return {
+        suite: suite.key,
+        input: suite.input.parse(f.input),
+        tags: ["feedback", f.rating === "DOWN" ? "thumbs down" : "thumbs up"],
+        split: "DEV",
+        status: "DRAFT",
+        source: "FEEDBACK",
+        sourceRef: f.id,
+        note,
+        labels: proposed === null
+            ? []
+            : [
+                {
+                    kind: "EXPECTED",
+                    status: "PENDING",
+                    value: proposed,
+                    raterRole: "FEEDBACK",
+                    raterId: null,
+                    note: "The model's answer, rated thumbs up.",
+                },
+            ],
+    };
+}
+/**
+ * Ratings per feature since a date, most thumbs down first. A feature is
+ * flagged when it has at least THUMBS_MIN ratings and at least
+ * THUMBS_DOWN_SHARE of them are down.
+ */
+export function thumbsSummary(feedback, since) {
+    const by = new Map();
+    for (const f of feedback) {
+        if (f.updatedAt < since)
+            continue;
+        const s = by.get(f.operation) ?? {
+            operation: f.operation,
+            up: 0,
+            down: 0,
+            downShare: null,
+            alert: false,
+            open: 0,
+        };
+        if (f.rating === "UP")
+            s.up++;
+        else {
+            s.down++;
+            if (f.status === "OPEN")
+                s.open++;
+        }
+        by.set(f.operation, s);
+    }
+    return [...by.values()]
+        .map((s) => ({
+        ...s,
+        downShare: s.up + s.down ? s.down / (s.up + s.down) : null,
+        alert: thumbsAlert(s.up, s.down),
+    }))
+        .sort((a, b) => b.down - a.down || a.operation.localeCompare(b.operation));
+}
+//# sourceMappingURL=feedback.js.map

package/dist/feedback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"feedback.js","sourceRoot":"","sources":["../src/feedback.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAkF/C,yEAAyE;AACzE,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,CAAC;AAChC,sCAAsC;AACtC,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC;AAEjC,yFAAyF;AACzF,MAAM,UAAU,WAAW,CAAC,CAAY;IACtC,MAAM,IAAI,GAAG,CAAC,CAA4B,EAAE,GAAW,EAAE,EAAE;QACzD,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAClC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtB,CAAC,CAAC;IACF,OAAO;QACL,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAClC,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,MAAM,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;QAC/D,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,WAAW,CAAC;QACrC,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI;QACtB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;KACzB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,CAAiB,EACjB,KAA2B;IAE3B,IAAI,CAAC,KAAK;QAAE,OAAO,kCAAkC,CAAC;IACtD,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;QAC3C,OAAO,uEAAuE,CAAC;IACjF,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO;QACzC,OAAO,0DAA0D,CAAC;IACpE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAC9B,CAAiB,EACjB,KAAe;IAEf,IAAI,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,QAAQ,GACZ,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;QAC7D,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC;QACjC,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,IAAI,GACR,CAAC,CAAC,MAAM,KAAK,MAAM;QACjB,CAAC,CAAC,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,+BAA+B;QACxF,CAAC,CAAC,6EAA6E,CAAC;IACpF,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACjC,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC;QACrE,KAAK,EAAE,KAAK;QACZ,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,UAAU;QAClB,SAAS,EAAE,CAAC,CAAC,EAAE;QACf,IAAI;QACJ,MAAM,EACJ,QAAQ,KAAK,IAAI;YACf,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC;gBACE;oBACE,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,SAAS;oBACjB,KAAK,EAAE,QAAQ;oBACf,SAAS,EAAE,UAAU;oBACrB,OAAO,EAAE,IAAI;oBACb,IAAI,EAAE,sCAAsC;iBAC7C;aACF;KACR,CAAC;AACJ,CAAC;AAkBD;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,QAA0B,EAC1B,KAAW;IAEX,MAAM,EAAE,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,SAAS,GAAG,KAAK;YAAE,SAAS;QAClC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI;YAC/B,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,EAAE,EAAE,CAAC;YACL,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,IAAI;YACf,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,CAAC;SACR,CAAC;QACF,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;YAAE,CAAC,CAAC,EAAE,EAAE,CAAC;aACzB,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,CAAC;QACD,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;SACpB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,GAAG,CAAC;QACJ,SAAS,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1D,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC;KACjC,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AAC/E,CAAC"}

package/dist/format.d.ts

@@ -0,0 +1,6 @@
+/** Eval costs in dollars, for admin pages; server and client both use it. */
+export declare function money(usd: number): string;
+/** One answer's cost: in cents below a dollar, so cheap models still differ. */
+export declare function perAnswer(usd: number | null): string;
+/** Seconds per answer. */
+export declare function seconds(ms: number | null): string;

package/dist/format.js

@@ -0,0 +1,22 @@
+/** Eval costs in dollars, for admin pages; server and client both use it. */
+export function money(usd) {
+    if (usd === 0)
+        return "$0";
+    if (usd < 0.01)
+        return "under 1¢";
+    return `$${usd.toFixed(usd < 10 ? 2 : 0)}`;
+}
+/** One answer's cost: in cents below a dollar, so cheap models still differ. */
+export function perAnswer(usd) {
+    if (usd === null)
+        return "–";
+    if (usd >= 1)
+        return `$${usd.toFixed(2)}`;
+    const cents = usd * 100;
+    return `${cents >= 10 ? cents.toFixed(0) : cents.toPrecision(2)}¢`;
+}
+/** Seconds per answer. */
+export function seconds(ms) {
+    return ms === null ? "–" : `${(ms / 1000).toFixed(1)}s`;
+}
+//# sourceMappingURL=format.js.map

package/dist/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,MAAM,UAAU,KAAK,CAAC,GAAW;IAC/B,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3B,IAAI,GAAG,GAAG,IAAI;QAAE,OAAO,UAAU,CAAC;IAClC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC7C,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CAAC,GAAkB;IAC1C,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAC7B,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,GAAG,GAAG,GAAG,CAAC;IACxB,OAAO,GAAG,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,0BAA0B;AAC1B,MAAM,UAAU,OAAO,CAAC,EAAiB;IACvC,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC1D,CAAC"}

package/dist/glossary.d.ts

@@ -0,0 +1,46 @@
+/**
+ * What each word on an eval page means, in plain words: one source for
+ * tooltips, pages, and docs, so they never disagree. Keep each entry to a
+ * sentence or two a person can read while hovering.
+ */
+export declare const GLOSSARY: {
+    readonly score: "How much the model got right on every case in use. With partial credit, parts right of parts expected; otherwise answers with every check passed.";
+    readonly seriousMistakes: "Failed serious checks: mistakes that change what your product does. Models are compared on these first.";
+    readonly time: "Average seconds per answer, from answers the model actually made (reused answers keep their original time).";
+    readonly cost: "Average cost per answer on this model, from your catalog's prices.";
+    readonly againstLive: "A paired comparison with the live model on the same cases: more accurate, about as accurate, less accurate, or too few cases to tell.";
+    readonly live: "The model this feature runs on now.";
+    readonly fallback: "The model that answers when the live model's provider is down. It comes from another provider.";
+    readonly retiring: "The provider has announced a date it stops serving this model. It's never suggested, and a retiring live model gets replaced.";
+    readonly outOfDate: "Cases, answers, or instructions changed since this test ran. Test again, or rescore for free if only answers changed.";
+    readonly serious: "A mistake that changes what the product does. Serious checks should pass 95% of the time unless the check sets its own bar.";
+    readonly minor: "A fair difference of judgment. Minor checks should pass 75% of the time unless the check sets its own bar.";
+    readonly bar: "The pass rate this check should reach.";
+    readonly confirmed: "A person said this answer is right. Tests grade against it.";
+    readonly provisional: "Put to use before anyone reviewed it. Tests use it, and every score says it rests on unreviewed answers.";
+    readonly pending: "Proposed and not reviewed yet. Tests never grade against it until a person confirms it.";
+    readonly dev: "Cases you tune against: change prompts and checks while looking at these.";
+    readonly test: "Cases you only check: never tune against them, so they show how a change does on cases it wasn't fitted to.";
+    readonly suggestion: "Fixed rules pick it, so the reason given is the real one: accuracy first, then the suite's priority. Nothing changes until you choose.";
+    readonly priority: "What this suite cares about after accuracy: speed (someone waits), cost (it runs often in the background), or quality (only accuracy counts).";
+    readonly rescore: "Regrades stored answers with today's checks and answers. No model calls, so it's free.";
+    readonly tags: "What a case tests. The quick set picks cases that cover the most tags.";
+    readonly improve: "The suite whose best tested model scores lowest: where better cases, checks, or instructions would help most.";
+    readonly slowest: "The live model with the longest average answer time, across suites.";
+    readonly costliest: "The live model with the highest cost per answer, across suites.";
+    readonly thumbs: "Thumbs up and down people gave the feature's answers in your product, over the last 7 days.";
+    readonly keepInput: "Keeps what you typed, without your name, so it can become a test case for future models. Without it, a rating still counts but can't become a case.";
+    readonly makeCase: "Turns this rating into a draft case on its real input. A thumbs down needs a person to write the right answer; a thumbs up proposes the model's answer for a person to confirm.";
+    readonly needsAnswer: "A draft case with no answer yet. Write what a good answer is; until then, tests leave it out.";
+    readonly weeklyCheck: "Each suite's quick set, answered again on its live model every week. A drop of 15 points or more against last week's check of the same model, instructions, and grading is drift.";
+    readonly versions: "Each numbered version of this feature's settings (model, fallback, reasoning, output limit, temperature), with who changed it, when, why, and its evidence. Any version can be made live again.";
+    readonly suites: "One suite per AI feature: its cases, the answers people confirmed, and the checks that grade any model against them. Open one to compare models and choose the live one.";
+    readonly answerKey: "The cases and the answers tests grade against. Only confirmed answers count: a model can draft answers, but they stay pending until a person confirms them. Open a case to see how each model did on it.";
+    readonly checks: "What a good answer gets right, one thing per check. Serious checks catch mistakes that change what your product does; minor ones, fair differences of judgment. Each has a bar its pass rate should reach.";
+    readonly probabilityQuality: "How good the probabilities were: 100% minus the Brier score. High means sure when right and unsure when wrong; picking right with careless probabilities scores lower.";
+    readonly calibrationError: "How far the model's confidence is from how often it's right: 0% means its 80%-sure answers are right 80% of the time. It needs more cases than accuracy to be trusted.";
+    readonly nativeProbabilities: "The model returns its own probabilities for every option, so they can be scored and compared.";
+    readonly promptedProbabilities: "The model writes out its own estimate. Choices come without a distribution, so only true-or-false questions get probability scores.";
+};
+/** A word the glossary defines, for tooltips. */
+export type GlossaryTerm = keyof typeof GLOSSARY;

package/dist/glossary.js

@@ -0,0 +1,45 @@
+/**
+ * What each word on an eval page means, in plain words: one source for
+ * tooltips, pages, and docs, so they never disagree. Keep each entry to a
+ * sentence or two a person can read while hovering.
+ */
+export const GLOSSARY = {
+    score: "How much the model got right on every case in use. With partial credit, parts right of parts expected; otherwise answers with every check passed.",
+    seriousMistakes: "Failed serious checks: mistakes that change what your product does. Models are compared on these first.",
+    time: "Average seconds per answer, from answers the model actually made (reused answers keep their original time).",
+    cost: "Average cost per answer on this model, from your catalog's prices.",
+    againstLive: "A paired comparison with the live model on the same cases: more accurate, about as accurate, less accurate, or too few cases to tell.",
+    live: "The model this feature runs on now.",
+    fallback: "The model that answers when the live model's provider is down. It comes from another provider.",
+    retiring: "The provider has announced a date it stops serving this model. It's never suggested, and a retiring live model gets replaced.",
+    outOfDate: "Cases, answers, or instructions changed since this test ran. Test again, or rescore for free if only answers changed.",
+    serious: "A mistake that changes what the product does. Serious checks should pass 95% of the time unless the check sets its own bar.",
+    minor: "A fair difference of judgment. Minor checks should pass 75% of the time unless the check sets its own bar.",
+    bar: "The pass rate this check should reach.",
+    confirmed: "A person said this answer is right. Tests grade against it.",
+    provisional: "Put to use before anyone reviewed it. Tests use it, and every score says it rests on unreviewed answers.",
+    pending: "Proposed and not reviewed yet. Tests never grade against it until a person confirms it.",
+    dev: "Cases you tune against: change prompts and checks while looking at these.",
+    test: "Cases you only check: never tune against them, so they show how a change does on cases it wasn't fitted to.",
+    suggestion: "Fixed rules pick it, so the reason given is the real one: accuracy first, then the suite's priority. Nothing changes until you choose.",
+    priority: "What this suite cares about after accuracy: speed (someone waits), cost (it runs often in the background), or quality (only accuracy counts).",
+    rescore: "Regrades stored answers with today's checks and answers. No model calls, so it's free.",
+    tags: "What a case tests. The quick set picks cases that cover the most tags.",
+    improve: "The suite whose best tested model scores lowest: where better cases, checks, or instructions would help most.",
+    slowest: "The live model with the longest average answer time, across suites.",
+    costliest: "The live model with the highest cost per answer, across suites.",
+    thumbs: "Thumbs up and down people gave the feature's answers in your product, over the last 7 days.",
+    keepInput: "Keeps what you typed, without your name, so it can become a test case for future models. Without it, a rating still counts but can't become a case.",
+    makeCase: "Turns this rating into a draft case on its real input. A thumbs down needs a person to write the right answer; a thumbs up proposes the model's answer for a person to confirm.",
+    needsAnswer: "A draft case with no answer yet. Write what a good answer is; until then, tests leave it out.",
+    weeklyCheck: "Each suite's quick set, answered again on its live model every week. A drop of 15 points or more against last week's check of the same model, instructions, and grading is drift.",
+    versions: "Each numbered version of this feature's settings (model, fallback, reasoning, output limit, temperature), with who changed it, when, why, and its evidence. Any version can be made live again.",
+    suites: "One suite per AI feature: its cases, the answers people confirmed, and the checks that grade any model against them. Open one to compare models and choose the live one.",
+    answerKey: "The cases and the answers tests grade against. Only confirmed answers count: a model can draft answers, but they stay pending until a person confirms them. Open a case to see how each model did on it.",
+    checks: "What a good answer gets right, one thing per check. Serious checks catch mistakes that change what your product does; minor ones, fair differences of judgment. Each has a bar its pass rate should reach.",
+    probabilityQuality: "How good the probabilities were: 100% minus the Brier score. High means sure when right and unsure when wrong; picking right with careless probabilities scores lower.",
+    calibrationError: "How far the model's confidence is from how often it's right: 0% means its 80%-sure answers are right 80% of the time. It needs more cases than accuracy to be trusted.",
+    nativeProbabilities: "The model returns its own probabilities for every option, so they can be scored and compared.",
+    promptedProbabilities: "The model writes out its own estimate. Choices come without a distribution, so only true-or-false questions get probability scores.",
+};
+//# sourceMappingURL=glossary.js.map

package/dist/glossary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"glossary.js","sourceRoot":"","sources":["../src/glossary.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,KAAK,EACH,mJAAmJ;IACrJ,eAAe,EACb,yGAAyG;IAC3G,IAAI,EAAE,6GAA6G;IACnH,IAAI,EAAE,oEAAoE;IAC1E,WAAW,EACT,uIAAuI;IACzI,IAAI,EAAE,qCAAqC;IAC3C,QAAQ,EACN,gGAAgG;IAClG,QAAQ,EACN,+HAA+H;IACjI,SAAS,EACP,uHAAuH;IACzH,OAAO,EACL,6HAA6H;IAC/H,KAAK,EACH,4GAA4G;IAC9G,GAAG,EAAE,wCAAwC;IAC7C,SAAS,EAAE,6DAA6D;IACxE,WAAW,EACT,0GAA0G;IAC5G,OAAO,EACL,yFAAyF;IAC3F,GAAG,EAAE,2EAA2E;IAChF,IAAI,EAAE,6GAA6G;IACnH,UAAU,EACR,wIAAwI;IAC1I,QAAQ,EACN,+IAA+I;IACjJ,OAAO,EACL,wFAAwF;IAC1F,IAAI,EAAE,wEAAwE;IAC9E,OAAO,EACL,+GAA+G;IACjH,OAAO,EACL,qEAAqE;IACvE,SAAS,EAAE,iEAAiE;IAC5E,MAAM,EACJ,6FAA6F;IAC/F,SAAS,EACP,qJAAqJ;IACvJ,QAAQ,EACN,iLAAiL;IACnL,WAAW,EACT,+FAA+F;IACjG,WAAW,EACT,mLAAmL;IACrL,QAAQ,EACN,iMAAiM;IACnM,MAAM,EACJ,0KAA0K;IAC5K,SAAS,EACP,0MAA0M;IAC5M,MAAM,EACJ,4MAA4M;IAC9M,kBAAkB,EAChB,wKAAwK;IAC1K,gBAAgB,EACd,wKAAwK;IAC1K,mBAAmB,EACjB,+FAA+F;IACjG,qBAAqB,EACnB,qIAAqI;CAC/H,CAAC"}