@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/dist/types.d.ts

@@ -0,0 +1,225 @@
+import type { z } from "zod";
+import type { ReasoningEffort } from "./models.js";
+/**
+ * An eval suite: how to run one AI operation on a frozen case and how to
+ * grade what comes back. Code says how to evaluate; the database holds the
+ * cases and labels (EvalCase, EvalCaseLabel). See docs/evals.md.
+ */
+/** One check on one output. null: doesn't apply to this case. */
+export type CheckResult = {
+    key: string;
+    pass: boolean | null;
+    /** Why it failed (or a detail worth showing), in plain words. */
+    note?: string | undefined;
+    /** For matching checks: which expected item matched which output, and why. */
+    matches?: {
+        expected: string;
+        actual: string;
+        why: string;
+    }[] | undefined;
+    /** "meaning": a judge model matched them; "words": shared words did. */
+    matchedBy?: "meaning" | "words" | undefined;
+    /**
+     * Partial credit: of the parts this answer should have (e.g. expected
+     * items), how many it got right. The headline score sums these.
+     */
+    units?: {
+        right: number;
+        total: number;
+    } | undefined;
+    /**
+     * Numbers measured on this answer, averaged across a test: a decision's
+     * `brier`, `logLoss`, `confidence`, and `correct` (see decisionCheck).
+     */
+    measures?: Record<string, number> | undefined;
+};
+/**
+ * One check a suite runs: what it checks, whether it needs an answer, and how serious a
+ * miss is.
+ */
+export type CheckDef = {
+    key: string;
+    label: string;
+    /** What it checks, for the page that explains the suite. */
+    description: string;
+    /** "code": needs no label. "label": compares with a confirmed label. */
+    kind: "code" | "label";
+    /**
+     * Serious: changes what the product does (which queue a ticket lands
+     * in, what a summary claims). Minor: a fair difference of judgment (a category, how finely
+     * to split). Models are ranked on serious mistakes.
+     */
+    severity: "serious" | "minor";
+    /** Pass rate it should reach; default by severity (checkBar). */
+    bar?: number;
+};
+/** The pass rate a check should reach: its own, else 95% serious, 75% minor. */
+export declare function checkBar(c: CheckDef): number;
+/** Meets the bar, is within 10 points of it, or is further below. */
+export declare function barStatus(rate: number, bar: number): "meets" | "close" | "below";
+/**
+ * What a suite cares about once accuracy holds. Accuracy is always the
+ * floor; this is the one thing worth switching models for beyond it.
+ */
+export type SuitePriority = {
+    /** speed: someone waits on it. cost: runs in the background, often.
+     * quality: people read and act on it, so only accuracy counts. */
+    favor: "speed" | "cost" | "quality";
+    /** Models slower than this on average are never suggested. */
+    maxSeconds?: number;
+};
+/** Each priority, explained for the suite page. */
+export declare const FAVOR_TEXT: Record<SuitePriority["favor"], string>;
+/**
+ * An eval suite: how to call one AI feature on a model and how to grade what comes back.
+ * Code says how to evaluate; the store holds the cases and answers.
+ */
+export type Suite<Input, Output, Label> = {
+    key: string;
+    /** The AI feature it tests, as your app names it ("summarize_ticket"). */
+    operation: string;
+    priority: SuitePriority;
+    label: string;
+    description: string;
+    /** Bump when checks change, so runs say which grading they used. */
+    graderVersion: string;
+    input: z.ZodType<Input>;
+    /** The EXPECTED label's value. */
+    labelValue: z.ZodType<Label>;
+    checks: CheckDef[];
+    /** Calls the operation on exactly this model, recorded as an eval. */
+    run(input: Input, ctx: {
+        model: string;
+        /** A reasoning setting to test; undefined: the operation's own. */
+        reasoningEffort?: ReasoningEffort | null;
+        /**
+         * The temperature the test runs at: the operation's live one when the
+         * test was started. Null: send none. Undefined from runners before
+         * temperature was versioned.
+         */
+        temperature?: number | null;
+        onCost: (usd: number) => void;
+    }): Promise<Output>;
+    /**
+     * Grades one output. `label` is the confirmed one, or null. A check that
+     * asks a judge model reports its cost through `ctx.onCost`.
+     */
+    grade(input: Input, output: Output, label: Label | null, ctx?: {
+        onCost: (usd: number) => void;
+    }): Promise<CheckResult[]>;
+    /**
+     * Checks that compare a test's answers with each other, such as ranking.
+     * Gets every graded answer of one repeat; returns extra checks for each,
+     * in the same order. Pure; runs after a test and on every regrade.
+     */
+    gradeTogether?(answers: {
+        input: Input;
+        output: Output;
+        label: Label | null;
+    }[]): CheckResult[][];
+    /** Reads a case's input from the add/edit form; throws on bad input. */
+    inputFromForm(form: FormData): Input;
+    /** Reads an answer from the add/edit form; throws on bad input. */
+    labelFromForm?(form: FormData): Label;
+    /**
+     * The operation's output as an answer, so a thumbs-up can propose it
+     * (pending until a person confirms it). Omit when outputs and answers
+     * differ too much to propose one.
+     */
+    labelFromOutput?(output: Output): Label;
+    /**
+     * The form's fields, so pages can render one for any suite. Every field
+     * has a plain label and help: people confirming answers weeks later
+     * won't remember what a field means.
+     */
+    fields?: {
+        input: FormField[];
+        label: FormField[];
+    };
+    /** Drafted cases to start from, added as drafts with PENDING labels. */
+    starter?: {
+        input: Input;
+        label: Label;
+        tags: string[];
+        split: "DEV" | "TEST";
+        smoke?: boolean;
+        note: string;
+    }[];
+    /**
+     * Drafts new cases aimed at what the answer key is missing, for a person
+     * to review (they arrive with PENDING answers).
+     */
+    draftMore?(args: {
+        existing: {
+            input: Input;
+            tags: string[];
+        }[];
+        count: number;
+        focus: string | null;
+    }): Promise<{
+        input: Input;
+        label: Label;
+        tags: string[];
+        note: string;
+    }[]>;
+    /**
+     * How long one answer may take before it counts as failed. Default 60s;
+     * multi-step agent turns need more. Keep it under the run budget.
+     */
+    answerTimeoutMs?: number;
+    /**
+     * How many answers a test asks for at once. Default 6; slow operations
+     * need more to finish before the platform's time limit.
+     */
+    concurrency?: number;
+    /**
+     * Graded as decisions (typed answers with probabilities, as from the AI
+     * SDK's `experimental_decide`), so pages show probability quality and
+     * calibration next to accuracy, and which models' probabilities are
+     * native or prompted.
+     */
+    decisions?: boolean;
+    /** What the headline counts: "items", "tickets", … Default "answers". */
+    unitNoun?: string;
+    /**
+     * "typed": cases are text someone types (added by hand, or drafted).
+     * "imported": cases are frozen from the app's own data, then rated by a
+     * person.
+     * "fixture": cases are structured inputs, shown with describe() and
+     * edited as JSON.
+     */
+    caseKind?: "typed" | "imported" | "fixture";
+    /** A short line naming the case in lists. */
+    title(input: Input): string;
+    /** Fixture suites: the whole input, readable, for reviewing a case. */
+    describe?(input: Input): string;
+    /** Fixture suites: what the expected answer's fields mean. */
+    labelHelp?: string;
+};
+/**
+ * One field of a suite's add/edit form. `name` is the form key (and the
+ * input or answer property it fills). "lines": one item per line, read as
+ * a list.
+ */
+export type FormField = {
+    name: string;
+    label: string;
+    help: string;
+    kind: "text" | "textarea" | "select" | "lines";
+    /** For "select". */
+    options?: readonly string[];
+    required?: boolean;
+};
+/** A value as plain text: strings as they are, other values as JSON. */
+export declare function asText(v: unknown): string;
+/** A form value as text: "" when missing or a file. */
+export declare function formText(form: FormData, name: string): string;
+/** A field's value from a stored input or answer, as the form shows it. */
+export declare function fieldValue(field: FormField, value: unknown): string;
+/** A "lines" field's value as a list: one item per non-empty line. */
+export declare function linesOf(form: FormData, name: string): string[];
+/**
+ * A suite of any shape, for registries. Its methods take and return
+ * `unknown`; a suite's own code keeps its precise types.
+ */
+export type AnySuite = Suite<unknown, unknown, unknown>;

package/dist/types.js

@@ -0,0 +1,46 @@
+/** The pass rate a check should reach: its own, else 95% serious, 75% minor. */
+export function checkBar(c) {
+    return c.bar ?? (c.severity === "serious" ? 0.95 : 0.75);
+}
+/** Meets the bar, is within 10 points of it, or is further below. */
+export function barStatus(rate, bar) {
+    return rate >= bar ? "meets" : rate >= bar - 0.1 ? "close" : "below";
+}
+/** Each priority, explained for the suite page. */
+export const FAVOR_TEXT = {
+    speed: "speed: someone waits on this, so a model that scores as well and answers clearly faster is worth switching to, and so is one about as accurate (within 10 points) in half the time",
+    cost: "cost: this runs in the background, often, so a model that scores as well and is clearly cheaper is worth switching to, and so is one about as accurate (within 10 points) at half the cost",
+    quality: "quality: people read and act on this, so only clearly better accuracy or fewer serious mistakes is worth switching for",
+};
+/** A value as plain text: strings as they are, other values as JSON. */
+export function asText(v) {
+    if (v === undefined || v === null)
+        return "";
+    if (typeof v === "string")
+        return v;
+    if (typeof v === "number" || typeof v === "boolean")
+        return String(v);
+    return JSON.stringify(v);
+}
+/** A form value as text: "" when missing or a file. */
+export function formText(form, name) {
+    const v = form.get(name);
+    return typeof v === "string" ? v.trim() : "";
+}
+/** A field's value from a stored input or answer, as the form shows it. */
+export function fieldValue(field, value) {
+    const v = value && typeof value === "object"
+        ? value[field.name]
+        : undefined;
+    if (Array.isArray(v))
+        return v.map(asText).join("\n");
+    return asText(v);
+}
+/** A "lines" field's value as a list: one item per non-empty line. */
+export function linesOf(form, name) {
+    return formText(form, name)
+        .split("\n")
+        .map((l) => l.trim())
+        .filter(Boolean);
+}
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAoDA,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,CAAW;IAClC,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3D,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,GAAW;IAEX,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AACvE,CAAC;AAcD,mDAAmD;AACnD,MAAM,CAAC,MAAM,UAAU,GAA2C;IAChE,KAAK,EACH,oLAAoL;IACtL,IAAI,EAAE,4LAA4L;IAClM,OAAO,EACL,wHAAwH;CAC3H,CAAC;AAyIF,wEAAwE;AACxE,MAAM,UAAU,MAAM,CAAC,CAAU;IAC/B,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACtE,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,QAAQ,CAAC,IAAc,EAAE,IAAY;IACnD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,UAAU,CAAC,KAAgB,EAAE,KAAc;IACzD,MAAM,CAAC,GACL,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAChC,CAAC,CAAE,KAAiC,CAAC,KAAK,CAAC,IAAI,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,OAAO,CAAC,IAAc,EAAE,IAAY;IAClD,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;SACxB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}

package/dist/upkeepRules.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Rules for the weekly eval check, pure so they're tested: when a quick-set
+ * score has drifted, when a model is about to retire, and when people's
+ * thumbs-down rate needs a look.
+ */
+/** A drop this big on the quick set is worth a look. */
+export declare const DRIFT_DROP = 0.15;
+/** Warn about a live or fallback model this long before it retires. */
+export declare const RETIRE_WARN_DAYS = 60;
+/** Thumbs: at least this many ratings, at least this share down. */
+export declare const THUMBS_MIN = 5;
+/**
+ * The share of thumbs down that flags a feature (with at least THUMBS_MIN ratings).
+ */
+export declare const THUMBS_DOWN_SHARE = 0.2;
+/** This week's quick-set score against last week's: a drop worth a look. */
+export declare function drifted(previous: number, latest: number): boolean;
+/**
+ * Days until a "YYYY-MM" or "YYYY-MM-DD" retirement; a month counts from
+ * its first day. Negative once it's passed.
+ */
+export declare function daysUntil(retiring: string, today: Date): number;
+/** Whether a model retires within the warning window, or already has. */
+export declare function retiresSoon(retiring: string | undefined, today: Date): boolean;
+/**
+ * Whether a feature's ratings need a look: enough of them, and enough down.
+ */
+export declare function thumbsAlert(up: number, down: number): boolean;

package/dist/upkeepRules.js

@@ -0,0 +1,40 @@
+/**
+ * Rules for the weekly eval check, pure so they're tested: when a quick-set
+ * score has drifted, when a model is about to retire, and when people's
+ * thumbs-down rate needs a look.
+ */
+/** A drop this big on the quick set is worth a look. */
+export const DRIFT_DROP = 0.15;
+/** Warn about a live or fallback model this long before it retires. */
+export const RETIRE_WARN_DAYS = 60;
+/** Thumbs: at least this many ratings, at least this share down. */
+export const THUMBS_MIN = 5;
+/**
+ * The share of thumbs down that flags a feature (with at least THUMBS_MIN ratings).
+ */
+export const THUMBS_DOWN_SHARE = 0.2;
+/** This week's quick-set score against last week's: a drop worth a look. */
+export function drifted(previous, latest) {
+    return previous - latest >= DRIFT_DROP;
+}
+/**
+ * Days until a "YYYY-MM" or "YYYY-MM-DD" retirement; a month counts from
+ * its first day. Negative once it's passed.
+ */
+export function daysUntil(retiring, today) {
+    const [y = 1970, m = 1, d = 1] = retiring.split("-").map(Number);
+    const when = Date.UTC(y, (m || 1) - 1, d || 1);
+    return Math.floor((when - today.getTime()) / 86_400_000);
+}
+/** Whether a model retires within the warning window, or already has. */
+export function retiresSoon(retiring, today) {
+    return (retiring !== undefined && daysUntil(retiring, today) <= RETIRE_WARN_DAYS);
+}
+/**
+ * Whether a feature's ratings need a look: enough of them, and enough down.
+ */
+export function thumbsAlert(up, down) {
+    const n = up + down;
+    return n >= THUMBS_MIN && down / n >= THUMBS_DOWN_SHARE;
+}
+//# sourceMappingURL=upkeepRules.js.map

package/dist/upkeepRules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"upkeepRules.js","sourceRoot":"","sources":["../src/upkeepRules.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,wDAAwD;AACxD,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAC/B,uEAAuE;AACvE,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,CAAC;AACnC,oEAAoE;AACpE,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAC5B;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAErC,4EAA4E;AAC5E,MAAM,UAAU,OAAO,CAAC,QAAgB,EAAE,MAAc;IACtD,OAAO,QAAQ,GAAG,MAAM,IAAI,UAAU,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAW;IACrD,MAAM,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;AAC3D,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,WAAW,CAAC,QAA4B,EAAE,KAAW;IACnE,OAAO,CACL,QAAQ,KAAK,SAAS,IAAI,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,gBAAgB,CACzE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU,EAAE,IAAY;IAClD,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IACpB,OAAO,CAAC,IAAI,UAAU,IAAI,IAAI,GAAG,CAAC,IAAI,iBAAiB,CAAC;AAC1D,CAAC"}

package/dist/util.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The item at `i`, for indexes the caller has already checked (a loop
+ * bound, a non-empty list). Throws instead of returning undefined, so a
+ * wrong index fails loudly rather than as NaN or a missing row.
+ */
+export declare function at<T>(items: readonly T[], i: number): T;
+/** A map's value for a key the caller knows is there. Throws otherwise. */
+export declare function mustGet<K, V>(map: ReadonlyMap<K, V>, key: K): V;

package/dist/util.js

@@ -0,0 +1,18 @@
+/**
+ * The item at `i`, for indexes the caller has already checked (a loop
+ * bound, a non-empty list). Throws instead of returning undefined, so a
+ * wrong index fails loudly rather than as NaN or a missing row.
+ */
+export function at(items, i) {
+    if (i < 0 || i >= items.length)
+        throw new RangeError(`Index ${i} is outside 0..${items.length - 1}`);
+    return items[i];
+}
+/** A map's value for a key the caller knows is there. Throws otherwise. */
+export function mustGet(map, key) {
+    const v = map.get(key);
+    if (v === undefined)
+        throw new Error(`Missing ${String(key)}`);
+    return v;
+}
+//# sourceMappingURL=util.js.map

package/dist/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,UAAU,EAAE,CAAI,KAAmB,EAAE,CAAS;IAClD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;QAC5B,MAAM,IAAI,UAAU,CAAC,SAAS,CAAC,kBAAkB,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,KAAK,CAAC,CAAC,CAAM,CAAC;AACvB,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,OAAO,CAAO,GAAsB,EAAE,GAAM;IAC1D,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,CAAC,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/weekly.d.ts

@@ -0,0 +1,80 @@
+import type { ThumbsSummary } from "./feedback.js";
+import { type ModelCatalog } from "./models.js";
+import type { RunMetrics, RunRecord } from "./store.js";
+/**
+ * The weekly check: each suite's quick set on its live model, with fresh
+ * answers, so a provider quietly changing a model under the same name shows
+ * up as drift. The summary also flags models retiring soon and features
+ * people rate poorly. It only reports; a person decides what to change.
+ * Pure.
+ */
+/** `startedBy` on tests the weekly check starts, so they're told apart. */
+export declare const SCHEDULED = "schedule";
+/**
+ * What the last weekly check found: drift, unfinished checks, retiring models, and
+ * thumbs-down, plus when it ran and what it cost.
+ */
+export type WeeklySummary = {
+    /** When the last weekly check started; null before the first. */
+    lastAt: Date | null;
+    /** What last week's quick sets cost (reused answers are free). */
+    costUsd: number;
+    /** Quick sets still running. */
+    running: number;
+    drift: {
+        suite: string;
+        label: string;
+        model: string;
+        previous: number;
+        latest: number;
+    }[];
+    /** Quick sets that didn't finish, and why. */
+    failed: {
+        suite: string;
+        label: string;
+        error: string;
+    }[];
+    /** Live and fallback models retiring within the warning window. */
+    retiring: {
+        suite: string;
+        label: string;
+        model: string;
+        role: "live" | "fallback";
+        when: string;
+        days: number;
+    }[];
+    /** Features with a high thumbs-down share this week. */
+    thumbs: ThumbsSummary[];
+    /** Ran, and nothing needs a look. */
+    clear: boolean;
+};
+/** A test's headline score: parts right where it gives partial credit. */
+export declare function headline(m: RunMetrics | null): number | null;
+/**
+ * What the last weekly check found, from every suite's tests.
+ *
+ * @example
+ * ```ts
+ * const runs = (await Promise.all(SUITES.map((s) => store.listRuns(s.key, { limit: 50 })))).flat();
+ * const weekly = weeklySummary({ runs, suites: SUITES, routes, models: MODELS, thumbs });
+ * if (!weekly.clear) console.log(weekly.drift, weekly.retiring);
+ * ```
+ */
+export declare function weeklySummary(args: {
+    /** Every suite's tests, newest first or not. */
+    runs: RunRecord[];
+    suites: {
+        key: string;
+        label: string;
+        operation: string;
+    }[];
+    /** Each operation's live and fallback model. */
+    routes: Record<string, {
+        model: string;
+        fallbackModel: string | null;
+    }>;
+    models: ModelCatalog;
+    /** From thumbsSummary over the last week; only flagged ones are shown. */
+    thumbs?: ThumbsSummary[];
+    today?: Date;
+}): WeeklySummary;

package/dist/weekly.js

@@ -0,0 +1,113 @@
+import { retiresLabel } from "./models.js";
+import { daysUntil, drifted, retiresSoon } from "./upkeepRules.js";
+/**
+ * The weekly check: each suite's quick set on its live model, with fresh
+ * answers, so a provider quietly changing a model under the same name shows
+ * up as drift. The summary also flags models retiring soon and features
+ * people rate poorly. It only reports; a person decides what to change.
+ * Pure.
+ */
+/** `startedBy` on tests the weekly check starts, so they're told apart. */
+export const SCHEDULED = "schedule";
+const DAY = 86_400_000;
+/** A test's headline score: parts right where it gives partial credit. */
+export function headline(m) {
+    if (!m)
+        return null;
+    return m.units && m.units.total
+        ? m.units.right / m.units.total
+        : (m.allPass.rate ?? 0);
+}
+/**
+ * What the last weekly check found, from every suite's tests.
+ *
+ * @example
+ * ```ts
+ * const runs = (await Promise.all(SUITES.map((s) => store.listRuns(s.key, { limit: 50 })))).flat();
+ * const weekly = weeklySummary({ runs, suites: SUITES, routes, models: MODELS, thumbs });
+ * if (!weekly.clear) console.log(weekly.drift, weekly.retiring);
+ * ```
+ */
+export function weeklySummary(args) {
+    const today = args.today ?? new Date();
+    const label = (key) => args.suites.find((s) => s.key === key)?.label ?? key;
+    const scheduled = args.runs
+        .filter((r) => r.startedBy === SCHEDULED && r.set === "SMOKE")
+        .sort((a, b) => +b.startedAt - +a.startedAt);
+    const lastAt = scheduled[0]?.startedAt ?? null;
+    // The last weekly batch: everything started within a day of the newest.
+    const batch = lastAt
+        ? scheduled.filter((r) => +lastAt - +r.startedAt < DAY)
+        : [];
+    const drift = [];
+    const failed = [];
+    for (const r of batch) {
+        if (r.status === "FAILED") {
+            failed.push({
+                suite: r.suite,
+                label: label(r.suite),
+                error: r.error ?? "unknown error",
+            });
+            continue;
+        }
+        if (r.status !== "SUCCEEDED")
+            continue;
+        // The last check of the same model, temperature, instructions, and
+        // grading, a day or more earlier: anything else changed for a known
+        // reason.
+        const prior = scheduled.find((p) => p.suite === r.suite &&
+            p.status === "SUCCEEDED" &&
+            p.model === r.model &&
+            (p.temperature ?? null) === (r.temperature ?? null) &&
+            p.promptVersion === r.promptVersion &&
+            p.graderVersion === r.graderVersion &&
+            +r.startedAt - +p.startedAt >= DAY);
+        const latest = headline(r.metrics);
+        const previous = headline(prior?.metrics ?? null);
+        if (latest !== null && previous !== null && drifted(previous, latest))
+            drift.push({
+                suite: r.suite,
+                label: label(r.suite),
+                model: args.models[r.model]?.label ?? r.model,
+                previous,
+                latest,
+            });
+    }
+    const retiring = [];
+    for (const s of args.suites) {
+        const route = args.routes[s.operation];
+        if (!route)
+            continue;
+        for (const [model, role] of [
+            [route.model, "live"],
+            [route.fallbackModel, "fallback"],
+        ]) {
+            const when = model ? args.models[model]?.retiring : undefined;
+            if (model && when && retiresSoon(when, today))
+                retiring.push({
+                    suite: s.key,
+                    label: s.label,
+                    model: args.models[model]?.label ?? model,
+                    role,
+                    when: retiresLabel(args.models, model) ?? when,
+                    days: daysUntil(when, today),
+                });
+        }
+    }
+    const thumbs = (args.thumbs ?? []).filter((t) => t.alert);
+    return {
+        lastAt,
+        costUsd: batch.reduce((t, r) => t + (r.metrics?.costUsd ?? 0), 0),
+        running: batch.filter((r) => r.status === "RUNNING").length,
+        drift,
+        failed,
+        retiring,
+        thumbs,
+        clear: lastAt !== null &&
+            !drift.length &&
+            !failed.length &&
+            !retiring.length &&
+            !thumbs.length,
+    };
+}
+//# sourceMappingURL=weekly.js.map

package/dist/weekly.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"weekly.js","sourceRoot":"","sources":["../src/weekly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;GAMG;AAEH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC;AAEpC,MAAM,GAAG,GAAG,UAAU,CAAC;AAqCvB,0EAA0E;AAC1E,MAAM,UAAU,QAAQ,CAAC,CAAoB;IAC3C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK;QAC7B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK;QAC/B,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,IAU7B;IACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC;SAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;IAC/C,wEAAwE;IACxE,MAAM,KAAK,GAAG,MAAM;QAClB,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC;QACvD,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,eAAe;aAClC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACvC,mEAAmE;QACnE,oEAAoE;QACpE,UAAU;QACV,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,MAAM,KAAK,WAAW;YACxB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC;YACnD,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CACrC,CAAC;QACF,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,CAAC;QAClD,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK;gBAC7C,QAAQ;gBACR,MAAM;aACP,CAAC,CAAC;IACP,CAAC;IAED,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI;YAC1B,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;YACrB,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,CAAC;SACzB,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9D,IAAI,KAAK,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC;gBAC3C,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,CAAC,CAAC,GAAG;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,KAAK;oBACzC,IAAI;oBACJ,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI;oBAC9C,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;iBAC7B,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,MAAM;QACN,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC3D,KAAK;QACL,MAAM;QACN,QAAQ;QACR,MAAM;QACN,KAAK,EACH,MAAM,KAAK,IAAI;YACf,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,MAAM,CAAC,MAAM;YACd,CAAC,QAAQ,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,MAAM;KACjB,CAAC;AACJ,CAAC"}