@crowandraven/evalkeeper 0.0.0-stage → 0.1.1

package/dist/upkeepRules.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Rules for the weekly eval check, pure so they're tested: when a quick-set
+ * score has drifted, when a model is about to retire, and when people's
+ * thumbs-down rate needs a look.
+ */
+/** A drop this big on the quick set is worth a look. */
+export declare const DRIFT_DROP = 0.15;
+/** Warn about a live or fallback model this long before it retires. */
+export declare const RETIRE_WARN_DAYS = 60;
+/** Thumbs: at least this many ratings, at least this share down. */
+export declare const THUMBS_MIN = 5;
+/**
+ * The share of thumbs down that flags a feature (with at least THUMBS_MIN ratings).
+ */
+export declare const THUMBS_DOWN_SHARE = 0.2;
+/** This week's quick-set score against last week's: a drop worth a look. */
+export declare function drifted(previous: number, latest: number): boolean;
+/**
+ * Days until a "YYYY-MM" or "YYYY-MM-DD" retirement; a month counts from
+ * its first day. Negative once it's passed.
+ */
+export declare function daysUntil(retiring: string, today: Date): number;
+/** Whether a model retires within the warning window, or already has. */
+export declare function retiresSoon(retiring: string | undefined, today: Date): boolean;
+/**
+ * Whether a feature's ratings need a look: enough of them, and enough down.
+ */
+export declare function thumbsAlert(up: number, down: number): boolean;

package/dist/upkeepRules.js

@@ -0,0 +1,40 @@
+/**
+ * Rules for the weekly eval check, pure so they're tested: when a quick-set
+ * score has drifted, when a model is about to retire, and when people's
+ * thumbs-down rate needs a look.
+ */
+/** A drop this big on the quick set is worth a look. */
+export const DRIFT_DROP = 0.15;
+/** Warn about a live or fallback model this long before it retires. */
+export const RETIRE_WARN_DAYS = 60;
+/** Thumbs: at least this many ratings, at least this share down. */
+export const THUMBS_MIN = 5;
+/**
+ * The share of thumbs down that flags a feature (with at least THUMBS_MIN ratings).
+ */
+export const THUMBS_DOWN_SHARE = 0.2;
+/** This week's quick-set score against last week's: a drop worth a look. */
+export function drifted(previous, latest) {
+    return previous - latest >= DRIFT_DROP;
+}
+/**
+ * Days until a "YYYY-MM" or "YYYY-MM-DD" retirement; a month counts from
+ * its first day. Negative once it's passed.
+ */
+export function daysUntil(retiring, today) {
+    const [y = 1970, m = 1, d = 1] = retiring.split("-").map(Number);
+    const when = Date.UTC(y, (m || 1) - 1, d || 1);
+    return Math.floor((when - today.getTime()) / 86_400_000);
+}
+/** Whether a model retires within the warning window, or already has. */
+export function retiresSoon(retiring, today) {
+    return (retiring !== undefined && daysUntil(retiring, today) <= RETIRE_WARN_DAYS);
+}
+/**
+ * Whether a feature's ratings need a look: enough of them, and enough down.
+ */
+export function thumbsAlert(up, down) {
+    const n = up + down;
+    return n >= THUMBS_MIN && down / n >= THUMBS_DOWN_SHARE;
+}
+//# sourceMappingURL=upkeepRules.js.map

package/dist/upkeepRules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"upkeepRules.js","sourceRoot":"","sources":["../src/upkeepRules.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,wDAAwD;AACxD,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAC/B,uEAAuE;AACvE,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,CAAC;AACnC,oEAAoE;AACpE,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAC5B;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAErC,4EAA4E;AAC5E,MAAM,UAAU,OAAO,CAAC,QAAgB,EAAE,MAAc;IACtD,OAAO,QAAQ,GAAG,MAAM,IAAI,UAAU,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAW;IACrD,MAAM,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;AAC3D,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,WAAW,CAAC,QAA4B,EAAE,KAAW;IACnE,OAAO,CACL,QAAQ,KAAK,SAAS,IAAI,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,gBAAgB,CACzE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU,EAAE,IAAY;IAClD,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IACpB,OAAO,CAAC,IAAI,UAAU,IAAI,IAAI,GAAG,CAAC,IAAI,iBAAiB,CAAC;AAC1D,CAAC"}

package/dist/util.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The item at `i`, for indexes the caller has already checked (a loop
+ * bound, a non-empty list). Throws instead of returning undefined, so a
+ * wrong index fails loudly rather than as NaN or a missing row.
+ */
+export declare function at<T>(items: readonly T[], i: number): T;
+/** A map's value for a key the caller knows is there. Throws otherwise. */
+export declare function mustGet<K, V>(map: ReadonlyMap<K, V>, key: K): V;

package/dist/util.js

@@ -0,0 +1,18 @@
+/**
+ * The item at `i`, for indexes the caller has already checked (a loop
+ * bound, a non-empty list). Throws instead of returning undefined, so a
+ * wrong index fails loudly rather than as NaN or a missing row.
+ */
+export function at(items, i) {
+    if (i < 0 || i >= items.length)
+        throw new RangeError(`Index ${i} is outside 0..${items.length - 1}`);
+    return items[i];
+}
+/** A map's value for a key the caller knows is there. Throws otherwise. */
+export function mustGet(map, key) {
+    const v = map.get(key);
+    if (v === undefined)
+        throw new Error(`Missing ${String(key)}`);
+    return v;
+}
+//# sourceMappingURL=util.js.map

package/dist/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,UAAU,EAAE,CAAI,KAAmB,EAAE,CAAS;IAClD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;QAC5B,MAAM,IAAI,UAAU,CAAC,SAAS,CAAC,kBAAkB,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,KAAK,CAAC,CAAC,CAAM,CAAC;AACvB,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,OAAO,CAAO,GAAsB,EAAE,GAAM;IAC1D,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,CAAC,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/weekly.d.ts

@@ -0,0 +1,80 @@
+import type { ThumbsSummary } from "./feedback.js";
+import { type ModelCatalog } from "./models.js";
+import type { RunMetrics, RunRecord } from "./store.js";
+/**
+ * The weekly check: each suite's quick set on its live model, with fresh
+ * answers, so a provider quietly changing a model under the same name shows
+ * up as drift. The summary also flags models retiring soon and features
+ * people rate poorly. It only reports; a person decides what to change.
+ * Pure.
+ */
+/** `startedBy` on tests the weekly check starts, so they're told apart. */
+export declare const SCHEDULED = "schedule";
+/**
+ * What the last weekly check found: drift, unfinished checks, retiring models, and
+ * thumbs-down, plus when it ran and what it cost.
+ */
+export type WeeklySummary = {
+    /** When the last weekly check started; null before the first. */
+    lastAt: Date | null;
+    /** What last week's quick sets cost (reused answers are free). */
+    costUsd: number;
+    /** Quick sets still running. */
+    running: number;
+    drift: {
+        suite: string;
+        label: string;
+        model: string;
+        previous: number;
+        latest: number;
+    }[];
+    /** Quick sets that didn't finish, and why. */
+    failed: {
+        suite: string;
+        label: string;
+        error: string;
+    }[];
+    /** Live and fallback models retiring within the warning window. */
+    retiring: {
+        suite: string;
+        label: string;
+        model: string;
+        role: "live" | "fallback";
+        when: string;
+        days: number;
+    }[];
+    /** Features with a high thumbs-down share this week. */
+    thumbs: ThumbsSummary[];
+    /** Ran, and nothing needs a look. */
+    clear: boolean;
+};
+/** A test's headline score: parts right where it gives partial credit. */
+export declare function headline(m: RunMetrics | null): number | null;
+/**
+ * What the last weekly check found, from every suite's tests.
+ *
+ * @example
+ * ```ts
+ * const runs = (await Promise.all(SUITES.map((s) => store.listRuns(s.key, { limit: 50 })))).flat();
+ * const weekly = weeklySummary({ runs, suites: SUITES, routes, models: MODELS, thumbs });
+ * if (!weekly.clear) console.log(weekly.drift, weekly.retiring);
+ * ```
+ */
+export declare function weeklySummary(args: {
+    /** Every suite's tests, newest first or not. */
+    runs: RunRecord[];
+    suites: {
+        key: string;
+        label: string;
+        operation: string;
+    }[];
+    /** Each operation's live and fallback model. */
+    routes: Record<string, {
+        model: string;
+        fallbackModel: string | null;
+    }>;
+    models: ModelCatalog;
+    /** From thumbsSummary over the last week; only flagged ones are shown. */
+    thumbs?: ThumbsSummary[];
+    today?: Date;
+}): WeeklySummary;

package/dist/weekly.js

@@ -0,0 +1,111 @@
+import { retiresLabel } from "./models.js";
+import { daysUntil, drifted, retiresSoon } from "./upkeepRules.js";
+/**
+ * The weekly check: each suite's quick set on its live model, with fresh
+ * answers, so a provider quietly changing a model under the same name shows
+ * up as drift. The summary also flags models retiring soon and features
+ * people rate poorly. It only reports; a person decides what to change.
+ * Pure.
+ */
+/** `startedBy` on tests the weekly check starts, so they're told apart. */
+export const SCHEDULED = "schedule";
+const DAY = 86_400_000;
+/** A test's headline score: parts right where it gives partial credit. */
+export function headline(m) {
+    if (!m)
+        return null;
+    return m.units && m.units.total
+        ? m.units.right / m.units.total
+        : (m.allPass.rate ?? 0);
+}
+/**
+ * What the last weekly check found, from every suite's tests.
+ *
+ * @example
+ * ```ts
+ * const runs = (await Promise.all(SUITES.map((s) => store.listRuns(s.key, { limit: 50 })))).flat();
+ * const weekly = weeklySummary({ runs, suites: SUITES, routes, models: MODELS, thumbs });
+ * if (!weekly.clear) console.log(weekly.drift, weekly.retiring);
+ * ```
+ */
+export function weeklySummary(args) {
+    const today = args.today ?? new Date();
+    const label = (key) => args.suites.find((s) => s.key === key)?.label ?? key;
+    const scheduled = args.runs
+        .filter((r) => r.startedBy === SCHEDULED && r.set === "SMOKE")
+        .sort((a, b) => +b.startedAt - +a.startedAt);
+    const lastAt = scheduled[0]?.startedAt ?? null;
+    // The last weekly batch: everything started within a day of the newest.
+    const batch = lastAt
+        ? scheduled.filter((r) => +lastAt - +r.startedAt < DAY)
+        : [];
+    const drift = [];
+    const failed = [];
+    for (const r of batch) {
+        if (r.status === "FAILED") {
+            failed.push({
+                suite: r.suite,
+                label: label(r.suite),
+                error: r.error ?? "unknown error",
+            });
+            continue;
+        }
+        if (r.status !== "SUCCEEDED")
+            continue;
+        // The last check of the same model, instructions, and grading, a day or
+        // more earlier: anything else changed for a known reason.
+        const prior = scheduled.find((p) => p.suite === r.suite &&
+            p.status === "SUCCEEDED" &&
+            p.model === r.model &&
+            p.promptVersion === r.promptVersion &&
+            p.graderVersion === r.graderVersion &&
+            +r.startedAt - +p.startedAt >= DAY);
+        const latest = headline(r.metrics);
+        const previous = headline(prior?.metrics ?? null);
+        if (latest !== null && previous !== null && drifted(previous, latest))
+            drift.push({
+                suite: r.suite,
+                label: label(r.suite),
+                model: args.models[r.model]?.label ?? r.model,
+                previous,
+                latest,
+            });
+    }
+    const retiring = [];
+    for (const s of args.suites) {
+        const route = args.routes[s.operation];
+        if (!route)
+            continue;
+        for (const [model, role] of [
+            [route.model, "live"],
+            [route.fallbackModel, "fallback"],
+        ]) {
+            const when = model ? args.models[model]?.retiring : undefined;
+            if (model && when && retiresSoon(when, today))
+                retiring.push({
+                    suite: s.key,
+                    label: s.label,
+                    model: args.models[model]?.label ?? model,
+                    role,
+                    when: retiresLabel(args.models, model) ?? when,
+                    days: daysUntil(when, today),
+                });
+        }
+    }
+    const thumbs = (args.thumbs ?? []).filter((t) => t.alert);
+    return {
+        lastAt,
+        costUsd: batch.reduce((t, r) => t + (r.metrics?.costUsd ?? 0), 0),
+        running: batch.filter((r) => r.status === "RUNNING").length,
+        drift,
+        failed,
+        retiring,
+        thumbs,
+        clear: lastAt !== null &&
+            !drift.length &&
+            !failed.length &&
+            !retiring.length &&
+            !thumbs.length,
+    };
+}
+//# sourceMappingURL=weekly.js.map

package/dist/weekly.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"weekly.js","sourceRoot":"","sources":["../src/weekly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;GAMG;AAEH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC;AAEpC,MAAM,GAAG,GAAG,UAAU,CAAC;AAqCvB,0EAA0E;AAC1E,MAAM,UAAU,QAAQ,CAAC,CAAoB;IAC3C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK;QAC7B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK;QAC/B,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,IAU7B;IACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC;SAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;IAC/C,wEAAwE;IACxE,MAAM,KAAK,GAAG,MAAM;QAClB,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC;QACvD,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,eAAe;aAClC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACvC,wEAAwE;QACxE,0DAA0D;QAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,MAAM,KAAK,WAAW;YACxB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CACrC,CAAC;QACF,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,CAAC;QAClD,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK;gBAC7C,QAAQ;gBACR,MAAM;aACP,CAAC,CAAC;IACP,CAAC;IAED,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI;YAC1B,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;YACrB,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,CAAC;SACzB,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9D,IAAI,KAAK,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC;gBAC3C,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,CAAC,CAAC,GAAG;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,KAAK;oBACzC,IAAI;oBACJ,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI;oBAC9C,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;iBAC7B,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,MAAM;QACN,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC3D,KAAK;QACL,MAAM;QACN,QAAQ;QACR,MAAM;QACN,KAAK,EACH,MAAM,KAAK,IAAI;YACf,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,MAAM,CAAC,MAAM;YACd,CAAC,QAAQ,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,MAAM;KACjB,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,83 @@
 {
   "name": "@crowandraven/evalkeeper",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Pure eval logic: suites, checks, stats, model comparisons, and model suggestions on evidence.",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./runner": {
+      "types": "./dist/runner.d.ts",
+      "default": "./dist/runner.js"
+    },
+    "./memory": {
+      "types": "./dist/memoryStore.d.ts",
+      "default": "./dist/memoryStore.js"
+    },
+    "./testing": {
+      "types": "./dist/testing.d.ts",
+      "default": "./dist/testing.js"
+    },
+    "./cli": {
+      "types": "./dist/cli/main.d.ts",
+      "default": "./dist/cli/main.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "!src/**/*.test.ts",
+    "bin",
+    "templates",
+    "starter"
+  ],
+  "sideEffects": false,
+  "scripts": {
+    "build": "node scripts/build-templates.mjs && tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run --coverage"
+  },
+  "peerDependencies": {
+    "zod": "^3.25.0",
+    "vitest": ">=3"
+  },
+  "devDependencies": {
+    "zod": "^3.25.76"
+  },
+  "keywords": [
+    "evals",
+    "llm",
+    "ai",
+    "testing",
+    "model-selection"
+  ],
+  "peerDependenciesMeta": {
+    "vitest": {
+      "optional": true
+    }
+  },
+  "author": "Sean Hudson",
+  "homepage": "https://github.com/seanhudson/evalkeeper#readme",
+  "bugs": {
+    "url": "https://github.com/seanhudson/evalkeeper/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/seanhudson/evalkeeper.git",
+    "directory": "packages/core"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "bin": {
+    "evalkeeper": "./bin/evalkeeper.mjs"
+  }
+}

package/src/attention.ts

@@ -0,0 +1,88 @@
+import type { BoardRow } from "./board.js";
+
+/**
+ * What needs attention across suites, for the evals hub: the suite with
+ * the most room to improve, the slowest and costliest live models, and the
+ * answers waiting for review. From the same boards the suite pages show;
+ * only current, complete tests count. Pure.
+ */
+
+/** One suite's board, as attention reads it. */
+export type SuiteBoard = {
+  key: string;
+  label: string;
+  rows: BoardRow[];
+  /** Cases waiting for a person: drafts, and provisional answers. */
+  toReview: number;
+};
+
+/**
+ * What needs a look across suites: the most room to improve, the slowest and costliest
+ * live models, and the answers waiting for review.
+ */
+export type Attention = {
+  improve: { key: string; label: string; best: number; model: string } | null;
+  slowest: { key: string; label: string; model: string; ms: number } | null;
+  costliest: { key: string; label: string; model: string; usd: number } | null;
+  review: {
+    total: number;
+    key: string | null;
+    label: string | null;
+    most: number;
+  };
+};
+
+const current = (r: BoardRow) =>
+  r.test && !r.test.stale && !r.test.failedAnswers ? r.test : null;
+
+/**
+ * Finds what needs a look across suites, from their boards. Only current, complete tests count.
+ */
+export function attention(boards: SuiteBoard[]): Attention {
+  let improve: Attention["improve"] = null;
+  let slowest: Attention["slowest"] = null;
+  let costliest: Attention["costliest"] = null;
+  for (const b of boards) {
+    const tested = b.rows.filter((r) => current(r));
+    if (tested.length) {
+      const rateOf = (r: BoardRow) => {
+        const t = current(r);
+        return t ? t.right / t.total : -1;
+      };
+      const best = tested.reduce((a, r) => (rateOf(r) > rateOf(a) ? r : a));
+      const rate = rateOf(best);
+      if (!improve || rate < improve.best)
+        improve = { key: b.key, label: b.label, best: rate, model: best.label };
+    }
+    const live = b.rows.find((r) => r.live && current(r));
+    const t = live && current(live);
+    if (live && t?.avgMs && (!slowest || t.avgMs > slowest.ms))
+      slowest = { key: b.key, label: b.label, model: live.label, ms: t.avgMs };
+    if (
+      live &&
+      t?.costPerAnswer &&
+      (!costliest || t.costPerAnswer > costliest.usd)
+    )
+      costliest = {
+        key: b.key,
+        label: b.label,
+        model: live.label,
+        usd: t.costPerAnswer,
+      };
+  }
+  const most = boards.reduce<SuiteBoard | null>(
+    (a, b) => (b.toReview > (a?.toReview ?? 0) ? b : a),
+    null,
+  );
+  return {
+    improve,
+    slowest,
+    costliest,
+    review: {
+      total: boards.reduce((t, b) => t + b.toReview, 0),
+      key: most?.key ?? null,
+      label: most?.label ?? null,
+      most: most?.toReview ?? 0,
+    },
+  };
+}

package/src/board.ts

@@ -0,0 +1,56 @@
+import type { Verdict } from "./compare.js";
+import type { SuiteMetrics } from "./stats.js";
+
+/**
+ * One row of a suite's model board: a model (and reasoning setting) with
+ * its latest test on every case in use, and how it compares with the live
+ * model. Storage adapters build these; health, attention, and suggest read
+ * them.
+ */
+export type BoardRow = {
+  /** Model and reasoning setting together. */
+  key: string;
+  model: string;
+  /** The reasoning setting; null: the model's own default. */
+  effort: string | null;
+  /** At a setting other than the operation's (a "Test a setting" row). */
+  variant: boolean;
+  label: string;
+  live: boolean;
+  fallback: boolean;
+  testing: boolean;
+  /** The newest test failed: why, in a line. Older results still show. */
+  failed: string | null;
+  /** Estimated from real usage, for models not tested yet. */
+  estCostPerAnswer: number | null;
+  test: {
+    runId: string;
+    at: Date;
+    /** Parts right of parts expected (or answers, where a suite has no parts). */
+    right: number;
+    total: number;
+    /** Answers with every check passed. */
+    answersRight: number;
+    answers: number;
+    /** Pass rate of each check, for the breakdown table. */
+    byCheck: SuiteMetrics["byCheck"];
+    /** Per check, probability measures (decision suites); empty otherwise. */
+    measures: NonNullable<SuiteMetrics["measures"]>;
+    avgMs: number | null;
+    costPerAnswer: number | null;
+    /** Cases, answers, or instructions changed since: test again. */
+    stale: boolean;
+    /** Answers that failed (no score), and the usual reason. */
+    failedAnswers: number;
+    failureReason: string | null;
+  } | null;
+  /** Against the live model's test, when both are current. */
+  vsLive: Verdict | null;
+  canMakeLive: boolean;
+  /** Tested, current, and from another provider than the live model. */
+  canMakeFallback: boolean;
+  /** Can be live but not the fallback (a model that only answers typed questions). */
+  liveOnly: boolean;
+  /** Can be the fallback but not live. */
+  fallbackOnly: boolean;
+};