@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/dist/health.d.ts

@@ -0,0 +1,40 @@
+import type { BoardRow } from "./board.js";
+import type { ModelCatalog } from "./models.js";
+import { type CheckDef, type SuitePriority } from "./types.js";
+/**
+ * Where a suite stands, for its card on the evals hub: one status and the
+ * reasons, from the same board and suggestion its own page shows. Pure.
+ */
+/** Where a suite stands, from not tested to good. */
+export type HealthStatus = "untested" | "stale" | "few" | "improve" | "fair" | "good";
+/**
+ * A suite's status for its card, with the live model's score and the reasons, most important first.
+ */
+export type Health = {
+    status: HealthStatus;
+    title: string;
+    /** The live model, its score, and how many models were tested. */
+    live: {
+        label: string;
+        rate: number;
+    } | null;
+    testedModels: number;
+    /** What to do or watch, most important first. */
+    details: string[];
+};
+/**
+ * Where a suite stands, for its card on the hub, from the same board and suggestion its
+ * page shows.
+ *
+ * @example
+ * ```ts
+ * const board = await runner.board("ticket_triage");
+ * const h = suiteHealth(board.rows, suite.checks, suite.priority, board.cases, board.provisional, MODELS);
+ * // h.status: "good" | "fair" | "improve" | "few" | "stale" | "untested"
+ * ```
+ */
+export declare function suiteHealth(rows: BoardRow[], checks: CheckDef[], priority: SuitePriority, cases: number, 
+/** Cases in use on unreviewed answers. */
+provisional?: number, 
+/** Your model catalog, for retirements and the fallback's provider. */
+models?: ModelCatalog): Health;

package/dist/health.js

@@ -0,0 +1,101 @@
+import { confidenceFor, TARGET_CASES } from "./summary.js";
+import { suggest } from "./suggest.js";
+import { checkBar } from "./types.js";
+const pct = (x) => `${Math.round(x * 100)}%`;
+/**
+ * Where a suite stands, for its card on the hub, from the same board and suggestion its
+ * page shows.
+ *
+ * @example
+ * ```ts
+ * const board = await runner.board("ticket_triage");
+ * const h = suiteHealth(board.rows, suite.checks, suite.priority, board.cases, board.provisional, MODELS);
+ * // h.status: "good" | "fair" | "improve" | "few" | "stale" | "untested"
+ * ```
+ */
+export function suiteHealth(rows, checks, priority, cases, 
+/** Cases in use on unreviewed answers. */
+provisional = 0, 
+/** Your model catalog, for retirements and the fallback's provider. */
+models = {}) {
+    const liveRow = rows.find((r) => r.live) ?? null;
+    const testedModels = rows.filter((r) => !r.baseline && r.test && !r.test.stale).length;
+    const t = liveRow?.test;
+    const live = liveRow && t ? { label: liveRow.label, rate: t.right / t.total } : null;
+    const base = { live, testedModels };
+    const fewer = cases < TARGET_CASES ? `${cases} of ${TARGET_CASES} cases` : "";
+    if (!t)
+        return {
+            ...base,
+            status: "untested",
+            title: "Not tested yet",
+            details: [
+                `Test the models to see how ${liveRow?.label ?? "the live model"} does.`,
+            ],
+        };
+    if (t.stale)
+        return {
+            ...base,
+            status: "stale",
+            title: "Test again",
+            details: ["Cases, answers, or instructions changed since the last test."],
+        };
+    const details = [];
+    if (provisional)
+        details.push(`${provisional} of ${cases} answers are provisional: review them before trusting these scores.`);
+    // Live checks under their bars, serious first.
+    const below = checks
+        .map((c) => ({ c, rate: t.byCheck[c.key]?.rate ?? null }))
+        .filter((x) => x.rate !== null && x.rate < checkBar(x.c))
+        .sort((a, b) => Number(b.c.severity === "serious") -
+        Number(a.c.severity === "serious") || a.rate - b.rate);
+    for (const { c, rate } of below.slice(0, 3))
+        details.push(`${c.label}: ${pct(rate)}, under the ${pct(checkBar(c))} bar${c.severity === "serious" ? " (serious)" : ""}.`);
+    if (below.length > 3)
+        details.push(`${below.length - 3} more under the bar.`);
+    const s = suggest(rows, checks, priority, models);
+    const change = s && !s.same;
+    if (change)
+        details.push(s.keepLive
+            ? `A better fallback is suggested: ${s.fallback?.label}.`
+            : `${s.live.label} is suggested over ${liveRow.label}.`);
+    if (testedModels < 2)
+        details.push("Only one model tested: test others to compare.");
+    const confidence = confidenceFor(cases, live?.rate ?? 0.5);
+    if (confidence.level === "too-few")
+        return {
+            ...base,
+            status: "few",
+            title: "Too few cases to trust",
+            details: [
+                `${fewer}: a score could be off by about ${confidence.margin} points.`,
+                ...details,
+            ],
+        };
+    if (below.length || change)
+        return {
+            ...base,
+            status: "improve",
+            title: "Room to improve",
+            details: fewer
+                ? [...details, `${fewer}: add more to firm it up.`]
+                : details,
+        };
+    if (confidence.level === "rough")
+        return {
+            ...base,
+            status: "fair",
+            title: "Good so far",
+            details: [
+                `Meets every bar, but with ${fewer} a score could be off by about ${confidence.margin} points.`,
+                ...details,
+            ],
+        };
+    return {
+        ...base,
+        status: "good",
+        title: "In good shape",
+        details: details.length ? details : ["Meets every bar on enough cases."],
+    };
+}
+//# sourceMappingURL=health.js.map

package/dist/health.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"health.js","sourceRoot":"","sources":["../src/health.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,OAAO,EAAE,QAAQ,EAAqC,MAAM,YAAY,CAAC;AAwBzE,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;AAErD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CACzB,IAAgB,EAChB,MAAkB,EAClB,QAAuB,EACvB,KAAa;AACb,0CAA0C;AAC1C,WAAW,GAAG,CAAC;AACf,uEAAuE;AACvE,SAAuB,EAAE;IAEzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IACjD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAC9C,CAAC,MAAM,CAAC;IACT,MAAM,CAAC,GAAG,OAAO,EAAE,IAAI,CAAC;IACxB,MAAM,IAAI,GACR,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,YAAY,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9E,IAAI,CAAC,CAAC;QACJ,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,gBAAgB;YACvB,OAAO,EAAE;gBACP,8BAA8B,OAAO,EAAE,KAAK,IAAI,gBAAgB,QAAQ;aACzE;SACF,CAAC;IACJ,IAAI,CAAC,CAAC,KAAK;QACT,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,OAAO;YACf,KAAK,EAAE,YAAY;YACnB,OAAO,EAAE,CAAC,8DAA8D,CAAC;SAC1E,CAAC;IAEJ,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,WAAW;QACb,OAAO,CAAC,IAAI,CACV,GAAG,WAAW,OAAO,KAAK,qEAAqE,CAChG,CAAC;IACJ,+CAA+C;IAC/C,MAAM,KAAK,GAAG,MAAM;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;SACzD,MAAM,CACL,CAAC,CAAC,EAAsC,EAAE,CACxC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAC5C;SACA,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC;QAChC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAC1D,CAAC;IACJ,KAAK,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,CAC9G,CAAC;IACJ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAE9E,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5B,IAAI,MAAM;QACR,OAAO,CAAC,IAAI,CACV,CAAC,CAAC,QAAQ;YACR,CAAC,CAAC,mCAAmC,CAAC,CAAC,QAAQ,EAAE,KAAK,GAAG;YACzD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,sBAAsB,OAAO,CAAC,KAAK,GAAG,CAC1D,CAAC;IACJ,IAAI,YAAY,GAAG,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAEjE,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC;IAC3D,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;QAChC,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,wBAAwB;YAC/B,OAAO,EAAE;gBACP,GAAG,KAAK,mCAAmC,UAAU,CAAC,MAAM,UAAU;gBACtE,GAAG,OAAO;aACX;SACF,CAAC;IACJ,IAAI,KAAK,CAAC,MAAM,IAAI,MAAM;QACxB,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,iBAAiB;YACxB,OAAO,EAAE,KAAK;gBACZ,CAAC,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,KAAK,2BAA2B,CAAC;gBACnD,CAAC,CAAC,OAAO;SACZ,CAAC;IACJ,IAAI,UAAU,CAAC,KAAK,KAAK,OAAO;QAC9B,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,aAAa;YACpB,OAAO,EAAE;gBACP,6BAA6B,KAAK,kCAAkC,UAAU,CAAC,MAAM,UAAU;gBAC/F,GAAG,OAAO;aACX;SACF,CAAC;IACJ,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,eAAe;QACtB,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,kCAAkC,CAAC;KACzE,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * The core: suites, checks, stats, comparisons, suggestions, health, feedback,
+ * versioned settings, and the store interfaces. Pure: no database,
+ * framework, or provider.
+ *
+ * @module evalkeeper
+ */
+export * from "./types.js";
+export * from "./models.js";
+export * from "./board.js";
+export * from "./stats.js";
+export * from "./compare.js";
+export * from "./summary.js";
+export * from "./suggest.js";
+export * from "./health.js";
+export * from "./attention.js";
+export * from "./common.js";
+export * from "./legs.js";
+export * from "./timeout.js";
+export * from "./prune.js";
+export * from "./quickSet.js";
+export * from "./stableJson.js";
+export * from "./upkeepRules.js";
+export * from "./format.js";
+export * from "./glossary.js";
+export * from "./feedback.js";
+export * from "./weekly.js";
+export * from "./settings.js";
+export * from "./decisions.js";
+export * from "./store.js";
+export * from "./labels.js";

package/dist/index.js

@@ -0,0 +1,34 @@
+/**
+ * The core: suites, checks, stats, comparisons, suggestions, health, feedback,
+ * versioned settings, and the store interfaces. Pure: no database,
+ * framework, or provider.
+ *
+ * @module evalkeeper
+ */
+export * from "./types.js";
+export * from "./models.js";
+export * from "./board.js";
+export * from "./stats.js";
+export * from "./compare.js";
+export * from "./summary.js";
+export * from "./suggest.js";
+export * from "./health.js";
+export * from "./attention.js";
+export * from "./common.js";
+export * from "./legs.js";
+export * from "./timeout.js";
+export * from "./prune.js";
+export * from "./quickSet.js";
+export * from "./stableJson.js";
+export * from "./upkeepRules.js";
+export * from "./format.js";
+export * from "./glossary.js";
+export * from "./feedback.js";
+export * from "./weekly.js";
+export * from "./settings.js";
+export * from "./decisions.js";
+export * from "./store.js";
+export * from "./labels.js";
+// Server-only (node:crypto): import from "@crowandraven/evalkeeper/runner".
+// In-memory store for tests and demos: "@crowandraven/evalkeeper/memory".
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,4EAA4E;AAC5E,0EAA0E"}

package/dist/labels.d.ts

@@ -0,0 +1,8 @@
+import type { LabelRecord } from "./store.js";
+/**
+ * The EXPECTED answer tests use: the newest confirmed one, or failing that a
+ * provisional one (put to use before review). Pending answers never count.
+ */
+export declare function confirmedLabel(labels: LabelRecord[]): LabelRecord | null;
+/** In use on an answer nobody has reviewed yet. */
+export declare function isProvisional(labels: LabelRecord[]): boolean;

package/dist/labels.js

@@ -0,0 +1,15 @@
+/**
+ * The EXPECTED answer tests use: the newest confirmed one, or failing that a
+ * provisional one (put to use before review). Pending answers never count.
+ */
+export function confirmedLabel(labels) {
+    const newest = (status) => labels
+        .filter((l) => l.status === status)
+        .sort((a, b) => +b.updatedAt - +a.updatedAt)[0] ?? null;
+    return newest("CONFIRMED") ?? newest("PROVISIONAL");
+}
+/** In use on an answer nobody has reviewed yet. */
+export function isProvisional(labels) {
+    return confirmedLabel(labels)?.status === "PROVISIONAL";
+}
+//# sourceMappingURL=labels.js.map

package/dist/labels.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"labels.js","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAqB;IAClD,MAAM,MAAM,GAAG,CAAC,MAA6B,EAAE,EAAE,CAC/C,MAAM;SACH,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;SAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IAC5D,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC;AACtD,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,aAAa,CAAC,MAAqB;IACjD,OAAO,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAC1D,CAAC"}

package/dist/legs.d.ts

@@ -0,0 +1,23 @@
+/**
+ * A test longer than one request's time limit runs in legs: each leg
+ * answers what it can, then hands the rest to a fresh request. Pure.
+ */
+/** At most this many legs (about half an hour) before a test gives up. */
+export declare const MAX_LEGS = 6;
+/**
+ * Whether a leg that ran out of time should hand off: only when it made
+ * progress (so a stuck case can't loop) and legs remain.
+ */
+export declare function shouldContinue(opts: {
+    /** Answers left for a later leg. */
+    left: number;
+    /** Answers this leg gave. */
+    done: number;
+    leg: number;
+    outOfCredits: boolean;
+}): boolean;
+/**
+ * Whether a test still marked running was stopped by the server: nothing
+ * started or answered within `stuckMs`. Legs keep it alive.
+ */
+export declare function isStuck(startedAt: Date, lastAnswerAt: Date | null, now: number, stuckMs: number): boolean;

package/dist/legs.js

@@ -0,0 +1,22 @@
+/**
+ * A test longer than one request's time limit runs in legs: each leg
+ * answers what it can, then hands the rest to a fresh request. Pure.
+ */
+/** At most this many legs (about half an hour) before a test gives up. */
+export const MAX_LEGS = 6;
+/**
+ * Whether a leg that ran out of time should hand off: only when it made
+ * progress (so a stuck case can't loop) and legs remain.
+ */
+export function shouldContinue(opts) {
+    return (opts.left > 0 && opts.done > 0 && !opts.outOfCredits && opts.leg < MAX_LEGS);
+}
+/**
+ * Whether a test still marked running was stopped by the server: nothing
+ * started or answered within `stuckMs`. Legs keep it alive.
+ */
+export function isStuck(startedAt, lastAnswerAt, now, stuckMs) {
+    const last = Math.max(+startedAt, lastAnswerAt ? +lastAnswerAt : 0);
+    return now - last > stuckMs;
+}
+//# sourceMappingURL=legs.js.map

package/dist/legs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"legs.js","sourceRoot":"","sources":["../src/legs.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAC;AAE1B;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAO9B;IACC,OAAO,CACL,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,GAAG,QAAQ,CAC5E,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CACrB,SAAe,EACf,YAAyB,EACzB,GAAW,EACX,OAAe;IAEf,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,OAAO,GAAG,GAAG,IAAI,GAAG,OAAO,CAAC;AAC9B,CAAC"}

package/dist/memoryStore.d.ts

@@ -0,0 +1,21 @@
+/**
+ * An in-memory store, for tests, demos, and trying Evalkeeper before adding
+ * tables.
+ *
+ * @module evalkeeper/memory
+ */
+import type { FeedbackRecord, FeedbackStore } from "./feedback.js";
+import { type SettingsChange, type SettingsStore, type SettingsVersion } from "./settings.js";
+import type { CaseRecord, EvalStore, OutputRecord, RunRecord } from "./store.js";
+/**
+ * An EvalStore in memory: for tests, demos, and trying Evalkeeper before
+ * adding tables. Everything is gone when the process ends.
+ */
+export declare function memoryStore(): EvalStore & FeedbackStore & SettingsStore & {
+    cases: CaseRecord[];
+    runs: RunRecord[];
+    outputs: OutputRecord[];
+    feedback: FeedbackRecord[];
+    versions: SettingsVersion[];
+    changes: SettingsChange[];
+};

package/dist/memoryStore.js

@@ -0,0 +1,228 @@
+import { cleanRating } from "./feedback.js";
+import { SettingsConflict, } from "./settings.js";
+/**
+ * An EvalStore in memory: for tests, demos, and trying Evalkeeper before
+ * adding tables. Everything is gone when the process ends.
+ */
+export function memoryStore() {
+    const cases = [];
+    const runs = [];
+    const outputs = [];
+    const feedback = [];
+    const versions = [];
+    const changes = [];
+    const live = new Map();
+    let n = 0;
+    const id = (prefix) => `${prefix}_${(++n).toString(36)}`;
+    const now = () => new Date();
+    return {
+        cases,
+        runs,
+        outputs,
+        feedback,
+        versions,
+        changes,
+        async liveVersion(operation) {
+            const id = live.get(operation);
+            return versions.find((v) => v.id === id) ?? null;
+        },
+        async listVersions(operation) {
+            return versions
+                .filter((v) => v.operation === operation)
+                .sort((a, b) => b.number - a.number);
+        },
+        async listChanges(operation, opts = {}) {
+            // Kept in the order they happened, so newest first is the reverse.
+            return changes
+                .filter((c) => c.operation === operation)
+                .reverse()
+                .slice(0, opts.limit ?? Infinity);
+        },
+        async switchTo(operation, sw) {
+            // Synchronous from here on, so it's atomic in memory.
+            const fromId = live.get(operation) ?? null;
+            if (fromId !== sw.expectLive)
+                throw new SettingsConflict(operation);
+            let to;
+            if (sw.create) {
+                const last = versions
+                    .filter((v) => v.operation === operation)
+                    .reduce((n, v) => Math.max(n, v.number), 0);
+                to = {
+                    ...sw.create.settings,
+                    id: id("ver"),
+                    operation,
+                    number: last + 1,
+                    promptVersion: sw.create.promptVersion,
+                    createdAt: now(),
+                };
+                versions.push(to);
+            }
+            else {
+                to = versions.find((v) => v.id === sw.versionId && v.operation === operation);
+                if (!to)
+                    throw new Error("No such version for this feature");
+            }
+            live.set(operation, to.id);
+            changes.push({
+                id: id("chg"),
+                operation,
+                kind: sw.kind,
+                fromVersionId: fromId,
+                toVersionId: to.id,
+                reason: sw.reason,
+                by: sw.by,
+                evidence: sw.evidence ?? null,
+                createdAt: now(),
+            });
+            return to;
+        },
+        async rate(r) {
+            const c = cleanRating(r);
+            const at = now();
+            const prior = feedback.find((f) => f.raterId === c.raterId &&
+                f.operation === c.operation &&
+                f.targetId === c.targetId);
+            if (prior) {
+                // A changed rating needs a fresh look.
+                if (prior.rating !== c.rating) {
+                    prior.status = "OPEN";
+                    prior.caseId = null;
+                }
+                Object.assign(prior, c, { updatedAt: at });
+                return prior;
+            }
+            const f = {
+                ...c,
+                id: id("fb"),
+                status: "OPEN",
+                caseId: null,
+                createdAt: at,
+                updatedAt: at,
+            };
+            feedback.push(f);
+            return f;
+        },
+        async clearRating(raterId, operation, targetId) {
+            const i = feedback.findIndex((f) => f.raterId === raterId &&
+                f.operation === operation &&
+                f.targetId === targetId);
+            if (i >= 0)
+                feedback.splice(i, 1);
+        },
+        async listFeedback(opts = {}) {
+            return feedback
+                .filter((f) => (!opts.operation || f.operation === opts.operation) &&
+                (!opts.rating || f.rating === opts.rating) &&
+                (!opts.status || f.status === opts.status) &&
+                (!opts.since || f.updatedAt >= opts.since))
+                .sort((a, b) => +b.updatedAt - +a.updatedAt)
+                .slice(0, opts.limit ?? Infinity);
+        },
+        async setFeedbackStatus(feedbackId, status, caseId = null) {
+            const f = feedback.find((x) => x.id === feedbackId);
+            if (f) {
+                f.status = status;
+                f.caseId = caseId;
+                f.updatedAt = now();
+            }
+        },
+        async listCases(suite) {
+            return cases
+                .filter((c) => c.suite === suite && c.status !== "RETIRED")
+                .sort((a, b) => +a.createdAt - +b.createdAt);
+        },
+        async createCase({ labels = [], ...data }) {
+            const at = now();
+            const c = {
+                ...data,
+                id: id("case"),
+                createdAt: at,
+                updatedAt: at,
+                labels: labels.map((l) => ({
+                    ...l,
+                    id: id("label"),
+                    createdAt: at,
+                    updatedAt: at,
+                })),
+            };
+            cases.push(c);
+            return c;
+        },
+        async addLabel(caseId, data) {
+            const c = cases.find((x) => x.id === caseId);
+            if (!c)
+                throw new Error(`Unknown case ${caseId}`);
+            const at = now();
+            const l = { ...data, id: id("label"), createdAt: at, updatedAt: at };
+            c.labels.push(l);
+            c.updatedAt = at;
+            return l;
+        },
+        async setLabelStatus(labelId, status) {
+            for (const c of cases)
+                for (const l of c.labels)
+                    if (l.id === labelId) {
+                        l.status = status;
+                        l.updatedAt = now();
+                    }
+        },
+        async setCaseStatus(caseId, status) {
+            const c = cases.find((x) => x.id === caseId);
+            if (c) {
+                c.status = status;
+                c.updatedAt = now();
+            }
+        },
+        async getRun(runId) {
+            return runs.find((r) => r.id === runId) ?? null;
+        },
+        async listRuns(suite, opts = {}) {
+            return runs
+                .filter((r) => r.suite === suite && (!opts.status || r.status === opts.status))
+                .sort((a, b) => +b.startedAt - +a.startedAt)
+                .slice(0, opts.limit ?? Infinity);
+        },
+        async createRun(data) {
+            const r = {
+                ...data,
+                id: id("run"),
+                status: "RUNNING",
+                metrics: null,
+                error: null,
+                startedAt: now(),
+                finishedAt: null,
+            };
+            runs.push(r);
+            return r;
+        },
+        async updateRun(runId, patch, onlyIf) {
+            const r = runs.find((x) => x.id === runId);
+            if (!r || (onlyIf && r.status !== onlyIf.status))
+                return;
+            Object.assign(r, patch);
+        },
+        async listOutputs(runId) {
+            return outputs.filter((o) => o.runId === runId);
+        },
+        async findReusableOutput(requestHash, since) {
+            return (outputs
+                .filter((o) => o.requestHash === requestHash &&
+                !o.reused &&
+                !o.error &&
+                o.createdAt >= since)
+                .sort((a, b) => +b.createdAt - +a.createdAt)[0] ?? null);
+        },
+        async createOutput(data) {
+            const o = { ...data, id: id("out"), createdAt: now() };
+            outputs.push(o);
+            return o;
+        },
+        async setOutputScores(outputId, scores) {
+            const o = outputs.find((x) => x.id === outputId);
+            if (o)
+                o.scores = scores;
+        },
+    };
+}
+//# sourceMappingURL=memoryStore.js.map

package/dist/memoryStore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memoryStore.js","sourceRoot":"","sources":["../src/memoryStore.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EACL,gBAAgB,GAIjB,MAAM,eAAe,CAAC;AAQvB;;;GAGG;AACH,MAAM,UAAU,WAAW;IAUzB,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,EAAE,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;IACjE,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;IAE7B,OAAO;QACL,KAAK;QACL,IAAI;QACJ,OAAO;QACP,QAAQ;QACR,QAAQ;QACR,OAAO;QACP,KAAK,CAAC,WAAW,CAAC,SAAS;YACzB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC;QACnD,CAAC;QACD,KAAK,CAAC,YAAY,CAAC,SAAS;YAC1B,OAAO,QAAQ;iBACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC;iBACxC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,GAAG,EAAE;YACpC,mEAAmE;YACnE,OAAO,OAAO;iBACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC;iBACxC,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACtC,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE;YAC1B,sDAAsD;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;YAC3C,IAAI,MAAM,KAAK,EAAE,CAAC,UAAU;gBAAE,MAAM,IAAI,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACpE,IAAI,EAA+B,CAAC;YACpC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;gBACd,MAAM,IAAI,GAAG,QAAQ;qBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC;qBACxC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC9C,EAAE,GAAG;oBACH,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ;oBACrB,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC;oBACb,SAAS;oBACT,MAAM,EAAE,IAAI,GAAG,CAAC;oBAChB,aAAa,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa;oBACtC,SAAS,EAAE,GAAG,EAAE;iBACjB,CAAC;gBACF,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,EAAE,GAAG,QAAQ,CAAC,IAAI,CAChB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,CAC1D,CAAC;gBACF,IAAI,CAAC,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YAC/D,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC;gBACb,SAAS;gBACT,IAAI,EAAE,EAAE,CAAC,IAAI;gBACb,aAAa,EAAE,MAAM;gBACrB,WAAW,EAAE,EAAE,CAAC,EAAE;gBAClB,MAAM,EAAE,EAAE,CAAC,MAAM;gBACjB,EAAE,EAAE,EAAE,CAAC,EAAE;gBACT,QAAQ,EAAE,EAAE,CAAC,QAAQ,IAAI,IAAI;gBAC7B,SAAS,EAAE,GAAG,EAAE;aACjB,CAAC,CAAC;YACH,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,CAAC;YACV,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACzB,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CACzB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;gBACvB,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS;gBAC3B,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,CAC5B,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACV,uCAAuC;gBACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;oBAC9B,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;oBACtB,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;gBACtB,CAAC;gBACD,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC3C,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,CAAC,GAAmB;gBACxB,GAAG,CAAC;gBACJ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,EAAE;gBACb,SAAS,EAAE,EAAE;aACd,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ;YAC5C,MAAM,CAAC,GAAG,QAAQ,CAAC,SAAS,CAC1B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,OAAO,KAAK,OAAO;gBACrB,CAAC,CAAC,SAAS,KAAK,SAAS;gBACzB,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAC1B,CAAC;YACF,IAAI,CAAC,IAAI,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,EAAE;YAC1B,OAAO,QAAQ;iBACZ,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC;gBACnD,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;gBAC1C,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;gBAC1C,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,CAC7C;iBACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;iBAC3C,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACtC,CAAC;QACD,KAAK,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;YACvD,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC;YACpD,IAAI,CAAC,EAAE,CAAC;gBACN,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;gBAClB,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;gBAClB,CAAC,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,KAAK;YACnB,OAAO,KAAK;iBACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC;iBAC1D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjD,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE;YACvC,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;YACjB,MAAM,CAAC,GAAe;gBACpB,GAAG,IAAI;gBACP,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC;gBACd,SAAS,EAAE,EAAE;gBACb,SAAS,EAAE,EAAE;gBACb,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACzB,GAAG,CAAC;oBACJ,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC;oBACf,SAAS,EAAE,EAAE;oBACb,SAAS,EAAE,EAAE;iBACd,CAAC,CAAC;aACJ,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACd,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI;YACzB,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,MAAM,EAAE,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;YACrE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM;YAClC,KAAK,MAAM,CAAC,IAAI,KAAK;gBACnB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM;oBACtB,IAAI,CAAC,CAAC,EAAE,KAAK,OAAO,EAAE,CAAC;wBACrB,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;wBAClB,CAAC,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;oBACtB,CAAC;QACP,CAAC;QACD,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM;YAChC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,EAAE,CAAC;gBACN,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;gBAClB,CAAC,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,KAAK;YAChB,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC;QAClD,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE;YAC7B,OAAO,IAAI;iBACR,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,CAClE;iBACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;iBAC3C,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACtC,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,IAAI;YAClB,MAAM,CAAC,GAAc;gBACnB,GAAG,IAAI;gBACP,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC;gBACb,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,GAAG,EAAE;gBAChB,UAAU,EAAE,IAAI;aACjB,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM;YAClC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC;gBAAE,OAAO;YACzD,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,WAAW,CAAC,KAAK;YACrB,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAClD,CAAC;QACD,KAAK,CAAC,kBAAkB,CAAC,WAAW,EAAE,KAAK;YACzC,OAAO,CACL,OAAO;iBACJ,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,WAAW,KAAK,WAAW;gBAC7B,CAAC,CAAC,CAAC,MAAM;gBACT,CAAC,CAAC,CAAC,KAAK;gBACR,CAAC,CAAC,SAAS,IAAI,KAAK,CACvB;iBACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAC1D,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,YAAY,CAAC,IAAI;YACrB,MAAM,CAAC,GAAiB,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC;YACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM;YACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;YACjD,IAAI,CAAC;gBAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;QAC3B,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/models.d.ts

@@ -0,0 +1,76 @@
+/**
+ * What Evalkeeper needs to know about the models your app can use. You
+ * pass a catalog in; Evalkeeper never picks a provider itself. Pure.
+ */
+/** How hard a reasoning model thinks, where the model takes a setting. */
+export declare const REASONING_EFFORTS: readonly ["none", "low", "medium", "high"];
+/** A reasoning setting a model may accept. */
+export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
+/**
+ * What Evalkeeper needs to know about one model: its label, provider, prices, reasoning
+ * settings, and retirement date.
+ */
+export type ModelInfo = {
+    label: string;
+    /** Who serves it ("anthropic", "openai", …): the fallback must differ. */
+    provider: string;
+    /** USD per 1M tokens, for estimates and the cost of each answer. */
+    inputPer1M?: number;
+    outputPer1M?: number;
+    /**
+     * USD per 1M input tokens read from the provider's prompt cache. Unset:
+     * cached input costs `inputPer1M`, which overstates rather than flatters.
+     */
+    cachedInputPer1M?: number;
+    /**
+     * USD per 1M input tokens written to the prompt cache (Anthropic bills
+     * these above the input price). Unset: they cost `inputPer1M`.
+     */
+    cacheWriteInputPer1M?: number;
+    /** Reasoning efforts it accepts, lowest first. */
+    efforts?: ReasoningEffort[];
+    /**
+     * When the provider retires it, as "YYYY-MM" or "YYYY-MM-DD". A retiring
+     * model is never suggested, and a retiring live model gets replaced.
+     */
+    retiring?: string;
+    /** Where the price came from, so it can be checked. */
+    source?: string;
+    /**
+     * As a decision model: "native" probabilities come from the model itself
+     * (a full distribution for choices and scales); "prompted" ones are the
+     * model's own estimate written out, with no distribution for choices.
+     */
+    decisions?: "native" | "prompted";
+    /**
+     * Code your app runs instead of a model (a matcher, a rules engine) or
+     * alongside it, tested so the board shows whether a model beats it. A
+     * baseline is compared like any model but never suggested as live or as
+     * the fallback. Give it a label that says what it is.
+     */
+    baseline?: boolean;
+};
+/** Your app's models, by id. */
+export type ModelCatalog = Record<string, ModelInfo>;
+/** Whether a catalog entry is a baseline (code, not a model): never suggested. */
+export declare function isBaseline(models: ModelCatalog, model: string): boolean;
+/** "January 2027" for a retiring model; null when it isn't retiring. */
+export declare function retiresLabel(models: ModelCatalog, model: string): string | null;
+/**
+ * The cost of a call from its token counts; null when the price is unknown.
+ * `cachedInput` and `cacheWriteInput` are the parts of `input` read from
+ * and written to the provider's prompt cache, priced at the model's cache
+ * prices when it has them.
+ */
+export declare function priceCall(models: ModelCatalog, model: string, tokens: {
+    input: number;
+    output: number;
+    cachedInput?: number;
+    cacheWriteInput?: number;
+}): number | null;
+/**
+ * The effort a model actually gets for a requested one: the same if it
+ * takes it; for "none" on a model without it, its lowest; otherwise its own
+ * default. Undefined: send nothing.
+ */
+export declare function effortFor(models: ModelCatalog, model: string, requested: ReasoningEffort | null | undefined): ReasoningEffort | undefined;