@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/src/index.ts

@@ -0,0 +1,33 @@
+/**
+ * The core: suites, checks, stats, comparisons, suggestions, health, feedback,
+ * versioned settings, and the store interfaces. Pure: no database,
+ * framework, or provider.
+ *
+ * @module evalkeeper
+ */
+export * from "./types.js";
+export * from "./models.js";
+export * from "./board.js";
+export * from "./stats.js";
+export * from "./compare.js";
+export * from "./summary.js";
+export * from "./suggest.js";
+export * from "./health.js";
+export * from "./attention.js";
+export * from "./common.js";
+export * from "./legs.js";
+export * from "./timeout.js";
+export * from "./prune.js";
+export * from "./quickSet.js";
+export * from "./stableJson.js";
+export * from "./upkeepRules.js";
+export * from "./format.js";
+export * from "./glossary.js";
+export * from "./feedback.js";
+export * from "./weekly.js";
+export * from "./settings.js";
+export * from "./decisions.js";
+export * from "./store.js";
+export * from "./labels.js";
+// Server-only (node:crypto): import from "@crowandraven/evalkeeper/runner".
+// In-memory store for tests and demos: "@crowandraven/evalkeeper/memory".

package/src/labels.ts

@@ -0,0 +1,18 @@
+import type { LabelRecord } from "./store.js";
+
+/**
+ * The EXPECTED answer tests use: the newest confirmed one, or failing that a
+ * provisional one (put to use before review). Pending answers never count.
+ */
+export function confirmedLabel(labels: LabelRecord[]): LabelRecord | null {
+  const newest = (status: LabelRecord["status"]) =>
+    labels
+      .filter((l) => l.status === status)
+      .sort((a, b) => +b.updatedAt - +a.updatedAt)[0] ?? null;
+  return newest("CONFIRMED") ?? newest("PROVISIONAL");
+}
+
+/** In use on an answer nobody has reviewed yet. */
+export function isProvisional(labels: LabelRecord[]): boolean {
+  return confirmedLabel(labels)?.status === "PROVISIONAL";
+}

package/src/legs.ts

@@ -0,0 +1,38 @@
+/**
+ * A test longer than one request's time limit runs in legs: each leg
+ * answers what it can, then hands the rest to a fresh request. Pure.
+ */
+
+/** At most this many legs (about half an hour) before a test gives up. */
+export const MAX_LEGS = 6;
+
+/**
+ * Whether a leg that ran out of time should hand off: only when it made
+ * progress (so a stuck case can't loop) and legs remain.
+ */
+export function shouldContinue(opts: {
+  /** Answers left for a later leg. */
+  left: number;
+  /** Answers this leg gave. */
+  done: number;
+  leg: number;
+  outOfCredits: boolean;
+}): boolean {
+  return (
+    opts.left > 0 && opts.done > 0 && !opts.outOfCredits && opts.leg < MAX_LEGS
+  );
+}
+
+/**
+ * Whether a test still marked running was stopped by the server: nothing
+ * started or answered within `stuckMs`. Legs keep it alive.
+ */
+export function isStuck(
+  startedAt: Date,
+  lastAnswerAt: Date | null,
+  now: number,
+  stuckMs: number,
+): boolean {
+  const last = Math.max(+startedAt, lastAnswerAt ? +lastAnswerAt : 0);
+  return now - last > stuckMs;
+}

package/src/memoryStore.ts

@@ -0,0 +1,268 @@
+/**
+ * An in-memory store, for tests, demos, and trying Evalkeeper before adding
+ * tables.
+ *
+ * @module evalkeeper/memory
+ */
+import type { FeedbackRecord, FeedbackStore } from "./feedback.js";
+import { cleanRating } from "./feedback.js";
+import {
+  SettingsConflict,
+  type SettingsChange,
+  type SettingsStore,
+  type SettingsVersion,
+} from "./settings.js";
+import type {
+  CaseRecord,
+  EvalStore,
+  OutputRecord,
+  RunRecord,
+} from "./store.js";
+
+/**
+ * An EvalStore in memory: for tests, demos, and trying Evalkeeper before
+ * adding tables. Everything is gone when the process ends.
+ */
+export function memoryStore(): EvalStore &
+  FeedbackStore &
+  SettingsStore & {
+    cases: CaseRecord[];
+    runs: RunRecord[];
+    outputs: OutputRecord[];
+    feedback: FeedbackRecord[];
+    versions: SettingsVersion[];
+    changes: SettingsChange[];
+  } {
+  const cases: CaseRecord[] = [];
+  const runs: RunRecord[] = [];
+  const outputs: OutputRecord[] = [];
+  const feedback: FeedbackRecord[] = [];
+  const versions: SettingsVersion[] = [];
+  const changes: SettingsChange[] = [];
+  const live = new Map<string, string>();
+  let n = 0;
+  const id = (prefix: string) => `${prefix}_${(++n).toString(36)}`;
+  const now = () => new Date();
+
+  return {
+    cases,
+    runs,
+    outputs,
+    feedback,
+    versions,
+    changes,
+    async liveVersion(operation) {
+      const id = live.get(operation);
+      return versions.find((v) => v.id === id) ?? null;
+    },
+    async listVersions(operation) {
+      return versions
+        .filter((v) => v.operation === operation)
+        .sort((a, b) => b.number - a.number);
+    },
+    async listChanges(operation, opts = {}) {
+      // Kept in the order they happened, so newest first is the reverse.
+      return changes
+        .filter((c) => c.operation === operation)
+        .reverse()
+        .slice(0, opts.limit ?? Infinity);
+    },
+    async switchTo(operation, sw) {
+      // Synchronous from here on, so it's atomic in memory.
+      const fromId = live.get(operation) ?? null;
+      if (fromId !== sw.expectLive) throw new SettingsConflict(operation);
+      let to: SettingsVersion | undefined;
+      if (sw.create) {
+        const last = versions
+          .filter((v) => v.operation === operation)
+          .reduce((n, v) => Math.max(n, v.number), 0);
+        to = {
+          ...sw.create.settings,
+          id: id("ver"),
+          operation,
+          number: last + 1,
+          promptVersion: sw.create.promptVersion,
+          createdAt: now(),
+        };
+        versions.push(to);
+      } else {
+        to = versions.find(
+          (v) => v.id === sw.versionId && v.operation === operation,
+        );
+        if (!to) throw new Error("No such version for this feature");
+      }
+      live.set(operation, to.id);
+      changes.push({
+        id: id("chg"),
+        operation,
+        kind: sw.kind,
+        fromVersionId: fromId,
+        toVersionId: to.id,
+        reason: sw.reason,
+        by: sw.by,
+        evidence: sw.evidence ?? null,
+        createdAt: now(),
+      });
+      return to;
+    },
+    async rate(r) {
+      const c = cleanRating(r);
+      const at = now();
+      const prior = feedback.find(
+        (f) =>
+          f.raterId === c.raterId &&
+          f.operation === c.operation &&
+          f.targetId === c.targetId,
+      );
+      if (prior) {
+        // A changed rating needs a fresh look.
+        if (prior.rating !== c.rating) {
+          prior.status = "OPEN";
+          prior.caseId = null;
+        }
+        Object.assign(prior, c, { updatedAt: at });
+        return prior;
+      }
+      const f: FeedbackRecord = {
+        ...c,
+        id: id("fb"),
+        status: "OPEN",
+        caseId: null,
+        createdAt: at,
+        updatedAt: at,
+      };
+      feedback.push(f);
+      return f;
+    },
+    async clearRating(raterId, operation, targetId) {
+      const i = feedback.findIndex(
+        (f) =>
+          f.raterId === raterId &&
+          f.operation === operation &&
+          f.targetId === targetId,
+      );
+      if (i >= 0) feedback.splice(i, 1);
+    },
+    async listFeedback(opts = {}) {
+      return feedback
+        .filter(
+          (f) =>
+            (!opts.operation || f.operation === opts.operation) &&
+            (!opts.rating || f.rating === opts.rating) &&
+            (!opts.status || f.status === opts.status) &&
+            (!opts.since || f.updatedAt >= opts.since),
+        )
+        .sort((a, b) => +b.updatedAt - +a.updatedAt)
+        .slice(0, opts.limit ?? Infinity);
+    },
+    async setFeedbackStatus(feedbackId, status, caseId = null) {
+      const f = feedback.find((x) => x.id === feedbackId);
+      if (f) {
+        f.status = status;
+        f.caseId = caseId;
+        f.updatedAt = now();
+      }
+    },
+    async listCases(suite) {
+      return cases
+        .filter((c) => c.suite === suite && c.status !== "RETIRED")
+        .sort((a, b) => +a.createdAt - +b.createdAt);
+    },
+    async createCase({ labels = [], ...data }) {
+      const at = now();
+      const c: CaseRecord = {
+        ...data,
+        id: id("case"),
+        createdAt: at,
+        updatedAt: at,
+        labels: labels.map((l) => ({
+          ...l,
+          id: id("label"),
+          createdAt: at,
+          updatedAt: at,
+        })),
+      };
+      cases.push(c);
+      return c;
+    },
+    async addLabel(caseId, data) {
+      const c = cases.find((x) => x.id === caseId);
+      if (!c) throw new Error(`Unknown case ${caseId}`);
+      const at = now();
+      const l = { ...data, id: id("label"), createdAt: at, updatedAt: at };
+      c.labels.push(l);
+      c.updatedAt = at;
+      return l;
+    },
+    async setLabelStatus(labelId, status) {
+      for (const c of cases)
+        for (const l of c.labels)
+          if (l.id === labelId) {
+            l.status = status;
+            l.updatedAt = now();
+          }
+    },
+    async setCaseStatus(caseId, status) {
+      const c = cases.find((x) => x.id === caseId);
+      if (c) {
+        c.status = status;
+        c.updatedAt = now();
+      }
+    },
+    async getRun(runId) {
+      return runs.find((r) => r.id === runId) ?? null;
+    },
+    async listRuns(suite, opts = {}) {
+      return runs
+        .filter(
+          (r) =>
+            r.suite === suite && (!opts.status || r.status === opts.status),
+        )
+        .sort((a, b) => +b.startedAt - +a.startedAt)
+        .slice(0, opts.limit ?? Infinity);
+    },
+    async createRun(data) {
+      const r: RunRecord = {
+        ...data,
+        id: id("run"),
+        status: "RUNNING",
+        metrics: null,
+        error: null,
+        startedAt: now(),
+        finishedAt: null,
+      };
+      runs.push(r);
+      return r;
+    },
+    async updateRun(runId, patch, onlyIf) {
+      const r = runs.find((x) => x.id === runId);
+      if (!r || (onlyIf && r.status !== onlyIf.status)) return;
+      Object.assign(r, patch);
+    },
+    async listOutputs(runId) {
+      return outputs.filter((o) => o.runId === runId);
+    },
+    async findReusableOutput(requestHash, since) {
+      return (
+        outputs
+          .filter(
+            (o) =>
+              o.requestHash === requestHash &&
+              !o.reused &&
+              !o.error &&
+              o.createdAt >= since,
+          )
+          .sort((a, b) => +b.createdAt - +a.createdAt)[0] ?? null
+      );
+    },
+    async createOutput(data) {
+      const o: OutputRecord = { ...data, id: id("out"), createdAt: now() };
+      outputs.push(o);
+      return o;
+    },
+    async setOutputScores(outputId, scores) {
+      const o = outputs.find((x) => x.id === outputId);
+      if (o) o.scores = scores;
+    },
+  };
+}

package/src/models.ts

@@ -0,0 +1,127 @@
+/**
+ * What Evalkeeper needs to know about the models your app can use. You
+ * pass a catalog in; Evalkeeper never picks a provider itself. Pure.
+ */
+
+/** How hard a reasoning model thinks, where the model takes a setting. */
+export const REASONING_EFFORTS = ["none", "low", "medium", "high"] as const;
+/** A reasoning setting a model may accept. */
+export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
+
+/**
+ * What Evalkeeper needs to know about one model: its label, provider, prices, reasoning
+ * settings, and retirement date.
+ */
+export type ModelInfo = {
+  label: string;
+  /** Who serves it ("anthropic", "openai", …): the fallback must differ. */
+  provider: string;
+  /** USD per 1M tokens, for estimates and the cost of each answer. */
+  inputPer1M?: number;
+  outputPer1M?: number;
+  /**
+   * USD per 1M input tokens read from the provider's prompt cache. Unset:
+   * cached input costs `inputPer1M`, which overstates rather than flatters.
+   */
+  cachedInputPer1M?: number;
+  /**
+   * USD per 1M input tokens written to the prompt cache (Anthropic bills
+   * these above the input price). Unset: they cost `inputPer1M`.
+   */
+  cacheWriteInputPer1M?: number;
+  /** Reasoning efforts it accepts, lowest first. */
+  efforts?: ReasoningEffort[];
+  /**
+   * When the provider retires it, as "YYYY-MM" or "YYYY-MM-DD". A retiring
+   * model is never suggested, and a retiring live model gets replaced.
+   */
+  retiring?: string;
+  /** Where the price came from, so it can be checked. */
+  source?: string;
+  /**
+   * As a decision model: "native" probabilities come from the model itself
+   * (a full distribution for choices and scales); "prompted" ones are the
+   * model's own estimate written out, with no distribution for choices.
+   */
+  decisions?: "native" | "prompted";
+  /**
+   * Code your app runs instead of a model (a matcher, a rules engine) or
+   * alongside it, tested so the board shows whether a model beats it. A
+   * baseline is compared like any model but never suggested as live or as
+   * the fallback. Give it a label that says what it is.
+   */
+  baseline?: boolean;
+};
+
+/** Your app's models, by id. */
+export type ModelCatalog = Record<string, ModelInfo>;
+
+/** Whether a catalog entry is a baseline (code, not a model): never suggested. */
+export function isBaseline(models: ModelCatalog, model: string): boolean {
+  return models[model]?.baseline === true;
+}
+
+/** "January 2027" for a retiring model; null when it isn't retiring. */
+export function retiresLabel(
+  models: ModelCatalog,
+  model: string,
+): string | null {
+  const when = models[model]?.retiring;
+  if (!when) return null;
+  const [y = 1970, m = 1] = when.split("-").map(Number);
+  return new Date(Date.UTC(y, (m || 1) - 1, 1)).toLocaleDateString("en-US", {
+    month: "long",
+    year: "numeric",
+    timeZone: "UTC",
+  });
+}
+
+/**
+ * The cost of a call from its token counts; null when the price is unknown.
+ * `cachedInput` and `cacheWriteInput` are the parts of `input` read from
+ * and written to the provider's prompt cache, priced at the model's cache
+ * prices when it has them.
+ */
+export function priceCall(
+  models: ModelCatalog,
+  model: string,
+  tokens: {
+    input: number;
+    output: number;
+    cachedInput?: number;
+    cacheWriteInput?: number;
+  },
+): number | null {
+  const m = models[model];
+  if (m?.inputPer1M === undefined || m.outputPer1M === undefined) return null;
+  // The cache parts are subsets of input: never count more than it has.
+  const cached = Math.min(Math.max(tokens.cachedInput ?? 0, 0), tokens.input);
+  const written = Math.min(
+    Math.max(tokens.cacheWriteInput ?? 0, 0),
+    tokens.input - cached,
+  );
+  const plain = tokens.input - cached - written;
+  return (
+    (plain * m.inputPer1M +
+      cached * (m.cachedInputPer1M ?? m.inputPer1M) +
+      written * (m.cacheWriteInputPer1M ?? m.inputPer1M) +
+      tokens.output * m.outputPer1M) /
+    1e6
+  );
+}
+
+/**
+ * The effort a model actually gets for a requested one: the same if it
+ * takes it; for "none" on a model without it, its lowest; otherwise its own
+ * default. Undefined: send nothing.
+ */
+export function effortFor(
+  models: ModelCatalog,
+  model: string,
+  requested: ReasoningEffort | null | undefined,
+): ReasoningEffort | undefined {
+  const efforts = models[model]?.efforts;
+  if (!requested || !efforts) return undefined;
+  if (efforts.includes(requested)) return requested;
+  return requested === "none" ? efforts[0] : undefined;
+}

package/src/prune.ts

@@ -0,0 +1,95 @@
+import { stableJson } from "./stableJson.js";
+import { at } from "./util.js";
+
+/**
+ * Keeping an answer key useful: cases every model gets right (easy), cases
+ * models split on (worth a review), near-duplicates, drafts nobody
+ * reviewed, and how the tags spread. Pure; the suite page shows these and
+ * the person decides, except that stale drafts retire themselves weekly
+ * (retiring is reversible).
+ */
+
+/** Easy needs at least this many current models, all fully right. */
+export const EASY_MIN_MODELS = 3;
+/** Split: this many models or more, with 40–60% fully right. */
+export const SPLIT_MIN_MODELS = 4;
+/** Easy cases kept as a regression floor, as a share of cases in use. */
+export const EASY_FLOOR = 0.1;
+/** Drafts unreviewed this long retire. */
+export const STALE_DRAFT_DAYS = 30;
+/** Inputs this similar (shared words) are near-duplicates. */
+export const DUPLICATE_SIMILARITY = 0.9;
+
+/** How one case did across the models tested on it. */
+export type CaseResult = { passed: number; models: number };
+
+/**
+ * A case every tested model got right (with enough models): it no longer tells them apart.
+ */
+export function isEasy(r: CaseResult | undefined): boolean {
+  return (
+    r !== undefined && r.models >= EASY_MIN_MODELS && r.passed === r.models
+  );
+}
+
+/** A case some models get right and others wrong: the most useful kind. */
+export function isSplit(r: CaseResult | undefined): boolean {
+  if (!r || r.models < SPLIT_MIN_MODELS) return false;
+  const share = r.passed / r.models;
+  return share >= 0.4 && share <= 0.6;
+}
+
+/**
+ * Easy cases to retire, keeping a floor (smoke cases first, then the
+ * oldest) so the suite still catches a regression on the basics.
+ */
+export function easyToRetire<
+  C extends { id: string; smoke: boolean; createdAt: Date },
+>(easy: C[], inUse: number): C[] {
+  const keep = Math.max(1, Math.ceil(inUse * EASY_FLOOR));
+  const ordered = [...easy].sort(
+    (a, b) => Number(b.smoke) - Number(a.smoke) || +a.createdAt - +b.createdAt,
+  );
+  return ordered.slice(keep);
+}
+
+const words = (input: unknown) =>
+  new Set(
+    stableJson(input)
+      .toLowerCase()
+      .match(/[a-z0-9]{3,}/g) ?? [],
+  );
+
+/** Pairs of cases whose inputs share nearly all their words. */
+export function nearDuplicates<C extends { id: string; input: unknown }>(
+  cases: C[],
+): [C, C][] {
+  const sets = cases.map((c) => ({ c, w: words(c.input) }));
+  const pairs: [C, C][] = [];
+  for (let i = 0; i < sets.length; i++)
+    for (let j = i + 1; j < sets.length; j++) {
+      const x = at(sets, i);
+      const y = at(sets, j);
+      const a = x.w;
+      const b = y.w;
+      let shared = 0;
+      for (const w of a) if (b.has(w)) shared++;
+      const union = a.size + b.size - shared;
+      if (union && shared / union >= DUPLICATE_SIMILARITY)
+        pairs.push([x.c, y.c]);
+    }
+  return pairs;
+}
+
+/** A draft nobody has reviewed for long enough to retire. */
+export function isStaleDraft(createdAt: Date, today: Date): boolean {
+  return +today - +createdAt > STALE_DRAFT_DAYS * 86_400_000;
+}
+
+/** How many cases in use carry each tag, most first. */
+export function tagCounts(cases: { tags: string[] }[]): [string, number][] {
+  const counts = new Map<string, number>();
+  for (const c of cases)
+    for (const t of c.tags) counts.set(t, (counts.get(t) ?? 0) + 1);
+  return [...counts].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+}

package/src/quickSet.ts

@@ -0,0 +1,55 @@
+import { at } from "./util.js";
+/**
+ * The quick-check set (internally SMOKE): a few cases for a cheap first
+ * look, picked by the app rather than by hand. Greedy: confirmed cases
+ * first, each pick covering the most tags not yet covered, oldest first
+ * on ties, so the set is stable and spans what the dataset tests. Pure.
+ */
+
+/** How many cases the quick set holds. */
+export const QUICK_SET_SIZE = 8;
+
+/**
+ * Picks the quick set: confirmed cases first, each pick covering the most tags not yet
+ * covered, oldest first on ties, so the set is stable.
+ */
+export function pickQuickSet<
+  T extends { id: string; tags: string[]; createdAt: Date; confirmed: boolean },
+>(cases: T[], size = QUICK_SET_SIZE): T[] {
+  const pool = [...cases].sort(
+    (a, b) =>
+      Number(b.confirmed) - Number(a.confirmed) ||
+      +a.createdAt - +b.createdAt ||
+      a.id.localeCompare(b.id),
+  );
+  const covered = new Set<string>();
+  const picked: T[] = [];
+  while (picked.length < size && pool.length) {
+    let best = 0;
+    let bestGain = -1;
+    pool.forEach((c, i) => {
+      const gain =
+        (c.confirmed ? 100 : 0) + c.tags.filter((t) => !covered.has(t)).length;
+      if (gain > bestGain) {
+        bestGain = gain;
+        best = i;
+      }
+    });
+    const c = at(pool.splice(best, 1), 0);
+    c.tags.forEach((t) => covered.add(t));
+    picked.push(c);
+  }
+  return picked;
+}
+
+/**
+ * DEV or TEST for a new case: keeps about 40% of a suite's cases in TEST,
+ * the set decisions are made on and instructions are never tuned against.
+ */
+export function nextSplit(counts: {
+  DEV: number;
+  TEST: number;
+}): "DEV" | "TEST" {
+  const total = counts.DEV + counts.TEST;
+  return counts.TEST < Math.round((total + 1) * 0.4) ? "TEST" : "DEV";
+}