@crowandraven/evalkeeper 0.0.0-stage → 0.1.1

package/src/common.ts

@@ -0,0 +1,338 @@
+import type { CheckDef, CheckResult } from "./types.js";
+
+/**
+ * Checks shared by suites: rules many operations' instructions state the
+ * same way (lib/ai/voice.ts, "don't invent numbers"). Pure, so they're
+ * unit-tested and rerun on stored outputs for free.
+ */
+
+// He, she, and the like never belong in text addressed to the person.
+// They, them, and their often mean things, so they count only when they
+// open a sentence the way writing about the person would ("They want").
+const PRONOUNS = /\b(he|she|him|her|his|hers|himself|herself)\b/i;
+const THIRD_PERSON_START = /(^|[.!?]\s+)(they|their|them)\b/i;
+
+/** Text that refers to the person as he, she, or they; null when none. */
+export function thirdPerson(text: string): string | null {
+  const m = text.match(PRONOUNS) ?? text.match(THIRD_PERSON_START);
+  return m ? m[0].trim().replace(/^[.!?]\s*/, "") : null;
+}
+
+/**
+ * Standard phrasing whose numbers aren't claims about the person: a "first
+ * 90 days" plan, "30-60-90", "2 to 3 minutes", a "top 5", a quarter (Q3),
+ * fact ids ("[f9]"), and list markers ("(1)", "2.").
+ */
+export function withoutStockPhrases(text: string): string {
+  return (
+    text
+      .replace(/\b30\s*[-–/]\s*60\s*[-–/]\s*90\b/g, " ")
+      .replace(/\b(?:first|next|initial)\s+(?:30|60|90|100)\s+days\b/gi, " ")
+      .replace(/\b(?:30|60|90)\s+days?\s*[:\-–]/gi, " ")
+      .replace(
+        /\b\d+\s*(?:to|-|–|or)\s*\d+\s*(?:minutes?|mins?|sentences?|seconds?)\b/gi,
+        " ",
+      )
+      .replace(
+        /\b(?:under|about|in|within)\s+\d+\s+(?:minutes?|seconds?)\b/gi,
+        " ",
+      )
+      .replace(/\btop\s+(?:3|5|10)\b/gi, " ")
+      .replace(/\bQ[1-4]\b/g, " ")
+      // References, not figures: fact ids ("[f9]", "f10") and list markers.
+      .replace(/\bf\d+\b/g, " ")
+      .replace(/\(\d{1,2}\)/g, " ")
+      .replace(/(^|\n)\s*\d{1,2}[.)]\s/g, "$1 ")
+  );
+}
+
+/** Nouns a number counts in a claim ("twenty years", "five reports"). */
+const COUNTED =
+  "years?|months?|weeks?|hours?|people|employees|staff|clients|customers|direct reports|reports|students|patients|accounts|projects|sites|schools|percent";
+
+const NUMBER_WORDS =
+  "(?:(?:twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety)(?:[\\s-](?:one|two|three|four|five|six|seven|eight|nine))?|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|two|three|four|five|six|seven|eight|nine|ten)";
+
+/**
+ * Number words that count something in an answer ("twenty years", "a
+ * decade", "five direct reports") as digits, so an invented figure written
+ * out is caught too. Number words that count nothing ("one idea", "two
+ * things to try") stay words.
+ */
+export function withCountedWords(text: string): string {
+  return text
+    .replace(/\b(?:a|one)\s+decade\b/gi, "10 years")
+    .replace(
+      new RegExp(`\\b${NUMBER_WORDS}(?=\\s+(?:${COUNTED})\\b)`, "gi"),
+      (w) => withNumberWords(w),
+    );
+}
+
+/**
+ * Numbers in `text` that appear in none of the sources (150k and 150,000
+ * count as the same). Years and small counts are numbers too: a letter
+ * that says "12 years" when the profile says 10 has invented one.
+ */
+/**
+ * A number with an optional "k" or "m" ("150k", "1.5m"). It starts only
+ * where a number starts and reads it once (the lookahead and backreference
+ * keep the digits from being retried), so long digit runs in model output
+ * stay linear. A number glued to a word ("3rd", "10am") doesn't count.
+ */
+const NUMBER = /(?<![\d,.])(?=(\d[\d,]*(?:\.\d+)?))\1\s*([km])?\b/gi;
+
+/**
+ * Numbers the text states that appear in none of the sources: likely invented.
+ * Standard phrasing (a 30-60-90 day plan, list markers, fact ids) doesn't count.
+ */
+export function inventedNumbers(text: string, sources: string[]): string[] {
+  const known = new Set(
+    sources.flatMap((s) => [...numbersIn(withNumberWords(s))]),
+  );
+  const out: string[] = [];
+  const claims = withCountedWords(withoutStockPhrases(text));
+  for (const m of claims.matchAll(NUMBER)) {
+    const found = numbersIn(m[0]);
+    if (![...found].some((n) => known.has(n))) out.push(m[0].trim());
+  }
+  return [...new Set(out)];
+}
+
+/** How many words the text has. */
+export function wordCount(text: string): number {
+  return text.split(/\s+/).filter(Boolean).length;
+}
+
+/** Sentences, counted by their ending punctuation (a last one without it counts). */
+export function sentenceCount(text: string): number {
+  const t = text.trim();
+  if (!t) return 0;
+  // The last mark before a space or the end, so "?!" counts once. Linear:
+  // a run of marks is never retried from each position.
+  const ends = t.match(/[.!?](?=\s|$)/g)?.length ?? 0;
+  return /[.!?]["')\]]?$/.test(t) ? ends : ends + 1;
+}
+
+/** Pass when n is within [min, max]; the note says what it was. */
+export function inRange(
+  key: string,
+  n: number,
+  min: number,
+  max: number,
+  noun: string,
+): CheckResult {
+  const pass = n >= min && n <= max;
+  return {
+    key,
+    pass,
+    ...(pass
+      ? {}
+      : {
+          note: `${n} ${noun}; should be ${min === max ? min : `${min} to ${max}`}`,
+        }),
+  };
+}
+
+/** The "addressed as you" check over several pieces of text. */
+export function voiceCheck(key: string, texts: string[]): CheckResult {
+  for (const t of texts) {
+    const hit = thirdPerson(t);
+    if (hit) return { key, pass: false, note: `"${hit}" in: ${t}` };
+  }
+  return { key, pass: true };
+}
+
+/** The "no invented numbers" check over several pieces of text. */
+export function numbersCheck(
+  key: string,
+  texts: string[],
+  sources: string[],
+): CheckResult {
+  const invented = [
+    ...new Set(texts.flatMap((t) => inventedNumbers(t, sources))),
+  ];
+  return invented.length
+    ? {
+        key,
+        pass: false,
+        note: `Not in what it was given: ${invented.join(", ")}`,
+      }
+    : { key, pass: true };
+}
+
+/**
+ * A minor check that answers speak to the person ("you"), not about them.
+ */
+export const VOICE_CHECK: CheckDef = {
+  key: "voice",
+  label: 'Addressed as "you"',
+  description:
+    'Never "he", "she", or "they" for the person (lib/ai/voice.ts): they read it about themselves.',
+  kind: "code",
+  severity: "serious",
+};
+
+/**
+ * A serious check that every number in an answer appears in what the model was given.
+ */
+export const NUMBERS_CHECK: CheckDef = {
+  key: "numbers",
+  label: "No invented numbers",
+  description:
+    "Every number appears in what the model was given (150k and 150,000 count as the same).",
+  kind: "code",
+  severity: "serious",
+};
+
+/** Numbers in text, with "150k" and "1.5m" also read as 150000 and 1500000. */
+export function numbersIn(text: string): Set<string> {
+  const out = new Set<string>();
+  for (const m of text.matchAll(NUMBER)) {
+    const n = Number((m[1] ?? "").replace(/,/g, ""));
+    if (!Number.isFinite(n)) continue;
+    out.add(String(n));
+    const suffix = m[2]?.toLowerCase();
+    if (suffix === "k") out.add(String(Math.round(n * 1_000)));
+    if (suffix === "m") out.add(String(Math.round(n * 1_000_000)));
+  }
+  return out;
+}
+
+const ONES = [
+  "zero",
+  "one",
+  "two",
+  "three",
+  "four",
+  "five",
+  "six",
+  "seven",
+  "eight",
+  "nine",
+  "ten",
+  "eleven",
+  "twelve",
+  "thirteen",
+  "fourteen",
+  "fifteen",
+  "sixteen",
+  "seventeen",
+  "eighteen",
+  "nineteen",
+];
+const TENS = [
+  "",
+  "",
+  "twenty",
+  "thirty",
+  "forty",
+  "fifty",
+  "sixty",
+  "seventy",
+  "eighty",
+  "ninety",
+];
+
+/**
+ * Number words up to ninety-nine as digits ("sixteen weeks" → "16 weeks",
+ * "twenty-one" → "21"), for the evidence side of a number check: sources
+ * often spell numbers out where answers use digits.
+ */
+export function withNumberWords(text: string): string {
+  const tens = TENS.filter(Boolean).join("|");
+  return text
+    .replace(
+      new RegExp(`\\b(${tens})[\\s-](${ONES.slice(1, 10).join("|")})\\b`, "gi"),
+      (_, t: string, o: string) =>
+        String(
+          TENS.indexOf(t.toLowerCase()) * 10 + ONES.indexOf(o.toLowerCase()),
+        ),
+    )
+    .replace(new RegExp(`\\b(${tens})\\b`, "gi"), (t: string) =>
+      String(TENS.indexOf(t.toLowerCase()) * 10),
+    )
+    .replace(new RegExp(`\\b(${ONES.join("|")})\\b`, "gi"), (o: string) =>
+      String(ONES.indexOf(o.toLowerCase())),
+    );
+}
+
+/**
+ * A word's stem for matching: words of 5 letters or more lose one common
+ * ending ("training", "trained" → "train"); shorter words and codes ("AR",
+ * "CPT") match as written, so "AR" never matches "are".
+ */
+export function stem(word: string): string {
+  const w = word.toLowerCase();
+  if (w.length < 5) return w;
+  const cut = w
+    .replace(/(ings|ing|ers|er|ed|es|ions|ion|ly|s)$/, "")
+    // "drone" and "drones" both become "dron".
+    .replace(/(.{4,})e$/, "$1");
+  return cut.length >= 4 ? cut : w;
+}
+
+const wordsOf = (text: string) => text.toLowerCase().match(/[a-z0-9]+/g) ?? [];
+
+/**
+ * Whether every word of `phrase` appears in `text`, by stem ("trained new
+ * staff" has "training"). "a|b" accepts either phrase.
+ */
+export function hasPhrase(text: string, phrase: string): boolean {
+  const have = new Set(wordsOf(text).map(stem));
+  return phrase
+    .split("|")
+    .some((p) => wordsOf(p).every((w) => have.has(stem(w))));
+}
+
+/** Words that carry meaning, stemmed: for matching by shared words. */
+export function contentWords(text: string): Set<string> {
+  return new Set(
+    (text.toLowerCase().match(/[a-z0-9]+/g) ?? [])
+      .filter((w) => w.length > 2 && !STOP_WORDS.has(w))
+      .map(stem),
+  );
+}
+
+const STOP_WORDS = new Set(
+  "the and for with that this from into onto over under about after before their there them they you your our its are was were will would should could has have had not but all any can each per out off than then when what who how why".split(
+    " ",
+  ),
+);
+
+/**
+ * Pairs expected items with actual ones by shared content words, best
+ * overlap first, one to one. The fallback when no judge model is set: it
+ * misses paraphrases, so a meaning judge is better.
+ */
+export function wordPairs(
+  expected: string[],
+  actual: string[],
+  /** Share of an expected item's words the actual one must have. */
+  minShare = 0.5,
+): { expected: number; actual: number; why: string }[] {
+  const scored: { e: number; a: number; share: number }[] = [];
+  expected.forEach((et, e) => {
+    const ew = contentWords(et);
+    if (!ew.size) return;
+    actual.forEach((at, a) => {
+      const aw = contentWords(at);
+      const shared = [...ew].filter((w) => aw.has(w)).length;
+      if (shared / ew.size >= minShare)
+        scored.push({ e, a, share: shared / ew.size });
+    });
+  });
+  const usedE = new Set<number>();
+  const usedA = new Set<number>();
+  const out: { expected: number; actual: number; why: string }[] = [];
+  for (const s of scored.sort((x, y) => y.share - x.share)) {
+    if (usedE.has(s.e) || usedA.has(s.a)) continue;
+    usedE.add(s.e);
+    usedA.add(s.a);
+    out.push({
+      expected: s.e,
+      actual: s.a,
+      why: `${Math.round(s.share * 100)}% of the words match`,
+    });
+  }
+  return out.sort((x, y) => x.expected - y.expected);
+}

package/src/compare.ts

@@ -0,0 +1,261 @@
+import { answerScore, wilson, type Rate } from "./stats.js";
+import type { CheckResult } from "./types.js";
+import { at } from "./util.js";
+import { mustGet } from "./util.js";
+
+/**
+ * Comparing two runs of a suite, case by case: the champion (usually the
+ * live version) and a challenger (another model, effort, or instructions).
+ * Each case scores the share of its repeats where every check passed; the
+ * challenger's average difference gets a 95% range from a paired
+ * bootstrap, and the verdict is read off that range. Pure; no database.
+ */
+
+/** One stored answer, as a comparison reads it. */
+export type RunOutput = {
+  caseId: string;
+  checks: CheckResult[] | null;
+  costUsd: number;
+  ms: number;
+  reused: boolean;
+};
+
+/**
+ * How a challenger compares with the model in place: more accurate, about as accurate,
+ * less accurate, or too few cases to tell.
+ */
+export type Verdict = "BETTER" | "SAME" | "WORSE" | "TOO_FEW";
+
+/**
+ * Two tests compared case by case: the verdict, the difference and its range, the cases
+ * the challenger fixed and broke, and each check's pass rates.
+ */
+export type Comparison = {
+  /** Cases both runs have. */
+  n: number;
+  /** Challenger minus champion, in pass share (−1 to 1). */
+  mean: number;
+  low: number;
+  high: number;
+  verdict: Verdict;
+  /** Passed every time on the challenger, not on the champion. */
+  fixed: string[];
+  /** Passed every time on the champion, not on the challenger. */
+  broke: string[];
+  byCheck: Record<string, { champion: Rate; challenger: Rate }>;
+  /** Per output, called (not reused). */
+  cost: { champion: number | null; challenger: number | null };
+  avgMs: { champion: number | null; challenger: number | null };
+};
+
+/** Fewer shared cases than this: no verdict. */
+export const MIN_CASES = 5;
+
+/** A small seeded generator, so the same runs always give the same range. */
+function mulberry32(seed: number) {
+  let a = seed;
+  return () => {
+    a |= 0;
+    a = (a + 0x6d2b79f5) | 0;
+    let t = Math.imul(a ^ (a >>> 15), 1 | a);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+/** 95% range of the mean of paired differences, by resampling cases. */
+export function bootstrapMean(
+  diffs: number[],
+  iterations = 4000,
+  seed = 1,
+): { mean: number; low: number; high: number } {
+  const n = diffs.length;
+  const mean = n ? diffs.reduce((t, d) => t + d, 0) / n : 0;
+  if (n < 2) return { mean, low: mean, high: mean };
+  const rand = mulberry32(seed);
+  const means: number[] = [];
+  for (let i = 0; i < iterations; i++) {
+    let sum = 0;
+    for (let j = 0; j < n; j++) sum += at(diffs, Math.floor(rand() * n));
+    means.push(sum / n);
+  }
+  means.sort((a, b) => a - b);
+  return {
+    mean,
+    low: at(means, Math.floor(iterations * 0.025)),
+    high: at(means, Math.ceil(iterations * 0.975) - 1),
+  };
+}
+
+/**
+ * The verdict for a difference's 95% range on `n` shared cases: too few under 5, else
+ * better or worse only when the whole range is on one side of zero.
+ */
+export function verdictFor(n: number, low: number, high: number): Verdict {
+  if (n < MIN_CASES) return "TOO_FEW";
+  if (low > 0) return "BETTER";
+  if (high < 0) return "WORSE";
+  return "SAME";
+}
+
+/** Each case's average score over its repeats (partial credit if any). */
+function passShareByCase(outputs: RunOutput[]): Map<string, number> {
+  const by = new Map<string, number[]>();
+  for (const o of outputs)
+    by.set(o.caseId, [...(by.get(o.caseId) ?? []), answerScore(o.checks)]);
+  return new Map(
+    [...by].map(([id, r]) => [id, r.reduce((t, x) => t + x, 0) / r.length]),
+  );
+}
+
+/**
+ * Each case's parts right and parts in all, averaged over its repeats, or
+ * null when the suite gives no partial credit.
+ */
+function unitsByCase(
+  outputs: RunOutput[],
+): Map<string, { right: number; total: number }> | null {
+  const by = new Map<string, { right: number; total: number; n: number }>();
+  for (const o of outputs) {
+    const u = o.checks?.find((c) => c.units)?.units;
+    if (o.checks && !u) return null;
+    const cur = by.get(o.caseId) ?? { right: 0, total: 0, n: 0 };
+    // A failed call: nothing right, as many parts as the case has (known
+    // from another repeat or run; 0 here, set below).
+    by.set(o.caseId, {
+      right: cur.right + (u?.right ?? 0),
+      total: cur.total + (u?.total ?? 0),
+      n: cur.n + 1,
+    });
+  }
+  return new Map(
+    [...by].map(([id, v]) => [
+      id,
+      { right: v.right / v.n, total: v.total / v.n },
+    ]),
+  );
+}
+
+/**
+ * 95% range of the difference in parts-right rate (b minus a), resampling
+ * whole cases: the same measure as the headline (parts right).
+ */
+export function bootstrapRate(
+  cases: { a: number; b: number; total: number }[],
+  iterations = 4000,
+  seed = 1,
+): { mean: number; low: number; high: number } {
+  const rate = (cs: typeof cases) => {
+    const total = cs.reduce((t, c) => t + c.total, 0);
+    return total ? cs.reduce((t, c) => t + c.b - c.a, 0) / total : 0;
+  };
+  const mean = rate(cases);
+  if (cases.length < 2) return { mean, low: mean, high: mean };
+  const rand = mulberry32(seed);
+  const rates: number[] = [];
+  for (let i = 0; i < iterations; i++) {
+    const sample = Array.from({ length: cases.length }, () =>
+      at(cases, Math.floor(rand() * cases.length)),
+    );
+    rates.push(rate(sample));
+  }
+  rates.sort((x, y) => x - y);
+  return {
+    mean,
+    low: at(rates, Math.floor(iterations * 0.025)),
+    high: at(rates, Math.ceil(iterations * 0.975) - 1),
+  };
+}
+
+function rates(outputs: RunOutput[], key: string): Rate {
+  const applied = outputs
+    .map((o) => o.checks?.find((c) => c.key === key))
+    .filter((c): c is CheckResult => c !== undefined && c.pass !== null);
+  return wilson(applied.filter((c) => c.pass).length, applied.length);
+}
+
+function perCall(outputs: RunOutput[], pick: (o: RunOutput) => number) {
+  const called = outputs.filter((o) => !o.reused);
+  return called.length
+    ? called.reduce((t, o) => t + pick(o), 0) / called.length
+    : null;
+}
+
+/**
+ * Compares two tests on the cases they share, with a paired bootstrap, using the same
+ * partial credit as the headline.
+ */
+export function compareRuns(
+  champion: RunOutput[],
+  challenger: RunOutput[],
+  checkKeys: string[],
+): Comparison {
+  const a = passShareByCase(champion);
+  const b = passShareByCase(challenger);
+  const shared = [...a.keys()].filter((id) => b.has(id)).sort();
+  // Parts right where the suite gives partial credit (the headline
+  // measure), else each case's share of passing answers.
+  const ua = unitsByCase(champion);
+  const ub = unitsByCase(challenger);
+  const { mean, low, high } =
+    ua && ub
+      ? bootstrapRate(
+          shared.map((id) => ({
+            a: mustGet(ua, id).right,
+            b: mustGet(ub, id).right,
+            total: Math.max(mustGet(ua, id).total, mustGet(ub, id).total),
+          })),
+        )
+      : bootstrapMean(shared.map((id) => mustGet(b, id) - mustGet(a, id)));
+  const inShared = (os: RunOutput[]) =>
+    os.filter((o) => shared.includes(o.caseId));
+  return {
+    n: shared.length,
+    mean,
+    low,
+    high,
+    verdict: verdictFor(shared.length, low, high),
+    fixed: shared.filter((id) => b.get(id) === 1 && mustGet(a, id) < 1),
+    broke: shared.filter((id) => a.get(id) === 1 && mustGet(b, id) < 1),
+    byCheck: Object.fromEntries(
+      checkKeys.map((k) => [
+        k,
+        {
+          champion: rates(inShared(champion), k),
+          challenger: rates(inShared(challenger), k),
+        },
+      ]),
+    ),
+    cost: {
+      champion: perCall(champion, (o) => o.costUsd),
+      challenger: perCall(challenger, (o) => o.costUsd),
+    },
+    avgMs: {
+      champion: perCall(champion, (o) => o.ms),
+      challenger: perCall(challenger, (o) => o.ms),
+    },
+  };
+}
+
+/** The verdict in plain words, with what it costs. */
+export function verdictSentence(c: Comparison, challengerName: string): string {
+  const pts = (x: number) => `${Math.round(Math.abs(x) * 100)} points`;
+  const cost =
+    c.cost.champion && c.cost.challenger
+      ? c.cost.challenger < c.cost.champion * 0.9
+        ? `, and costs ${Math.round((1 - c.cost.challenger / c.cost.champion) * 100)}% less per call`
+        : c.cost.challenger > c.cost.champion * 1.1
+          ? `, and costs ${Math.round((c.cost.challenger / c.cost.champion - 1) * 100)}% more per call`
+          : ", at about the same cost"
+      : "";
+  switch (c.verdict) {
+    case "TOO_FEW":
+      return `Only ${c.n} cases in common: too few to say. Run both on the same set of at least ${MIN_CASES} cases.`;
+    case "BETTER":
+      return `${challengerName} is better: it passes ${pts(c.mean)} more often (likely ${pts(c.low)} to ${pts(c.high)})${cost}.`;
+    case "WORSE":
+      return `${challengerName} is worse: it passes ${pts(c.mean)} less often (likely ${pts(c.high)} to ${pts(c.low)})${cost}.`;
+    default:
+      return `No clear difference: ${challengerName} is somewhere between ${Math.round(c.low * 100)} and ${c.high > 0 ? "+" : ""}${Math.round(c.high * 100)} points${cost}.`;
+  }
+}