evals-lab 0.2.0 → 0.3.0

package/lab/kinds/list.mjs

@@ -332,7 +332,7 @@ const LIST_MODIFIERS                                       = {
 
 const LIST_KIND                         = {
   label: "List", noun: "items", offers: ["items"],
-  description: "Splits the reply into items (by commas, lines or a JSON array) that tests and later jobs read one by one.",
+  description: "Splits the reply into items (by commas, lines or a JSON array) that evals and later jobs read one by one.",
   settings: [
     { key: "parse", label: "Parse as", type: "select", choices: [
       { value: "csv", label: "CSV" }, { value: "lines", label: "One a line" }, { value: "json", label: "JSON array" }] },

package/lab/metrics/builtin.mjs

@@ -3,13 +3,13 @@
 // read the text, ones that read it against what production replied, and
 // model-graded ones that ask a grader. Each is one METRICS entry the core
 // registers, as it registers kinds/list.ts; a plugin adds more the same way.
-                                                                                                   
+                                                                                                                
 
 const ok = (pass         , reason        , score = pass ? 1 : 0)                => ({ pass, score, reason });
 const str = (v         )         => (v == null ? "" : String(v));
 const num = (v         )                => (typeof v === "number" ? v : str(v).trim() && !Number.isNaN(Number(v)) ? Number(v) : null);
-/** A list option: one entry a line. */
-const lines = (v         )           => str(v).split("\n").map((l) => l.trim()).filter(Boolean);
+/** A list option: one entry a line, or a list of them. */
+const lines = (v         )           => (Array.isArray(v) ? v.map(str) : str(v).split("\n")).map((l) => l.trim()).filter(Boolean);
 const short = (s        ) => (s.length > 60 ? `${s.slice(0, 57)}…` : s);
 const json = (s        )                                      => {
   try { return { value: JSON.parse(s) }; } catch (e) { return { error: e instanceof Error ? e.message : String(e) }; }
@@ -81,6 +81,45 @@ function distance(a        , b        )         {
   return d[b.length] ;
 }
 
+// ---- matching what a reply holds ---------------------------------------------
+// A kind that yields items (a list) is matched item by item, the way the lab
+// matches a term (docs/datasets.md § Matching): a value's words in some item,
+// in order and adjacent -- "cat" is in "tabby cat" and not in "cathedral" --
+// and ignoring case where the metric's Ignore case is on, as it is by default.
+// A reply the job threw away yields none. A reply read as text, or the API's
+// own text (`of: "said"`), is matched as text.
+
+/** The words of [s], lowercased unless [keepCase]: the lab's own reading of a term. */
+const words = (s        , keepCase = false)           => (keepCase ? s : s.toLowerCase()).match(/[\p{L}\p{N}]+/gu) || [];
+const termIn = (items                   , term        , ignoreCase = true)          => {
+  const t = words(term, !ignoreCase);
+  if (!t.length) return false;
+  return items.some((item) => {
+    const w = words(item, !ignoreCase);
+    for (let i = 0; i + t.length <= w.length; i++) if (t.every((x, j) => w[i + j] === x)) return true;
+    return false;
+  });
+};
+/** Whether [input] is matched item by item. */
+const byItem = (input             , m        ) => !input.plain && m.of !== "said";
+/** Whether [input] holds [value], as [m] says to match it: in its items, or
+    in its text -- there, outside any of [except]. */
+function holds(input             , m        , value        , except                    = [])          {
+  if (byItem(input, m)) {
+    const items = input.error ? [] : input.terms;
+    const ic = m.ignoreCase === true;
+    // An item holding an exception that holds the value does not count; the
+    // value anywhere else still does.
+    return items.some((item) => termIn([item], value, ic) && !except.some((e) => termIn([e], value, ic) && termIn([item], e, ic)));
+  }
+  const fold = (s        ) => (m.ignoreCase ? s.toLowerCase() : s);
+  let text = fold(input.text);
+  for (const e of except) if (fold(e).includes(fold(value))) text = text.split(fold(e)).join(" ");
+  return text.includes(fold(value));
+}
+/** A requirement as Results names it: a term, or the group it was. */
+const spoken = (g          )              => (g.length === 1 ? g[0]  : g);
+
 /** Production's reply, or the reason a metric that compares with it cannot. */
 const production = (input             )         => {
   if (input.production == null) throw new Error("this item has no production reply to compare with");
@@ -107,8 +146,12 @@ const gradedPass = (v                              , threshold        )
 };
 const REPLY = 'Reply with one JSON object and nothing else: {"pass": true or false, "score": a number from 0 to 1, "reason": "one sentence"}.';
 
+// The families the Add metric picker lists the metrics under: what each reads.
+const TEXT = "The reply's text", RESULT = "The result", PROD = "Production's reply", GRADED = "Model-graded";
+
 const metrics                              = {
   equals: {
+    family: TEXT,
     label: "Equals",
     description: "Passes when the reply is exactly the value; as JSON, the same value with keys in any order.",
     // As JSON, a reply equals the value as a value -- keys in any order;
@@ -123,44 +166,53 @@ const metrics                              = {
     },
   },
   contains: {
+    family: TEXT,
     label: "Contains",
-    description: "Passes when the reply contains the value.",
-    options: [{ key: "value", label: "Value", type: "text" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
-    defaults: () => ({ value: "", ignoreCase: false }),
+    description: "Passes when the reply contains the value; turned round, an exception excuses the value inside it.",
+    options: [{ key: "value", label: "Value", type: "text" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" },
+              { key: "except", label: "Except", type: "textarea" }],
+    defaults: () => ({ value: "", ignoreCase: true }),
     validate: (m, at, bad) => needs(m, ["value"], at, bad),
+    // Turned round (`not`), what it found is what the reply invented; a
+    // reading of items says so, for a run's totals.
     score: (input, m) => {
-      const fold = (s        ) => (m.ignoreCase ? s.toLowerCase() : s);
-      const hit = fold(input.text).includes(fold(str(m.value)));
-      return ok(hit, hit ? `has ${short(str(m.value))}` : `has no ${short(str(m.value))}`);
+      const v = str(m.value);
+      const hit = holds(input, m, v, lines(m.except));
+      const detail = !byItem(input, m) ? {} : m.not ? { invented: hit ? [v] : [] } : hit ? { found: [v], missed: [] } : { found: [], missed: [v] };
+      return { ...ok(hit, hit ? `has ${short(v)}` : `has no ${short(v)}`), ...detail };
     },
   },
   "contains-any": {
+    family: TEXT,
     label: "Contains any",
     description: "Passes when the reply contains at least one of the values, one a line.",
     options: [{ key: "values", label: "Values", type: "textarea" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
-    defaults: () => ({ values: "", ignoreCase: false }),
+    defaults: () => ({ values: "", ignoreCase: true }),
     validate: (m, at, bad) => needs(m, ["values"], at, bad),
     score: (input, m) => {
-      const fold = (s        ) => (m.ignoreCase ? s.toLowerCase() : s);
-      const hit = lines(m.values).find((v) => fold(input.text).includes(fold(v)));
-      return ok(!!hit, hit ? `has ${short(hit)}` : `has none of ${lines(m.values).join(", ")}`);
+      const all = lines(m.values);
+      const hit = all.find((v) => holds(input, m, v));
+      const detail = !byItem(input, m) ? {} : hit ? { found: [spoken(all)], missed: [] } : { found: [], missed: [spoken(all)] };
+      return { ...ok(!!hit, hit ? `has ${short(hit)}` : `has none of ${all.join(", ")}`), ...detail };
     },
   },
   "contains-all": {
+    family: TEXT,
     label: "Contains all",
     description: "Passes when the reply contains every value, one a line.",
     options: [{ key: "values", label: "Values", type: "textarea" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
-    defaults: () => ({ values: "", ignoreCase: false }),
+    defaults: () => ({ values: "", ignoreCase: true }),
     validate: (m, at, bad) => needs(m, ["values"], at, bad),
     score: (input, m) => {
-      const fold = (s        ) => (m.ignoreCase ? s.toLowerCase() : s);
       const all = lines(m.values);
-      const missing = all.filter((v) => !fold(input.text).includes(fold(v)));
-      return ok(!missing.length, missing.length ? `has no ${missing.join(", ")}` : "has them all",
-        all.length ? (all.length - missing.length) / all.length : 1);
+      const missing = all.filter((v) => !holds(input, m, v));
+      const detail = !byItem(input, m) ? {} : { found: all.filter((v) => !missing.includes(v)), missed: missing };
+      return { ...ok(!missing.length, missing.length ? `has no ${missing.join(", ")}` : "has them all",
+        all.length ? (all.length - missing.length) / all.length : 1), ...detail };
     },
   },
   regex: {
+    family: TEXT,
     label: "Matches",
     description: "Passes when the reply matches the regular expression.",
     options: [{ key: "pattern", label: "Pattern", type: "text" }, { key: "flags", label: "Flags", type: "text" }],
@@ -172,6 +224,7 @@ const metrics                              = {
     },
   },
   "starts-with": {
+    family: TEXT,
     label: "Starts with",
     description: "Passes when the reply starts with the value.",
     options: [{ key: "value", label: "Value", type: "text" }],
@@ -183,6 +236,7 @@ const metrics                              = {
     },
   },
   "is-json": {
+    family: TEXT,
     label: "Is JSON",
     description: "Passes when the reply is JSON and, with a schema, holds to it.",
     options: [{ key: "schema", label: "Schema", type: "textarea" }],
@@ -197,6 +251,7 @@ const metrics                              = {
     },
   },
   "json-field": {
+    family: TEXT,
     label: "JSON field",
     description: "Reads one field of a JSON reply and compares it.",
     options: [
@@ -219,7 +274,21 @@ const metrics                              = {
       return ok(pass, `${str(m.path)} is ${short(text)}`);
     },
   },
+  // A case that expects its answer thrown away: the reply a job's Reject
+  // rules discarded is the finding.
+  discarded: {
+    family: RESULT,
+    label: "Discarded",
+    description: "Passes only when the job's rules discarded the reply.",
+    options: [],
+    defaults: () => ({}),
+    score: (input) => {
+      const thrown = !!input.error && input.error.startsWith("discarded: ");
+      return ok(thrown, thrown ? "discarded" : input.error ? input.error : `kept ${input.terms.length} items`);
+    },
+  },
   "item-count": {
+    family: RESULT,
     label: "Item count",
     description: "Passes when the number of items is within the bounds.",
     options: [{ key: "min", label: "At least", type: "number" }, { key: "max", label: "At most", type: "number" }],
@@ -231,6 +300,7 @@ const metrics                              = {
     },
   },
   "count-matches": {
+    family: RESULT,
     label: "Count matches",
     description: "Counts matches of the pattern: points, for the weighted mode.",
     counts: true,
@@ -246,6 +316,7 @@ const metrics                              = {
     },
   },
   latency: {
+    family: RESULT,
     label: "Latency",
     description: "Passes when the reply came back within the time set.",
     options: [{ key: "max", label: "At most (ms)", type: "number" }],
@@ -254,6 +325,7 @@ const metrics                              = {
     score: (input, m) => ok(input.ms <= (num(m.max) ?? 0), `${input.ms} ms`),
   },
   levenshtein: {
+    family: TEXT,
     label: "Near",
     description: "Passes when the reply is within a few edits of the value.",
     options: [{ key: "value", label: "Value", type: "textarea" }, { key: "max", label: "Edits at most", type: "number" }],
@@ -267,6 +339,7 @@ const metrics                              = {
   },
   // ---- against production ----------------------------------------------------
   "equals-production": {
+    family: PROD,
     label: "Same as production",
     description: "Passes when the reply is what production replied to the same item.",
     options: [],
@@ -279,6 +352,7 @@ const metrics                              = {
     },
   },
   "fields-equal-production": {
+    family: PROD,
     label: "Fields as production",
     description: "Passes when the fields listed are what production's reply had.",
     options: [{ key: "fields", label: "Fields", type: "textarea" }],
@@ -295,6 +369,7 @@ const metrics                              = {
     },
   },
   "same-parse-outcome": {
+    family: PROD,
     label: "Parses as production",
     description: "Passes when the reply parses as JSON exactly when production's did.",
     options: [],
@@ -306,6 +381,7 @@ const metrics                              = {
   },
   // ---- model-graded -------------------------------------------------------------
   "llm-rubric": {
+    family: GRADED,
     label: "Rubric",
     description: "Asks the grader whether the reply meets the rubric.",
     graded: true,
@@ -317,6 +393,7 @@ const metrics                              = {
       num(m.threshold) ?? 0.5),
   },
   factuality: {
+    family: GRADED,
     label: "Factual",
     description: "Asks the grader whether the reply agrees with the reference, or production's reply.",
     graded: true,
@@ -329,6 +406,7 @@ const metrics                              = {
       + `${str(m.reference).trim() || production(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`)), num(m.threshold) ?? 0.5),
   },
   "judge-vs-production": {
+    family: GRADED,
     label: "Judged against production",
     description: "Asks the grader whether the reply is better than, the same as or worse than production's.",
     graded: true,

package/lab/run-evals.js

@@ -101,7 +101,7 @@ const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
                       job's tokenMappings. Default: none, so a prompt
                       naming a token is refused, naming it.
   --pipeline <file>   a run document (docs/pipeline-model.md) with one
-                      scenario and a graded test, graded case by case against
+                      scenario and a graded eval, graded case by case against
                       the set. Instead of --prompt, --model and --url. Each
                       profile's key comes from $EVAL_API_KEY_<ID>, never the
                       file; its content.files, when not empty, is the file
@@ -274,7 +274,7 @@ function readRun(file, dataset) {
   } catch (e) {
     broken(`${file}: ${e.message}`);
   }
-  const named = isObject(doc) ? core.testsDataset(doc) : null;
+  const named = isObject(doc) ? core.evalsDataset(doc) : null;
   const bad = core.validatePipeline(doc, { datasets: dataset && named ? [named] : [] });
   if (bad.length) broken(`${file} cannot be run:\n  ${bad.join("\n  ")}`);
   return doc;
@@ -296,8 +296,8 @@ function readDataset(file) {
     broken(`${file}: ${e.message}`);
   }
   if (isObject(doc) && doc.format === "evals-lab/dataset") {
-    if (![1, 2, 3, 4].includes(doc.version)) {
-      broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads versions 1 to 4`);
+    if (![1, 2, 3, 4, 5].includes(doc.version)) {
+      broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads versions 1 to 5`);
     }
     doc = isObject(doc.dataset) ? doc.dataset.body : null;
   }
@@ -315,11 +315,12 @@ function readDataset(file) {
   return doc;
 }
 
-// The cases a graded run is scored against, by the file each names.
+// The cases a graded run is scored against, by the item each names: exactly,
+// as the Source names it.
 function gradedBy(dataset) {
   const out = new Map();
   for (const c of core.gradedSetFrom(dataset || {})) {
-    if (!c.todo) out.set(c.filename, c);
+    if (!c.todo) out.set(c.item, c);
   }
   return out;
 }
@@ -612,12 +613,12 @@ function readUtf8(file) {
   return fs.readFileSync(file).toString("utf8");
 }
 
-// Every test's reading of each scenario, keyed by the test's id, exactly as
-// the page reads it: a whole-run test's verdict, settled once every item is
-// in, and a per-item test's counts. [settled] is false for a run that
-// stopped short, whose whole-run tests have not settled.
+// Every eval's reading of each scenario, keyed by the eval's id, exactly as
+// the page reads it: a whole-run eval's verdict, settled once every item is
+// in, and a per-item eval's counts. [settled] is false for a run that
+// stopped short, whose whole-run evals have not settled.
 const verdictsOf = (run, items, settled) => core.targetsOf(run).map((_, i) =>
-  Object.fromEntries(core.scenarioTests(run, i, items, settled).map(o => [o.id, {
+  Object.fromEntries(core.scenarioEvals(run, i, items, settled).map(o => [o.id, {
     name: o.label, skipped: o.skipped,
     ...(o.whole
       ? { pass: o.verdict ? o.verdict.pass : null, detail: o.verdict?.detail ?? null }
@@ -754,7 +755,7 @@ async function runSnapshot(o) {
     verdict: cancelled ? "cancelled" : (unrun.length ? "incomplete" : "done"),
     run: {
       name: run.name ?? null,
-      tests: run.tests,
+      evals: run.evals,
       verdicts: verdictsOf(run, results, !cancelled && !unrun.length),
       scenarios: scenariosOf(run),
       items: results.slice(0, total),
@@ -779,7 +780,7 @@ async function runSnapshot(o) {
 /** Re-score a run's stored results against the dataset --dataset hands over,
  * keeping the replies that were stored: what changes a verdict is the scorer
  * or the cases, never a new question to the model. The report is shaped like
- * a --run's, with the stored results as its items, and a whole-run test's
+ * a --run's, with the stored results as its items, and a whole-run eval's
  * verdict settled the same way. */
 async function runRescore(o){
   const dataset = readDataset(o.dataset);
@@ -801,7 +802,7 @@ async function runRescore(o){
   const graded = gradedBy(dataset);
 
   const only = o.only != null ? parseInt(o.only, 10) : null;
-  // A test that reads every item (the Metrics) re-reads one with no case
+  // An eval that reads every item (the Metrics) re-reads one with no case
   // too, against the production reply the item kept.
   const items = await Promise.all(results.map(async (it, i) => {
     if (!isObject(it) || only != null && i !== only || !Array.isArray(it.scenarios)) return it;
@@ -819,7 +820,7 @@ async function runRescore(o){
     rescored: true,
     run: {
       name: run.name ?? null,
-      tests: run.tests,
+      evals: run.evals,
       verdicts: verdictsOf(run, items, true),
       scenarios: scenariosOf(run),
       items,
@@ -856,8 +857,8 @@ function cliRun(o, dataset) {
   doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
   doc.targets = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
                    steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }];
-  doc.tests = [{ id: "cli", type: "metrics", name: "", continueOnFailure: true, dataset: { id: "cli", name: "" },
-                 mode: "all", threshold: null, grader: null, over: "item", metrics: [{ type: "case" }] }];
+  doc.evals = [{ id: "cli", type: "metrics", name: "", continueOnFailure: true, dataset: { id: "cli", name: "" },
+                 mode: "all", threshold: null, grader: null, over: "item", metrics: [] }];
   doc.profiles = { cli: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
   return doc;
 }
@@ -950,17 +951,19 @@ async function main() {
       broken(`${o.pipeline} has ${core.targetsOf(run).length} targets, and --pipeline grades one `
            + "against the set -- --run runs them all.");
     }
-    if (!core.testsDataset(run)) {
+    if (!core.evalsDataset(run)) {
       broken(`${o.pipeline} is not graded, and --pipeline grades a run case by case -- --run runs it.`);
     }
     if (core.contentOf(run).type !== "source") {
       broken(`${o.pipeline} is over inline text, and --pipeline grades the set's files -- --run runs it.`);
     }
   }
-  // Case by case: each item against its case, as the case metric of the
-  // test that names the dataset reads it (scoreCase).
+  // Case by case: each item against its case's metrics, as the eval that
+  // names the dataset reads them (core.readCase).
   const { stages, tokens, connections } = core.stagesFor(run, 0);
   const prompt = shown(stages[0].text, core.tokenSet(tokens, 0));
+  // A last stage that yields no items is matched as its text.
+  const plain = !core.OUTPUT_KINDS[stages.at(-1)?.kind ?? ""]?.terms;
 
   // The graded half, through the function the tab builds its own list with.
   const set = core.gradedSetFrom(dataset);
@@ -1050,11 +1053,11 @@ async function main() {
                        + "--samples or --source names a directory.");
     const allowed = itemsOnly ?? graded;
     // $snapshotSet above: only a Source run sees it; --replies reads no file.
-    const inSnapshot = c => !snapshotSet || snapshotSet.has(c.filename);
+    const inSnapshot = c => !snapshotSet || snapshotSet.has(c.item);
     // Only the run's own image items need ImageMagick: a text-only Source
     // -- the .txt, .csv case -- runs with nothing on PATH but the node
     // running this, which is the point of having text in the registry.
-    const pictures = allowed.filter(c => inSnapshot(c) && HANDLERS.get(extOf(c.filename)) === "image");
+    const pictures = allowed.filter(c => inSnapshot(c) && HANDLERS.get(extOf(c.item)) === "image");
     if (pictures.length) {
       bin = imageMagick();
       if (!bin) broken("a run over images needs ImageMagick, to cap each "
@@ -1087,7 +1090,7 @@ async function main() {
   }
 
   // Only files the snapshot names are read from the Source.
-  const wanted = c => !snapshotSet || snapshotSet.has(c.filename);
+  const wanted = c => !snapshotSet || snapshotSet.has(c.item);
   // The cancel marker is checked between items, so a cancel never loses the
   // reply in flight: the item that was being asked finishes, and the rest
   // are unrun rather than half-asked.
@@ -1111,18 +1114,18 @@ async function main() {
       calls = connections.map((c, i) => async () => ({ raw: said[i], ms: 0, conn: c.name }));
     } else {
       if (!wanted(kase)) {
-        unrun.push(`${kase.id}: ${kase.filename} is not in the run's file list`);
+        unrun.push(`${kase.id}: ${kase.item} is not in the run's file list`);
         continue;
       }
-      const file = path.join(samples, kase.filename);
+      const file = path.join(samples, kase.item);
       if (!fs.existsSync(file)) {
-        unrun.push(`${kase.id}: ${kase.filename} is not in ${samples}`);
+        unrun.push(`${kase.id}: ${kase.item} is not in ${samples}`);
         continue;
       }
       try {
         item = itemContent(bin, file, connections[0]);
       } catch (e) {
-        unrun.push(`${kase.id}: ${kase.filename} could not be prepared — ${e.message}`);
+        unrun.push(`${kase.id}: ${kase.item} could not be prepared — ${e.message}`);
         continue;
       }
       calls = callsFor(connections, links, item?.text ?? null);
@@ -1130,16 +1133,15 @@ async function main() {
 
     const res = await core.runPipeline(stages, item?.dataUrl ?? null, calls,
                                        { tokens, text: item?.text ?? null });
-    const s = core.scoreCase(kase, res);
+    const s = core.readCase(kase, res, plain);
     // A file with no mapper ran anyway; the flag keeps the transcript honest
     // about what the model was actually shown. §4: stated, not hidden.
     if (item?.flags.length) res.transcript.unshift({ flag: item.flags.join(" ") });
     rows.push({
-      id: kase.id, filename: kase.filename, half: kase.half,
+      id: kase.id, item: kase.item, half: kase.half,
       pass: s.pass, score: s.score, discarded: s.discarded, reasons: s.reasons,
-      found: s.found, missed: s.missed, invented: s.invented, unmet: s.unmet,
-      under: s.under, over: s.over, count: s.count,
-      watchFound: s.watchFound, watchTotal: s.watchTotal,
+      found: s.found, missed: s.missed, invented: s.invented, count: s.count,
+      metrics: s.metrics, watch: s.watch,
       terms: res.terms || [], ms: res.ms,
       // What a file type was to this run -- "image", "text", "unmapped".
       type: item?.kind ?? null,
@@ -1153,7 +1155,7 @@ async function main() {
     });
     if (progress) {
       progress.write(JSON.stringify({
-        event: "item", id: kase.id, filename: kase.filename, n: rows.length,
+        event: "item", id: kase.id, item: kase.item, n: rows.length,
         pass: s.pass, score: s.score, discarded: s.discarded,
         error: s.discarded ? s.reasons[0]?.replace(/^Error - /, "") ?? null : null,
         ms: res.ms,