evals-lab 0.4.1 → 0.6.0

package/lab/run-evals.js

@@ -6,15 +6,19 @@
 // what it did twice: a short human report, and a JSON one with every verdict
 // in it. The exit code is the third statement and the coarsest:
 //
-//   0  every graded case passed
-//   1  the set ran and at least one case failed
-//   2  the set did not run in full -- nothing is graded, or an image or a
-//      recorded reply was missing. NOT a pass. An eval set that scores well
-//      because it never ran is the failure this whole ticket is about.
+//   0  every eval passed
+//   1  the run ran and at least one eval failed
+//   2  the run did not run in full -- nothing is graded, an image or a
+//      recorded reply was missing, or it was cancelled. NOT a pass. An eval
+//      set that scores well because it never ran is the failure this whole
+//      ticket is about.
 //   3  the run could not be attempted: bad arguments, no ImageMagick, no set.
 //
-// The gating policy -- which of those a pull request may merge on -- is #250's
-// to decide, and it has the JSON to decide it from. This only reports.
+// The same in every mode, --run and --rescore included, so CI can gate on it
+// (#251): each eval is held to its own rule -- every item it reads passes, or
+// a whole-run eval's verdict -- and --min-pass relaxes the per-item rule to a
+// share. The report's `verdict` (pass, fail, incomplete) says the same as the
+// code, and --junit writes it for a CI test panel.
 //
 // Ollama is firewalled to a handful of hosts, so a live run has to happen on
 // one of them. That is a fact about the network and not something a flag here
@@ -64,7 +68,10 @@ const { execFileSync } = require("child_process");
 const core = require("./evals-core.mjs");
 
 const HERE = __dirname;
-const ROOT = path.join(HERE, "..", "..");
+const ROOT = HERE;
+
+// What a report names as its runner: this file, from the lab's root.
+const RUNNER = "run-evals.js";
 
 // Tagger.PIXELS_720P, as an area rather than a longest edge: llama.cpp slices
 // into 448px tiles and the tile COUNT is what costs. The same budget the tab
@@ -82,11 +89,12 @@ const EXIT = { passed: 0, failed: 1, incomplete: 2, broken: 3 };
 // single-flight queue for good; `--timeout` can shorten it, never lengthen.
 const REQUEST_CAP = 600;
 
-const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
+const USAGE = `Usage: node run-evals.js [options]
 
   --model <id>        the model to grade. Required for a live run.
   --url <base>        an OpenAI-shaped endpoint. Default: $OLLAMA_URL, else
-                      Ollama on this machine. A key comes from $EVAL_API_KEY, never argv.
+                      Ollama on this machine. A key comes from
+                      $EVALSLAB_API_KEY, never argv.
   --prompt <text>     the prompt to grade, as a template: its tokens resolve
                       under --tokens. Required without --pipeline: a dataset
                       holds no prompt (the lab's Prompt library does).
@@ -103,17 +111,22 @@ const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
   --pipeline <file>   a run document (docs/pipeline-model.md) with one
                       scenario and a graded eval, graded case by case against
                       the set. Instead of --prompt, --model and --url. Each
-                      profile's key comes from $EVAL_API_KEY_<ID>, never the
+                      profile's key comes from $EVALSLAB_API_KEY_<SLUG> -- its
+                      id's, in a file whose profile has no slug -- never the
                       file; its content.files, when not empty, is the file
                       list a case has to be in, as --files says.
   --samples <dir>     where the images are. Default: the lab's samples/.
-  --dataset <file>    the dataset a graded run is scored against: its body
-                      ({"cases"}), or the file the lab's Export writes.
+  --dataset <file>    the one eval group a graded run is scored against: its
+                      body ({"cases"}), or the file the lab's Export writes.
                       Versions 1 to 3 are read too (a prompt they hold is
                       not used: --prompt names it); a version-2
                       body's rules are what an older run's jobs, and a run
                       without --pipeline, read replies under. Its cases grade.
-                      Required for anything graded.
+                      Required for anything graded (or --groups, for a --run).
+  --groups <file>     the eval group bodies a --run kept, by "<id>@<n>": a run
+                      grades against each one its evals link (§17), so a run
+                      linking several groups is handed them here instead of the
+                      one --dataset. For --run and --rescore; not with --dataset.
   --source <dir>      the same, named the way a run names it: the directory a
                       Source is stored at. --samples and --source are one
                       flag by two names; both together are refused.
@@ -153,6 +166,12 @@ const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
                       alone. No model, no files, no network.
   --json <path|->     write the machine-readable report. "-" means stdout, and
                       sends the human report to stderr.
+  --min-pass <0..1>   an eval whose items are read one by one passes when at
+                      least this share of them pass. It relaxes each eval's
+                      own rule -- every item passes -- and never tightens it;
+                      a whole-run eval keeps its own verdict.
+  --junit <file>      write JUnit XML: one testsuite per target and eval, one
+                      testcase per item, a failure carrying its reason.
 `;
 
 // NOTHING HERE CALLS process.exit(). Node's stdout is asynchronous down a
@@ -189,10 +208,10 @@ function broken(msg) {
 
 function parseArgs(argv) {
   const o = { model: "", url: "", prompt: null, pipeline: "", samples: "", dataset: "",
-              replies: "", json: "", source: "", files: "", item: "", resume: "",
+              groups: "", replies: "", json: "", source: "", files: "", item: "", resume: "",
               timeout: null, progress: false, cancelFile: "",
               run: "", progressFile: "", resultsFile: "", from: null, only: null,
-              rescore: false, tokens: "", plugins: "" };
+              rescore: false, tokens: "", plugins: "", minPass: null, junit: "" };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     const value = () => {
@@ -210,6 +229,7 @@ function parseArgs(argv) {
       case "--pipeline": o.pipeline = value(); break;
       case "--samples": o.samples = value(); break;
       case "--dataset": o.dataset = value(); break;
+      case "--groups": o.groups = value(); break;
       case "--replies": o.replies = value(); break;
       case "--source": o.source = value(); break;
       case "--files": o.files = value(); break;
@@ -225,6 +245,13 @@ function parseArgs(argv) {
       case "--only": o.only = value(); break;
       case "--rescore": o.rescore = true; break;
       case "--json": o.json = value(); break;
+      case "--min-pass": {
+        const v = value(), n = Number(v);
+        if (v.trim() === "" || !(n >= 0 && n <= 1)) broken(`--min-pass is a share from 0 to 1, and ${v} is not one`);
+        o.minPass = n;
+        break;
+      }
+      case "--junit": o.junit = value(); break;
       case "-h": case "--help":
         process.stdout.write(USAGE);
         throw new Stop("", EXIT.passed);
@@ -259,10 +286,10 @@ const inRepo = p => {
 const isObject = v => v != null && typeof v === "object" && !Array.isArray(v);
 const isText = v => typeof v === "string";
 
-// [dataset] is the body --dataset handed over, or null: the one dataset a
-// graded run can name, whatever id it names it by, since the queue hands the
-// worker the body of exactly the dataset the run was submitted against.
-function readRun(file, dataset) {
+// [gctx] is the grading context (gradingContext): the eval group bodies a run
+// grades against -- one, as --dataset, or a body per group kept with the run,
+// as --groups (§17). The run reads each link's body by its reference.
+function readRun(file, gctx) {
   let doc;
   try {
     // A run queued before the current version is read as one of today's:
@@ -270,12 +297,20 @@ function readRun(file, dataset) {
     // submitted with. A v2 run's list jobs read their replies under the
     // rules of the dataset it was graded against -- the body handed over.
     doc = core.upgradePipeline(JSON.parse(fs.readFileSync(file, "utf8")),
-                               { rulesFor: () => rulesOf(dataset) });
+                               { rulesFor: () => rulesOf(gctx.primary()) });
   } catch (e) {
     broken(`${file}: ${e.message}`);
   }
-  const named = isObject(doc) ? core.evalsDataset(doc) : null;
-  const bad = core.validatePipeline(doc, { datasets: dataset && named ? [named] : [] });
+  // Every Library group the evals name -- a link's group, a private group's
+  // Cases from -- each as validatePipeline reads one. Pins were resolved at
+  // submit (the reference carries `n`), so they are not checked again here.
+  const groups = [], seen = new Set();
+  for (const ref of (isObject(doc) ? core.evalsOf(doc).map(core.casesRef).filter(Boolean) : [])) {
+    if (seen.has(ref.id)) continue;
+    seen.add(ref.id);
+    groups.push({ id: ref.id, name: ref.name, source: gctx.resolve(ref)?.source ?? null });
+  }
+  const bad = core.validatePipeline(doc, { groups });
   if (bad.length) broken(`${file} cannot be run:\n  ${bad.join("\n  ")}`);
   return doc;
 }
@@ -283,9 +318,25 @@ function readRun(file, dataset) {
 // The rules a version-2 body held, by the body readDataset made of it: what
 // an older run's jobs read replies under. A body of today's holds none.
 const heldRules = new WeakMap();
-const rulesOf = (dataset) => (dataset && heldRules.get(dataset)) ?? null;
+const rulesOf = (body) => (body && heldRules.get(body)) ?? null;
+
+// An eval group's body as one of today's, with its version-2 rules taken
+// first: a bare body, or the body the lab's Export wraps, unwrapped by its
+// caller. [where] names it in a refusal.
+function bodyFrom(doc, where) {
+  if (isObject(doc) && doc.rules != null && !(isObject(doc.rules) && Array.isArray(doc.rules.rules))) {
+    broken(`${where}: rules has to be null or { "rules": [...] }`);
+  }
+  const rules = core.datasetRules(doc);
+  const body = core.upgradeDatasetBody(doc);
+  if (!isObject(body) || !Array.isArray(body.cases)) {
+    broken(`${where} is not an eval group: its body has a cases list, or it is the lab's Export of one`);
+  }
+  if (rules) heldRules.set(body, rules);
+  return body;
+}
 
-// The dataset --dataset names: a dataset's body, or the file the lab's
+// The dataset --dataset names: an eval group's body, or the file the lab's
 // Export writes for one, as one of today's.
 function readDataset(file) {
   if (!file) return null;
@@ -300,26 +351,57 @@ function readDataset(file) {
       broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads versions 1 to 7`);
     }
     doc = isObject(doc.dataset) ? doc.dataset.body : null;
+  } else if (isObject(doc) && doc.format === "evals-lab/eval-group") {
+    // The lab's Export of an eval group (docs/pipeline-model.md §17), which
+    // began at version 7.
+    if (doc.version !== 7) broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads version 7`);
+    doc = isObject(doc.group) ? doc.group.body : null;
   }
-  if (isObject(doc) && doc.rules != null && !(isObject(doc.rules) && Array.isArray(doc.rules.rules))) {
-    broken(`${file}: rules has to be null or { "rules": [...] }`);
+  return bodyFrom(doc, file);
+}
+
+// The bodies a run kept, as --groups hands them (§17): a JSON object of
+// `<id>@<n>` to body -- the group's id and the version it graded with, its id
+// alone for a run from before the lab numbered them -- each read as today's.
+function readGroups(file) {
+  let raw;
+  try {
+    raw = JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (e) {
+    broken(`${file}: ${e.message}`);
   }
-  // A body from an earlier version -- a run queued then, resumed now -- reads
-  // as one of today's, its rules taken first.
-  const rules = core.datasetRules(doc);
-  doc = core.upgradeDatasetBody(doc);
-  if (!isObject(doc) || !Array.isArray(doc.cases)) {
-    broken(`${file} is not a dataset: its body has a cases list, or it is the lab's Export of one`);
+  if (!isObject(raw)) broken(`${file}: the kept groups are a JSON object of "<id>@<n>": body`);
+  const bodies = new Map();
+  for (const [key, body] of Object.entries(raw)) bodies.set(key, bodyFrom(body, `${file} [${key}]`));
+  return bodies;
+}
+
+// The key a run keeps a group's body under, mirroring the server's group_key:
+// `<id>@<n>`, or the id alone for a reference from before versions.
+const groupKey = (ref) => (ref && Number.isInteger(ref.n) ? `${ref.id}@${ref.n}` : `${ref && ref.id}`);
+
+// The eval groups a --run grades against: one body, as --dataset, or a body
+// per group, as --groups -- never both. `resolve` gives a link's body by its
+// reference; `primary` the one a single-group run names, for a v2 run's rules.
+function gradingContext(o) {
+  if (o.groups && o.dataset) broken("--groups and --dataset name the groups two ways; pass one.");
+  if (o.groups) {
+    const bodies = readGroups(o.groups);
+    const first = bodies.values().next().value ?? null;
+    return { single: null, bodies, primary: () => first,
+             resolve: (ref) => bodies.get(groupKey(ref)) ?? bodies.get(String(ref && ref.id)) ?? undefined };
   }
-  if (rules) heldRules.set(doc, rules);
-  return doc;
+  const single = readDataset(o.dataset);
+  return { single, bodies: null, primary: () => single, resolve: () => single ?? undefined };
 }
 
 // The cases a graded run is scored against, by the item each names: exactly,
-// as the Source names it.
-function gradedBy(dataset) {
+// as the Source names it. Built from the one group a single-group run names,
+// for an eval type of its own that reads a case (a plugin's); a `group` eval
+// reads each group's own case, by the item's name, as it grades.
+function gradedBy(body) {
   const out = new Map();
-  for (const c of core.gradedSetFrom(dataset || {})) {
+  for (const c of core.gradedSetFrom(body || {})) {
     if (!c.todo) out.set(c.item, c);
   }
   return out;
@@ -505,7 +587,7 @@ function graderFor(run) {
       const c = run.profiles?.[ref.id];
       if (!c) return undefined;
       if (!made.has(ref.id)) {
-        const link = reach(c, core.keyVar(ref.id));
+        const link = reach(c, core.keyVar(ref.id, c.slug));
         made.set(ref.id, async prompt => {
           const r = await ask(link, { id: ref.id, ...c }, prompt, null);
           if (r.error) throw new Error(`the grader ${c.name || ref.id} failed: ${r.error}`);
@@ -613,17 +695,138 @@ function readUtf8(file) {
   return fs.readFileSync(file).toString("utf8");
 }
 
-// Every eval's reading of each scenario, keyed by the eval's id, exactly as
-// the page reads it: a whole-run eval's verdict, settled once every item is
-// in, and a per-item eval's counts. [settled] is false for a run that
-// stopped short, whose whole-run evals have not settled.
-const verdictsOf = (run, items, settled) => core.targetsOf(run).map((_, i) =>
-  Object.fromEntries(core.scenarioEvals(run, i, items, settled).map(o => [o.id, {
-    name: o.label, skipped: o.skipped,
-    ...(o.whole
-      ? { pass: o.verdict ? o.verdict.pass : null, detail: o.verdict?.detail ?? null }
-      : { ran: o.ran, passed: o.passed, skippedItems: o.skippedItems }),
-  }])));
+// Every eval's reading of each scenario, exactly as the page reads it.
+// [settled] is false for a run that stopped short, whose whole-run evals
+// have not settled.
+const outcomesOf = (run, items, settled) =>
+  core.targetsOf(run).map((_, i) => core.scenarioEvals(run, i, items, settled));
+
+// A link's Whole run verdict, per Target, keyed by the eval's id: a Library
+// group's `run` metrics read over the replies so far (readGroupRun). A private
+// group's Whole run is scenarioEvals' own (it holds the body); a link's body
+// is held apart, in --groups, so reading it beside the link's items is the
+// runner's (§17). [resolve] gives a link's body by its reference.
+function wholeRunsOf(run, items, resolve) {
+  const kind = core.lastKind(run);
+  return core.targetsOf(run).map((_, i) => {
+    const out = {};
+    const ress = (items || []).map(it => (it && !it.unrun ? it.scenarios?.[i]?.res : undefined));
+    for (const t of core.evalsOf(run)) {
+      // A private whole-run group is already scenarioEvals' verdict; here is
+      // the run beside a link's (or private group's) own items.
+      if (t.type !== "group" || core.isWholeRun(core.EVAL_TYPES.group, t)) continue;
+      const body = core.groupOf(t, { group: resolve });
+      if (body && Array.isArray(body.run) && body.run.length) out[t.id] = core.readGroupRun(body, ress, kind);
+    }
+    return out;
+  });
+}
+
+/**
+ * One eval's verdict on one target, for the build: its own rule -- every
+ * item it read passed, or a whole-run eval's verdict -- with [minPass] the
+ * share of items that is enough instead. It only relaxes: an eval whose
+ * every item passed passes whatever the share. One an earlier failure
+ * stopped is skipped -- that failure is the run's verdict already -- and one
+ * that had nothing to read, a group grading none of the run's items, says
+ * none: the run ran in full, which is what the server's status reads from
+ * the exit code, so it is not incomplete either. [wholeRun] is a link's Whole
+ * run verdict, where its group reads one beside its items (§17): the group
+ * passes when both its items and its Whole run do, and either failing fails it.
+ */
+function evalVerdict(o, settled, minPass, wholeRun) {
+  if (o.skipped) return "skipped";
+  if (!settled) return "incomplete";
+  if (o.whole) return !o.verdict?.ran ? "none" : o.verdict.pass ? "pass" : "fail";
+  const item = !o.ran ? (o.skippedItems ? "skipped" : "none")
+    : o.passed === o.ran ? "pass"
+    : minPass != null && o.passed / o.ran >= minPass ? "pass" : "fail";
+  if (!wholeRun) return item;
+  const whole = !wholeRun.ran ? "none" : wholeRun.pass ? "pass" : "fail";
+  if (item === "fail" || whole === "fail") return "fail";
+  if (item === "pass" || whole === "pass") return "pass";
+  return item === "skipped" || whole === "skipped" ? "skipped" : "none";
+}
+
+// Each target's evals, keyed by the eval's id: a whole-run eval's verdict, a
+// per-item eval's counts -- and, where a link reads a Whole run beside its
+// items, that verdict too -- with the build's verdict on each. [wholes] is
+// wholeRunsOf, one map per target.
+const verdictsOf = (outcomes, settled, minPass, wholes = []) => outcomes.map((evals, i) =>
+  Object.fromEntries(evals.map(o => {
+    const wr = wholes[i] && wholes[i][o.id];
+    return [o.id, {
+      name: o.label, skipped: o.skipped, verdict: evalVerdict(o, settled, minPass, wr),
+      ...(o.whole
+        ? { pass: o.verdict ? o.verdict.pass : null, detail: o.verdict?.detail ?? null }
+        : { ran: o.ran, passed: o.passed, skippedItems: o.skippedItems,
+            ...(wr ? { wholeRun: { pass: wr.pass, detail: wr.detail, ran: wr.ran } } : {}) }),
+    }];
+  })));
+
+/** Each Target's verdict under the overall pass rule (§17), from its groups'
+    verdicts: a group that passed counts, one that failed counts against, and
+    one that read nothing or was skipped is neither. */
+const passTargets = (verdicts, pass) => verdicts.map(v => core.passVerdict(pass,
+  Object.values(v).map(e => (e.verdict === "pass" ? true : e.verdict === "fail" ? false : null))));
+
+/** The run's verdict from its evals' and the overall rule: a run that did not
+    finish is incomplete, whatever its evals read so far; then it passes when
+    every Target does under the rule. */
+function runVerdict(verdicts, complete, pass) {
+  if (!complete) return "incomplete";
+  return passTargets(verdicts, pass).every(Boolean) ? "pass" : "fail";
+}
+
+const exitOf = verdict => ({ pass: EXIT.passed, fail: EXIT.failed })[verdict] ?? EXIT.incomplete;
+
+// ---- JUnit -----------------------------------------------------------------
+// What a CI test panel reads, written by the core's junitXml.
+
+/** Why a score failed, in a line: its failing metrics, else what it missed. */
+function reasonOf(score) {
+  const off = (score.metrics || []).filter(m => !m.pass).map(m => `${m.label}: ${m.reason}`);
+  if (off.length) return off.join(" · ");
+  const missed = (score.missed || []).map(r => (Array.isArray(r) ? r.join(" or ") : r));
+  return missed.length ? `missed ${missed.join(", ")}` : "failed";
+}
+
+/** A --run's or a --rescore's suites: each target's evals over [items], with a
+    Whole run case where a link reads one beside its items ([wholes]). */
+function runSuites(run, outcomes, items, settled, minPass, wholes = []) {
+  return outcomes.flatMap((evals, i) => evals.map(o => {
+    const name = `${core.targetLabel(run, i)} › ${o.label}`;
+    const wr = wholes[i] && wholes[i][o.id];
+    const v = evalVerdict(o, settled, minPass, wr);
+    if (o.whole) {
+      // Settled over the run rather than item by item: one case, the run.
+      const c = { name: o.label };
+      if (v === "skipped") c.skipped = "an earlier eval failed";
+      else if (v === "none") c.skipped = "nothing to read";
+      else if (v === "incomplete") c.error = "the run did not finish";
+      else if (v === "fail") c.failure = o.verdict.detail || "failed";
+      return { name, cases: [c] };
+    }
+    const cases = items.map((it, x) => {
+      const c = { name: it?.name ?? (items.length > 1 ? `item ${x + 1}` : "the text") };
+      const read = o.items[x];
+      if (!it) c.error = "not run";
+      else if (it.unrun) c.error = it.unrun;
+      else if (!read) c.skipped = "not graded";
+      else if (core.isSkipped(read)) c.skipped = "an earlier eval failed";
+      else if (!read.pass) c.failure = reasonOf(read);
+      return c;
+    });
+    if (wr) {
+      const c = { name: "Whole run" };
+      if (!settled) c.error = "the run did not finish";
+      else if (!wr.ran) c.skipped = "nothing to read";
+      else if (!wr.pass) c.failure = wr.detail || "failed";
+      cases.push(c);
+    }
+    return { name, cases };
+  }));
+}
 
 // The run as the report restates it: each scenario's stages with the
 // connection each one asked.
@@ -639,8 +842,8 @@ const scenariosOf = run => core.targetsOf(run).map((sc, i) => {
 });
 
 async function runSnapshot(o) {
-  const dataset = readDataset(o.dataset);
-  const run = readRun(o.run, dataset);
+  const gctx = gradingContext(o);
+  const run = readRun(o.run, gctx);
   // One item a pipeline holds itself (Text, or Prompt only's bare one), or
   // the Source's files.
   const content = core.contentOf(run);
@@ -652,8 +855,9 @@ async function runSnapshot(o) {
     broken(`${o.run} names no files and no text, so there is nothing to run`);
   }
 
-  // The dataset the run was submitted against, for scoring its items.
-  const graded = gradedBy(dataset);
+  // The one group a single-group run names, for an eval type of its own that
+  // reads a case; a `group` eval reads each group's case by the item's name.
+  const graded = gradedBy(gctx.primary());
 
   const items = bare ? [{ name: null, kind: "text", text: null, bare: true }]
     : text.trim() ? [{ name: null, kind: "text", text }]
@@ -678,7 +882,7 @@ async function runSnapshot(o) {
   // profile's id.
   const plans = core.targetsOf(run).map((_, i) => {
     const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
-    const links = connections.map(c => reach(c, core.keyVar(c.id)));
+    const links = connections.map(c => reach(c, core.keyVar(c.id, c.slug)));
     return { stages, tokens, connections, links, calls, cells };
   });
 
@@ -728,6 +932,12 @@ async function runSnapshot(o) {
           break;
         }
       }
+      // A text file the Source does not hold is unrun, as an image is: the
+      // run is incomplete rather than broken.
+      if (item.kind === "text" && !item.bare && item.text == null && !fs.existsSync(path.join(o.source, item.name))) {
+        failed = `the file ${item.name} is not in the source`;
+        break;
+      }
       const textOf = item.kind === "text" && !item.bare
         ? (item.text != null ? item.text : readUtf8(path.join(o.source, item.name))) : null;
       const res = await core.runPipeline(plan.stages, dataUrl, callsFor(plan.connections, plan.links, textOf, plan, record),
@@ -736,7 +946,8 @@ async function runSnapshot(o) {
       // text item a dataset grades is graded like an image.
       const kase = item.name ? graded.get(item.name) ?? null : null;
       production = core.productionOf(run, record);
-      sides.push({ res, scores: await core.itemScoresAsync(run, kase, res, { production, grader: graderFor(run) }) });
+      sides.push({ res, scores: await core.itemScoresAsync(run, kase, res,
+        { production, grader: graderFor(run), group: gctx.resolve, item: item.name }) });
     }
     // Production's reply to the item, where its record holds one: what a
     // metric compares with, kept so a re-score reads it again.
@@ -749,16 +960,27 @@ async function runSnapshot(o) {
   }
   if (progressFd != null) fs.closeSync(progressFd);
 
+  // Every item in, none of them unrun: only then has the run settled. A
+  // re-run of one item reads the rest from the results it was handed.
+  const ranItems = results.slice(0, total);
+  const complete = !cancelled && !unrun.length
+    && ranItems.length === total && ranItems.every(it => it && !it.unrun);
+  const outcomes = outcomesOf(run, ranItems, complete);
+  const wholes = wholeRunsOf(run, ranItems, gctx.resolve);
+  const verdicts = verdictsOf(outcomes, complete, o.minPass, wholes);
   const report = {
-    runner: "tools/prompt-lab/run-evals.js",
+    runner: RUNNER,
     ranAt: new Date().toISOString(),
-    verdict: cancelled ? "cancelled" : (unrun.length ? "incomplete" : "done"),
+    verdict: runVerdict(verdicts, complete, run.pass),
+    ...(cancelled ? { cancelled: true } : {}),
+    ...(o.minPass != null ? { minPass: o.minPass } : {}),
     run: {
       name: run.name ?? null,
       evals: run.evals,
-      verdicts: verdictsOf(run, results, !cancelled && !unrun.length),
+      verdicts,
+      pass: { rule: run.pass ?? { mode: "all" }, targets: passTargets(verdicts, run.pass) },
       scenarios: scenariosOf(run),
-      items: results.slice(0, total),
+      items: ranItems,
     },
     unrun,
   };
@@ -767,14 +989,37 @@ async function runSnapshot(o) {
   say(`${report.runner} — run ${report.run.name ?? ""}`);
   for (const u of unrun.slice(0, 10)) say(`----  ${u}`);
   if (unrun.length > 10) say(`----  and ${unrun.length - 10} more, all of them in the JSON report`);
-  say(`${results.filter(Boolean).length} of ${total} items ran · ${cancelled ? "cancelled" : report.verdict}`);
+  sayVerdicts(say, run, verdicts, report.run.pass);
+  say(`${results.filter(Boolean).length} of ${total} items ran · ${cancelled ? "cancelled · " : ""}${report.verdict}`);
+
+  writeReports(o, report, () => runSuites(run, outcomes, ranItems, complete, o.minPass, wholes));
+  process.exitCode = exitOf(report.verdict);
+}
+
+/** Each target's groups and their verdicts, a line each, and the overall rule
+    beneath them where [pass] gives a figure per target. */
+function sayVerdicts(say, run, verdicts, pass) {
+  verdicts.forEach((evals, i) => {
+    for (const e of Object.values(evals)) {
+      const n = e.ran != null ? ` · ${e.passed} of ${e.ran} passed` : e.detail ? ` · ${e.detail}` : "";
+      const whole = e.wholeRun ? ` · whole run ${e.wholeRun.pass ? "passed" : "failed"}` : "";
+      say(`${e.verdict.padEnd(10)} ${core.targetLabel(run, i)} › ${e.name}${n}${whole}`);
+    }
+    if (pass) {
+      const rule = pass.rule.mode === "atLeast" ? `at least ${pass.rule.count}` : "all groups";
+      say(`${(pass.targets[i] ? "pass" : "fail").padEnd(10)} ${core.targetLabel(run, i)} › overall (${rule})`);
+    }
+  });
+}
 
+/** The JSON report, and the JUnit one from [suites], where each was asked for. */
+function writeReports(o, report, suites) {
   if (o.json) {
-    const text2 = JSON.stringify(report, null, 2);
-    if (o.json === "-") process.stdout.write(text2 + "\n");
-    else fs.writeFileSync(o.json, text2 + "\n");
+    const text = JSON.stringify(report, null, 2);
+    if (o.json === "-") process.stdout.write(text + "\n");
+    else fs.writeFileSync(o.json, text + "\n");
   }
-  process.exitCode = cancelled ? EXIT.passed : (unrun.length ? EXIT.incomplete : EXIT.passed);
+  if (o.junit) fs.writeFileSync(o.junit, core.junitXml(suites(), RUNNER));
 }
 
 /** Re-score a run's stored results against the dataset --dataset hands over,
@@ -783,8 +1028,8 @@ async function runSnapshot(o) {
  * a --run's, with the stored results as its items, and a whole-run eval's
  * verdict settled the same way. */
 async function runRescore(o){
-  const dataset = readDataset(o.dataset);
-  const run = readRun(o.run, dataset);
+  const gctx = gradingContext(o);
+  const run = readRun(o.run, gctx);
   let results;
   try {
     results = JSON.parse(fs.readFileSync(o.resultsFile, "utf8"));
@@ -796,50 +1041,53 @@ async function runRescore(o){
   }
   results = core.upgradeResults(results);
 
-  // The cases the run's items are scored against. A file the set does
-  // not grade keeps its reply and has no score -- honestly: nothing else would
-  // say what the set covers.
-  const graded = gradedBy(dataset);
+  // The one group a single-group run names, for an eval type of its own that
+  // reads a case. A file no group grades keeps its reply and has no score --
+  // honestly: nothing else would say what the groups cover.
+  const graded = gradedBy(gctx.primary());
 
   const only = o.only != null ? parseInt(o.only, 10) : null;
-  // An eval that reads every item (the Metrics) re-reads one with no case
+  // An eval that reads every item (a group) re-reads one with no case
   // too, against the production reply the item kept.
   const items = await Promise.all(results.map(async (it, i) => {
     if (!isObject(it) || only != null && i !== only || !Array.isArray(it.scenarios)) return it;
     const kase = isText(it.name) ? graded.get(it.name) ?? null : null;
-    const more = { production: it.production ?? null, grader: graderFor(run) };
+    const more = { production: it.production ?? null, grader: graderFor(run), group: gctx.resolve, item: it.name ?? null };
     return { ...it, scenarios: await Promise.all(it.scenarios.map(async side =>
       (isObject(side) && side.res)
         ? { ...side, scores: await core.itemScoresAsync(run, kase, side.res, more) } : side)) };
   }));
 
+  // A stored run that stopped short re-scores as what it is: incomplete.
+  const complete = items.every(it => isObject(it) && !it.unrun);
+  const outcomes = outcomesOf(run, items, complete);
+  const wholes = wholeRunsOf(run, items, gctx.resolve);
+  const verdicts = verdictsOf(outcomes, complete, o.minPass, wholes);
   const report = {
-    runner: "tools/prompt-lab/run-evals.js",
+    runner: RUNNER,
     ranAt: new Date().toISOString(),
-    verdict: "done",
+    verdict: runVerdict(verdicts, complete, run.pass),
+    ...(o.minPass != null ? { minPass: o.minPass } : {}),
     rescored: true,
     run: {
       name: run.name ?? null,
       evals: run.evals,
-      verdicts: verdictsOf(run, items, true),
+      verdicts,
+      pass: { rule: run.pass ?? { mode: "all" }, targets: passTargets(verdicts, run.pass) },
       scenarios: scenariosOf(run),
       items,
     },
   };
 
-  if (o.json) {
-    const text = JSON.stringify(report, null, 2);
-    if (o.json === "-") process.stdout.write(text + "\n");
-    else fs.writeFileSync(o.json, text + "\n");
-  }
-  process.exitCode = EXIT.passed;
+  writeReports(o, report, () => runSuites(run, outcomes, items, complete, o.minPass, wholes));
+  process.exitCode = exitOf(report.verdict);
 }
 
 /**
  * The run --prompt, --model and --url describe, as a run document: one job
  * that sees the image and answers in the default kind, one scenario,
  * graded against --dataset, asking --prompt. Its profile has no name, so the transcript names no
- * connection, as it never did, and its key is $EVAL_API_KEY.
+ * connection, as it never did, and its key is $EVALSLAB_API_KEY.
  */
 function cliRun(o, dataset) {
   let doc = core.blankPipeline();
@@ -857,8 +1105,7 @@ function cliRun(o, dataset) {
   doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
   doc.targets = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
                    steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }];
-  doc.evals = [{ id: "cli", type: "metrics", name: "", continueOnFailure: true, dataset: { id: "cli", name: "" },
-                 mode: "all", threshold: null, grader: null, over: "item", metrics: [] }];
+  doc.evals = [{ id: "cli", type: "group", name: "", continueOnFailure: true, group: { id: "cli", name: "" }, pin: null }];
   doc.profiles = { cli: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
   return doc;
 }
@@ -933,19 +1180,22 @@ async function main() {
   }
 
   // The dataset the run is graded against. Everything below grades, so there
-  // is no run without one.
+  // is no run without one. --groups hands a --run the bodies it kept; a
+  // --pipeline or --prompt grades one scenario against one, named by --dataset.
+  if (o.groups) broken("--groups hands the bodies a --run kept; grade a --pipeline or --prompt against --dataset.");
   if (!o.dataset) broken(`--dataset names the dataset to grade against.\n\n${USAGE}`);
   const dataset = readDataset(o.dataset);
+  const gctx = { single: dataset, bodies: null, primary: () => dataset, resolve: () => dataset ?? undefined };
 
   // The run to grade. A --pipeline document is one scenario graded against
   // its dataset; without one it is the stage this script always ran -- the
-  // prompt, seeing the image, on --url with $EVAL_API_KEY -- stated as
+  // prompt, seeing the image, on --url with $EVALSLAB_API_KEY -- stated as
   // the same kind of document, so both go through one reading of it.
   if (!o.pipeline && !(o.prompt ?? "").trim()) {
     broken("--prompt or --prompt-file names the prompt to grade: a dataset holds none, "
          + "the lab's Prompt library does.");
   }
-  const run = o.pipeline ? readRun(o.pipeline, dataset) : cliRun(o, dataset);
+  const run = o.pipeline ? readRun(o.pipeline, gctx) : cliRun(o, dataset);
   if (o.pipeline) {
     if (core.targetsOf(run).length !== 1) {
       broken(`${o.pipeline} has ${core.targetsOf(run).length} targets, and --pipeline grades one `
@@ -1038,7 +1288,7 @@ async function main() {
     if (!o.pipeline && !o.model) broken(`--model names the model to grade.\n\n${USAGE}`);
     links = connections.map((c, i) => {
       if (!String(c.model || "").trim()) broken(`stage ${i + 1}'s connection names no model.`);
-      const to = reach(c, o.pipeline ? core.keyVar(c.id) : "EVAL_API_KEY");
+      const to = reach(c, o.pipeline ? core.keyVar(c.id, c.slug) : "EVALSLAB_API_KEY");
       // Once here rather than once per image: a llama.cpp stage on a
       // hosted model is a request that cannot be built at all.
       try {
@@ -1186,10 +1436,12 @@ async function main() {
   // passed, and 49 items absent with the fiftieth green is exactly how
   // that happens.
   const complete = graded.length > 0 && unrun.length === 0;
-  const verdict = !complete ? "incomplete" : tally.passed === tally.ran ? "pass" : "fail";
+  const enough = tally.passed === tally.ran
+    || (o.minPass != null && tally.passed / tally.ran >= o.minPass);
+  const verdict = !complete ? "incomplete" : enough ? "pass" : "fail";
 
   const report = {
-    runner: "tools/prompt-lab/run-evals.js",
+    runner: RUNNER,
     ranAt: new Date().toISOString(),
     dataset: inRepo(o.dataset),
     prompt,
@@ -1217,6 +1469,7 @@ async function main() {
                         filesListed: snapshotSet.size } : {}),
     ...(o.source ? { source: inRepo(o.source) } : {}),
     verdict,
+    ...(o.minPass != null ? { minPass: o.minPass } : {}),
     graded: (itemsOnly ?? graded).length,
     ungraded: set.length - graded.length,
     ran: tally.ran,
@@ -1269,14 +1522,16 @@ async function main() {
         + `so this is a partial run and not a result.`);
   }
 
-  if (o.json) {
-    const text = JSON.stringify(report, null, 2);
-    if (toStdout) process.stdout.write(text + "\n");
-    else fs.writeFileSync(o.json, text + "\n");
-  }
-
-  process.exitCode = !complete ? EXIT.incomplete
-                   : report.failed ? EXIT.failed : EXIT.passed;
+  // The one eval this mode grades by, as a suite of its cases.
+  const suites = () => {
+    const j = Math.max(0, core.evalsOf(run).findIndex(t => core.evalsDataset({ evals: [t] })));
+    return [{ name: `${core.targetLabel(run, 0)} › ${core.evalLabel(run, j)}`, cases: [
+      ...rows.map(r => ({ name: r.id, ...(r.pass ? {} : { failure: r.reasons.join(" · ") || "failed" }) })),
+      ...unrun.map(u => { const at = u.indexOf(": "); return { name: u.slice(0, at), error: u.slice(at + 2) }; }),
+    ] }];
+  };
+  writeReports(o, report, suites);
+  process.exitCode = exitOf(verdict);
 }
 
 main().catch(e => {