evals-lab 0.5.0 → 0.7.0

package/lab/run-evals.js

@@ -116,13 +116,17 @@ const USAGE = `Usage: node run-evals.js [options]
                       file; its content.files, when not empty, is the file
                       list a case has to be in, as --files says.
   --samples <dir>     where the images are. Default: the lab's samples/.
-  --dataset <file>    the dataset a graded run is scored against: its body
-                      ({"cases"}), or the file the lab's Export writes.
+  --dataset <file>    the one eval group a graded run is scored against: its
+                      body ({"cases"}), or the file the lab's Export writes.
                       Versions 1 to 3 are read too (a prompt they hold is
                       not used: --prompt names it); a version-2
                       body's rules are what an older run's jobs, and a run
                       without --pipeline, read replies under. Its cases grade.
-                      Required for anything graded.
+                      Required for anything graded (or --groups, for a --run).
+  --groups <file>     the eval group bodies a --run kept, by "<id>@<n>": a run
+                      grades against each one its evals link (§17), so a run
+                      linking several groups is handed them here instead of the
+                      one --dataset. For --run and --rescore; not with --dataset.
   --source <dir>      the same, named the way a run names it: the directory a
                       Source is stored at. --samples and --source are one
                       flag by two names; both together are refused.
@@ -204,7 +208,7 @@ function broken(msg) {
 
 function parseArgs(argv) {
   const o = { model: "", url: "", prompt: null, pipeline: "", samples: "", dataset: "",
-              replies: "", json: "", source: "", files: "", item: "", resume: "",
+              groups: "", replies: "", json: "", source: "", files: "", item: "", resume: "",
               timeout: null, progress: false, cancelFile: "",
               run: "", progressFile: "", resultsFile: "", from: null, only: null,
               rescore: false, tokens: "", plugins: "", minPass: null, junit: "" };
@@ -225,6 +229,7 @@ function parseArgs(argv) {
       case "--pipeline": o.pipeline = value(); break;
       case "--samples": o.samples = value(); break;
       case "--dataset": o.dataset = value(); break;
+      case "--groups": o.groups = value(); break;
       case "--replies": o.replies = value(); break;
       case "--source": o.source = value(); break;
       case "--files": o.files = value(); break;
@@ -281,10 +286,10 @@ const inRepo = p => {
 const isObject = v => v != null && typeof v === "object" && !Array.isArray(v);
 const isText = v => typeof v === "string";
 
-// [dataset] is the body --dataset handed over, or null: the one dataset a
-// graded run can name, whatever id it names it by, since the queue hands the
-// worker the body of exactly the dataset the run was submitted against.
-function readRun(file, dataset) {
+// [gctx] is the grading context (gradingContext): the eval group bodies a run
+// grades against -- one, as --dataset, or a body per group kept with the run,
+// as --groups (§17). The run reads each link's body by its reference.
+function readRun(file, gctx) {
   let doc;
   try {
     // A run queued before the current version is read as one of today's:
@@ -292,12 +297,20 @@ function readRun(file, dataset) {
     // submitted with. A v2 run's list jobs read their replies under the
     // rules of the dataset it was graded against -- the body handed over.
     doc = core.upgradePipeline(JSON.parse(fs.readFileSync(file, "utf8")),
-                               { rulesFor: () => rulesOf(dataset) });
+                               { rulesFor: () => rulesOf(gctx.primary()) });
   } catch (e) {
     broken(`${file}: ${e.message}`);
   }
-  const named = isObject(doc) ? core.evalsDataset(doc) : null;
-  const bad = core.validatePipeline(doc, { groups: dataset && named ? [named] : [] });
+  // Every Library group the evals name -- a link's group, a private group's
+  // Cases from -- each as validatePipeline reads one. Pins were resolved at
+  // submit (the reference carries `n`), so they are not checked again here.
+  const groups = [], seen = new Set();
+  for (const ref of (isObject(doc) ? core.evalsOf(doc).map(core.casesRef).filter(Boolean) : [])) {
+    if (seen.has(ref.id)) continue;
+    seen.add(ref.id);
+    groups.push({ id: ref.id, name: ref.name, source: gctx.resolve(ref)?.source ?? null });
+  }
+  const bad = core.validatePipeline(doc, { groups });
   if (bad.length) broken(`${file} cannot be run:\n  ${bad.join("\n  ")}`);
   return doc;
 }
@@ -305,9 +318,25 @@ function readRun(file, dataset) {
 // The rules a version-2 body held, by the body readDataset made of it: what
 // an older run's jobs read replies under. A body of today's holds none.
 const heldRules = new WeakMap();
-const rulesOf = (dataset) => (dataset && heldRules.get(dataset)) ?? null;
+const rulesOf = (body) => (body && heldRules.get(body)) ?? null;
+
+// An eval group's body as one of today's, with its version-2 rules taken
+// first: a bare body, or the body the lab's Export wraps, unwrapped by its
+// caller. [where] names it in a refusal.
+function bodyFrom(doc, where) {
+  if (isObject(doc) && doc.rules != null && !(isObject(doc.rules) && Array.isArray(doc.rules.rules))) {
+    broken(`${where}: rules has to be null or { "rules": [...] }`);
+  }
+  const rules = core.datasetRules(doc);
+  const body = core.upgradeDatasetBody(doc);
+  if (!isObject(body) || !Array.isArray(body.cases)) {
+    broken(`${where} is not an eval group: its body has a cases list, or it is the lab's Export of one`);
+  }
+  if (rules) heldRules.set(body, rules);
+  return body;
+}
 
-// The dataset --dataset names: a dataset's body, or the file the lab's
+// The dataset --dataset names: an eval group's body, or the file the lab's
 // Export writes for one, as one of today's.
 function readDataset(file) {
   if (!file) return null;
@@ -328,29 +357,51 @@ function readDataset(file) {
     if (doc.version !== 7) broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads version 7`);
     doc = isObject(doc.group) ? doc.group.body : null;
   }
-  if (isObject(doc) && doc.rules != null && !(isObject(doc.rules) && Array.isArray(doc.rules.rules))) {
-    broken(`${file}: rules has to be null or { "rules": [...] }`);
-  }
-  // A body from an earlier version -- a run queued then, resumed now -- reads
-  // as one of today's, its rules taken first.
-  const rules = core.datasetRules(doc);
-  doc = core.upgradeDatasetBody(doc);
-  if (!isObject(doc) || !Array.isArray(doc.cases)) {
-    broken(`${file} is not a dataset: its body has a cases list, or it is the lab's Export of one`);
+  return bodyFrom(doc, file);
+}
+
+// The bodies a run kept, as --groups hands them (§17): a JSON object of
+// `<id>@<n>` to body -- the group's id and the version it graded with, its id
+// alone for a run from before the lab numbered them -- each read as today's.
+function readGroups(file) {
+  let raw;
+  try {
+    raw = JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (e) {
+    broken(`${file}: ${e.message}`);
   }
-  if (rules) heldRules.set(doc, rules);
-  return doc;
+  if (!isObject(raw)) broken(`${file}: the kept groups are a JSON object of "<id>@<n>": body`);
+  const bodies = new Map();
+  for (const [key, body] of Object.entries(raw)) bodies.set(key, bodyFrom(body, `${file} [${key}]`));
+  return bodies;
 }
 
-// A link's eval group, by reference: the body --dataset handed over, the one
-// Library group a run reads, whatever id it names it by.
-const groupOf = (dataset) => () => dataset ?? undefined;
+// The key a run keeps a group's body under, mirroring the server's group_key:
+// `<id>@<n>`, or the id alone for a reference from before versions.
+const groupKey = (ref) => (ref && Number.isInteger(ref.n) ? `${ref.id}@${ref.n}` : `${ref && ref.id}`);
+
+// The eval groups a --run grades against: one body, as --dataset, or a body
+// per group, as --groups -- never both. `resolve` gives a link's body by its
+// reference; `primary` the one a single-group run names, for a v2 run's rules.
+function gradingContext(o) {
+  if (o.groups && o.dataset) broken("--groups and --dataset name the groups two ways; pass one.");
+  if (o.groups) {
+    const bodies = readGroups(o.groups);
+    const first = bodies.values().next().value ?? null;
+    return { single: null, bodies, primary: () => first,
+             resolve: (ref) => bodies.get(groupKey(ref)) ?? bodies.get(String(ref && ref.id)) ?? undefined };
+  }
+  const single = readDataset(o.dataset);
+  return { single, bodies: null, primary: () => single, resolve: () => single ?? undefined };
+}
 
 // The cases a graded run is scored against, by the item each names: exactly,
-// as the Source names it.
-function gradedBy(dataset) {
+// as the Source names it. Built from the one group a single-group run names,
+// for an eval type of its own that reads a case (a plugin's); a `group` eval
+// reads each group's own case, by the item's name, as it grades.
+function gradedBy(body) {
   const out = new Map();
-  for (const c of core.gradedSetFrom(dataset || {})) {
+  for (const c of core.gradedSetFrom(body || {})) {
     if (!c.todo) out.set(c.item, c);
   }
   return out;
@@ -499,8 +550,25 @@ function callsFor(connections, links, text, plan, record) {
     if (core.STEP_TYPES[step?.type]?.asks === "request") {
       return () => send(links[k], c, step, plan.cells[k], record ?? textRecord(text));
     }
+    // The Recorded target replays each call's recorded reply, read as the flow
+    // reads one (like send, but sending nothing); it does not go through the
+    // model Read-as wrapper below -- the recorded body is read directly.
+    if (core.CONNECTION_TYPES[core.typeOf(c)]?.replays) {
+      return () => {
+        const r = record ?? textRecord(text);
+        const result = r && r.result;
+        if (!result || typeof result.status !== "number") return { raw: "", said: "", ms: 0, conn: null };
+        try {
+          const read = core.httpReplyOf(step, plan.cells[k], result.body, result.status,
+                                        { scope: r.scope || {}, now: r.at ?? null });
+          return { raw: read.raw, said: read.said, ms: 0, conn: null };
+        } catch (e) {
+          return { error: `the recorded reply could not be read as the flow reads it -- ${e.message}`, ms: 0, conn: null };
+        }
+      };
+    }
     const call = core.CONNECTION_TYPES[core.typeOf(c)]?.local
-      ? async sent => core.localAnswer(c, text, sent)
+      ? async sent => core.localAnswer(c, text, sent, record)
       : (sent, url) => ask(links[k], c, sent, url);
     if (!step?.readAs || !step?.step) return call;
     return async (sent, url) => {
@@ -650,40 +718,81 @@ function readUtf8(file) {
 const outcomesOf = (run, items, settled) =>
   core.targetsOf(run).map((_, i) => core.scenarioEvals(run, i, items, settled));
 
+// A link's Whole run verdict, per Target, keyed by the eval's id: a Library
+// group's `run` metrics read over the replies so far (readGroupRun). A private
+// group's Whole run is scenarioEvals' own (it holds the body); a link's body
+// is held apart, in --groups, so reading it beside the link's items is the
+// runner's (§17). [resolve] gives a link's body by its reference.
+function wholeRunsOf(run, items, resolve) {
+  const kind = core.lastKind(run);
+  return core.targetsOf(run).map((_, i) => {
+    const out = {};
+    const ress = (items || []).map(it => (it && !it.unrun ? it.scenarios?.[i]?.res : undefined));
+    for (const t of core.evalsOf(run)) {
+      // A private whole-run group is already scenarioEvals' verdict; here is
+      // the run beside a link's (or private group's) own items.
+      if (t.type !== "group" || core.isWholeRun(core.EVAL_TYPES.group, t)) continue;
+      const body = core.groupOf(t, { group: resolve });
+      if (body && Array.isArray(body.run) && body.run.length) out[t.id] = core.readGroupRun(body, ress, kind);
+    }
+    return out;
+  });
+}
+
 /**
  * One eval's verdict on one target, for the build: its own rule -- every
  * item it read passed, or a whole-run eval's verdict -- with [minPass] the
  * share of items that is enough instead. It only relaxes: an eval whose
  * every item passed passes whatever the share. One an earlier failure
  * stopped is skipped -- that failure is the run's verdict already -- and one
- * that had nothing to read, a dataset grading none of the run's items, says
+ * that had nothing to read, a group grading none of the run's items, says
  * none: the run ran in full, which is what the server's status reads from
- * the exit code, so it is not incomplete either.
+ * the exit code, so it is not incomplete either. [wholeRun] is a link's Whole
+ * run verdict, where its group reads one beside its items (§17): the group
+ * passes when both its items and its Whole run do, and either failing fails it.
  */
-function evalVerdict(o, settled, minPass) {
+function evalVerdict(o, settled, minPass, wholeRun) {
   if (o.skipped) return "skipped";
   if (!settled) return "incomplete";
   if (o.whole) return !o.verdict?.ran ? "none" : o.verdict.pass ? "pass" : "fail";
-  if (!o.ran) return o.skippedItems ? "skipped" : "none";
-  if (o.passed === o.ran) return "pass";
-  return minPass != null && o.passed / o.ran >= minPass ? "pass" : "fail";
+  const item = !o.ran ? (o.skippedItems ? "skipped" : "none")
+    : o.passed === o.ran ? "pass"
+    : minPass != null && o.passed / o.ran >= minPass ? "pass" : "fail";
+  if (!wholeRun) return item;
+  const whole = !wholeRun.ran ? "none" : wholeRun.pass ? "pass" : "fail";
+  if (item === "fail" || whole === "fail") return "fail";
+  if (item === "pass" || whole === "pass") return "pass";
+  return item === "skipped" || whole === "skipped" ? "skipped" : "none";
 }
 
-// Each target's evals, keyed by the eval's id: a whole-run eval's verdict,
-// a per-item eval's counts, and the build's verdict on each.
-const verdictsOf = (outcomes, settled, minPass) => outcomes.map(evals =>
-  Object.fromEntries(evals.map(o => [o.id, {
-    name: o.label, skipped: o.skipped, verdict: evalVerdict(o, settled, minPass),
-    ...(o.whole
-      ? { pass: o.verdict ? o.verdict.pass : null, detail: o.verdict?.detail ?? null }
-      : { ran: o.ran, passed: o.passed, skippedItems: o.skippedItems }),
-  }])));
-
-/** The run's verdict from its evals': a run that did not finish is
-    incomplete, whatever its evals read so far; then any eval failing fails it. */
-function runVerdict(verdicts, complete) {
+// Each target's evals, keyed by the eval's id: a whole-run eval's verdict, a
+// per-item eval's counts -- and, where a link reads a Whole run beside its
+// items, that verdict too -- with the build's verdict on each. [wholes] is
+// wholeRunsOf, one map per target.
+const verdictsOf = (outcomes, settled, minPass, wholes = []) => outcomes.map((evals, i) =>
+  Object.fromEntries(evals.map(o => {
+    const wr = wholes[i] && wholes[i][o.id];
+    return [o.id, {
+      name: o.label, skipped: o.skipped, verdict: evalVerdict(o, settled, minPass, wr),
+      ...(o.whole
+        ? { pass: o.verdict ? o.verdict.pass : null, detail: o.verdict?.detail ?? null }
+        : { ran: o.ran, passed: o.passed, skippedItems: o.skippedItems,
+            ...(wr ? { wholeRun: { pass: wr.pass, detail: wr.detail, ran: wr.ran } } : {}) }),
+    }];
+  })));
+
+/** Each Target's verdict under the overall pass rule (§17), from its groups'
+    verdicts: a group that passed counts, one that failed counts against, and
+    one that read nothing or was skipped is neither. */
+const passTargets = (verdicts, pass) => verdicts.map(v => core.passVerdict(pass,
+  Object.values(v).map(e => (e.verdict === "pass" ? true : e.verdict === "fail" ? false : null))));
+
+/** The run's verdict from its evals' and the overall rule: a run that did not
+    finish is incomplete, whatever its evals read so far; then it passes when
+    every Target does under the rule. */
+function runVerdict(verdicts, complete, pass) {
   if (!complete) return "incomplete";
-  return verdicts.some(v => Object.values(v).some(e => e.verdict === "fail")) ? "fail" : "pass";
+  return passTargets(verdicts, pass).every(Boolean) ? "pass" : "fail";
 }
 
 const exitOf = verdict => ({ pass: EXIT.passed, fail: EXIT.failed })[verdict] ?? EXIT.incomplete;
@@ -699,11 +808,13 @@ function reasonOf(score) {
   return missed.length ? `missed ${missed.join(", ")}` : "failed";
 }
 
-/** A --run's or a --rescore's suites: each target's evals over [items]. */
-function runSuites(run, outcomes, items, settled, minPass) {
+/** A --run's or a --rescore's suites: each target's evals over [items], with a
+    Whole run case where a link reads one beside its items ([wholes]). */
+function runSuites(run, outcomes, items, settled, minPass, wholes = []) {
   return outcomes.flatMap((evals, i) => evals.map(o => {
     const name = `${core.targetLabel(run, i)} › ${o.label}`;
-    const v = evalVerdict(o, settled, minPass);
+    const wr = wholes[i] && wholes[i][o.id];
+    const v = evalVerdict(o, settled, minPass, wr);
     if (o.whole) {
       // Settled over the run rather than item by item: one case, the run.
       const c = { name: o.label };
@@ -713,7 +824,7 @@ function runSuites(run, outcomes, items, settled, minPass) {
       else if (v === "fail") c.failure = o.verdict.detail || "failed";
       return { name, cases: [c] };
     }
-    return { name, cases: items.map((it, x) => {
+    const cases = items.map((it, x) => {
       const c = { name: it?.name ?? (items.length > 1 ? `item ${x + 1}` : "the text") };
       const read = o.items[x];
       if (!it) c.error = "not run";
@@ -722,7 +833,15 @@ function runSuites(run, outcomes, items, settled, minPass) {
       else if (core.isSkipped(read)) c.skipped = "an earlier eval failed";
       else if (!read.pass) c.failure = reasonOf(read);
       return c;
-    }) };
+    });
+    if (wr) {
+      const c = { name: "Whole run" };
+      if (!settled) c.error = "the run did not finish";
+      else if (!wr.ran) c.skipped = "nothing to read";
+      else if (!wr.pass) c.failure = wr.detail || "failed";
+      cases.push(c);
+    }
+    return { name, cases };
   }));
 }
 
@@ -740,8 +859,8 @@ const scenariosOf = run => core.targetsOf(run).map((sc, i) => {
 });
 
 async function runSnapshot(o) {
-  const dataset = readDataset(o.dataset);
-  const run = readRun(o.run, dataset);
+  const gctx = gradingContext(o);
+  const run = readRun(o.run, gctx);
   // One item a pipeline holds itself (Text, or Prompt only's bare one), or
   // the Source's files.
   const content = core.contentOf(run);
@@ -753,8 +872,9 @@ async function runSnapshot(o) {
     broken(`${o.run} names no files and no text, so there is nothing to run`);
   }
 
-  // The dataset the run was submitted against, for scoring its items.
-  const graded = gradedBy(dataset);
+  // The one group a single-group run names, for an eval type of its own that
+  // reads a case; a `group` eval reads each group's case by the item's name.
+  const graded = gradedBy(gctx.primary());
 
   const items = bare ? [{ name: null, kind: "text", text: null, bare: true }]
     : text.trim() ? [{ name: null, kind: "text", text }]
@@ -780,7 +900,10 @@ async function runSnapshot(o) {
   const plans = core.targetsOf(run).map((_, i) => {
     const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
     const links = connections.map(c => reach(c, core.keyVar(c.id, c.slug)));
-    return { stages, tokens, connections, links, calls, cells };
+    // The Recorded target replays the recorded reply, so a metric that compares
+    // with the recorded reply has nothing to say of it (n/a).
+    const recordedTarget = connections.some(c => core.CONNECTION_TYPES[core.typeOf(c)]?.replays);
+    return { stages, tokens, connections, links, calls, cells, recordedTarget };
   });
 
   let results = [];
@@ -842,8 +965,9 @@ async function runSnapshot(o) {
       // A case is found by its file's name, whatever kind of file it is: a
       // text item a dataset grades is graded like an image.
       const kase = item.name ? graded.get(item.name) ?? null : null;
-      production = core.productionOf(run, record);
-      sides.push({ res, scores: await core.itemScoresAsync(run, kase, res, { production, grader: graderFor(run), group: groupOf(dataset) }) });
+      production = core.recordedReplyOf(run, record);
+      sides.push({ res, scores: await core.itemScoresAsync(run, kase, res,
+        { production, grader: graderFor(run), group: gctx.resolve, item: item.name, recordedTarget: plan.recordedTarget }) });
     }
     // Production's reply to the item, where its record holds one: what a
     // metric compares with, kept so a re-score reads it again.
@@ -862,17 +986,19 @@ async function runSnapshot(o) {
   const complete = !cancelled && !unrun.length
     && ranItems.length === total && ranItems.every(it => it && !it.unrun);
   const outcomes = outcomesOf(run, ranItems, complete);
-  const verdicts = verdictsOf(outcomes, complete, o.minPass);
+  const wholes = wholeRunsOf(run, ranItems, gctx.resolve);
+  const verdicts = verdictsOf(outcomes, complete, o.minPass, wholes);
   const report = {
     runner: RUNNER,
     ranAt: new Date().toISOString(),
-    verdict: runVerdict(verdicts, complete),
+    verdict: runVerdict(verdicts, complete, run.pass),
     ...(cancelled ? { cancelled: true } : {}),
     ...(o.minPass != null ? { minPass: o.minPass } : {}),
     run: {
       name: run.name ?? null,
       evals: run.evals,
       verdicts,
+      pass: { rule: run.pass ?? { mode: "all" }, targets: passTargets(verdicts, run.pass) },
       scenarios: scenariosOf(run),
       items: ranItems,
     },
@@ -883,19 +1009,25 @@ async function runSnapshot(o) {
   say(`${report.runner} — run ${report.run.name ?? ""}`);
   for (const u of unrun.slice(0, 10)) say(`----  ${u}`);
   if (unrun.length > 10) say(`----  and ${unrun.length - 10} more, all of them in the JSON report`);
-  sayVerdicts(say, run, verdicts);
+  sayVerdicts(say, run, verdicts, report.run.pass);
   say(`${results.filter(Boolean).length} of ${total} items ran · ${cancelled ? "cancelled · " : ""}${report.verdict}`);
 
-  writeReports(o, report, () => runSuites(run, outcomes, ranItems, complete, o.minPass));
+  writeReports(o, report, () => runSuites(run, outcomes, ranItems, complete, o.minPass, wholes));
   process.exitCode = exitOf(report.verdict);
 }
 
-/** Each target's evals and their verdicts, a line each. */
-function sayVerdicts(say, run, verdicts) {
+/** Each target's groups and their verdicts, a line each, and the overall rule
+    beneath them where [pass] gives a figure per target. */
+function sayVerdicts(say, run, verdicts, pass) {
   verdicts.forEach((evals, i) => {
     for (const e of Object.values(evals)) {
       const n = e.ran != null ? ` · ${e.passed} of ${e.ran} passed` : e.detail ? ` · ${e.detail}` : "";
-      say(`${e.verdict.padEnd(10)} ${core.targetLabel(run, i)} › ${e.name}${n}`);
+      const whole = e.wholeRun ? ` · whole run ${e.wholeRun.pass ? "passed" : "failed"}` : "";
+      say(`${e.verdict.padEnd(10)} ${core.targetLabel(run, i)} › ${e.name}${n}${whole}`);
+    }
+    if (pass) {
+      const rule = pass.rule.mode === "atLeast" ? `at least ${pass.rule.count}` : "all groups";
+      say(`${(pass.targets[i] ? "pass" : "fail").padEnd(10)} ${core.targetLabel(run, i)} › overall (${rule})`);
     }
   });
 }
@@ -916,8 +1048,8 @@ function writeReports(o, report, suites) {
  * a --run's, with the stored results as its items, and a whole-run eval's
  * verdict settled the same way. */
 async function runRescore(o){
-  const dataset = readDataset(o.dataset);
-  const run = readRun(o.run, dataset);
+  const gctx = gradingContext(o);
+  const run = readRun(o.run, gctx);
   let results;
   try {
     results = JSON.parse(fs.readFileSync(o.resultsFile, "utf8"));
@@ -929,18 +1061,18 @@ async function runRescore(o){
   }
   results = core.upgradeResults(results);
 
-  // The cases the run's items are scored against. A file the set does
-  // not grade keeps its reply and has no score -- honestly: nothing else would
-  // say what the set covers.
-  const graded = gradedBy(dataset);
+  // The one group a single-group run names, for an eval type of its own that
+  // reads a case. A file no group grades keeps its reply and has no score --
+  // honestly: nothing else would say what the groups cover.
+  const graded = gradedBy(gctx.primary());
 
   const only = o.only != null ? parseInt(o.only, 10) : null;
-  // An eval that reads every item (the Metrics) re-reads one with no case
+  // An eval that reads every item (a group) re-reads one with no case
   // too, against the production reply the item kept.
   const items = await Promise.all(results.map(async (it, i) => {
     if (!isObject(it) || only != null && i !== only || !Array.isArray(it.scenarios)) return it;
     const kase = isText(it.name) ? graded.get(it.name) ?? null : null;
-    const more = { production: it.production ?? null, grader: graderFor(run), group: groupOf(dataset) };
+    const more = { production: it.production ?? null, grader: graderFor(run), group: gctx.resolve, item: it.name ?? null };
     return { ...it, scenarios: await Promise.all(it.scenarios.map(async side =>
       (isObject(side) && side.res)
         ? { ...side, scores: await core.itemScoresAsync(run, kase, side.res, more) } : side)) };
@@ -949,23 +1081,25 @@ async function runRescore(o){
   // A stored run that stopped short re-scores as what it is: incomplete.
   const complete = items.every(it => isObject(it) && !it.unrun);
   const outcomes = outcomesOf(run, items, complete);
-  const verdicts = verdictsOf(outcomes, complete, o.minPass);
+  const wholes = wholeRunsOf(run, items, gctx.resolve);
+  const verdicts = verdictsOf(outcomes, complete, o.minPass, wholes);
   const report = {
     runner: RUNNER,
     ranAt: new Date().toISOString(),
-    verdict: runVerdict(verdicts, complete),
+    verdict: runVerdict(verdicts, complete, run.pass),
     ...(o.minPass != null ? { minPass: o.minPass } : {}),
     rescored: true,
     run: {
       name: run.name ?? null,
       evals: run.evals,
       verdicts,
+      pass: { rule: run.pass ?? { mode: "all" }, targets: passTargets(verdicts, run.pass) },
       scenarios: scenariosOf(run),
       items,
     },
   };
 
-  writeReports(o, report, () => runSuites(run, outcomes, items, complete, o.minPass));
+  writeReports(o, report, () => runSuites(run, outcomes, items, complete, o.minPass, wholes));
   process.exitCode = exitOf(report.verdict);
 }
 
@@ -1066,9 +1200,12 @@ async function main() {
   }
 
   // The dataset the run is graded against. Everything below grades, so there
-  // is no run without one.
+  // is no run without one. --groups hands a --run the bodies it kept; a
+  // --pipeline or --prompt grades one scenario against one, named by --dataset.
+  if (o.groups) broken("--groups hands the bodies a --run kept; grade a --pipeline or --prompt against --dataset.");
   if (!o.dataset) broken(`--dataset names the dataset to grade against.\n\n${USAGE}`);
   const dataset = readDataset(o.dataset);
+  const gctx = { single: dataset, bodies: null, primary: () => dataset, resolve: () => dataset ?? undefined };
 
   // The run to grade. A --pipeline document is one scenario graded against
   // its dataset; without one it is the stage this script always ran -- the
@@ -1078,7 +1215,7 @@ async function main() {
     broken("--prompt or --prompt-file names the prompt to grade: a dataset holds none, "
          + "the lab's Prompt library does.");
   }
-  const run = o.pipeline ? readRun(o.pipeline, dataset) : cliRun(o, dataset);
+  const run = o.pipeline ? readRun(o.pipeline, gctx) : cliRun(o, dataset);
   if (o.pipeline) {
     if (core.targetsOf(run).length !== 1) {
       broken(`${o.pipeline} has ${core.targetsOf(run).length} targets, and --pipeline grades one `