evals-lab 0.1.2

package/lab/run-evals.js

@@ -0,0 +1,1262 @@
+#!/usr/bin/env node
+// Runs the graded set headlessly. The second of the three callers of
+// evals-core.ts -- the Prompt Lab tab is the first, CI is the third.
+//
+// An agent is a first-class caller here, not an afterthought, so this says
+// what it did twice: a short human report, and a JSON one with every verdict
+// in it. The exit code is the third statement and the coarsest:
+//
+//   0  every graded case passed
+//   1  the set ran and at least one case failed
+//   2  the set did not run in full -- nothing is graded, or an image or a
+//      recorded reply was missing. NOT a pass. An eval set that scores well
+//      because it never ran is the failure this whole ticket is about.
+//   3  the run could not be attempted: bad arguments, no ImageMagick, no set.
+//
+// The gating policy -- which of those a pull request may merge on -- is #250's
+// to decide, and it has the JSON to decide it from. This only reports.
+//
+// Ollama is firewalled to a handful of hosts, so a live run has to happen on
+// one of them. That is a fact about the network and not something a flag here
+// can arrange; `--replies` is the mode that needs no model at all.
+//
+// The server-side-runs modes (#525, design §4 and §11.4) are here rather
+// than in a second executor, because the executor *is* the CLI: a run the
+// server queues and a run a terminal starts have to be the same thing or the
+// parity check measures nothing.
+//
+//   --source/--files  the files come out of a Source: the snapshot's file
+//                     list, read off the directory the orchestrator handed
+//                     over. A case whose image missed the snapshot is
+//                     unrun, not guessed around -- a report that cannot be
+//                     reproduced is not a report.
+//   --progress        JSON lines, one item per line, on a stream of their
+//                     own: stdout, unless the final report is on stdout,
+//                     in which case stderr -- so one cannot swallow the
+//                     other. With --progress the human report moves to
+//                     stderr, keeping stdout pure lines for the tail.
+//   --timeout         a per-request cap, like the relay's, so a mute model
+//                     becomes a failed item rather than a frozen queue.
+//   --resume          continue a run from its first unfinished item: the
+//                     cases the earlier report scored are carried whole and
+//                     the rest are run. Verdicts are never recomputed.
+//   --item            one case against the run's snapshot.
+//   --rescore         a run's stored results, re-scored against the dataset
+//                     it names with --dataset: the replies are kept -- they
+//                     happened, and asking again is a different question --
+//                     and only the verdicts are recomputed. No model, no
+//                     files. A run whose target results are for text is the
+//                     one item a --run names; this mode needs the snapshot
+//                     (`--run`), the stored results (`--results-file`) and,
+//                     for a graded run, the dataset (`--dataset`), and takes
+//                     `--only <n>` for one item.
+//
+// A dataset is never read from the lab's own directories. The queue hands the
+// worker the body a run was submitted with (--dataset), so a run grades
+// against exactly that, whatever the dataset holds by the time it runs.
+//
+// What a file IS to a run is decided by the type→handler registry, not the
+// queue: text joins at {text}, images go through the pixel budget, and a
+// file with no handler runs anyway, flagged. See HANDLERS below.
+const fs = require("fs");
+const path = require("path");
+const { execFileSync } = require("child_process");
+const core = require("./evals-core.mjs");
+
+const HERE = __dirname;
+const ROOT = path.join(HERE, "..", "..");
+
+// Tagger.PIXELS_720P, as an area rather than a longest edge: llama.cpp slices
+// into 448px tiles and the tile COUNT is what costs. The same budget the tab
+// offers by default and the container bakes its copies to.
+const PIXELS = 921600;
+
+// The connection a caller that names no address means, exactly as in
+// server.py -- Ollama on this machine.
+const OLLAMA = process.env.OLLAMA_URL || "http://127.0.0.1:11434";
+
+const EXIT = { passed: 0, failed: 1, incomplete: 2, broken: 3 };
+
+// The relay's own per-request cap (server.py `_proxy`). A request that
+// outlives it is a failed item rather than a model that mutes and holds a
+// single-flight queue for good; `--timeout` can shorten it, never lengthen.
+const REQUEST_CAP = 600;
+
+const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
+
+  --model <id>        the model to grade. Required for a live run.
+  --url <base>        an OpenAI-shaped endpoint. Default: $OLLAMA_URL, else
+                      Ollama on this machine. A key comes from $EVAL_API_KEY, never argv.
+  --prompt <text>     the prompt to grade, as a template: its tokens resolve
+                      under --tokens. Required without --pipeline: a dataset
+                      holds no prompt (the lab's Prompt library does).
+  --prompt-file <p>   the same, read from a file.
+  --plugins <dir>     where the lab keeps its plugins: <dir>/<id>/<version>/,
+                      each with its manifest.json and entry. A --run loads
+                      the plugins it recorded; anything else loads every one
+                      there. Each registers into the core before the run is
+                      read, as the page's plugins do (docs/packs.md).
+  --tokens <file>     the token mappings that prompt is resolved with, as
+                      JSON: a list of { name, type, value | enabled }, as a
+                      job's tokenMappings. Default: none, so a prompt
+                      naming a token is refused, naming it.
+  --pipeline <file>   a run document (docs/pipeline-model.md) with one
+                      scenario and a graded test, graded case by case against
+                      the set. Instead of --prompt, --model and --url. Each
+                      profile's key comes from $EVAL_API_KEY_<ID>, never the
+                      file; its content.files, when not empty, is the file
+                      list a case has to be in, as --files says.
+  --samples <dir>     where the images are. Default: the lab's samples/.
+  --dataset <file>    the dataset a graded run is scored against: its body
+                      ({"cases"}), or the file the lab's Export writes.
+                      Versions 1 to 3 are read too (a prompt they hold is
+                      not used: --prompt names it); a version-2
+                      body's rules are what an older run's jobs, and a run
+                      without --pipeline, read replies under. Its cases grade.
+                      Required for anything graded.
+  --source <dir>      the same, named the way a run names it: the directory a
+                      Source is stored at. --samples and --source are one
+                      flag by two names; both together are refused.
+  --files <file>      the run's file-list snapshot, as JSON: a list of names,
+                      or {"files": [...]}. Only files it names are read from
+                      the Source; a case whose file missed the list is unrun.
+  --replies <file>    score recorded replies instead of calling a model:
+                      {"<case id>": "<reply>"}, or for a pipeline one reply per
+                      stage: {"<case id>": ["<stage 1>", "<stage 2>"]}. No
+                      network, no images.
+  --item <id>         run one case from the set, nothing else.
+  --resume <file>     a report from an earlier attempt: the cases it scored
+                      are carried whole, the rest run from the first
+                      unfinished one.
+  --timeout <s>       cap each request at this many seconds, at most 600 --
+                      the relay's cap, and a longer value cannot extend it.
+                      Default: 600.
+  --progress          write one JSON line per finished item, so a caller
+                      can tail a run in progress. On stdout unless the
+                      report is on stdout too, in which case stderr; and
+                      then the human report moves to stderr.
+  --cancel-file <p>   a path whose existence is checked between items: the run
+                      stops after the item in flight, never mid-reply.
+  --run <file>        a server run: the run document the queue hands the
+                      worker, every scenario over every item of its content;
+                      the files themselves live in --source. Replaces
+                      --pipeline/--model/--replies. Keys as --pipeline.
+  --progress-file <p> JSON-lines progress for a --run, one line per item as it
+                      finishes, on a stream of its own -- never the stream the
+                      final report takes, so one cannot swallow the other.
+  --results-file <f>  a --run's results so far (resume and re-run item).
+  --from <n>          a --run starts at item n (resume), keeping earlier results.
+  --only <n>          a --run runs item n alone (re-run item), replacing it.
+  --rescore           re-score a run's stored results against --dataset.
+                      Needs --run and --results-file, and --dataset for a
+                      graded run; an optional --only <n> re-scores item n
+                      alone. No model, no files, no network.
+  --json <path|->     write the machine-readable report. "-" means stdout, and
+                      sends the human report to stderr.
+`;
+
+// NOTHING HERE CALLS process.exit(). Node's stdout is asynchronous down a
+// pipe, and exiting discards whatever has not drained: measured, a report of
+// 78,872 bytes came out of `--json - | ...` cut to 65,431 -- the pipe buffer
+// -- and unparseable, while the same run redirected to a file was whole. An
+// agent piping the machine-readable report is the caller this script was
+// written for, so the code is set and the process is left to finish writing
+// and leave on its own.
+//
+// Whether the loss shows up depends on how fast the reader drains -- the same
+// report piped to `wc -c` arrived whole -- so it is not a fault a run can be
+// relied on to reveal. `runner-check.js` asserts the absence of the call
+// instead, and that is why.
+//
+// The price is that a live run sits for a few seconds after printing its
+// report: undici keeps a socket to the endpoint alive and that holds the event
+// loop open. Measured at 6.3s against an endpoint still listening and 0.3s
+// once it closes; a --replies run makes no request and leaves at once. It is a
+// wait and not a hang, the exit code is already set by then, and a run that
+// spent minutes on the images will not notice. Do not "fix" it with a
+// process.exit() -- that is the truncation above, straight back.
+//
+// Which means stopping early has to be a throw rather than an exit, since a
+// `broken()` that returned would let its caller carry on with the argument it
+// had just refused.
+class Stop extends Error {
+  constructor(message, code) { super(message); this.code = code; }
+}
+
+function broken(msg) {
+  throw new Stop(msg, EXIT.broken);
+}
+
+function parseArgs(argv) {
+  const o = { model: "", url: "", prompt: null, pipeline: "", samples: "", dataset: "",
+              replies: "", json: "", source: "", files: "", item: "", resume: "",
+              timeout: null, progress: false, cancelFile: "",
+              run: "", progressFile: "", resultsFile: "", from: null, only: null,
+              rescore: false, tokens: "", plugins: "" };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    const value = () => {
+      const v = argv[++i];
+      if (v === undefined) broken(`${a} needs a value\n\n${USAGE}`);
+      return v;
+    };
+    switch (a) {
+      case "--model": o.model = value(); break;
+      case "--url": o.url = value(); break;
+      case "--prompt": o.prompt = value(); break;
+      case "--prompt-file": o.prompt = fs.readFileSync(value(), "utf8").trim(); break;
+      case "--tokens": o.tokens = value(); break;
+      case "--plugins": o.plugins = value(); break;
+      case "--pipeline": o.pipeline = value(); break;
+      case "--samples": o.samples = value(); break;
+      case "--dataset": o.dataset = value(); break;
+      case "--replies": o.replies = value(); break;
+      case "--source": o.source = value(); break;
+      case "--files": o.files = value(); break;
+      case "--item": o.item = value(); break;
+      case "--resume": o.resume = value(); break;
+      case "--timeout": o.timeout = value(); break;
+      case "--progress": o.progress = true; break;
+      case "--cancel-file": o.cancelFile = value(); break;
+      case "--run": o.run = value(); break;
+      case "--progress-file": o.progressFile = value(); break;
+      case "--results-file": o.resultsFile = value(); break;
+      case "--from": o.from = value(); break;
+      case "--only": o.only = value(); break;
+      case "--rescore": o.rescore = true; break;
+      case "--json": o.json = value(); break;
+      case "-h": case "--help":
+        process.stdout.write(USAGE);
+        throw new Stop("", EXIT.passed);
+      default: broken(`unknown option ${a}\n\n${USAGE}`);
+    }
+  }
+  return o;
+}
+
+// api_base in server.py, in JavaScript. A bare host and port, a base URL, a
+// /v1 and the full chat path are all the same endpoint, and all four are
+// reasonable things to type. The tab reaches a model through the relay, which
+// does this for it; this script does not, so it does it here -- one copy in
+// the core, shared with the page's Test connection.
+const apiBase = core.apiBase;
+
+const firstFile = (...paths) => paths.find(p => fs.existsSync(p)) || "";
+
+// Repo-relative where it is in the repo, and as given where it is not: a
+// report that names the set as ../../../../tmp/x.json is a report nobody can
+// tell was run against the committed one.
+const inRepo = p => {
+  const rel = path.relative(ROOT, p);
+  return rel && !rel.startsWith("..") ? rel : p;
+};
+
+// ---- A run document -------------------------------------------------------
+// What the queue hands the worker and what CI commits: a pipeline resolved
+// (docs/pipeline-model.md §5). evals-core.ts validates it -- every field on a
+// list, a key refused by name -- so this script, the page and #29's import
+// refuse the same file with the same sentence.
+const isObject = v => v != null && typeof v === "object" && !Array.isArray(v);
+const isText = v => typeof v === "string";
+
+// [dataset] is the body --dataset handed over, or null: the one dataset a
+// graded run can name, whatever id it names it by, since the queue hands the
+// worker the body of exactly the dataset the run was submitted against.
+function readRun(file, dataset) {
+  let doc;
+  try {
+    // A run queued before the current version is read as one of today's:
+    // resuming or re-running an old row works on the document it was
+    // submitted with. A v2 run's list jobs read their replies under the
+    // rules of the dataset it was graded against -- the body handed over.
+    doc = core.upgradePipeline(JSON.parse(fs.readFileSync(file, "utf8")),
+                               { rulesFor: () => rulesOf(dataset) });
+  } catch (e) {
+    broken(`${file}: ${e.message}`);
+  }
+  const named = isObject(doc) ? core.testsDataset(doc) : null;
+  const bad = core.validatePipeline(doc, { datasets: dataset && named ? [named] : [] });
+  if (bad.length) broken(`${file} cannot be run:\n  ${bad.join("\n  ")}`);
+  return doc;
+}
+
+// The rules a version-2 body held, by the body readDataset made of it: what
+// an older run's jobs read replies under. A body of today's holds none.
+const heldRules = new WeakMap();
+const rulesOf = (dataset) => (dataset && heldRules.get(dataset)) ?? null;
+
+// The dataset --dataset names: a dataset's body, or the file the lab's
+// Export writes for one, as one of today's.
+function readDataset(file) {
+  if (!file) return null;
+  let doc;
+  try {
+    doc = JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (e) {
+    broken(`${file}: ${e.message}`);
+  }
+  if (isObject(doc) && doc.format === "evals-lab/dataset") {
+    if (![1, 2, 3, 4].includes(doc.version)) {
+      broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads versions 1 to 4`);
+    }
+    doc = isObject(doc.dataset) ? doc.dataset.body : null;
+  }
+  if (isObject(doc) && doc.rules != null && !(isObject(doc.rules) && Array.isArray(doc.rules.rules))) {
+    broken(`${file}: rules has to be null or { "rules": [...] }`);
+  }
+  // A body from an earlier version -- a run queued then, resumed now -- reads
+  // as one of today's, its rules taken first.
+  const rules = core.datasetRules(doc);
+  doc = core.upgradeDatasetBody(doc);
+  if (!isObject(doc) || !Array.isArray(doc.cases)) {
+    broken(`${file} is not a dataset: its body has a cases list, or it is the lab's Export of one`);
+  }
+  if (rules) heldRules.set(doc, rules);
+  return doc;
+}
+
+// The cases a graded run is scored against, by the file each names.
+function gradedBy(dataset) {
+  const out = new Map();
+  for (const c of core.gradedSetFrom(dataset || {})) {
+    if (!c.todo) out.set(c.filename, c);
+  }
+  return out;
+}
+
+// ---- Preparing an image ----------------------------------------------
+// The tab does this on a canvas, which is the one thing in the run that cannot
+// be shared: there is no canvas in Node and adding one would mean a native
+// dependency in a lab whose whole premise is that it has none.
+//
+// So the arithmetic is shared instead, by being the same arithmetic:
+// core.preparedSize is the tab's own capping -- including the 448-longest-edge
+// mode an area cannot express -- and the conversion that follows matches the
+// canvas's toDataURL.
+//
+// The stage's profile says how far an image is capped and by which
+// encoding; absent, the 720p JPEG quality 92 the container bakes its copies
+// to, which is what this stage always ran at.
+function imageMagick() {
+  // Not `convert` on Windows: that name is the system's own disk tool there.
+  for (const bin of process.platform === "win32" ? ["magick"] : ["magick", "convert"]) {
+    try {
+      execFileSync(bin, ["-version"], { stdio: "ignore" });
+      return bin;
+    } catch { /* not installed under that name */ }
+  }
+  return "";
+}
+
+function prepare(bin, file, c) {
+  const out = execFileSync(bin, [file, "-auto-orient", "-format", "%w %h", "info:"],
+                           { maxBuffer: 1 << 20 }).toString().trim();
+  let [w, h] = out.split(/\s+/).map(Number);
+  if (!(w > 0 && h > 0)) throw new Error(`${path.basename(file)}: no dimensions`);
+  // What the connection says, from the one place that reads its fields.
+  const image = core.connectionImage(c);
+  [w, h] = core.preparedSize(w, h, image.px ?? PIXELS);
+  const fmt = { "image/jpeg": ["jpg", "image/jpeg"], "image/png": ["png", "image/png"],
+                "image/webp": ["webp", "image/webp"] }[image.format];
+  const quality = String(core.encoderQuality(image.quality));
+  const jpeg = execFileSync(
+    bin, [file, "-auto-orient", "-strip", "-resize", `${w}x${h}!`, "-quality", quality, `${fmt[0]}:-`],
+    { maxBuffer: 64 << 20 });
+  return { dataUrl: `data:${fmt[1]};base64,` + jpeg.toString("base64"), dims: `${w}x${h}` };
+}
+
+// ---- The type → handler registry ------------------------------------------
+// Design §4. What a file *is* to a run is decided here and only here, so the
+// queue and the page never learn about file types, and pdf and xlsx are
+// later entries that touch nothing but this map.
+//
+// A file with no mapper is passed through anyway -- decoded as UTF-8 with
+// replacement and joined at {text} like the text handler, and flagged in the
+// transcript. Stating the failure mode beats hiding it: a PDF's text layer
+// is not extracted, so its streams arrive as arbitrary bytes, and a run
+// that read them "well" would be a finding about nothing.
+const HANDLERS = new Map();
+[
+  // Images: prepared to the profile's pixel budget by ImageMagick and sent
+  // as the image content part. The formats are server.py's FILE_TYPES
+  // image half, so nothing accepted for upload has no executor to run it.
+  ["image", "jpg", "jpeg", "png", "heic", "heif", "dng", "tif", "tiff"],
+  // Text-ish: decoded UTF-8, joining the stage at {text} -- the text-content
+  // semantics the canvas already has (core.textPrompt).
+  ["text", "txt", "md", "csv"],
+].forEach((row) => {
+  const [kind, ...exts] = row;
+  for (const ext of exts) HANDLERS.set(ext, kind);
+});
+// The extension of a file in the Source, lower-cased. Names arrive already
+// clean from the upload path; this only classifies.
+const extOf = name => (path.extname(String(name)) || "").slice(1).toLowerCase();
+
+const UNMAPPED = "the file has no handler, so it was passed through as raw "
+               + "bytes decoded as UTF-8 with replacement -- whatever a "
+               + "real reader of this type would have extracted was not "
+               + "extracted, and the content is mojibake";
+
+/**
+ * One file of the Source made into what the job consumes: an image as the
+ * dataUrl at [conn]'s budget, or text at {text}. A file with no mapper
+ * still runs, and says so.
+ */
+function itemContent(bin, file, conn) {
+  const kind = HANDLERS.get(extOf(file));
+  if (kind === "image") {
+    const { dataUrl, dims } = prepare(bin, file, conn);
+    return { kind, dataUrl, text: null, flags: [] };
+  }
+  // TextDecoder with fatal unset replaces malformed bytes with U+FFFD --
+  // what "decoded as UTF-8 with replacement" means, for a mapped-text file
+  // and an unmapped one alike.
+  const text = new TextDecoder().decode(fs.readFileSync(file));
+  return { kind: kind ?? "unmapped", dataUrl: null, text, flags: kind ? [] : [UNMAPPED] };
+}
+
+// ---- Reaching a model ----------------------------------------------------
+
+/**
+ * Where a connection goes and the key it takes, refused before anything is
+ * sent if the key cannot go there. [from] is the variable the key is read
+ * from, named in the report in place of the key.
+ */
+function reach(c, from) {
+  const base = core.connectionBase({ ...c, url: c.url || OLLAMA });
+  const key = (process.env[from] || "").trim();
+  if (key) {
+    // server.py's header_safe: fetch rejects such a header with the value in
+    // its message, which would put the key in the report.
+    if (![...key].every(ch => ch >= " " && ch <= "~")) {
+      broken(`$${from} has characters a header cannot carry, so the key was not sent.`);
+    }
+    // server.py's rule, for server.py's reason: a key that bills a real card
+    // should not cross a network in the clear. Loopback never leaves the
+    // machine.
+    const host = new URL(base).hostname;
+    if (!base.startsWith("https://") && !["127.0.0.1", "[::1]", "localhost"].includes(host)) {
+      broken(`$${from} is set and ${base} is not https, so the key was not sent.`);
+    }
+  }
+  return { name: c.name, base, key, from };
+}
+
+// The page's callModel, less the relay: the same body, and on llama.cpp the
+// same check for a reply that looped. A request that outlives the cap is
+// aborted and comes back as the stage's error, so the queue it belongs to
+// keeps moving instead of waiting on a model that mutes.
+let seconds = REQUEST_CAP; // the run's per-request cap, set in main()
+
+// Each stage's transport for one item. A type that answers from the item
+// itself (Echo, #97) is asked nothing: its reply is the item's own text --
+// or, over Prompt only, the prompt -- and the stage reads it as it would a
+// model's.
+//
+// A call that asks for a whole request (an HTTP Request, #flows) builds it
+// from its step, the scenario's cell and the item's record, and is handed
+// only its words: the transport is where the three meet.
+function callsFor(connections, links, text, plan, record) {
+  return connections.map((c, k) => {
+    const step = plan?.calls?.[k];
+    if (core.STEP_TYPES[step?.type]?.asks === "request") {
+      return () => send(links[k], c, step, plan.cells[k], record ?? textRecord(text));
+    }
+    return core.CONNECTION_TYPES[core.typeOf(c)]?.local
+      ? async sent => core.localAnswer(c, text, sent)
+      : (sent, url) => ask(links[k], c, sent, url);
+  });
+}
+
+/** A Setup profile the run carries, asked as a grader: its reply's text, or
+    the failure as an error the metric reports. One transport per profile. */
+const graders = new WeakMap();
+function graderFor(run) {
+  if (!graders.has(run)) {
+    const made = new Map();
+    graders.set(run, ref => {
+      const c = run.profiles?.[ref.id];
+      if (!c) return undefined;
+      if (!made.has(ref.id)) {
+        const link = reach(c, core.keyVar(ref.id));
+        made.set(ref.id, async prompt => {
+          const r = await ask(link, { id: ref.id, ...c }, prompt, null);
+          if (r.error) throw new Error(`the grader ${c.name || ref.id} failed: ${r.error}`);
+          return r.raw ?? "";
+        });
+      }
+      return made.get(ref.id);
+    });
+  }
+  return graders.get(run);
+}
+
+/** A text item as a record's scope: its text is the trigger's body. */
+const textRecord = (text) => ({ scope: { trigger: { body: text } }, at: null });
+
+/** An HTTP Request: the flow's request rebuilt from [record], sent to [to],
+    and the reply read as the flow reads it. */
+async function send(to, c, step, cell, record) {
+  const t0 = Date.now();
+  const said = more => ({ ms: Date.now() - t0, conn: to.name, ...more });
+  const ctx = { scope: record.scope || {}, now: record.at || null };
+  let req;
+  try {
+    req = core.connectionSend(c, core.httpRequestOf(step, cell, ctx), to.base, to.key);
+  } catch (e) {
+    return said({ error: `the request could not be built -- ${e.message}` });
+  }
+  // A body that is text -- an Any endpoint flow's -- goes as it is written;
+  // anything else as JSON, the way the flow sent it.
+  const isText = typeof req.body === "string";
+  const payload = req.method === "GET" ? null : isText ? req.body : JSON.stringify(req.body);
+  const sent = `${req.method || "POST"} ${req.url}${payload == null ? "" : `\n\n${payload}`}`;
+  const done = more => said({ sent, ...more });
+  try {
+    const r = await fetch(req.url, {
+      method: req.method || "POST",
+      headers: { "Content-Type": isText ? "text/plain; charset=utf-8" : "application/json", ...req.headers },
+      redirect: "error",
+      signal: AbortSignal.timeout(seconds * 1000),
+      ...(payload == null ? {} : { body: payload }),
+    });
+    const text = await r.text();
+    let j = text;
+    try { j = JSON.parse(text); } catch { /* the body as text */ }
+    if (!r.ok) return done({ error: j?.error?.message || (typeof j?.error === "string" ? j.error : null) || `HTTP ${r.status}` });
+    const read = core.httpReplyOf(step, cell, j, r.status, ctx);
+    return done({ raw: read.raw, said: read.said });
+  } catch (e) {
+    if (e?.name === "TimeoutError" || e?.name === "AbortError") {
+      return done({ error: `the endpoint gave no reply within ${seconds}s — timed out` });
+    }
+    return done({ error: String(e?.message || e) });
+  }
+}
+
+async function ask(to, c, prompt, dataUrl) {
+  const t0 = Date.now();
+  const said = more => ({ ms: Date.now() - t0, conn: to.name, ...more });
+  try {
+    // The connection's type builds the whole request: the URL, the headers
+    // (Anthropic's key travels in x-api-key), and the body.
+    const req = core.connectionRequest(c, prompt, dataUrl, to.base, to.key);
+    const r = await fetch(req.url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...req.headers },
+      signal: AbortSignal.timeout(seconds * 1000),
+      body: JSON.stringify(req.body),
+    });
+    const j = await r.json().catch(() => ({}));
+    if (!r.ok) return said({ error: j.error?.message || j.error || `HTTP ${r.status}` });
+    const { raw, finishReason } = core.connectionReply(c, j);
+    const error = core.connectionReplyProblem(c, raw, finishReason);
+    return said(error ? { raw, error } : { raw });
+  } catch (e) {
+    if (e?.name === "TimeoutError" || e?.name === "AbortError") {
+      return said({ error: `the model gave no reply within ${seconds}s — timed out` });
+    }
+    return said({ error: String(e?.message || e) });
+  }
+}
+
+// ---- The run -------------------------------------------------------------
+
+// ---- A server run: what the queue hands the worker (#531) ------------------
+// The run document carries everything a run was submitted with -- the
+// scenarios, the profiles they resolved to, and the content: a file list into
+// --source, in order and with its repeats, or the inline text. Each item runs
+// through every scenario's jobs as an image does on the canvas; a
+// Source deleted between submit and here shows as a failed run, and a file
+// missing from it is an incomplete item, never a pass.
+//
+// Progress is JSON lines on a stream of its own -- --progress-file, never the
+// report -- so the queue can tail it into the run row as items finish. The
+// report is written whole at the end, from the results the run held plus what
+// this invocation did, so resume and re-run item agree with a fresh run.
+
+// What a file *is* to a run, the same registry §4 gives the graded set: an
+// image is prepared to the profile's pixel budget; a text file is decoded
+// UTF-8 and joined at {text}; anything else is passed through as UTF-8 with
+// replacement and flagged, so the failure mode is stated rather than hidden.
+const RUN_IMAGE_TYPES = new Set([".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff"]);
+const RUN_TEXT_TYPES = new Set([".txt", ".md", ".csv"]);
+
+function readUtf8(file) {
+  return fs.readFileSync(file).toString("utf8");
+}
+
+// Every test's reading of each scenario, keyed by the test's id, exactly as
+// the page reads it: a whole-run test's verdict, settled once every item is
+// in, and a per-item test's counts. [settled] is false for a run that
+// stopped short, whose whole-run tests have not settled.
+const verdictsOf = (run, items, settled) => run.scenarios.map((_, i) =>
+  Object.fromEntries(core.scenarioTests(run, i, items, settled).map(o => [o.id, {
+    name: o.label, skipped: o.skipped,
+    ...(o.whole
+      ? { pass: o.verdict ? o.verdict.pass : null, detail: o.verdict?.detail ?? null }
+      : { ran: o.ran, passed: o.passed, skippedItems: o.skippedItems }),
+  }])));
+
+// The run as the report restates it: each scenario's stages with the
+// connection each one asked.
+const scenariosOf = run => run.scenarios.map((sc, i) => {
+  const { stages, connections } = core.stagesFor(run, i);
+  return {
+    n: i + 1, name: sc.name || null,
+    stages: stages.map((s, k) => {
+      const { id, ...connection } = connections[k];
+      return { n: k + 1, prompt: s.text, kind: s.kind, withImage: s.withImage, profile: id, connection };
+    }),
+  };
+});
+
+async function runSnapshot(o) {
+  const dataset = readDataset(o.dataset);
+  const run = readRun(o.run, dataset);
+  // One item a pipeline holds itself (Text, or Prompt only's bare one), or
+  // the Source's files.
+  const content = core.contentOf(run);
+  const inline = core.CONTENT_TYPES[content.type];
+  const bare = !!inline?.bare;
+  const text = inline?.inline && !bare ? inline.text?.(content, {}) ?? "" : "";
+  const files = content.type === "source" ? content.files || [] : [];
+  if (!bare && !files.length && !text.trim()) {
+    broken(`${o.run} names no files and no text, so there is nothing to run`);
+  }
+
+  // The dataset the run was submitted against, for scoring its items.
+  const graded = gradedBy(dataset);
+
+  const items = bare ? [{ name: null, kind: "text", text: null, bare: true }]
+    : text.trim() ? [{ name: null, kind: "text", text }]
+    : files.map(name => {
+        const ext = path.extname(name).toLowerCase();
+        if (RUN_IMAGE_TYPES.has(ext)) return { name, kind: "image" };
+        // A record (flows/record.ts): one call of a Power Automate step, what
+        // an HTTP Request is rebuilt from.
+        if (ext === ".json") return { name, kind: "record" };
+        return { name, kind: "text", unmapped: !RUN_TEXT_TYPES.has(ext) };
+      });
+  const total = items.length;
+  // Only the run's own image items need ImageMagick: a text-only Source runs
+  // with nothing on PATH but the node running this.
+  const bin = items.some(it => it.kind === "image") ? imageMagick() : null;
+  if (items.some(it => it.kind === "image") && !bin) {
+    broken("a run over images needs ImageMagick, to cap each image to the pixel budget the tab caps it to on a canvas.");
+  }
+
+  // Each scenario once: its stages, the jobs' token sets, and a transport
+  // per stage to the profile that stage resolves to, keyed by that
+  // profile's id.
+  const plans = run.scenarios.map((_, i) => {
+    const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
+    const links = connections.map(c => reach(c, core.keyVar(c.id)));
+    return { stages, tokens, connections, links, calls, cells };
+  });
+
+  let results = [];
+  if (o.resultsFile) {
+    try { results = JSON.parse(fs.readFileSync(o.resultsFile, "utf8")); } catch { /* fresh run */ }
+    // Kept as they were written; read as today's.
+    results = Array.isArray(results) ? core.upgradeResults(results) : [];
+  }
+  const only = o.only != null ? parseInt(o.only, 10) : null;
+  const from = only != null ? only : (o.from != null ? parseInt(o.from, 10) : 0);
+
+  const progressFd = o.progressFile ? fs.openSync(o.progressFile, "a") : null;
+  const progress = obj => { if (progressFd != null) fs.writeSync(progressFd, JSON.stringify(obj) + "\n"); };
+
+  const unrun = [];
+  let cancelled = false;
+
+  for (let i = 0; i < total; i++) {
+    if (only == null && i < from) continue;
+    if (o.cancelFile && fs.existsSync(o.cancelFile)) { cancelled = true; break; }
+    const item = items[i];
+    // Said before the work, so a reader knows what is in flight rather than
+    // what last finished; the row below it is the same item, done.
+    progress({ event: "start", item: i, name: item.name });
+    const t0 = Date.now();
+    const sides = [];
+    let production = null;
+    let failed = null;
+    for (const plan of plans) {
+      let dataUrl = null;
+      if (item.kind === "image") {
+        const file = path.join(o.source, item.name);
+        if (!fs.existsSync(file)) { failed = `the file ${item.name} is not in the source`; break; }
+        try {
+          ({ dataUrl } = prepare(bin, file, plan.connections[0]));
+        } catch (e) {
+          failed = `${item.name} could not be prepared — ${e.message}`;
+          break;
+        }
+      }
+      let record = null;
+      if (item.kind === "record") {
+        const file = path.join(o.source, item.name);
+        try { record = JSON.parse(readUtf8(file)); } catch (e) {
+          failed = fs.existsSync(file) ? `${item.name} is not a record -- ${e.message}` : `the file ${item.name} is not in the source`;
+          break;
+        }
+      }
+      const textOf = item.kind === "text" && !item.bare
+        ? (item.text != null ? item.text : readUtf8(path.join(o.source, item.name))) : null;
+      const res = await core.runPipeline(plan.stages, dataUrl, callsFor(plan.connections, plan.links, textOf, plan, record),
+                                         { tokens: plan.tokens, text: textOf });
+      // A case is found by its file's name, whatever kind of file it is: a
+      // text item a dataset grades is graded like an image.
+      const kase = item.name ? graded.get(item.name) ?? null : null;
+      production = core.productionOf(run, record);
+      sides.push({ res, scores: await core.itemScoresAsync(run, kase, res, { production, grader: graderFor(run) }) });
+    }
+    // Production's reply to the item, where its record holds one: what a
+    // metric compares with, kept so a re-score reads it again.
+    const out = { item: i, name: item.name, kind: item.kind, ...(production != null ? { production } : {}),
+                  unmapped: item.unmapped || null, ms: Date.now() - t0,
+                  scenarios: sides, ...(failed ? { unrun: failed } : {}) };
+    results[i] = out;
+    progress(out);
+    if (failed) unrun.push(`${item.name || "the text"}: ${failed}`);
+  }
+  if (progressFd != null) fs.closeSync(progressFd);
+
+  const report = {
+    runner: "tools/prompt-lab/run-evals.js",
+    ranAt: new Date().toISOString(),
+    verdict: cancelled ? "cancelled" : (unrun.length ? "incomplete" : "done"),
+    run: {
+      name: run.name ?? null,
+      tests: run.tests,
+      verdicts: verdictsOf(run, results, !cancelled && !unrun.length),
+      scenarios: scenariosOf(run),
+      items: results.slice(0, total),
+    },
+    unrun,
+  };
+
+  const say = o.json === "-" ? console.error : console.log;
+  say(`${report.runner} — run ${report.run.name ?? ""}`);
+  for (const u of unrun.slice(0, 10)) say(`----  ${u}`);
+  if (unrun.length > 10) say(`----  and ${unrun.length - 10} more, all of them in the JSON report`);
+  say(`${results.filter(Boolean).length} of ${total} items ran · ${cancelled ? "cancelled" : report.verdict}`);
+
+  if (o.json) {
+    const text2 = JSON.stringify(report, null, 2);
+    if (o.json === "-") process.stdout.write(text2 + "\n");
+    else fs.writeFileSync(o.json, text2 + "\n");
+  }
+  process.exitCode = cancelled ? EXIT.passed : (unrun.length ? EXIT.incomplete : EXIT.passed);
+}
+
+/** Re-score a run's stored results against the dataset --dataset hands over,
+ * keeping the replies that were stored: what changes a verdict is the scorer
+ * or the cases, never a new question to the model. The report is shaped like
+ * a --run's, with the stored results as its items, and a whole-run test's
+ * verdict settled the same way. */
+async function runRescore(o){
+  const dataset = readDataset(o.dataset);
+  const run = readRun(o.run, dataset);
+  let results;
+  try {
+    results = JSON.parse(fs.readFileSync(o.resultsFile, "utf8"));
+  } catch (e) {
+    broken(`${e.path ?? "a read"}: ${e.message}`);
+  }
+  if (!Array.isArray(results)) {
+    broken(`${o.resultsFile}: a run's results are a list of items`);
+  }
+  results = core.upgradeResults(results);
+
+  // The cases the run's items are scored against. A file the set does
+  // not grade keeps its reply and has no score -- honestly: nothing else would
+  // say what the set covers.
+  const graded = gradedBy(dataset);
+
+  const only = o.only != null ? parseInt(o.only, 10) : null;
+  // A test that reads every item (the Metrics) re-reads one with no case
+  // too, against the production reply the item kept.
+  const items = await Promise.all(results.map(async (it, i) => {
+    if (!isObject(it) || only != null && i !== only || !Array.isArray(it.scenarios)) return it;
+    const kase = isText(it.name) ? graded.get(it.name) ?? null : null;
+    const more = { production: it.production ?? null, grader: graderFor(run) };
+    return { ...it, scenarios: await Promise.all(it.scenarios.map(async side =>
+      (isObject(side) && side.res)
+        ? { ...side, scores: await core.itemScoresAsync(run, kase, side.res, more) } : side)) };
+  }));
+
+  const report = {
+    runner: "tools/prompt-lab/run-evals.js",
+    ranAt: new Date().toISOString(),
+    verdict: "done",
+    rescored: true,
+    run: {
+      name: run.name ?? null,
+      tests: run.tests,
+      verdicts: verdictsOf(run, items, true),
+      scenarios: scenariosOf(run),
+      items,
+    },
+  };
+
+  if (o.json) {
+    const text = JSON.stringify(report, null, 2);
+    if (o.json === "-") process.stdout.write(text + "\n");
+    else fs.writeFileSync(o.json, text + "\n");
+  }
+  process.exitCode = EXIT.passed;
+}
+
+/**
+ * The run --prompt, --model and --url describe, as a run document: one job
+ * that sees the image and answers in the default kind, one scenario,
+ * graded against --dataset, asking --prompt. Its profile has no name, so the transcript names no
+ * connection, as it never did, and its key is $EVAL_API_KEY.
+ */
+function cliRun(o, dataset) {
+  let doc = core.blankPipeline();
+  // Graded, so the job answers with a list: the configuration that reads a
+  // reply as this command always has (legacyTagsOut), under the dataset's
+  // rules. A --pipeline states its own.
+  doc.jobs[0] = core.withOut(doc.jobs[0], core.legacyTagsOut(rulesOf(dataset), "LOWER"));
+  if (o.tokens) {
+    try {
+      doc.jobs[0] = core.withCall(doc.jobs[0], { tokenMappings: JSON.parse(fs.readFileSync(o.tokens, "utf8")) });
+    } catch (e) {
+      broken(`${o.tokens}: ${e.message}`);
+    }
+  }
+  doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
+  doc.scenarios = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
+                     stages: [{ prompt: o.prompt ?? "" }] }];
+  doc.tests = [{ id: "cli", type: "metrics", name: "", continueOnFailure: true, dataset: { id: "cli", name: "" },
+                 mode: "all", threshold: null, grader: null, over: "item", metrics: [{ type: "case" }] }];
+  doc.profiles = { cli: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
+  return doc;
+}
+
+// The plugins a run is read under, registered before anything reads it: a
+// --run's recorded list, at the versions it recorded, or every plugin in the
+// folder. A plugin that will not load stops the run, naming it -- a run read
+// without the code it was submitted under grades something else.
+async function loadPlugins(o) {
+  if (!o.plugins) return;
+  const { pathToFileURL } = require("url");
+  let wanted = null;
+  if (o.run) {
+    try { wanted = JSON.parse(fs.readFileSync(o.run, "utf8")).plugins ?? []; } catch { wanted = []; }
+  }
+  const dirs = wanted
+    ? wanted.map(p => path.join(o.plugins, String(p.id), String(p.version)))
+    : fs.readdirSync(o.plugins, { withFileTypes: true }).filter(d => d.isDirectory() && !d.name.startsWith("tmp"))
+        .flatMap(d => fs.readdirSync(path.join(o.plugins, d.name)).map(v => path.join(o.plugins, d.name, v)));
+  for (const dir of dirs) {
+    let manifest;
+    try {
+      manifest = JSON.parse(fs.readFileSync(path.join(dir, "manifest.json"), "utf8"));
+      const mod = await import(pathToFileURL(path.join(dir, manifest.entry)).href);
+      await (mod.default ?? mod.register)(core.pluginHost(manifest.id));
+    } catch (e) {
+      broken(`the plugin in ${dir} did not load: ${e.message}`);
+    }
+  }
+}
+
+async function main() {
+  const o = parseArgs(process.argv.slice(2));
+  await loadPlugins(o);
+  if (o.rescore) {
+    if (!o.run || !o.resultsFile) {
+      broken("--rescore re-scores a stored run, so it needs --run and --results-file, "
+           + "and for a graded run the dataset it scores against, --dataset.");
+    }
+    if (o.pipeline || o.prompt != null || o.tokens || o.model || o.url || o.replies
+        || o.samples || o.source || o.files || o.resume || o.item || o.progress) {
+      broken("--rescore re-scores what the run already holds, so it cannot be given "
+           + "with --pipeline, --prompt, --tokens, --model, --url, --replies, --samples, --source, "
+           + "--files, --resume, --item or --progress.");
+    }
+    return runRescore(o);
+  }
+  if (o.run) {
+    if (o.pipeline || o.prompt != null || o.tokens || o.model || o.url || o.replies || o.samples || o.files) {
+      broken("--run names everything a run takes -- scenarios, files and text -- so it "
+           + "cannot be given with --pipeline, --prompt, --tokens, --model, --url, --replies, --samples or --files.");
+    }
+    const n = Number(o.timeout ?? REQUEST_CAP);
+    if (!(n > 0)) broken(`--timeout has to be a number of seconds above zero`);
+    seconds = Math.min(n, REQUEST_CAP);
+    return runSnapshot(o);
+  }
+  if (o.pipeline && (o.prompt != null || o.tokens || o.model || o.url)) {
+    broken("--pipeline names every stage's prompt, tokens and connection, so it cannot be "
+         + "given with --prompt, --prompt-file, --tokens, --model or --url.");
+  }
+  if (o.pipeline && o.files) {
+    broken("--pipeline's run document carries its file list in content.files, so --files "
+         + "would say it twice.");
+  }
+  if (o.samples && o.source) {
+    broken("--samples and --source name the same directory, so both together "
+         + "say two things.");
+  }
+  if (o.resume && o.item) {
+    broken("--resume and --item say different things about what to run.");
+  }
+
+  // The dataset the run is graded against. Everything below grades, so there
+  // is no run without one.
+  if (!o.dataset) broken(`--dataset names the dataset to grade against.\n\n${USAGE}`);
+  const dataset = readDataset(o.dataset);
+
+  // The run to grade. A --pipeline document is one scenario graded against
+  // its dataset; without one it is the stage this script always ran -- the
+  // prompt, seeing the image, on --url with $EVAL_API_KEY -- stated as
+  // the same kind of document, so both go through one reading of it.
+  if (!o.pipeline && !(o.prompt ?? "").trim()) {
+    broken("--prompt or --prompt-file names the prompt to grade: a dataset holds none, "
+         + "the lab's Prompt library does.");
+  }
+  const run = o.pipeline ? readRun(o.pipeline, dataset) : cliRun(o, dataset);
+  if (o.pipeline) {
+    if (run.scenarios.length !== 1) {
+      broken(`${o.pipeline} has ${run.scenarios.length} scenarios, and --pipeline grades one `
+           + "against the set -- --run runs them all.");
+    }
+    if (!core.testsDataset(run)) {
+      broken(`${o.pipeline} is not graded, and --pipeline grades a run case by case -- --run runs it.`);
+    }
+    if (core.contentOf(run).type !== "source") {
+      broken(`${o.pipeline} is over inline text, and --pipeline grades the set's files -- --run runs it.`);
+    }
+  }
+  // Case by case: each item against its case, as the case metric of the
+  // test that names the dataset reads it (scoreCase).
+  const { stages, tokens, connections } = core.stagesFor(run, 0);
+  const prompt = core.resolvePrompt(stages[0].text, core.tokenSet(tokens, 0));
+
+  // The graded half, through the function the tab builds its own list with.
+  const set = core.gradedSetFrom(dataset);
+  let graded = set.filter(c => !c.todo);
+
+  const limit = (() => {
+    // The relay's cap applies no matter what the caller asked for: a request
+    // that outlives it is a failed item rather than a model that mutes and
+    // holds a single-flight queue for good.
+    const n = Number(o.timeout ?? REQUEST_CAP);
+    if (!(n > 0)) broken(`--timeout has to be a number of seconds above zero`);
+    return Math.min(n, REQUEST_CAP);
+  })();
+  seconds = limit;
+
+  const itemsOnly = (() => {
+    if (!o.item) return null;
+    const hit = graded.filter(c => c.id === o.item);
+    if (!hit.length) broken(`no graded case is named ${o.item}`);
+    return hit;
+  })();
+
+  // The earlier report a resume carries its finished items from. Both shapes
+  // a caller can leave behind: a full --json report, or the bare list of
+  // scores of rows. The earlier attempt's unrun is deliberately not honoured:
+  // those are the unfinished items a resume exists to run.
+  const carried = (() => {
+    if (!o.resume) return null;
+    let doc;
+    try {
+      doc = JSON.parse(fs.readFileSync(o.resume, "utf8"));
+    } catch (e) {
+      broken(`${o.resume}: ${e.message}`);
+    }
+    const rows = Array.isArray(doc) ? doc : doc?.cases;
+    if (!Array.isArray(rows) || !rows.every(r => isObject(r) && isText(r.id))) {
+      broken(`${o.resume}: a report carries its results under "cases" as a list of rows`);
+    }
+    const out = new Map();
+    for (const r of rows) if (!out.has(r.id)) out.set(r.id, r);
+    return out;
+  })();
+
+  // The snapshot a run was submitted with: only files it names are read, so
+  // files added to the Source afterwards cannot join midway and a report
+  // names exactly what it graded. A run document's own list is that
+  // snapshot; an empty one names none, and every graded case runs.
+  const snapshotSet = (() => {
+    if (o.pipeline) return core.contentOf(run).files?.length ? new Set(core.contentOf(run).files) : null;
+    if (!o.files) return null;
+    let doc;
+    try {
+      doc = JSON.parse(fs.readFileSync(o.files, "utf8"));
+    } catch (e) {
+      broken(`${o.files}: ${e.message}`);
+    }
+    const list = Array.isArray(doc) ? doc : isObject(doc) && Array.isArray(doc.files) ? doc.files : null;
+    if (!list || !list.every(isText)) {
+      broken(`${o.files}: a file-list snapshot is a list of names, or {"files": [...]}`);
+    }
+    return new Set(list);
+  })();
+
+  let replies = null, bin = "", samples = "", links = [];
+  if (o.replies) {
+    try {
+      replies = JSON.parse(fs.readFileSync(o.replies, "utf8"));
+    } catch (e) {
+      broken(`${o.replies}: ${e.message}`);
+    }
+  } else {
+    if (!o.pipeline && !o.model) broken(`--model names the model to grade.\n\n${USAGE}`);
+    links = connections.map((c, i) => {
+      if (!String(c.model || "").trim()) broken(`stage ${i + 1}'s connection names no model.`);
+      const to = reach(c, o.pipeline ? core.keyVar(c.id) : "EVAL_API_KEY");
+      // Once here rather than once per image: a llama.cpp stage on a
+      // hosted model is a request that cannot be built at all.
+      try {
+        core.connectionRequest(c, "", null, to.base, to.key);
+      } catch (e) {
+        broken(`stage ${i + 1}: ${e.message}`);
+      }
+      return to;
+    });
+    samples = o.samples || o.source || firstFile(path.join(HERE, "samples"));
+    if (!samples) broken("no images: there is no samples/ beside this script. "
+                       + "--samples or --source names a directory.");
+    const allowed = itemsOnly ?? graded;
+    // $snapshotSet above: only a Source run sees it; --replies reads no file.
+    const inSnapshot = c => !snapshotSet || snapshotSet.has(c.filename);
+    // Only the run's own image items need ImageMagick: a text-only Source
+    // -- the .txt, .csv case -- runs with nothing on PATH but the node
+    // running this, which is the point of having text in the registry.
+    const pictures = allowed.filter(c => inSnapshot(c) && HANDLERS.get(extOf(c.filename)) === "image");
+    if (pictures.length) {
+      bin = imageMagick();
+      if (!bin) broken("a run over images needs ImageMagick, to cap each "
+                     + "one to the pixel budget the tab caps it to on a canvas. "
+                     + "Install it, or use --replies to score recorded replies "
+                     + "without a model.");
+    }
+  }
+
+  // ---- Progress: items on a stream of their own ---------------------------
+  // §5's JSON lines, one per finished item, for a caller to tail. Which
+  // stream is decided by where the final report goes: if the report is on
+  // stdout, the lines take the other one, so neither can swallow the other.
+  // With --progress the human report moves to stderr regardless, so a line
+  // on stdout is an item and nothing else.
+  const progress = o.progress ? (o.json === "-" ? process.stderr : process.stdout) : null;
+
+  const rows = [];
+  const unrun = [];
+
+  // The work: everything graded except a single-item run's one case, and,
+  // on a resume, minus the earlier attempt's finished items. Those are
+  // carried whole -- their verdicts already happened and a resume that
+  // recomputed them would be deciding the numbers twice.
+  const toRun = (itemsOnly ?? graded).filter(c => !carried?.has(c.id));
+  if (carried) {
+    for (const [id, r] of carried) {
+      if (graded.some(c => c.id === id)) rows.push(r);
+    }
+  }
+
+  // Only files the snapshot names are read from the Source.
+  const wanted = c => !snapshotSet || snapshotSet.has(c.filename);
+  // The cancel marker is checked between items, so a cancel never loses the
+  // reply in flight: the item that was being asked finishes, and the rest
+  // are unrun rather than half-asked.
+  let cancelled = false, done = 0;
+  for (const kase of toRun) {
+    if (o.cancelFile && fs.existsSync(o.cancelFile)) { cancelled = true; break; }
+    done++;
+    let item, calls;
+    if (replies) {
+      const got = replies[kase.id];
+      const said = isText(got) ? [got] : got;
+      if (!Array.isArray(said) || !said.every(isText)) {
+        unrun.push(`${kase.id}: no recorded reply`);
+        continue;
+      }
+      if (said.length !== stages.length) {
+        unrun.push(`${kase.id}: ${said.length} recorded repl${said.length === 1 ? "y" : "ies"} `
+                 + `for ${stages.length} stages`);
+        continue;
+      }
+      calls = connections.map((c, i) => async () => ({ raw: said[i], ms: 0, conn: c.name }));
+    } else {
+      if (!wanted(kase)) {
+        unrun.push(`${kase.id}: ${kase.filename} is not in the run's file list`);
+        continue;
+      }
+      const file = path.join(samples, kase.filename);
+      if (!fs.existsSync(file)) {
+        unrun.push(`${kase.id}: ${kase.filename} is not in ${samples}`);
+        continue;
+      }
+      try {
+        item = itemContent(bin, file, connections[0]);
+      } catch (e) {
+        unrun.push(`${kase.id}: ${kase.filename} could not be prepared — ${e.message}`);
+        continue;
+      }
+      calls = callsFor(connections, links, item?.text ?? null);
+    }
+
+    const res = await core.runPipeline(stages, item?.dataUrl ?? null, calls,
+                                       { tokens, text: item?.text ?? null });
+    const s = core.scoreCase(kase, res);
+    // A file with no mapper ran anyway; the flag keeps the transcript honest
+    // about what the model was actually shown. §4: stated, not hidden.
+    if (item?.flags.length) res.transcript.unshift({ flag: item.flags.join(" ") });
+    rows.push({
+      id: kase.id, filename: kase.filename, half: kase.half,
+      pass: s.pass, score: s.score, discarded: s.discarded, reasons: s.reasons,
+      found: s.found, missed: s.missed, invented: s.invented, unmet: s.unmet,
+      under: s.under, over: s.over, count: s.count,
+      watchFound: s.watchFound, watchTotal: s.watchTotal,
+      terms: res.terms || [], ms: res.ms,
+      // What a file type was to this run -- "image", "text", "unmapped".
+      type: item?.kind ?? null,
+      // What the model actually said. #248 wants a failure written out as a
+      // task an agent can act on, and "missed water" without the reply that
+      // missed it is not one.
+      reply: res.transcript?.[res.transcript.length - 1]?.got ?? "",
+      // Every stage's prompt, reply and time, as the page's transcript shows
+      // them: the final reply alone cannot say which stage went wrong.
+      transcript: res.transcript,
+    });
+    if (progress) {
+      progress.write(JSON.stringify({
+        event: "item", id: kase.id, filename: kase.filename, n: rows.length,
+        pass: s.pass, score: s.score, discarded: s.discarded,
+        error: s.discarded ? s.reasons[0]?.replace(/^Error - /, "") ?? null : null,
+        ms: res.ms,
+      }) + "\n");
+    }
+  }
+  // A cancelled run says which items never ran, so the report names what it
+  // did not cover rather than letting the tail guess.
+  if (cancelled) {
+    for (const kase of toRun.slice(done)) {
+      unrun.push(`${kase.id}: not run — cancelled after the item in flight`);
+    }
+  }
+
+  // The sum is made here, from the rows, in one place: a fresh run's rows
+  // and a resume's carried ones are scored the same way, and no number is
+  // worked out twice.
+  const tally = core.emptyTally();
+  for (const r of rows) {
+    tally.ran++;
+    if (r.pass) tally.passed++;
+    tally.found += r.found.length;
+    tally.of += r.found.length + r.missed.length + r.invented.length;
+  }
+
+  // Anything short of the whole graded set is not a result. This is the
+  // constraint in the issue: a set that did not run must not read as one that
+  // passed, and 49 items absent with the fiftieth green is exactly how
+  // that happens.
+  const complete = graded.length > 0 && unrun.length === 0;
+  const verdict = !complete ? "incomplete" : tally.passed === tally.ran ? "pass" : "fail";
+
+  const report = {
+    runner: "tools/prompt-lab/run-evals.js",
+    ranAt: new Date().toISOString(),
+    dataset: inRepo(o.dataset),
+    prompt,
+    ...(o.pipeline ? { pipeline: {
+      file: inRepo(o.pipeline), name: run.name || null,
+      stages: stages.map((s, i) => {
+        const { id, ...connection } = connections[i];
+        return { n: i + 1, kind: s.kind, withImage: s.withImage,
+                 prompt: core.resolvePrompt(s.text, core.tokenSet(tokens, i)), profile: id, connection };
+      }),
+    } } : {}),
+    // Which variable a key came from, never the key.
+    transport: replies ? { kind: "replies", file: o.replies }
+      : o.pipeline ? { kind: "model", stages: links.map((to, i) => ({
+          n: i + 1, connection: to.name, model: connections[i].model,
+          endpoint: to.base, keyFrom: to.key ? to.from : null })) }
+      : { kind: "model", model: o.model, endpoint: links[0].base },
+    // How long a request could run, as given; a higher ask is capped at
+    // the relay's 600.
+    timeout: seconds,
+    // The modes a server-side run reaches the worker through.
+    ...(o.item ? { item: o.item } : {}),
+    ...(o.resume ? { resume: o.resume, carried: carried.size } : {}),
+    ...(snapshotSet ? { files: o.files || inRepo(o.pipeline),
+                        filesListed: snapshotSet.size } : {}),
+    ...(o.source ? { source: inRepo(o.source) } : {}),
+    verdict,
+    graded: (itemsOnly ?? graded).length,
+    ungraded: set.length - graded.length,
+    ran: tally.ran,
+    passed: tally.passed,
+    failed: tally.ran - tally.passed,
+    unrun,
+    terms: { found: tally.found, of: tally.of, percent: core.tallyPercent(tally) },
+    cases: rows,
+  };
+
+  const toStdout = o.json === "-";
+  // With --progress the chosen stream stays pure JSON, so what was prose
+  // moves to stderr whether the report took stdout or not.
+  const say = o.progress ? console.error : toStdout ? console.error : console.log;
+  const pct = core.tallyPercent(tally);
+
+  say(`${report.runner} — ${report.dataset}`);
+  if (o.pipeline) {
+    say(`pipeline ${report.pipeline.file}${run.name ? ` — ${run.name}` : ""}`);
+    if (replies) say(`recorded replies from ${o.replies}`);
+    for (const [i, s] of report.pipeline.stages.entries()) {
+      const to = links[i];
+      say(`stage ${s.n} · ${s.kind} · ${s.withImage ? "with the image" : "text only"}`
+        + (to ? ` · ${to.name}: ${s.connection.model} at ${to.base}`
+              + (to.key ? ` · key from $${to.from}` : "") : ""));
+      say(`  ${s.prompt.replace(/\n/g, "\n  ")}`);
+    }
+  } else {
+    say(replies ? `recorded replies from ${o.replies}`
+                : `${o.model} at ${links[0].base}`);
+    say(`prompt: ${prompt}`);
+  }
+  say("");
+  for (const r of rows.filter(r => !r.pass)) {
+    const n = r.score == null ? "   —" : `${String(Math.round(r.score * 100)).padStart(3)}%`;
+    say(`fail ${r.id.padEnd(16)} ${n}  ${r.reasons.join(" · ")}`);
+  }
+  // Capped: a run where nothing was found says so in one screen, and the
+  // JSON report carries all of them for anything that needs the list.
+  for (const u of unrun.slice(0, 10)) say(`----  ${u}`);
+  if (unrun.length > 10) say(`----  and ${unrun.length - 10} more, all of them in the JSON report`);
+  say("");
+  say(`${tally.passed} of ${report.graded} graded cases passed`
+    + ` · ${tally.found} of ${tally.of} terms${pct == null ? "" : ` (${pct}%)`}`
+    + ` · ${report.ungraded} not graded yet`);
+  if (!complete) {
+    say(rows.length === 0 && !unrun.length
+      ? "NOTHING RAN: the set grades no items. That is not a pass."
+      : `NOTHING SCORED for ${unrun.length} of ${report.graded} graded cases, `
+        + `so this is a partial run and not a result.`);
+  }
+
+  if (o.json) {
+    const text = JSON.stringify(report, null, 2);
+    if (toStdout) process.stdout.write(text + "\n");
+    else fs.writeFileSync(o.json, text + "\n");
+  }
+
+  process.exitCode = !complete ? EXIT.incomplete
+                   : report.failed ? EXIT.failed : EXIT.passed;
+}
+
+main().catch(e => {
+  if (e instanceof Stop) {
+    if (e.message) console.error(e.message);
+    process.exitCode = e.code;
+    return;
+  }
+  console.error(`the run failed: ${e?.stack || e}`);
+  process.exitCode = EXIT.broken;
+});