evals-lab 0.4.0 → 0.5.0

package/bin/run.js

@@ -0,0 +1,531 @@
+// `evals-lab run <bundle>`: an Export for CI bundle (#254, docs/pipeline-yaml.md)
+// run on this machine, with no lab (#255).
+//
+// The bundle is read into a run document by the core's readBundle -- the
+// import and the resolve the page does at submit -- and handed to the worker
+// a lab's queue runs, lab/run-evals.js --run, so a run here and the same run
+// in a lab are one code path. What this adds is what a queue would have:
+//
+//   THE ITEMS. --items, else the bundle's items/. Every file there runs, as
+//   every file of a Source does; an item a case names that is not there runs
+//   too, as Missing, so the run is incomplete (exit 2) rather than quietly
+//   grading fewer cases.
+//   THE PLUGINS. The bundle's plugins/, registered here before it is read and
+//   stamped on the run so the worker loads the same ones.
+//   THE KEYS. Nothing: the worker reads each from $EVALSLAB_API_KEY_<SLUG>.
+//
+// The exit code is the worker's: 0 pass, 1 an eval failed, 2 incomplete, 3
+// broken or refused. The table goes to stderr, so stdout is the JSON report's
+// alone when --json is -. --summary appends the same as Markdown, for a CI
+// page that shows it: GitHub's $GITHUB_STEP_SUMMARY is a file to append to.
+//
+// `evals-lab run --lab <url> --pipeline <name>` (#256) is the same command
+// against a running lab instead: the pipeline is the lab's, resolved by the
+// core as the page resolves it at submit, and queued there, so the run is in
+// that lab's History. The keys stay in the lab -- the run asks the lab's own
+// profiles, which the lab serves without their keys -- and with --wait the
+// exit code is the verdict the lab kept on the run's row (#252).
+const { spawn } = require("child_process");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { pathToFileURL } = require("url");
+
+const EXIT = { broken: 3 };
+
+const USAGE = `Usage: evals-lab run <bundle-dir | pipeline.yaml> [options]
+       evals-lab run --lab <url> --pipeline <name> [--wait] [options]
+
+  --items <dir>      the items the pipeline's Source reads. Default: the
+                     bundle's items/.
+  --json <file|->    write the JSON report. "-" means stdout.
+  --junit <file>     write JUnit XML: a testsuite per Target and eval, a
+                     testcase per item.
+  --summary <file>   add the results to <file> as Markdown: each Target and
+                     eval's pass rate, then the items that failed. In GitHub
+                     Actions, --summary "$GITHUB_STEP_SUMMARY".
+  --min-pass <0..1>  an eval passes when at least this share of its items
+                     pass. It relaxes each eval's own rule, never tightens it.
+  --progress         a line on stderr as each item finishes.
+  --lab <url>        run on a running lab instead: its pipeline, its profiles
+                     and their keys, and the run in its History. LAB_PASSWORD
+                     is sent when it is set.
+  --pipeline <name>  the lab's pipeline to run, by its name or its id.
+  --wait             wait for the lab's run to finish, and exit with its
+                     verdict. Without it the run's id is printed once it is
+                     queued. --json, --junit, --summary, --min-pass and
+                     --progress need it.
+  --help             this.
+
+A bundle's Target profile keys come from $EVALSLAB_API_KEY_<SLUG>. Exit
+codes: 0 pass (or queued, without --wait), 1 an eval failed, 2 incomplete (an
+item missing, one that did not run, or the run cancelled), 3 broken or
+refused.`;
+
+// A run refused before it starts; [usage] when it was the command line.
+class Refused extends Error {
+  constructor(message, usage = false) { super(message); this.usage = usage; }
+}
+
+// What the command line asks for, or a Refused saying what is wrong with it.
+function runOptions(argv) {
+  const o = { bundle: null, items: null, json: null, junit: null, summary: null, minPass: null, progress: false,
+              lab: null, pipeline: null, wait: false };
+  const value = { "--items": "items", "--json": "json", "--junit": "junit", "--summary": "summary", "--min-pass": "minPass",
+                  "--lab": "lab", "--pipeline": "pipeline" };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--help" || a === "-h") o.help = true;
+    else if (a === "--progress") o.progress = true;
+    else if (a === "--wait") o.wait = true;
+    else if (value[a]) {
+      if (argv[i + 1] === undefined) throw new Refused(`${a} needs a value`, true);
+      o[value[a]] = argv[++i];
+    } else if (a.startsWith("-") && a !== "-") throw new Refused(`unknown option ${a}`, true);
+    else if (o.bundle != null) throw new Refused(`one bundle at a time: ${o.bundle} and ${a}`, true);
+    else o.bundle = a;
+  }
+  if (o.help) return o;
+  if (o.lab != null || o.pipeline != null) {
+    if (o.bundle != null) throw new Refused(`a bundle or a lab, not both: ${o.bundle} and --lab`, true);
+    if (o.lab == null) throw new Refused("--pipeline names a lab's pipeline: name the lab with --lab", true);
+    if (o.pipeline == null) throw new Refused("name the lab's pipeline to run with --pipeline", true);
+    if (!/^https?:\/\/[^/]/i.test(o.lab)) throw new Refused(`--lab ${o.lab} is not an http or https address`, true);
+    if (o.items != null) throw new Refused("--items is a bundle's: a lab's run reads its own Source", true);
+    const waits = ["json", "junit", "summary", "minPass", "progress"].filter(k => o[k] != null && o[k] !== false);
+    if (waits.length && !o.wait) {
+      throw new Refused(`--${waits[0].replace("minPass", "min-pass")} reads the finished run: add --wait`, true);
+    }
+  } else if (o.wait) {
+    throw new Refused("--wait is for a lab's run: a bundle's always waits", true);
+  } else if (o.bundle == null) {
+    throw new Refused("name the bundle: its directory, or its pipeline.yaml", true);
+  }
+  if (o.minPass != null) {
+    const n = Number(o.minPass);
+    if (o.minPass.trim() === "" || !(n >= 0 && n <= 1)) throw new Refused(`--min-pass ${o.minPass} is not a share from 0 to 1`, true);
+  }
+  return o;
+}
+
+// The bundle's text files by path inside it -- what readBundle reads -- and
+// the directory they sit in. [where] is the directory or its pipeline.yaml.
+function bundleFiles(where) {
+  let stat;
+  try { stat = fs.statSync(where); } catch { throw new Refused(`${where} is not there`); }
+  const dir = stat.isDirectory() ? where : path.dirname(where);
+  const pipeline = stat.isDirectory() ? path.join(dir, "pipeline.yaml") : where;
+  if (!fs.existsSync(pipeline)) throw new Refused(`${dir} holds no pipeline.yaml: name an Export for CI bundle`);
+  const files = { "pipeline.yaml": fs.readFileSync(pipeline, "utf8") };
+  const profiles = path.join(dir, "profiles.yaml");
+  if (fs.existsSync(profiles)) files["profiles.yaml"] = fs.readFileSync(profiles, "utf8");
+  const datasets = path.join(dir, "datasets");
+  if (fs.existsSync(datasets)) {
+    for (const d of fs.readdirSync(datasets, { withFileTypes: true })) {
+      if (d.isFile()) files[`datasets/${d.name}`] = fs.readFileSync(path.join(datasets, d.name), "utf8");
+    }
+  }
+  return { dir, files };
+}
+
+// The bundle's plugins/<id>/<version>/, each registered into [core] as the
+// worker registers it, and the stamps that have the worker load the same.
+async function loadPlugins(core, dir) {
+  const root = path.join(dir, "plugins");
+  if (!fs.existsSync(root)) return [];
+  const stamps = [];
+  for (const id of fs.readdirSync(root, { withFileTypes: true }).filter(d => d.isDirectory())) {
+    for (const version of fs.readdirSync(path.join(root, id.name), { withFileTypes: true }).filter(d => d.isDirectory())) {
+      const at = path.join(root, id.name, version.name);
+      try {
+        const manifest = JSON.parse(fs.readFileSync(path.join(at, "manifest.json"), "utf8"));
+        const mod = await import(pathToFileURL(path.join(at, manifest.entry)).href);
+        await (mod.default ?? mod.register)(core.pluginHost(manifest.id));
+      } catch (e) {
+        throw new Refused(`the plugin in ${at} did not load: ${e.message}`);
+      }
+      stamps.push({ id: id.name, version: version.name, sha256: "" });
+    }
+  }
+  return stamps;
+}
+
+// The files of an items directory, as a Source names its own: by name, no
+// directories, nothing hidden.
+function itemsIn(dir) {
+  return fs.readdirSync(dir, { withFileTypes: true })
+    .filter(d => d.isFile() && !d.name.startsWith(".")).map(d => d.name).sort();
+}
+
+const word = v => v.charAt(0).toUpperCase() + v.slice(1);
+
+/** Why an item's score failed, in a line: its failing metrics, else what it missed. */
+function reasonOf(score) {
+  const off = (score.metrics || []).filter(m => !m.pass).map(m => `${m.label}: ${m.reason}`);
+  if (off.length) return off.join(" · ");
+  const missed = (score.missed || []).map(r => (Array.isArray(r) ? r.join(" or ") : r));
+  return missed.length ? `missed ${missed.join(", ")}` : "failed";
+}
+
+// The report as a person reads it: each Target's evals and their verdicts,
+// the items each failed, and the items that never ran.
+function table(core, run, report, missing) {
+  const lines = [`${run.name || "the pipeline"}`];
+  const items = report.run.items || [];
+  const MAX = 10;
+  (report.run.verdicts || []).forEach((evals, i) => {
+    for (const [id, e] of Object.entries(evals)) {
+      const n = e.ran != null ? `${e.passed} of ${e.ran} passed` : e.detail || "";
+      lines.push(`  ${word(e.verdict).padEnd(11)} ${core.targetLabel(run, i)} › ${e.name}${n ? `  ${n}` : ""}`);
+      if (e.verdict !== "fail" || e.ran == null) continue;
+      const failed = items.filter(it => {
+        const s = it?.scenarios?.[i]?.scores?.[id];
+        return s && !core.isSkipped(s) && !s.pass;
+      });
+      for (const it of failed.slice(0, MAX)) {
+        lines.push(`  ${"".padEnd(11)}   ${it.name ?? "the text"}: ${reasonOf(it.scenarios[i].scores[id])}`);
+      }
+      if (failed.length > MAX) lines.push(`  ${"".padEnd(11)}   and ${failed.length - MAX} more, in the JSON report`);
+    }
+  });
+  const gone = new Set(missing);
+  for (const name of missing) lines.push(`  ${"Missing".padEnd(11)} ${name}`);
+  for (const it of items) {
+    if (it?.unrun && !gone.has(it.name)) lines.push(`  ${"Unrun".padEnd(11)} ${it.name ?? "the text"}: ${it.unrun}`);
+  }
+  const ran = items.filter(it => it && !it.unrun).length;
+  lines.push(`${word(report.verdict)}${report.cancelled ? " (cancelled)" : ""} · ${ran} of ${items.length} items ran`
+    + (report.minPass != null ? ` · min pass ${report.minPass}` : ""));
+  return lines.join("\n");
+}
+
+// A table cell's text: a pipe or a line break would end the cell early.
+const cell = v => String(v).replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " ");
+
+// The report as a CI page shows it, in Markdown: the run's verdict, a row per
+// Target and eval with its pass rate, then the items each failed and those
+// that never ran -- the table's content, laid out for a page.
+function markdown(core, run, report, missing) {
+  const items = report.run.items || [];
+  const ran = items.filter(it => it && !it.unrun).length;
+  const lines = [`### ${cell(run.name || "the pipeline")}: ${word(report.verdict)}${report.cancelled ? " (cancelled)" : ""}`, "",
+    `${ran} of ${items.length} items ran` + (report.minPass != null ? ` · min pass ${report.minPass}` : ""), "",
+    "| Target | Eval | Verdict | Passed |", "|---|---|---|---|"];
+  const failures = [];
+  (report.run.verdicts || []).forEach((evals, i) => {
+    for (const [id, e] of Object.entries(evals)) {
+      const rate = e.ran ? `${e.passed} of ${e.ran} (${Math.round(100 * e.passed / e.ran)}%)` : cell(e.detail || "");
+      lines.push(`| ${cell(core.targetLabel(run, i))} | ${cell(e.name)} | ${word(e.verdict)} | ${rate} |`);
+      if (e.verdict !== "fail") continue;
+      const failed = e.ran == null ? [] : items.filter(it => {
+        const s = it?.scenarios?.[i]?.scores?.[id];
+        return s && !core.isSkipped(s) && !s.pass;
+      });
+      failures.push({ title: `${core.targetLabel(run, i)} › ${e.name}`, failed, i, id });
+    }
+  });
+  const MAX = 25;
+  for (const f of failures) {
+    if (!f.failed.length) continue;
+    lines.push("", `#### Failed: ${cell(f.title)}`, "");
+    for (const it of f.failed.slice(0, MAX)) {
+      lines.push(`- \`${cell(it.name ?? "the text")}\`: ${cell(reasonOf(it.scenarios[f.i].scores[f.id]))}`);
+    }
+    if (f.failed.length > MAX) lines.push(`- and ${f.failed.length - MAX} more, in the JSON report`);
+  }
+  const gone = new Set(missing);
+  const unrun = [...missing.map(name => `- \`${cell(name)}\`: Missing`),
+    ...items.filter(it => it?.unrun && !gone.has(it.name)).map(it => `- \`${cell(it.name ?? "the text")}\`: ${cell(it.unrun)}`)];
+  if (unrun.length) lines.push("", "#### Not run", "", ...unrun);
+  return lines.join("\n") + "\n\n";
+}
+
+// Each line the worker adds to [file] as an item finishes, said on stderr.
+function follow(file, total, say) {
+  let at = 0, rest = "";
+  const read = () => {
+    let text;
+    try {
+      const fd = fs.openSync(file, "r");
+      const buf = Buffer.alloc(Math.max(0, fs.fstatSync(fd).size - at));
+      fs.readSync(fd, buf, 0, buf.length, at);
+      fs.closeSync(fd);
+      at += buf.length;
+      text = rest + buf.toString("utf8");
+    } catch { return; }
+    const lines = text.split("\n");
+    rest = lines.pop();
+    for (const line of lines) {
+      let row;
+      try { row = JSON.parse(line); } catch { continue; }
+      if (row.event) continue;
+      say(`  ${row.item + 1} of ${total}  ${row.name ?? "the text"}${row.unrun ? `  ${row.unrun}` : ""}`);
+    }
+  };
+  const timer = setInterval(read, 250);
+  return () => { clearInterval(timer); read(); };
+}
+
+// ---- a lab's run (#256) -------------------------------------------------------
+
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+// How long between reads of a queued run, and how many reads in a row may
+// go unanswered -- a lab restarting under a long run -- before it is given up.
+const POLL_MS = 500;
+const POLL_MISSES = 20;
+
+// The lab's API at [base], as [env]'s LAB_PASSWORD opens it: HTTP Basic, the
+// password alone, as server.py takes it. A refusal says what the lab said.
+function labApi(base, env) {
+  const auth = env.LAB_PASSWORD
+    ? { Authorization: `Basic ${Buffer.from(`:${env.LAB_PASSWORD}`).toString("base64")}` } : {};
+  return async (method, route, body, { missing = false } = {}) => {
+    let res;
+    try {
+      res = await fetch(base + route, {
+        method, redirect: "manual",
+        headers: { ...auth, ...(body !== undefined ? { "Content-Type": "application/json" } : {}) },
+        ...(body !== undefined ? { body: JSON.stringify(body) } : {}),
+      });
+    } catch (e) {
+      throw Object.assign(new Refused(`${base} did not answer: ${e.cause?.message ?? e.message}`), { unanswered: true });
+    }
+    const text = await res.text();
+    if (res.status === 401) {
+      throw new Refused(env.LAB_PASSWORD ? `${base} did not take LAB_PASSWORD` : `${base} asks for a password: set LAB_PASSWORD`);
+    }
+    if (res.status === 404 && missing) return null;
+    let json = null;
+    try { json = JSON.parse(text); } catch { /* a plain body, said as it is */ }
+    if (!res.ok) throw new Refused(`${base} refused ${method} ${route}: ${json?.error ?? `${res.status} ${text.trim().slice(0, 200)}`}`);
+    if (json === null) throw new Refused(`${base} answered ${method} ${route} with no JSON: is it a lab?`);
+    return json;
+  };
+}
+
+// The lab's saved pipeline [name] names -- by name, else by id -- or a
+// Refused saying which ones it has.
+function pipelineNamed(saved, name, base) {
+  const byName = saved.filter(w => w?.name === name);
+  const found = byName.length ? byName : saved.filter(w => w?.id === name);
+  if (!found.length) {
+    throw new Refused(`${base} has no pipeline named ${JSON.stringify(name)}`
+      + (saved.length ? `: it has ${saved.map(w => JSON.stringify(w?.name)).join(", ")}` : ""));
+  }
+  if (found.length > 1) {
+    throw new Refused(`${base} has ${found.length} pipelines named ${JSON.stringify(name)}: `
+      + `name one by its id (${found.map(w => w.id).join(", ")})`);
+  }
+  return found[0];
+}
+
+// The row's verdicts with --min-pass relaxing each eval read item by item,
+// as the worker's --min-pass does: it only ever turns a fail into a pass.
+const relaxed = (verdicts, minPass) => minPass == null ? verdicts : verdicts.map(evals =>
+  Object.fromEntries(Object.entries(evals).map(([id, e]) =>
+    [id, e.verdict === "fail" && e.ran && e.passed / e.ran >= minPass ? { ...e, verdict: "pass" } : e])));
+
+// The JUnit suites of a lab's run, from its row: what the worker's runSuites
+// writes from its own outcomes, read here from the verdicts and scores the
+// row kept.
+function rowSuites(core, run, verdicts, items) {
+  return verdicts.flatMap((evals, i) => Object.entries(evals).map(([id, e]) => {
+    const name = `${core.targetLabel(run, i)} › ${e.name}`;
+    if (e.ran == null) {
+      // Settled over the run rather than item by item: one case, the run.
+      const c = { name: e.name };
+      if (e.verdict === "skipped") c.skipped = "an earlier eval failed";
+      else if (e.verdict === "none") c.skipped = "nothing to read";
+      else if (e.verdict === "incomplete") c.error = "the run did not finish";
+      else if (e.verdict === "fail") c.failure = e.detail || "failed";
+      return { name, cases: [c] };
+    }
+    return { name, cases: items.map((it, x) => {
+      const c = { name: it?.name ?? (items.length > 1 ? `item ${x + 1}` : "the text") };
+      const read = it?.scenarios?.[i]?.scores?.[id];
+      if (!it) c.error = "not run";
+      else if (it.unrun) c.error = it.unrun;
+      else if (!read) c.skipped = "not graded";
+      else if (core.isSkipped(read)) c.skipped = "an earlier eval failed";
+      else if (!read.pass) c.failure = reasonOf(read);
+      return c;
+    }) };
+  }));
+}
+
+/** [o]'s pipeline, queued on its lab; with --wait, followed to its verdict. */
+async function runOnLab(core, o, { env, stdout, say }) {
+  const base = o.lab.replace(/\/+$/, "");
+  const call = labApi(base, env);
+
+  // What the page reads before it submits: the lab's pipelines and Target
+  // profiles, the Source the pipeline reads and the eval groups it may link.
+  const docs = (await call("GET", "/api/state")).docs ?? {};
+  const setup = docs["promptlab.profiles"]?.body ?? {};
+  const profiles = Array.isArray(setup.list) ? setup.list : [];
+  const saved = docs["promptlab.workflows"]?.body?.list;
+  const found = pipelineNamed(Array.isArray(saved) ? saved : [], o.pipeline, base);
+  const types = new Map(profiles.map(p => [p.id, p.type]));
+  const doc = core.upgradePipeline(found.work, { profileType: id => types.get(id) });
+  const g = profiles.find(p => p.id === setup.grader);
+  const grader = g ? { id: g.id, name: g.name } : null;
+  const content = core.contentOf(doc);
+  let source = null;
+  if (content?.type === "source" && content.ref?.id) {
+    source = await call("GET", `/api/sources/${encodeURIComponent(content.ref.id)}`, undefined, { missing: true });
+  }
+  const files = source ? source.files.map(f => f.name) : undefined;
+  const { datasets } = await call("GET", "/api/datasets");
+  const problems = core.validatePipeline(doc, {
+    profiles: id => profiles.find(p => p.id === id) ?? null,
+    sources: id => (source && id === content.ref.id ? { name: source.name, type: source.type, files } : null),
+    grader, groups: datasets ?? [],
+  });
+  if (problems.length) throw new Refused(`${found.name} cannot run: ${problems.join("; ")}`);
+  // As RunsTab's submit resolves it: slugged, so the run carries the slug
+  // its keys are spelt from (#253), and keyless -- the lab fills each key in
+  // from its own store.
+  const slugged = core.withSlugs(profiles);
+  const run = core.resolvePipeline(doc, {
+    profiles: id => slugged.find(p => p.id === id) ?? null, grader, ...(files ? { files } : {}),
+  });
+
+  const queued = (await call("POST", "/api/queue", run)).run;
+  say(`evals-lab run: ${found.name} is queued on ${base} as run ${queued.id}`);
+  if (!o.wait) {
+    stdout.write(`${queued.id}\n`);
+    return 0;
+  }
+
+  let row = queued, misses = 0, said = null;
+  while (row.status === "queued" || row.status === "running") {
+    await sleep(POLL_MS);
+    try {
+      row = await call("GET", `/api/queue/${encodeURIComponent(queued.id)}`);
+      misses = 0;
+    } catch (e) {
+      if (!e.unanswered || ++misses >= POLL_MISSES) throw e;
+      continue;
+    }
+    const p = row.progress ?? {};
+    if (o.progress && row.status === "running" && p.n !== said) {
+      said = p.n;
+      say(`  ${p.n} of ${p.total}${p.current ? `  ${p.current}` : ""}`);
+    }
+  }
+
+  // A run the lab could not finish has no verdict to exit with.
+  if (row.status === "failed") throw new Refused(`run ${row.id} failed on ${base}: ${row.error || "the lab said no more"}`);
+  const finished = row.status === "done";
+  if (finished && !row.verdict) {
+    throw new Refused(`${base} keeps no verdict on its runs: it is older than evals-lab run --lab`);
+  }
+  const minPass = o.minPass == null ? null : Number(o.minPass);
+  const verdicts = row.verdicts ? relaxed(row.verdicts, minPass) : [];
+  // The lab's own verdict; --min-pass only turns a fail into a pass, and
+  // only once no eval fails under it.
+  const failing = verdicts.some(v => Object.values(v).some(e => e.verdict === "fail"));
+  const verdict = !finished ? "incomplete" : row.verdict === "fail" && !failing ? "pass" : row.verdict;
+  const items = core.upgradeResults(Array.isArray(row.results) ? row.results : []);
+  const report = {
+    runner: "evals-lab run --lab", lab: base, id: row.id, status: row.status, verdict,
+    ...(row.status === "cancelled" ? { cancelled: true } : {}),
+    ...(minPass != null ? { minPass } : {}),
+    run: { name: row.snapshot?.name ?? null, evals: row.snapshot?.evals ?? [], verdicts, items },
+  };
+  say(table(core, row.snapshot ?? run, report, []));
+  if (o.summary) fs.appendFileSync(o.summary, markdown(core, row.snapshot ?? run, report, []));
+  const text = JSON.stringify(report, null, 2) + "\n";
+  if (o.json === "-") stdout.write(text);
+  else if (o.json) fs.writeFileSync(o.json, text);
+  if (o.junit) {
+    fs.writeFileSync(o.junit, core.junitXml(rowSuites(core, row.snapshot ?? run, verdicts, items), report.runner));
+  }
+  return { pass: 0, fail: 1 }[verdict] ?? 2;
+}
+
+/**
+ * Runs [argv] -- what follows `evals-lab run` -- against the staged lab in
+ * [lab], and resolves to the exit code.
+ */
+async function run(argv, { lab, env = process.env, stdout = process.stdout, stderr = process.stderr } = {}) {
+  const say = text => stderr.write(text + "\n");
+  let tmp = null;
+  try {
+    const o = runOptions(argv);
+    if (o.help) { stdout.write(USAGE + "\n"); return 0; }
+    const corePath = path.join(lab, "evals-core.mjs");
+    if (!fs.existsSync(corePath)) {
+      throw new Refused(`no ${corePath} -- in a checkout the package is staged by \`node npm/build.js\``);
+    }
+    const core = require(corePath);
+    if (o.lab != null) return await runOnLab(core, o, { env, stdout, say });
+    const { dir, files } = bundleFiles(o.bundle);
+    const plugins = await loadPlugins(core, dir);
+
+    // Read once without items, for the items its cases name; then with the
+    // ones there and those, so a case's item that is not there runs as missing.
+    let read = core.readBundle(files);
+    if ("error" in read) throw new Refused(`${o.bundle}: ${read.error}`);
+    let items = null, missing = [];
+    if (core.contentOf(read.run)?.type === "source") {
+      items = o.items ?? path.join(dir, "items");
+      if (!fs.existsSync(items) || !fs.statSync(items).isDirectory()) {
+        throw new Refused(o.items ? `--items ${o.items} is not a directory`
+          : `the pipeline reads a Source's items, and ${dir} holds no items/: name them with --items`);
+      }
+      const there = itemsIn(items);
+      const named = core.contentOf(read.run).files || [];
+      missing = named.filter(n => !there.includes(n));
+      read = core.readBundle(files, { items: [...new Set([...there, ...named])] });
+      if ("error" in read) throw new Refused(`${o.bundle}: ${read.error}`);
+    }
+    const doc = plugins.length ? { ...read.run, plugins } : read.run;
+
+    tmp = fs.mkdtempSync(path.join(os.tmpdir(), "evals-lab-run-"));
+    fs.writeFileSync(path.join(tmp, "run.json"), JSON.stringify(doc));
+    const args = [path.join(lab, "run-evals.js"), "--run", path.join(tmp, "run.json"),
+                  "--json", path.join(tmp, "report.json")];
+    if (items) args.push("--source", items);
+    const ref = core.evalsDataset(read.run);
+    if (ref && read.datasets[ref.id]) {
+      fs.writeFileSync(path.join(tmp, "dataset.json"), JSON.stringify(read.datasets[ref.id]));
+      args.push("--dataset", path.join(tmp, "dataset.json"));
+    }
+    if (plugins.length) args.push("--plugins", path.join(dir, "plugins"));
+    if (o.minPass != null) args.push("--min-pass", o.minPass);
+    if (o.junit) args.push("--junit", path.resolve(o.junit));
+    if (o.progress) args.push("--progress-file", path.join(tmp, "progress.jsonl"));
+
+    const content = core.contentOf(read.run);
+    const total = content?.type === "source" ? (content.files || []).length : 1;
+    const stop = o.progress ? follow(path.join(tmp, "progress.jsonl"), total, say) : () => {};
+    // The worker's own lines are the table's raw form, so they are not
+    // shown; what it says on stderr -- a refusal -- is.
+    const code = await new Promise(resolve => {
+      const child = spawn(process.execPath, args, { stdio: ["ignore", "ignore", "inherit"], env, windowsHide: true });
+      child.on("error", e => { say(`evals-lab run: the worker did not start: ${e.message}`); resolve(EXIT.broken); });
+      child.on("exit", c => resolve(c ?? EXIT.broken));
+    });
+    stop();
+
+    const reportFile = path.join(tmp, "report.json");
+    if (!fs.existsSync(reportFile)) return code || EXIT.broken;
+    const text = fs.readFileSync(reportFile, "utf8");
+    const report = JSON.parse(text);
+    say(table(core, read.run, report, missing));
+    if (o.summary) fs.appendFileSync(o.summary, markdown(core, read.run, report, missing));
+    if (o.json === "-") stdout.write(text);
+    else if (o.json) fs.writeFileSync(o.json, text);
+    return code;
+  } catch (e) {
+    if (!(e instanceof Refused)) throw e;
+    say(`evals-lab run: ${e.message}`);
+    if (e.usage) say(`\n${USAGE}`);
+    return EXIT.broken;
+  } finally {
+    if (tmp) fs.rmSync(tmp, { recursive: true, force: true });
+  }
+}
+
+module.exports = { run, runOptions, USAGE };

package/lab/VERSION

@@ -1 +1 @@
-0.4.0 (2026.10.02-361)
+0.5.0 (2026.10.04-397)

package/lab/demo/pipelines/demo-1.json

@@ -1,5 +1,5 @@
 {
-  "version": 12,
+  "version": 13,
   "id": "demo-1",
   "name": "Demo 1",
   "jobs": [
@@ -94,17 +94,16 @@
     {
       "id": "t1",
       "name": "",
+      "type": "group",
       "continueOnFailure": true,
-      "type": "metrics",
-      "mode": "all",
-      "threshold": null,
-      "grader": null,
-      "dataset": {
+      "group": {
         "id": "demo-1",
         "name": "Demo 1"
       },
-      "over": "item",
-      "metrics": []
+      "pin": null
     }
-  ]
+  ],
+  "pass": {
+    "mode": "all"
+  }
 }

package/lab/demo/pipelines/demo-2.json

@@ -1,5 +1,5 @@
 {
-  "version": 12,
+  "version": 13,
   "id": "demo-2",
   "name": "Demo 2",
   "jobs": [
@@ -94,17 +94,16 @@
     {
       "id": "t1",
       "name": "",
+      "type": "group",
       "continueOnFailure": true,
-      "type": "metrics",
-      "mode": "all",
-      "threshold": null,
-      "grader": null,
-      "dataset": {
+      "group": {
         "id": "demo-2",
         "name": "Demo 2"
       },
-      "over": "item",
-      "metrics": []
+      "pin": null
     }
-  ]
+  ],
+  "pass": {
+    "mode": "all"
+  }
 }