evals-lab 0.4.1 → 0.5.0

package/lab/run-evals.js

@@ -6,15 +6,19 @@
 // what it did twice: a short human report, and a JSON one with every verdict
 // in it. The exit code is the third statement and the coarsest:
 //
-//   0  every graded case passed
-//   1  the set ran and at least one case failed
-//   2  the set did not run in full -- nothing is graded, or an image or a
-//      recorded reply was missing. NOT a pass. An eval set that scores well
-//      because it never ran is the failure this whole ticket is about.
+//   0  every eval passed
+//   1  the run ran and at least one eval failed
+//   2  the run did not run in full -- nothing is graded, an image or a
+//      recorded reply was missing, or it was cancelled. NOT a pass. An eval
+//      set that scores well because it never ran is the failure this whole
+//      ticket is about.
 //   3  the run could not be attempted: bad arguments, no ImageMagick, no set.
 //
-// The gating policy -- which of those a pull request may merge on -- is #250's
-// to decide, and it has the JSON to decide it from. This only reports.
+// The same in every mode, --run and --rescore included, so CI can gate on it
+// (#251): each eval is held to its own rule -- every item it reads passes, or
+// a whole-run eval's verdict -- and --min-pass relaxes the per-item rule to a
+// share. The report's `verdict` (pass, fail, incomplete) says the same as the
+// code, and --junit writes it for a CI test panel.
 //
 // Ollama is firewalled to a handful of hosts, so a live run has to happen on
 // one of them. That is a fact about the network and not something a flag here
@@ -64,7 +68,10 @@ const { execFileSync } = require("child_process");
 const core = require("./evals-core.mjs");
 
 const HERE = __dirname;
-const ROOT = path.join(HERE, "..", "..");
+const ROOT = HERE;
+
+// What a report names as its runner: this file, from the lab's root.
+const RUNNER = "run-evals.js";
 
 // Tagger.PIXELS_720P, as an area rather than a longest edge: llama.cpp slices
 // into 448px tiles and the tile COUNT is what costs. The same budget the tab
@@ -82,11 +89,12 @@ const EXIT = { passed: 0, failed: 1, incomplete: 2, broken: 3 };
 // single-flight queue for good; `--timeout` can shorten it, never lengthen.
 const REQUEST_CAP = 600;
 
-const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
+const USAGE = `Usage: node run-evals.js [options]
 
   --model <id>        the model to grade. Required for a live run.
   --url <base>        an OpenAI-shaped endpoint. Default: $OLLAMA_URL, else
-                      Ollama on this machine. A key comes from $EVAL_API_KEY, never argv.
+                      Ollama on this machine. A key comes from
+                      $EVALSLAB_API_KEY, never argv.
   --prompt <text>     the prompt to grade, as a template: its tokens resolve
                       under --tokens. Required without --pipeline: a dataset
                       holds no prompt (the lab's Prompt library does).
@@ -103,7 +111,8 @@ const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
   --pipeline <file>   a run document (docs/pipeline-model.md) with one
                       scenario and a graded eval, graded case by case against
                       the set. Instead of --prompt, --model and --url. Each
-                      profile's key comes from $EVAL_API_KEY_<ID>, never the
+                      profile's key comes from $EVALSLAB_API_KEY_<SLUG> -- its
+                      id's, in a file whose profile has no slug -- never the
                       file; its content.files, when not empty, is the file
                       list a case has to be in, as --files says.
   --samples <dir>     where the images are. Default: the lab's samples/.
@@ -153,6 +162,12 @@ const USAGE = `Usage: node tools/prompt-lab/run-evals.js [options]
                       alone. No model, no files, no network.
   --json <path|->     write the machine-readable report. "-" means stdout, and
                       sends the human report to stderr.
+  --min-pass <0..1>   an eval whose items are read one by one passes when at
+                      least this share of them pass. It relaxes each eval's
+                      own rule -- every item passes -- and never tightens it;
+                      a whole-run eval keeps its own verdict.
+  --junit <file>      write JUnit XML: one testsuite per target and eval, one
+                      testcase per item, a failure carrying its reason.
 `;
 
 // NOTHING HERE CALLS process.exit(). Node's stdout is asynchronous down a
@@ -192,7 +207,7 @@ function parseArgs(argv) {
               replies: "", json: "", source: "", files: "", item: "", resume: "",
               timeout: null, progress: false, cancelFile: "",
               run: "", progressFile: "", resultsFile: "", from: null, only: null,
-              rescore: false, tokens: "", plugins: "" };
+              rescore: false, tokens: "", plugins: "", minPass: null, junit: "" };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     const value = () => {
@@ -225,6 +240,13 @@ function parseArgs(argv) {
       case "--only": o.only = value(); break;
       case "--rescore": o.rescore = true; break;
       case "--json": o.json = value(); break;
+      case "--min-pass": {
+        const v = value(), n = Number(v);
+        if (v.trim() === "" || !(n >= 0 && n <= 1)) broken(`--min-pass is a share from 0 to 1, and ${v} is not one`);
+        o.minPass = n;
+        break;
+      }
+      case "--junit": o.junit = value(); break;
       case "-h": case "--help":
         process.stdout.write(USAGE);
         throw new Stop("", EXIT.passed);
@@ -275,7 +297,7 @@ function readRun(file, dataset) {
     broken(`${file}: ${e.message}`);
   }
   const named = isObject(doc) ? core.evalsDataset(doc) : null;
-  const bad = core.validatePipeline(doc, { datasets: dataset && named ? [named] : [] });
+  const bad = core.validatePipeline(doc, { groups: dataset && named ? [named] : [] });
   if (bad.length) broken(`${file} cannot be run:\n  ${bad.join("\n  ")}`);
   return doc;
 }
@@ -300,6 +322,11 @@ function readDataset(file) {
       broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads versions 1 to 7`);
     }
     doc = isObject(doc.dataset) ? doc.dataset.body : null;
+  } else if (isObject(doc) && doc.format === "evals-lab/eval-group") {
+    // The lab's Export of an eval group (docs/pipeline-model.md §17), which
+    // began at version 7.
+    if (doc.version !== 7) broken(`${file} is version ${JSON.stringify(doc.version)}, and this reads version 7`);
+    doc = isObject(doc.group) ? doc.group.body : null;
   }
   if (isObject(doc) && doc.rules != null && !(isObject(doc.rules) && Array.isArray(doc.rules.rules))) {
     broken(`${file}: rules has to be null or { "rules": [...] }`);
@@ -315,6 +342,10 @@ function readDataset(file) {
   return doc;
 }
 
+// A link's eval group, by reference: the body --dataset handed over, the one
+// Library group a run reads, whatever id it names it by.
+const groupOf = (dataset) => () => dataset ?? undefined;
+
 // The cases a graded run is scored against, by the item each names: exactly,
 // as the Source names it.
 function gradedBy(dataset) {
@@ -505,7 +536,7 @@ function graderFor(run) {
       const c = run.profiles?.[ref.id];
       if (!c) return undefined;
       if (!made.has(ref.id)) {
-        const link = reach(c, core.keyVar(ref.id));
+        const link = reach(c, core.keyVar(ref.id, c.slug));
         made.set(ref.id, async prompt => {
           const r = await ask(link, { id: ref.id, ...c }, prompt, null);
           if (r.error) throw new Error(`the grader ${c.name || ref.id} failed: ${r.error}`);
@@ -613,18 +644,88 @@ function readUtf8(file) {
   return fs.readFileSync(file).toString("utf8");
 }
 
-// Every eval's reading of each scenario, keyed by the eval's id, exactly as
-// the page reads it: a whole-run eval's verdict, settled once every item is
-// in, and a per-item eval's counts. [settled] is false for a run that
-// stopped short, whose whole-run evals have not settled.
-const verdictsOf = (run, items, settled) => core.targetsOf(run).map((_, i) =>
-  Object.fromEntries(core.scenarioEvals(run, i, items, settled).map(o => [o.id, {
-    name: o.label, skipped: o.skipped,
+// Every eval's reading of each scenario, exactly as the page reads it.
+// [settled] is false for a run that stopped short, whose whole-run evals
+// have not settled.
+const outcomesOf = (run, items, settled) =>
+  core.targetsOf(run).map((_, i) => core.scenarioEvals(run, i, items, settled));
+
+/**
+ * One eval's verdict on one target, for the build: its own rule -- every
+ * item it read passed, or a whole-run eval's verdict -- with [minPass] the
+ * share of items that is enough instead. It only relaxes: an eval whose
+ * every item passed passes whatever the share. One an earlier failure
+ * stopped is skipped -- that failure is the run's verdict already -- and one
+ * that had nothing to read, a dataset grading none of the run's items, says
+ * none: the run ran in full, which is what the server's status reads from
+ * the exit code, so it is not incomplete either.
+ */
+function evalVerdict(o, settled, minPass) {
+  if (o.skipped) return "skipped";
+  if (!settled) return "incomplete";
+  if (o.whole) return !o.verdict?.ran ? "none" : o.verdict.pass ? "pass" : "fail";
+  if (!o.ran) return o.skippedItems ? "skipped" : "none";
+  if (o.passed === o.ran) return "pass";
+  return minPass != null && o.passed / o.ran >= minPass ? "pass" : "fail";
+}
+
+// Each target's evals, keyed by the eval's id: a whole-run eval's verdict,
+// a per-item eval's counts, and the build's verdict on each.
+const verdictsOf = (outcomes, settled, minPass) => outcomes.map(evals =>
+  Object.fromEntries(evals.map(o => [o.id, {
+    name: o.label, skipped: o.skipped, verdict: evalVerdict(o, settled, minPass),
     ...(o.whole
       ? { pass: o.verdict ? o.verdict.pass : null, detail: o.verdict?.detail ?? null }
       : { ran: o.ran, passed: o.passed, skippedItems: o.skippedItems }),
   }])));
 
+/** The run's verdict from its evals': a run that did not finish is
+    incomplete, whatever its evals read so far; then any eval failing fails it. */
+function runVerdict(verdicts, complete) {
+  if (!complete) return "incomplete";
+  return verdicts.some(v => Object.values(v).some(e => e.verdict === "fail")) ? "fail" : "pass";
+}
+
+const exitOf = verdict => ({ pass: EXIT.passed, fail: EXIT.failed })[verdict] ?? EXIT.incomplete;
+
+// ---- JUnit -----------------------------------------------------------------
+// What a CI test panel reads, written by the core's junitXml.
+
+/** Why a score failed, in a line: its failing metrics, else what it missed. */
+function reasonOf(score) {
+  const off = (score.metrics || []).filter(m => !m.pass).map(m => `${m.label}: ${m.reason}`);
+  if (off.length) return off.join(" · ");
+  const missed = (score.missed || []).map(r => (Array.isArray(r) ? r.join(" or ") : r));
+  return missed.length ? `missed ${missed.join(", ")}` : "failed";
+}
+
+/** A --run's or a --rescore's suites: each target's evals over [items]. */
+function runSuites(run, outcomes, items, settled, minPass) {
+  return outcomes.flatMap((evals, i) => evals.map(o => {
+    const name = `${core.targetLabel(run, i)} › ${o.label}`;
+    const v = evalVerdict(o, settled, minPass);
+    if (o.whole) {
+      // Settled over the run rather than item by item: one case, the run.
+      const c = { name: o.label };
+      if (v === "skipped") c.skipped = "an earlier eval failed";
+      else if (v === "none") c.skipped = "nothing to read";
+      else if (v === "incomplete") c.error = "the run did not finish";
+      else if (v === "fail") c.failure = o.verdict.detail || "failed";
+      return { name, cases: [c] };
+    }
+    return { name, cases: items.map((it, x) => {
+      const c = { name: it?.name ?? (items.length > 1 ? `item ${x + 1}` : "the text") };
+      const read = o.items[x];
+      if (!it) c.error = "not run";
+      else if (it.unrun) c.error = it.unrun;
+      else if (!read) c.skipped = "not graded";
+      else if (core.isSkipped(read)) c.skipped = "an earlier eval failed";
+      else if (!read.pass) c.failure = reasonOf(read);
+      return c;
+    }) };
+  }));
+}
+
 // The run as the report restates it: each scenario's stages with the
 // connection each one asked.
 const scenariosOf = run => core.targetsOf(run).map((sc, i) => {
@@ -678,7 +779,7 @@ async function runSnapshot(o) {
   // profile's id.
   const plans = core.targetsOf(run).map((_, i) => {
     const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
-    const links = connections.map(c => reach(c, core.keyVar(c.id)));
+    const links = connections.map(c => reach(c, core.keyVar(c.id, c.slug)));
     return { stages, tokens, connections, links, calls, cells };
   });
 
@@ -728,6 +829,12 @@ async function runSnapshot(o) {
           break;
         }
       }
+      // A text file the Source does not hold is unrun, as an image is: the
+      // run is incomplete rather than broken.
+      if (item.kind === "text" && !item.bare && item.text == null && !fs.existsSync(path.join(o.source, item.name))) {
+        failed = `the file ${item.name} is not in the source`;
+        break;
+      }
       const textOf = item.kind === "text" && !item.bare
         ? (item.text != null ? item.text : readUtf8(path.join(o.source, item.name))) : null;
       const res = await core.runPipeline(plan.stages, dataUrl, callsFor(plan.connections, plan.links, textOf, plan, record),
@@ -736,7 +843,7 @@ async function runSnapshot(o) {
       // text item a dataset grades is graded like an image.
       const kase = item.name ? graded.get(item.name) ?? null : null;
       production = core.productionOf(run, record);
-      sides.push({ res, scores: await core.itemScoresAsync(run, kase, res, { production, grader: graderFor(run) }) });
+      sides.push({ res, scores: await core.itemScoresAsync(run, kase, res, { production, grader: graderFor(run), group: groupOf(dataset) }) });
     }
     // Production's reply to the item, where its record holds one: what a
     // metric compares with, kept so a re-score reads it again.
@@ -749,16 +856,25 @@ async function runSnapshot(o) {
   }
   if (progressFd != null) fs.closeSync(progressFd);
 
+  // Every item in, none of them unrun: only then has the run settled. A
+  // re-run of one item reads the rest from the results it was handed.
+  const ranItems = results.slice(0, total);
+  const complete = !cancelled && !unrun.length
+    && ranItems.length === total && ranItems.every(it => it && !it.unrun);
+  const outcomes = outcomesOf(run, ranItems, complete);
+  const verdicts = verdictsOf(outcomes, complete, o.minPass);
   const report = {
-    runner: "tools/prompt-lab/run-evals.js",
+    runner: RUNNER,
     ranAt: new Date().toISOString(),
-    verdict: cancelled ? "cancelled" : (unrun.length ? "incomplete" : "done"),
+    verdict: runVerdict(verdicts, complete),
+    ...(cancelled ? { cancelled: true } : {}),
+    ...(o.minPass != null ? { minPass: o.minPass } : {}),
     run: {
       name: run.name ?? null,
       evals: run.evals,
-      verdicts: verdictsOf(run, results, !cancelled && !unrun.length),
+      verdicts,
       scenarios: scenariosOf(run),
-      items: results.slice(0, total),
+      items: ranItems,
     },
     unrun,
   };
@@ -767,14 +883,31 @@ async function runSnapshot(o) {
   say(`${report.runner} — run ${report.run.name ?? ""}`);
   for (const u of unrun.slice(0, 10)) say(`----  ${u}`);
   if (unrun.length > 10) say(`----  and ${unrun.length - 10} more, all of them in the JSON report`);
-  say(`${results.filter(Boolean).length} of ${total} items ran · ${cancelled ? "cancelled" : report.verdict}`);
+  sayVerdicts(say, run, verdicts);
+  say(`${results.filter(Boolean).length} of ${total} items ran · ${cancelled ? "cancelled · " : ""}${report.verdict}`);
 
+  writeReports(o, report, () => runSuites(run, outcomes, ranItems, complete, o.minPass));
+  process.exitCode = exitOf(report.verdict);
+}
+
+/** Each target's evals and their verdicts, a line each. */
+function sayVerdicts(say, run, verdicts) {
+  verdicts.forEach((evals, i) => {
+    for (const e of Object.values(evals)) {
+      const n = e.ran != null ? ` · ${e.passed} of ${e.ran} passed` : e.detail ? ` · ${e.detail}` : "";
+      say(`${e.verdict.padEnd(10)} ${core.targetLabel(run, i)} › ${e.name}${n}`);
+    }
+  });
+}
+
+/** The JSON report, and the JUnit one from [suites], where each was asked for. */
+function writeReports(o, report, suites) {
   if (o.json) {
-    const text2 = JSON.stringify(report, null, 2);
-    if (o.json === "-") process.stdout.write(text2 + "\n");
-    else fs.writeFileSync(o.json, text2 + "\n");
+    const text = JSON.stringify(report, null, 2);
+    if (o.json === "-") process.stdout.write(text + "\n");
+    else fs.writeFileSync(o.json, text + "\n");
   }
-  process.exitCode = cancelled ? EXIT.passed : (unrun.length ? EXIT.incomplete : EXIT.passed);
+  if (o.junit) fs.writeFileSync(o.junit, core.junitXml(suites(), RUNNER));
 }
 
 /** Re-score a run's stored results against the dataset --dataset hands over,
@@ -807,39 +940,40 @@ async function runRescore(o){
   const items = await Promise.all(results.map(async (it, i) => {
     if (!isObject(it) || only != null && i !== only || !Array.isArray(it.scenarios)) return it;
     const kase = isText(it.name) ? graded.get(it.name) ?? null : null;
-    const more = { production: it.production ?? null, grader: graderFor(run) };
+    const more = { production: it.production ?? null, grader: graderFor(run), group: groupOf(dataset) };
     return { ...it, scenarios: await Promise.all(it.scenarios.map(async side =>
       (isObject(side) && side.res)
         ? { ...side, scores: await core.itemScoresAsync(run, kase, side.res, more) } : side)) };
   }));
 
+  // A stored run that stopped short re-scores as what it is: incomplete.
+  const complete = items.every(it => isObject(it) && !it.unrun);
+  const outcomes = outcomesOf(run, items, complete);
+  const verdicts = verdictsOf(outcomes, complete, o.minPass);
   const report = {
-    runner: "tools/prompt-lab/run-evals.js",
+    runner: RUNNER,
     ranAt: new Date().toISOString(),
-    verdict: "done",
+    verdict: runVerdict(verdicts, complete),
+    ...(o.minPass != null ? { minPass: o.minPass } : {}),
     rescored: true,
     run: {
       name: run.name ?? null,
       evals: run.evals,
-      verdicts: verdictsOf(run, items, true),
+      verdicts,
       scenarios: scenariosOf(run),
       items,
     },
   };
 
-  if (o.json) {
-    const text = JSON.stringify(report, null, 2);
-    if (o.json === "-") process.stdout.write(text + "\n");
-    else fs.writeFileSync(o.json, text + "\n");
-  }
-  process.exitCode = EXIT.passed;
+  writeReports(o, report, () => runSuites(run, outcomes, items, complete, o.minPass));
+  process.exitCode = exitOf(report.verdict);
 }
 
 /**
  * The run --prompt, --model and --url describe, as a run document: one job
  * that sees the image and answers in the default kind, one scenario,
  * graded against --dataset, asking --prompt. Its profile has no name, so the transcript names no
- * connection, as it never did, and its key is $EVAL_API_KEY.
+ * connection, as it never did, and its key is $EVALSLAB_API_KEY.
  */
 function cliRun(o, dataset) {
   let doc = core.blankPipeline();
@@ -857,8 +991,7 @@ function cliRun(o, dataset) {
   doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
   doc.targets = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
                    steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }];
-  doc.evals = [{ id: "cli", type: "metrics", name: "", continueOnFailure: true, dataset: { id: "cli", name: "" },
-                 mode: "all", threshold: null, grader: null, over: "item", metrics: [] }];
+  doc.evals = [{ id: "cli", type: "group", name: "", continueOnFailure: true, group: { id: "cli", name: "" }, pin: null }];
   doc.profiles = { cli: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
   return doc;
 }
@@ -939,7 +1072,7 @@ async function main() {
 
   // The run to grade. A --pipeline document is one scenario graded against
   // its dataset; without one it is the stage this script always ran -- the
-  // prompt, seeing the image, on --url with $EVAL_API_KEY -- stated as
+  // prompt, seeing the image, on --url with $EVALSLAB_API_KEY -- stated as
   // the same kind of document, so both go through one reading of it.
   if (!o.pipeline && !(o.prompt ?? "").trim()) {
     broken("--prompt or --prompt-file names the prompt to grade: a dataset holds none, "
@@ -1038,7 +1171,7 @@ async function main() {
     if (!o.pipeline && !o.model) broken(`--model names the model to grade.\n\n${USAGE}`);
     links = connections.map((c, i) => {
       if (!String(c.model || "").trim()) broken(`stage ${i + 1}'s connection names no model.`);
-      const to = reach(c, o.pipeline ? core.keyVar(c.id) : "EVAL_API_KEY");
+      const to = reach(c, o.pipeline ? core.keyVar(c.id, c.slug) : "EVALSLAB_API_KEY");
       // Once here rather than once per image: a llama.cpp stage on a
       // hosted model is a request that cannot be built at all.
       try {
@@ -1186,10 +1319,12 @@ async function main() {
   // passed, and 49 items absent with the fiftieth green is exactly how
   // that happens.
   const complete = graded.length > 0 && unrun.length === 0;
-  const verdict = !complete ? "incomplete" : tally.passed === tally.ran ? "pass" : "fail";
+  const enough = tally.passed === tally.ran
+    || (o.minPass != null && tally.passed / tally.ran >= o.minPass);
+  const verdict = !complete ? "incomplete" : enough ? "pass" : "fail";
 
   const report = {
-    runner: "tools/prompt-lab/run-evals.js",
+    runner: RUNNER,
     ranAt: new Date().toISOString(),
     dataset: inRepo(o.dataset),
     prompt,
@@ -1217,6 +1352,7 @@ async function main() {
                         filesListed: snapshotSet.size } : {}),
     ...(o.source ? { source: inRepo(o.source) } : {}),
     verdict,
+    ...(o.minPass != null ? { minPass: o.minPass } : {}),
     graded: (itemsOnly ?? graded).length,
     ungraded: set.length - graded.length,
     ran: tally.ran,
@@ -1269,14 +1405,16 @@ async function main() {
         + `so this is a partial run and not a result.`);
   }
 
-  if (o.json) {
-    const text = JSON.stringify(report, null, 2);
-    if (toStdout) process.stdout.write(text + "\n");
-    else fs.writeFileSync(o.json, text + "\n");
-  }
-
-  process.exitCode = !complete ? EXIT.incomplete
-                   : report.failed ? EXIT.failed : EXIT.passed;
+  // The one eval this mode grades by, as a suite of its cases.
+  const suites = () => {
+    const j = Math.max(0, core.evalsOf(run).findIndex(t => core.evalsDataset({ evals: [t] })));
+    return [{ name: `${core.targetLabel(run, 0)} › ${core.evalLabel(run, j)}`, cases: [
+      ...rows.map(r => ({ name: r.id, ...(r.pass ? {} : { failure: r.reasons.join(" · ") || "failed" }) })),
+      ...unrun.map(u => { const at = u.indexOf(": "); return { name: u.slice(0, at), error: u.slice(at + 2) }; }),
+    ] }];
+  };
+  writeReports(o, report, suites);
+  process.exitCode = exitOf(verdict);
 }
 
 main().catch(e => {