evals-lab 0.5.0 → 0.6.0

package/CHANGELOG.md

@@ -5,6 +5,43 @@ Newest first. Read a release's **Upgrade notes** before installing it: an
 upgrade can rewrite what the lab keeps in your data directory, and an older
 version cannot always read it back.
 
+## 0.6.0
+
+### Added
+
+- Library › Evals (was Library › Datasets) lists every eval group: its
+  Source, cases, scoring, and the pipelines that link it. A group's page
+  holds its Grades, Every item, Whole run and cases.
+- Runs › Evals links eval groups to a pipeline. Link eval group picks from
+  the Library and marks the groups made for this content. A link follows a
+  group's latest version, or is pinned to one. A pipeline's own group is
+  edited in place, and Move to Library shares it.
+- A pipeline can link several eval groups. Its pass rule says whether every
+  one must pass, or at least a number. Results, History and `evals-lab run`
+  all grade by it.
+- An Undo button in the header takes back the last 20 actions, newest
+  first, until the page reloads.
+- Choose pipeline lists the 5 used most recently, then the rest, searchable
+  and sortable.
+- A Profile, Source or Grader field opens a picker, most recent first.
+
+### Changed
+
+- Results shows a row per eval group, with each Target's share and verdict,
+  then the overall verdict. History's Outcome is the overall verdict.
+- A dialog is a draft: nothing reaches the lab until Save & Close. Cancel,
+  × and Esc discard it.
+- An unnamed target is Target A, B, C.
+- The run bar sits under the pipeline's header. It says "12 of 42" while a
+  run goes and "42 in 2m" when it is done.
+- Verdicts show as a check or a cross, in Results and History.
+- A metric's "How it counts" is Options, and Grades is Reads: Parsed reply
+  or Raw reply.
+
+### Fixed
+
+- Add profile, then ×, added a profile.
+
 ## 0.5.0
 
 ### Upgrade notes

package/bin/run.js

@@ -487,10 +487,21 @@ async function run(argv, { lab, env = process.env, stdout = process.stdout, stde
     const args = [path.join(lab, "run-evals.js"), "--run", path.join(tmp, "run.json"),
                   "--json", path.join(tmp, "report.json")];
     if (items) args.push("--source", items);
-    const ref = core.evalsDataset(read.run);
-    if (ref && read.datasets[ref.id]) {
-      fs.writeFileSync(path.join(tmp, "dataset.json"), JSON.stringify(read.datasets[ref.id]));
+    // The eval group bodies the run grades against: its one, as --dataset, or
+    // a body per group it links, as --groups (#233), keyed by the id its
+    // reference carries (a bundle's refs record no version).
+    const ids = [...new Set(core.evalsOf(read.run).map(core.casesRef).filter(Boolean).map(r => r.id))];
+    if (ids.length === 1 && read.datasets[ids[0]]) {
+      fs.writeFileSync(path.join(tmp, "dataset.json"), JSON.stringify(read.datasets[ids[0]]));
       args.push("--dataset", path.join(tmp, "dataset.json"));
+    } else if (ids.length > 1) {
+      const groups = {};
+      for (const id of ids) {
+        if (!read.datasets[id]) throw new Refused(`${o.bundle}: the bundle holds no eval group ${id}`);
+        groups[id] = read.datasets[id];
+      }
+      fs.writeFileSync(path.join(tmp, "groups.json"), JSON.stringify(groups));
+      args.push("--groups", path.join(tmp, "groups.json"));
     }
     if (plugins.length) args.push("--plugins", path.join(dir, "plugins"));
     if (o.minPass != null) args.push("--min-pass", o.minPass);

package/lab/VERSION

@@ -1 +1 @@
-0.5.0 (2026.10.04-397)
+0.6.0 (2026.10.05-430)

package/lab/evals-core.mjs

@@ -775,16 +775,18 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 
 // ---- the registries ----
 
-/** An option a modifier or an eval type exposes for editing. */
-                        
-                                                                                               
-                                                
+/** An option a modifier or an eval type exposes for editing. `hint` says
+    what it does in one short line, under its control, where the label
+    cannot: a switch's effect, not why it exists. */
+                                                                          
+                                                                   
+                    
                                                           
-                                                    
-                                                  
-                                                    
+                        
+                      
+                        
                                                                              
-                                                                            
+                                                 
 
 /** What an output kind made of a reply. */
                                 
@@ -966,6 +968,10 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                            
                        
                                                 
+                                                                              
+                                                                             
+                                                                         
+                       
  
 
 /** A job's stages, in the order they run (pipeline-model §16). A target's
@@ -3487,10 +3493,21 @@ EVAL_TYPES.group = {
   read: async (t, kase, res, more) => {
     const group = groupOf(t, more);
     const grader = isRef(group.grader) ? group.grader : isRef(t.grader) ? t.grader : null;
-    return readGroup({ ...group, grader }               , casesRef(t) ? kase : null, res, more);
+    const ref = casesRef(t);
+    // The item's case in this eval's own group: resolved from the group whose
+    // cases it reads where the runner names the item (grading several groups
+    // at once, --groups), else the case handed in (a single group).
+    const own = ref && isStr(more.item) ? caseIn(more.group?.(ref), more.item) : kase;
+    return readGroup({ ...group, grader }               , ref ? own : null, res, more);
   },
 };
 
+/** The case for an item in a group's body, by the name its file has, or null:
+    a non-todo case whose item matches, as a Library group reads one. */
+function caseIn(body                         , item        )              {
+  return (body?.cases ?? []).find(c => !c.todo && caseItem(c) === item) ?? null;
+}
+
 // ---- the lab's own scorers, as metrics ----------------------------------------
 // What the Single Test checks, one metric each, so an eval of it converts to
 // Metrics that read a run exactly as it did (metrics-parity-check.js). Here
@@ -3901,9 +3918,15 @@ function targetsOf(doc
   return Array.isArray(doc?.targets) ? doc .targets             : [];
 }
 
-/** Target [i]'s name, or the number it has always shown. */
+/** Target [i]'s letter, A for the first: the same in every job, so a
+    Target reads as one column through them. A number past Z. */
+function targetLetter(i        )         {
+  return i < 26 ? String.fromCharCode(65 + i) : String(i + 1);
+}
+
+/** Target [i]'s name, or its letter's: "Target A". */
 function targetLabel(doc                                                           , i        )         {
-  return (targetsOf(doc)[i]?.name || "").trim() || `Target ${i + 1}`;
+  return (targetsOf(doc)[i]?.name || "").trim() || `Target ${targetLetter(i)}`;
 }
 
 /** What target [i] sends in job [k]: its step there. */
@@ -4150,11 +4173,8 @@ STEP_TYPES.evals = {
                + `and the last job answers with ${OUTPUT_KINDS[lastKind] .noun || lastKind}`);
       }
     });
-    // The worker is handed one Library group's body to grade against
-    // (server-side-runs §4) until it reads the body the queue keeps per
-    // group (#233).
-    const named = new Set(evals.map(casesRef).filter(Boolean).map(r => r .id));
-    if (named.size > 1) bad.push("the evals grade against one Library eval group at a time");
+    // The worker reads a body per group now (#233), so the evals may grade
+    // against several Library groups at once; the queue keeps each body.
   },
 };
 
@@ -5390,22 +5410,40 @@ function scenarioEvals(run             , i        , items
   });
 }
 
+/** One eval group's verdict over a scenario (§17): a whole-run group's own
+    verdict, or a per-item group's "every item it read passed"; null where it
+    was skipped, or had nothing to read yet. */
+function groupPass(o             )                 {
+  if (o.skipped) return null;
+  if (o.whole) return o.verdict?.ran ? o.verdict.pass : null;
+  return o.ran ? o.passed === o.ran : null;
+}
+
 /** A scenario's pass or fail over every eval that read it: null where none
     has anything to say yet. */
 function scenarioPasses(outcomes                        )                 {
-  let said = false;
-  for (const o of outcomes) {
-    if (o.skipped) continue;
-    if (o.whole) {
-      if (!o.verdict?.ran) continue;
-      said = true;
-      if (!o.verdict.pass) return false;
-    } else if (o.ran) {
-      said = true;
-      if (o.passed < o.ran) return false;
-    }
-  }
-  return said ? true : null;
+  const passes = outcomes.map(groupPass);
+  if (!passes.some((p) => p != null)) return null;
+  return !passes.includes(false);
+}
+
+/** A scenario's overall verdict under a run's pass rule (§17): each linked
+    group's verdict folded together the way `pass` says -- every group passes,
+    or at least a number of them. Null where no group has a verdict yet, so a
+    run with no evals, or one still grading, reads as it does without a rule. */
+function overallVerdict(outcomes                        , pass                             )                 {
+  const passes = outcomes.map(groupPass);
+  if (!passes.some((p) => p != null)) return null;
+  return passVerdict(pass, passes);
+}
+
+/** Whether a run passes a Target under its overall pass rule (§17): `all`
+    passes when no linked group fails, `atLeast` when at least `count` pass.
+    [passes] is each group's pass (true), fail (false), or neither (null: it
+    graded nothing, or an earlier group skipped it). */
+function passVerdict(pass                             , passes                             )          {
+  if (pass?.mode === "atLeast") return passes.filter(p => p === true).length >= pass.count;
+  return !passes.includes(false);
 }
 
 /**
@@ -5610,10 +5648,10 @@ export {
     connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
-    targetsOf, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, withTarget,
+    targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
-    scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, isSkipped, failedScore,
+    scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, isWholeRun, EVAL_FIELDS,
     pipelineToYaml, importPipeline, yamlToPipeline, exportBundle, readBundle, bundleProblems, junitXml,
     pluginHost, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,

package/lab/kinds/list.mjs

@@ -336,7 +336,8 @@ const LIST_KIND                         = {
   settings: [
     { key: "parse", label: "Parse as", type: "select", choices: [
       { value: "csv", label: "CSV" }, { value: "lines", label: "One a line" }, { value: "json", label: "JSON array" }] },
-    { key: "preamble", label: "Ignore preamble", type: "checkbox" },
+    { key: "preamble", label: "Ignore preamble", type: "checkbox",
+      hint: "Discard any text in <think> tags and any text before the first colon" },
   ],
   settingDefaults: () => ({ parse: "csv", preamble: false }),
   validateSettings(out, at, bad) {