evals-lab 0.2.0 → 0.4.0

package/lab/evals-core.mjs

@@ -26,7 +26,7 @@
 // WHAT IS NOT HERE. Preparing an image: the tab has a canvas and the
 // script has not, so `prepare()` stays in the page and the script shells out
 // to ImageMagick with the same arithmetic. Rendering: the tab writes HTML and
-// the script writes JSON. Both read the same verdict out of `scoreCase`.
+// the script writes JSON. Both read the same verdict out of the same Metrics.
 //
 // `runner-check.js` runs one set through two of the callers and asserts the
 // verdicts and the totals are identical, because sharing a file is a claim
@@ -232,8 +232,8 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                       
  
 
-/** What every test carries whatever its type: an id minted once and never
-    shown, an optional name ("Test 2" when blank), and whether the tests after
+/** What every eval carries whatever its type: an id minted once and never
+    shown, an optional name ("Eval 2" when blank), and whether the evals after
     it still read what it failed on (docs/pipeline-model.md §3). */
                              
              
@@ -260,7 +260,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                   
  
 
-/** Checks of every reply (TEST_TYPES.metrics): its own for every item, and
+/** Checks of every reply (EVAL_TYPES.metrics): its own for every item, and
     a case's own for its item; all must pass, or weighted points reach the
     threshold. A model-graded one asks the grader. */
                                                  
@@ -275,11 +275,11 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                         
  
 
-/** A test, from version 9: Metrics. A Single Test or a Graded set is what an
+/** An eval, from version 9: Metrics. A Single Test or a Graded set is what an
     older document held (upgradePipeline reads it converted). */
                                
 
-/** A pipeline's tests, in the order they read a run. */
+/** A pipeline's evals, in the order they read a run. */
                            
 
                            
@@ -482,75 +482,47 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 
 // ---- grading ----
 
-/** A graded case, as cases.json writes one.
-
-    The item a case grades is named by `filename`: a dataset joins on a file
-    name, and nothing about that is an image -- the next dataset addresses
-    a row of a spreadsheet or a line of a log by the same key. The field had
-    an older name while the lab was one app's bench, and it is still *read*:
-    a set re-synced from that app's repository arrives spelled that way, and
-    a case that silently stopped matching would be worse than one that reads
-    both.
-    `caseFile` is the one reader; `evalsJson` is the one writer, and it writes
-    `filename`. */
+/** A graded case (dataset body version 5): an Item and the Metrics its
+    reply is held to.
+
+    The item is named by `item`: a dataset joins on an item's name in the
+    Source it grades, exactly, and nothing about that is an image -- the
+    next dataset addresses a row of a spreadsheet or a line of a log by the
+    same key. Version 4 called it `filename`, and the lab's first app had a
+    name of its own for it; `upgradeDatasetBody` reads both as `item`.
+
+    A case's metrics are what a good answer is: each a metric as a
+    pipeline's Metrics eval holds one, added to the eval's own for this item
+    when an eval names the dataset. One with weight 0 is watched -- reported,
+    never scored. */
                        
               
-                    
-                                                                    
-                                                                       
+                
+                                                                            
                  
-               
-                    
-                     
-                    
-                   
-                           
-                           
-                                                            
-                   
-                                                                          
+                                            
+                
                      
-                                                                           
-                                                                   
-                      
-                    
                          
  
 
-/** A graded set: cases.json. */
+/** A graded set: a dataset's body, or anything holding its cases. */
                           
                  
                          
  
 
-/** One row the Datasets tab's Cases group draws, as gradedSetFrom builds it:
-    a case of a set validateEvals accepts, so it has its id and file. */
+/** One row of the Library's Dataset group's cases table, as gradedSetFrom builds it:
+    a case of a set validateEvals accepts, so it has its id and item. */
                                          
              
-                   
+               
                
  
 
 /** A requirement as a score reports it: a term, or the group it was. */
                                             
 
-/** How a graded case read a reply: the same shape both engines agree on. */
-                            
-                
-                       
-                     
-                       
-                        
-                     
-                    
-                 
-                
-                
-                       
-                     
-                    
- 
-
 /** A run's totals, summed across cases. */
                         
               
@@ -559,7 +531,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
              
  
 
-/** The whole-run verdict of a test type, where it has one of its own. */
+/** The whole-run verdict of an eval type, where it has one of its own. */
                                   
                 
                   
@@ -570,7 +542,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                        
  
 
-/** One thing a test asserts, as its type names it: "Exact" "outdoor". */
+/** One thing an eval asserts, as its type names it: "Exact" "outdoor". */
                        
               
                 
@@ -599,7 +571,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                  
  
 
-/** A per-item test's score as a stored row carries it. */
+/** A per-item eval's score as a stored row carries it. */
                                                                                                               
                                                                              
                                                                                        
@@ -678,6 +650,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
     more entry, and no reader changes. */
                               
                 
+                                                                           
+                                                                               
+                  
                                                                        
                        
                                                                                 
@@ -698,12 +673,12 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                        
  
 
-/** What an earlier test that failed and does not continue leaves a later one. */
+/** What an earlier eval that failed and does not continue leaves a later one. */
                                           
 
                                               
 
-/** One test's reading of one scenario of a run (scenarioTests). */
+/** One eval's reading of one scenario of a run (scenarioEvals). */
                               
              
                 
@@ -760,7 +735,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 
 // ---- the registries ----
 
-/** An option a modifier or a test type exposes for editing. */
+/** An option a modifier or an eval type exposes for editing. */
                         
                                                                                                
                                                 
@@ -820,7 +795,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                    
                                     
                                             
-                                                                              
+                                                                               
                              
                                                                     
                                         
@@ -896,10 +871,10 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                             
  
 
-/** A test type: what it checks of a document, and how it scores a run. */
+/** An eval type: what it checks of a document, and how it scores a run. */
                                 
                 
-                                                                
+                                                                 
                        
                             
                                            
@@ -909,11 +884,11 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                   
                                                                 
                                                                                                    
-                                                                               
+                                                                                
                                                                                 
                                
                                            
-                                                                       
+                                                                         
                                                                         
                                                                       
                                                                                                                
@@ -934,7 +909,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                          
  
 
-/** What a test's `read` is handed besides the reply: production's reply to
+/** What an eval's `read` is handed besides the reply: production's reply to
     the item, and a grader, where the run has them. */
                            
                              
@@ -1004,42 +979,151 @@ const SLOTS                      = ["content", "target", "responses"];
  * submitted against, so nothing grades from a file.
  */
                               
-                          
+                                                                           
+                            
+                                       
+                                                                          
+                                                                              
+                     
+                                                                     
+                                                                   
+                        
+                                                                              
+                     
+                                                       
+                  
+                                                                
+                
+                                                                           
                 
  
 
+/** An eval group's scoring (docs/pipeline-model.md §17). */
+                               
+                           
+                           
+ 
+
+/** The dataset body's version: 7 is an eval group (§17); 6 marks itself; 5
+    named its Source; 4 and earlier, neither. */
+const DATASET_BODY_VERSION = 7         ;
+
+/** The metrics whose Ignore case version 6 made mean what it says for a
+    reply read as a list, as well as one read as text. */
+const CASE_FOLDING = ["contains", "contains-all", "contains-any"];
+
 /**
- * [body] as this version of a dataset (4), from any earlier one. Version 1
+ * [body] as this version of a dataset (7), from any earlier one. Version 1
  * held `imageCases`, each with `minTags`/`maxTags`, and the parser's
  * `replays` and `conformance` (now fixtures/replays.json beside the checks).
  * Version 2 held `rules`, which clean a job's answer and so belong to the
  * job (docs/pipeline-model.md §13): they leave, and the terms a case
  * watches for are its `watch`. Version 3 held the `prompt` a new scenario
- * started from, which the Prompt library holds now: it leaves. Every reader of a body calls this: the runner, the page, a
- * run's kept copy. A reader that needs a version-2 body's rules -- to upgrade
- * a pipeline graded against it -- takes them first (`datasetRules`).
- * Anything else comes back as it was.
+ * started from, which the Prompt library holds now: it leaves. Version 4's
+ * case named its item `filename` and said what a good answer is in
+ * expectations; each becomes the metric it is (`caseMetrics`), `why` is the
+ * `note`, `traits` go, and the body names no Source yet. Version 5 matched
+ * a list's items ignoring case whatever a Contains metric's Ignore case
+ * said, so each of its Contains metrics says Ignore case (`caseOfV5`), and
+ * the body says its version. Version 6 is a group's cases alone: version 7
+ * adds its scoring, grader, Every item and Whole run (`groupOfV6`), and a
+ * stored row is read that way rather than rewritten. Every reader of a
+ * body calls this: the runner, the page, a run's kept copy. A reader that
+ * needs a version-2 body's rules -- to upgrade a pipeline graded against
+ * it -- takes them first (`datasetRules`). Pure: the same body gives the
+ * same answer, and server.py's upgrade_body is its twin. Anything else
+ * comes back as it was.
  */
 function upgradeDatasetBody   (body   )                  {
   if (!isObj(body)) return body;
-  if (!("imageCases" in body || "rules" in body)) {
-    if (!("prompt" in body)) return body;
-    const { prompt: _library, ...rest } = body             ;
-    return rest               ;
-  }
+  if (body.version === DATASET_BODY_VERSION) return body;
   const b = body             ;
-  // vocab: the names older versions gave these fields
-  const RENAMED                         = { minTags: "minCount", maxTags: "maxCount", textInImage: "watch" };  // vocab: as above
-  const renamed = (c           ) => {
-    if (!isObj(c)) return c;
-    const out            = {};
-    for (const [k, v] of Object.entries(c)) out[RENAMED[k] ?? k] = v;
-    return out;
-  };
+  // A body saying any other version is one this lab does not read.
+  if ("version" in b) return b.version === 6 ? groupOfV6(b) : body;
+  // A body that names its Source, even as null, is version 5.
+  if ("source" in b) {
+    return groupOfV6({ ...b, ...(Array.isArray(b.cases) ? { cases: b.cases.map(caseOfV5) } : {}) });
+  }
   const raw = Array.isArray(b.cases) ? b.cases : Array.isArray(b.imageCases) ? b.imageCases : null;
   // Not a body of any version -- a copy kept while a dataset was an overlay.
   if (!raw) return body;
-  return { cases: (canonicalCases({ cases: raw.map(renamed) })                     ).cases };
+  return groupOfV6({ source: null, cases: raw.map(caseOfV4).map(caseOfV5) });
+}
+
+/** A version-6 body as a version-7 eval group: scored All, the lab's
+    grader, and no metrics of its own for every item or the whole run --
+    what a Metrics eval naming the dataset with none of its own graded, which
+    is what every Graded set converted to. */
+function groupOfV6(b           )              {
+  const { version: _v, ...rest } = b;
+  return { version: DATASET_BODY_VERSION, source: null, scoring: { mode: "all", threshold: null }, grader: null, every: [], run: [],
+           ...rest }               ;
+}
+
+/** A version-5 case as a version-6 one: each Contains metric says Ignore
+    case, as version 5 matched a list's items whatever it said. A reply read
+    as text did mind its setting, but a case's metrics were written for a
+    list -- each one converted from version 4, and each the case form made. */
+function caseOfV5(c         )          {
+  if (!isObj(c) || !Array.isArray(c.metrics)) return c;
+  return { ...c, metrics: c.metrics.map((m           ) => (isObj(m) && CASE_FOLDING.includes(m.type) && m.ignoreCase !== true
+    ? { ...m, ignoreCase: true } : m)) };
+}
+
+// vocab: the names older versions gave these fields
+const CASE_RENAMED                         = { minTags: "minCount", maxTags: "maxCount", textInImage: "watch", photo: "filename" };  // vocab: as above
+/** What a version-4 case said, which its metrics say now. */
+const CASE_V4 = ["filename", "why", "traits", "expect", "anyOf", "forbid", "allow", "minCount", "maxCount", "watch", "discarded"];
+
+/** One case of any earlier version as a version-5 one: its id, item, todo
+    and note, its expectations as metrics ahead of any metrics it held, and
+    anything else it carried kept as it was. */
+function caseOfV4(c         )          {
+  if (!isObj(c)) return c;
+  const was            = {};
+  for (const [k, v] of Object.entries(c)) {
+    const key = CASE_RENAMED[k] ?? k;
+    // A case naming its item both ways keeps the newer key's.
+    if (!(key in was) || key === k) was[key] = v;
+  }
+  if (isStr(was.item)) delete was.filename;
+  const out            = {};
+  if ("id" in was) out.id = was.id;
+  out.item = isStr(was.item) ? was.item : isStr(was.filename) ? was.filename : "";
+  out.todo = was.todo === true;
+  out.note = isStr(was.note) ? was.note : isStr(was.why) ? was.why : "";
+  out.metrics = [...caseMetrics(was), ...(Array.isArray(was.metrics) ? was.metrics : [])];
+  for (const [k, v] of Object.entries(was)) {
+    if (!(k in out) && !CASE_V4.includes(k)) out[k] = v;
+  }
+  return out;
+}
+
+/** A version-4 case's expectations as the metrics that say the same:
+    `expect` is Contains all, each `anyOf` group a Contains any, each
+    forbidden term a Contains turned round with the `allow` phrases that
+    excuse it, the count bounds an Item count, `discarded` the Discarded
+    metric, and each watched term a Contains any of weight 0 -- reported,
+    never scored. The order is the one scoreCase read them in, so a reason
+    reads in the order it did. */
+function caseMetrics(c           )           {
+  const list = (v         )           => (Array.isArray(v) ? v.filter(isStr) : []);
+  if (c.discarded === true) return [{ type: "discarded" }];
+  const out           = [];
+  const expect = list(c.expect), allow = list(c.allow);
+  if (expect.length) out.push({ type: "contains-all", values: expect.join("\n") });
+  for (const g of Array.isArray(c.anyOf) ? c.anyOf : []) {
+    if (list(g).length) out.push({ type: "contains-any", values: list(g).join("\n") });
+  }
+  for (const t of list(c.forbid)) {
+    // An exception excuses only the forbidden term inside it.
+    const except = allow.filter(a => termIn([a], t));
+    out.push({ type: "contains", value: t, not: true, ...(except.length ? { except: except.join("\n") } : {}) });
+  }
+  const min = Number.isInteger(c.minCount) ? c.minCount : null, max = Number.isInteger(c.maxCount) ? c.maxCount : null;
+  if (min != null || max != null) out.push({ type: "item-count", min, max });
+  for (const t of list(c.watch)) out.push({ type: "contains-any", values: t, weight: 0 });
+  return out;
 }
 
 /** The rules a version-1 or version-2 dataset body held, or null: what a
@@ -1055,6 +1139,9 @@ function datasetRules(body         )                  {
                                                 
                                             
                                             
+                                                                           
+                                                                        
+                                            
                                                 
                                         
  
@@ -1322,8 +1409,10 @@ function textPrompt(instruction        , text        )         {
 }
 
 // Mirrors TagRules' WORD. The unit the echo guard and the eval matcher both
-// work in, so that "New Zealand." and "new zealand" are the same two words.
-const words = (s         )           => String(s||"").toLowerCase().match(/[\p{L}\p{N}]+/gu) || [];
+// work in, so that "New Zealand." and "new zealand" are the same two words --
+// unless [keepCase], for a metric whose Ignore case is off.
+const words = (s         , keepCase = false)           =>
+  (keepCase ? String(s||"") : String(s||"").toLowerCase()).match(/[\p{L}\p{N}]+/gu) || [];
 
 // The request Tagger builds, field for field -- when the target reads those
 // fields. A llama.cpp server does; the hosted OpenAI-shaped providers do
@@ -2039,11 +2128,12 @@ function budgetLabel(target         )         {
 // A term is present when its words appear in some item, in order and adjacent:
 // "cat" is in "cat" and in "tabby cat", and is not in "cathedral". Anything
 // looser scores "new zealand" against "zealandia" and flatters every run.
-function termIn(items                   , term        )          {
-  const t = words(term);
+// Letter case counts only where a metric's Ignore case is off.
+function termIn(items                   , term        , ignoreCase = true)          {
+  const t = words(term, !ignoreCase);
   if (!t.length) return false;
   return items.some(item => {
-    const w = words(item);
+    const w = words(item, !ignoreCase);
     for (let i = 0; i + t.length <= w.length; i++) {
       if (t.every((x, j) => w[i + j] === x)) return true;
     }
@@ -2070,153 +2160,55 @@ function forbiddenIn(items                   , term        , allow
  * check green. `evals-check.js` calls this directly.
  */
 function gradedSetFrom(ev         )              {
-  return (ev.cases || []).map(c => ({ ...c, filename: caseFile(c), half: "cases" })             );
+  return (ev.cases || []).map(c => ({ ...c, item: caseItem(c), half: "cases" })             );
 }
 
-/**
- * The file a case grades, whichever of the two keys names it.
- *
- * One reader, so that accepting the older spelling is a fact about this
- * function rather than a branch every caller carries. Everything that joins a
- * case to an item -- the runner, the Datasets tab, a mapping against a Source
- * -- goes through here.
- */
-function caseFile(kase                                                            )         {  // vocab: the older spelling
-  const name = kase?.filename ?? kase?.photo;  // vocab: the older spelling
-  return typeof name === "string" ? name : "";
+/** The item a case grades: its `item`, or "" for none. The one reader, so
+    everything that joins a case to an item -- the runner, the Library, a
+    run's Results -- joins on the same key, exactly. */
+function caseItem(kase                                       )         {
+  return isStr(kase?.item) ? kase .item           : "";
 }
 
-/**
- * A graded set with every case naming its file as `filename`.
- *
- * What `evalsJson` writes, so the committed bytes carry one key and a set
- * re-synced from an app's own repository is normalised the first time it is
- * exported. The key takes the place the older one held, so normalising a set
- * changes the spelling of
- * one key and not the order of any.
- */
-function canonicalCases(ev         )          {
-  const OLD = "photo";  // vocab: the older spelling of filename
-  const set = ev                                ;
-  const cases = set && typeof set === "object" && Array.isArray(set.cases) ? set.cases : null;
-  if (!cases || !cases.some(c => c && typeof c === "object" && OLD in (c          ))) return ev;
-  return {
-    ...set,
-    cases: cases.map(c => {
-      if (!c || typeof c !== "object" || !(OLD in (c          ))) return c;
-      const out                          = {};
-      for (const [k, v] of Object.entries(c                           )) {
-        if (k === OLD) { if (!("filename" in (c          ))) out["filename"] = v; }
-        else out[k] = v;
-      }
-      return out;
-    }),
-  };
-}
+/** One graded case's reading of a reply, now: its own metrics, scored as a
+    Metrics eval scores them (all must pass; weight 0 is watched, never
+    scored), with what they found, missed and invented carried up, and
+    `watch` the readings of weight 0. What the page draws and the runner's
+    case-by-case report writes. A model-graded metric needs a grader and the
+    wait for one, so here it reads as not met: the runner's `--run` asks it. */
+                                                  
+                           
+                       
+                        
+                     
+                                      
+                     
+                                                        
+                    
+                                                       
+                         
+                                          
+                
+ 
 
-/**
- * One graded case, one result.
- *
- * Every expectation is a group, and a plain `expect` term is a group of one:
- * the requirements are the `expect` terms and the `anyOf` groups alike, and
- * the score is found requirements over all of them plus `forbid`. A count
- * bound stays pass/fail: an item that produced two perfect terms when five
- * were wanted has not done what was asked.
- *
- * Pure on purpose, and returning `reasons` as plain sentences rather than
- * markup. The dashboard was the first consumer; `run-evals.js` is the second
- * and CI the third, and neither can reach into a page for a rendered cell.
- * Issue #248 wants a failure written out as a task an agent can act on.
- */
-function scoreCase(kase      , res                                                              )            {
-  // A case that expects its answer discarded passes on a discard and on
-  // nothing else: the answer the job threw away is the finding.
-  if (kase.discarded === true) {
-    const thrown = !!res.error && res.error.startsWith("discarded: ");
-    const n = (res.terms || []).length;
-    return { pass: thrown, score: thrown ? 1 : 0, discarded: !!res.error, found: [], missed: [], invented: [],
-             unmet: [], under: false, over: false, count: res.error ? 0 : n, watchFound: [], watchTotal: 0,
-             reasons: thrown ? [] : [res.error ? `Error - ${res.error}` : `FAIL: kept ${n} items, and the case expects the answer discarded`] };
-  }
-  // A discarded answer is a failed item: whatever the pipeline produced
-  // along the way does not count.
-  const discarded = !!res.error;
-  const terms = discarded ? [] : (res.terms || []);
-  const expect = kase.expect || [], forbid = kase.forbid || [];
-
-  // `missed` stays populated for a discarded reply even though it reads as
-  // vacuous, because the score depends on it: docs/datasets.md has such a reply
-  // scoring zero against everything the entry asked for, groups included, and
-  // `addToTally` gets there through `found + missed + invented`. Clear it and
-  // a run that discarded every item would contribute nothing to the
-  // total instead of contributing a nought, which flatters it.
-  //
-  // One member of a group is enough. Without this rule the set manufactures
-  // failures out of synonyms.
-  //
-  // A requirement reads back as a term when it had one member and as the
-  // group itself where a synonym list was allowed, so `missed` carries just
-  // enough to state the reason: "FAIL: Missed dog" for a plain expect term,
-  // "none of dog / puppy" for a group.
-  //
-  // Nothing satisfies a group when nothing was stored, so a discarded reply
-  // misses every requirement rather than none -- the same cast that makes its
-  // count bounds not breached by one. `unmet` is a finding only now that
-  // `missed` names the unsatisfied groups itself, but the run report has
-  // always carried it, so it stays.
-  const met = (g          ) => g.some(t => termIn(terms, t));
-  const spoken = (g          )              => g.length === 1 ? g[0]  : g;
-  const requirements = [...expect.map(t => [t]), ...(kase.anyOf || [])];
-  const found = requirements.filter(met).map(spoken);
-  const missed = requirements.filter(g => !met(g)).map(spoken);
-  const invented = forbid.filter(t => forbiddenIn(terms, t, kase.allow));
-  const unmet = discarded ? []
-    : (kase.anyOf || []).filter(g => !g.some(t => termIn(terms, t)));
-
-  const n = terms.length;
-  // A null bound is not checked -- and neither is a bound on a reply that was
-  // discarded. Nothing was counted out and found wanting there; the reply was
-  // thrown away before it had a count, and saying "0 items, wanted at least 4"
-  // states a second failure that never happened.
-  const under = !discarded && kase.minCount != null && n < kase.minCount;
-  const over = !discarded && kase.maxCount != null && n > kase.maxCount;
-
-  const denom = requirements.length + invented.length;
-  // A case that passes is one whose whole expectation was met, not one that
-  // scored well: an unsatisfied group is in `missed` alongside any term
-  // missed, so pass needs nothing more than the terms already covered.
-  const pass = !discarded && !missed.length && !invented.length && !under && !over;
-
-  // `found` and `missed` are groups, not terms, so the reasons word them one
-  // by one: a bare term under one heading, a group on its own line.
-  const reasons           = [];
-  if (discarded) {
-    // Everything else would be derived from this one fact -- there are no
-    // terms -- and would bury it. `scoreHtml` above suppresses `missed` for
-    // the same reason: the reason that matters is already on the row.
-    reasons.push(`Error - ${res.error}`);
-  } else {
-    const plain = missed.filter(t => typeof t === "string");
-    if (plain.length) reasons.push(`FAIL: Missed ${plain.join(", ")}`);
-    for (const g of missed) {
-      if (Array.isArray(g)) reasons.push(`none of ${g.join(" / ")}`);
+function readCase(kase      , res                                   , plain = false)              {
+  const input = metricInput(res                      , kase, { plain });
+  const metrics = (Array.isArray(kase.metrics) ? kase.metrics : []).flatMap(m => {
+    const r = readMetric(m, input, {});
+    if (r && typeof (r                    ).then === "function") {
+      return [{ type: m.type, label: METRICS[m.type]?.label ?? m.type, weight: typeof m.weight === "number" ? m.weight : 1,
+                pass: false, score: 0, reason: "a model-graded metric needs a grader" }                 ];
     }
-    if (invented.length) reasons.push(`invented ${invented.join(", ")}`);
-    if (under) reasons.push(`${n} items, wanted at least ${kase.minCount}`);
-    if (over) reasons.push(`${n} items, wanted at most ${kase.maxCount}`);
-  }
-
-  // Observed rather than scored: a dataset that wants to know whether some
-  // terms turn up, without grading on them, lists them as `watch`.
-  const watch = kase.watch || [];
-  // `null`, not 1, when there are no requirements and nothing forbidden turned
-  // up: there is no score to report. Returning 1 there printed "fail 100%"
-  // beside an entry that asked for nothing -- a shape `evals-check.js` allows
-  // even though every graded case now names an expectation.
-  return { pass, score: denom ? found.length / denom : null, discarded,
-           found, missed, invented, unmet, under, over, count: n,
-           watchFound: watch.filter(t => termIn(terms, t)), watchTotal: watch.length,
-           reasons };
+    return r ? [r                 ] : [];
+  });
+  const s = scoreOf(metrics) ?? { pass: true, score: null, metrics };
+  const counted = metrics.filter(r => r.weight !== 0);
+  return { ...s, metrics, found: s.found ?? [], missed: s.missed ?? [], invented: s.invented ?? [],
+           discarded: !!res.error, count: res.error ? 0 : (res.terms || []).length,
+           // A reply the job threw away fails on that one fact, which would
+           // be buried under everything that derives from it.
+           reasons: res.error && !s.pass ? [`Error - ${res.error}`] : counted.filter(r => !r.pass).map(r => `${r.label}: ${r.reason}`),
+           watch: metrics.filter(r => r.weight === 0) };
 }
 
 /**
@@ -2257,11 +2249,12 @@ const emptyTally = ()        => ({ ran: 0, passed: 0, found: 0, of: 0 });
 // docs/datasets.md: an item naming more terms should weigh more than
 // one naming fewer, and averaging percentages lets the easiest case carry the
 // score.
-function addToTally(t       , s                                                           )       {
+function addToTally(t       , s                                                             )       {
+  const found = s.found?.length ?? 0;
   t.ran++;
   if (s.pass) t.passed++;
-  t.found += s.found.length;
-  t.of += s.found.length + s.missed.length + s.invented.length;
+  t.found += found;
+  t.of += found + (s.missed?.length ?? 0) + (s.invented?.length ?? 0);
 }
 
 // The headline percentage, in one place because it is a number and this file
@@ -2276,7 +2269,7 @@ function tallyPercent(t       )                {
 // A whole-run assertion with no graded set: All of / Any of / None of, a
 // parse, a count, an exact reply and a length bound. It lives in the shared
 // core because it is a pass/fail the tab, the worker and History all have to
-// agree on -- the same one-definition rule that keeps `scoreCase` here.
+// agree on -- the same one-definition rule that keeps the case reader here.
 function parseCount(raw        , parse                           )                                           {
   // An unparsed reply is one result, not no result. It used to return null,
   // which made a Count of 1 fail as "null results" against a reply that
@@ -2666,10 +2659,10 @@ function applyModifiers   (list                                 , kind        ,
 // queue and run-evals.js alike, and every one of them reads it through the
 // functions below rather than through a translation of its own.
 //
-// The lab is generic, so what a reply is, what a test scores and how a value
+// The lab is generic, so what a reply is, what an eval scores and how a value
 // is changed are registry entries. The ones here are the lab's own, and so
 // are kinds/list.ts's -- the List kind and its modifiers. A dataset
-// registers nothing: it is data a graded test names by id, and a run carries.
+// registers nothing: it is data a graded eval names by id, and a run carries.
 
 // 2: a job's mappings are a list of { name, type, … } (tokenMappings), where
 // version 1 kept them as two maps (tokens: { values, blocks }).
@@ -2685,14 +2678,16 @@ function applyModifiers   (list                                 , kind        ,
 // 7: `chains` are `jobs`: the field renames and each job's `type` is "job".
 // 8: a job is its steps -- job 1's Attach Content (the pipeline's content), a
 // Call (Prompt: the image flag and token mappings), Read Reply (the output).
-// 9: a test is Metrics. A Single Test and a Graded set are read as the Metrics
+// 9: an eval is Metrics. A Single Test and a Graded set are read as the Metrics
 // they convert to (LEGACY_TESTS' toMetrics, proven equal by
 // metrics-parity-check.js).
 // 10: a job's steps are its stages (pipeline-model §16) -- Content (Attach
 // Content, the flow step, Attach Image, Token mappings) and Responses (Read
 // as, Response Format Validation, modifiers) -- and its call is gone: each
 // scenario is a target, whose own step in each job is what it sends there.
-const PIPELINE_VERSION = 10         ;
+// 11: `tests` are `evals`: the key renames and nothing in an eval changes,
+// so a stored result's scores, keyed by eval id, read as they did.
+const PIPELINE_VERSION = 12         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -2700,7 +2695,7 @@ const STEP_TYPES                                = Object.create(null);
 const CONTENT_TYPES                                   = Object.create(null);
 const OUTPUT_KINDS                                  = Object.create(null);
 const MODIFIERS                                = Object.create(null);
-const TEST_TYPES                                = Object.create(null);
+const EVAL_TYPES                                = Object.create(null);
 const METRICS                              = Object.create(null);
 const SOURCE_TYPES                                  = Object.create(null);
 
@@ -2714,11 +2709,15 @@ let defaultOutputKind = "text";
 const defaultKind = () => defaultOutputKind;
 const kindOf = (st                               )         => st.kind ?? defaultOutputKind;
 
+/** A module's eval types, under either spelling (Kinds.testTypes). */
+const evalTypesOf = (k                                        )                                =>
+  ({ ...(k.testTypes || {}), ...(k.evalTypes || {}) });
+
 /** A module's entries into the registries, in one call. */
 function registerKinds(k       )       {
   Object.assign(OUTPUT_KINDS, k.outputKinds || {});
   Object.assign(MODIFIERS, k.modifiers || {});
-  Object.assign(TEST_TYPES, k.testTypes || {});
+  Object.assign(EVAL_TYPES, evalTypesOf(k));
   Object.assign(SOURCE_TYPES, k.sourceTypes || {});
   Object.assign(METRICS, k.metrics || {});
   if (k.defaultKind) defaultOutputKind = k.defaultKind;
@@ -2749,7 +2748,7 @@ function pluginHost(pluginId        )             {
     registerKinds(k) {
       taken(OUTPUT_KINDS, "output kind", Object.keys(k.outputKinds || {}));
       taken(MODIFIERS, "modifier", Object.keys(k.modifiers || {}));
-      taken(TEST_TYPES, "test type", Object.keys(k.testTypes || {}));
+      taken(EVAL_TYPES, "eval type", Object.keys(evalTypesOf(k)));
       taken(METRICS, "metric", Object.keys(k.metrics || {}));
       // The server keeps its own copy of the Source types, to refuse a row of
       // one it does not know, and a plugin never reaches the server's code.
@@ -3015,11 +3014,9 @@ LEGACY_TESTS.single = {
   },
 };
 
-// A dataset's cases, scored item by item with scoreCase. The test
-// references the dataset the way a pipeline references a Source, and the run
-// carries the body it was submitted against. It scores the terms a value
-// yields, so it accepts every kind that yields any: plain text has none, and
-// would fail every case.
+// A dataset's cases, scored item by item. The eval references the dataset
+// the way a pipeline references a Source, and the run carries the body it
+// was submitted against.
 LEGACY_TESTS.graded = {
   label: "Graded set",
   fields: ["type", "dataset"],
@@ -3027,16 +3024,15 @@ LEGACY_TESTS.graded = {
   validate(t, ctx, bad){
     const d = t.dataset;
     if (!isRef(d) || (d.version != null && !isStr(d.version))) {
-      return void bad.push("a graded test has to name its dataset as { id, name }");
+      return void bad.push("a graded eval has to name its dataset as { id, name }");
     }
     if (ctx.datasets && !ctx.datasets.some(x => x.id === d.id)) bad.push(`Dataset ${d.name || d.id} not found`);
   },
-  score: (t, kase, res) => scoreCase(kase, res),
 };
 
 // Metrics: checks of each reply, deterministic or model-graded, from the
 // METRICS registry (metrics/builtin.ts registers the lab's own), with the
-// test's own list for every item and a case's `metrics` for its item. Scored
+// eval's own list for every item and a case's `metrics` for its item. Scored
 // all-must-pass -- every metric passes -- or weighted: points, each metric's
 // score times its weight, against a threshold (#150's points, a negative
 // weight taking them away).
@@ -3045,6 +3041,13 @@ LEGACY_TESTS.graded = {
 const METRIC_FIELDS                    = ["type", "not", "weight", "metric", "of", "every"];
 const SCORING_MODES = ["all", "weighted"];
 
+/** A metric's list option -- Contains all's values, Contains's exceptions --
+    one entry a line, or a list of them. */
+function metricLines(v         )           {
+  const all = Array.isArray(v) ? v.map(x => String(x ?? "")) : String(v ?? "").split("\n");
+  return all.map(x => x.trim()).filter(Boolean);
+}
+
 /** What is wrong with a list of metrics, as sentences naming [at]. */
 function metricsProblems(list         , at        , bad          )       {
   if (!Array.isArray(list)) return void bad.push(`${at}: metrics has to be a list`);
@@ -3114,14 +3117,16 @@ function readMetric(m        , input             , ctx           )
   } catch (e) { return timed(failed(e)); }
 }
 
-/** A test's score from its metrics' readings, the way [mode] says: all must
+/** An eval's score from its metrics' readings, the way [mode] says: all must
     pass, or weighted points against [threshold]. What a case metric found,
     missed and invented is carried up, for a run's totals. Null for none. */
 function scoreOf(metrics                 , mode = "all", threshold                = null)                     {
   if (!metrics.length) return null;
-  const detail = metrics.some(r => r.found || r.missed || r.invented) ? {
-    found: metrics.flatMap(r => r.found ?? []), missed: metrics.flatMap(r => r.missed ?? []),
-    invented: metrics.flatMap(r => r.invented ?? []) } : {};
+  // A watched reading (weight 0) is reported, and counts toward nothing.
+  const scored = metrics.filter(r => r.weight !== 0);
+  const detail = scored.some(r => r.found || r.missed || r.invented) ? {
+    found: scored.flatMap(r => r.found ?? []), missed: scored.flatMap(r => r.missed ?? []),
+    invented: scored.flatMap(r => r.invented ?? []) } : {};
   if (mode === "weighted") {
     const points = metrics.reduce((sum, r) => sum + r.weight * r.score, 0);
     return { pass: points >= (threshold ?? 0), score: points, points: true, metrics, ...detail };
@@ -3164,7 +3169,7 @@ function readRun(list                   , run             )                  {
   });
 }
 
-TEST_TYPES.metrics = {
+EVAL_TYPES.metrics = {
   label: "Metrics",
   description: "Checks of each reply, or of the whole run, scored all-must-pass or in weighted points.",
   fields: ["type", "metrics", "mode", "threshold", "grader", "dataset", "over"],
@@ -3188,9 +3193,9 @@ TEST_TYPES.metrics = {
     if (t.mode === "weighted" && typeof t.threshold !== "number") bad.push("weighted Metrics need a threshold: the points an item has to reach");
     if (t.threshold != null && typeof t.threshold !== "number") bad.push("the Metrics' threshold has to be a number");
     if (t.grader != null && !isRef(t.grader)) bad.push("the Metrics name their grader as { id, name }");
-    // The test's own grader, or the lab's.
+    // The eval's own grader, or the lab's.
     const grader = isRef(t.grader) ? t.grader : isRef(ctx.grader) ? ctx.grader : null;
-    if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the test, or make a Target profile the lab's grader");
+    if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a Target profile the lab's grader");
     // A grader is asked words, and needs a model to ask.
     const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
     if (grader && ctx.profiles && !p) bad.push(`Target profile ${grader.name || grader.id} not found`);
@@ -3205,59 +3210,35 @@ TEST_TYPES.metrics = {
     want: [m.not ? "not" : "", metricSummary(m), typeof m.weight === "number" && m.weight !== 1 ? `×${m.weight}` : ""]
       .filter(Boolean).join(" ") })),
   profiles: t => (isRef(t.grader) ? [t.grader] : []),
-  // A run carries the lab's grader on a test that names none and may ask one:
+  // A run carries the lab's grader on an eval that names none and may ask one:
   // a model-graded metric of its own, or a case's, over a dataset.
   resolve: (t, ctx) => (!isRef(t.grader) && isRef(ctx.grader) && (gradedIn(t.metrics) || isRef(t.dataset))
     ? { ...t, grader: { id: ctx.grader.id, name: ctx.grader.name } } : t),
   wholeRun: t => t.over === "run",
   // Over the whole run: every metric over the replies together, or each alone.
-  verdict(t, ress, kind){
-    const run = runInput(ress, !(kind != null && OUTPUT_KINDS[kind]?.terms));
-    const metrics = readRun(t.metrics || [], run);
-    const s = scoreOf(metrics, t.mode, t.threshold);
-    const off = metrics.filter(r => !r.pass);
-    return { pass: s ? s.pass : true, detail: off.length ? off.map(r => `${r.label}: ${r.reason}`).join(" · ") : "all matched",
-             ran: run.replies.length,
-             checks: metrics.map((r, x) => ({ key: `m${x}`, label: r.label, want: "", pass: r.pass, ...(r.pass ? {} : { note: r.reason }) })) };
-  },
-  // Rule m{x} is the test's own metric x, read in that place on every item.
+  verdict: (t, ress, kind) => wholeRunVerdict(t.metrics || [], ress, kind, t.mode, t.threshold),
+  // Rule m{x} is the eval's own metric x, read in that place on every item.
   // A score stored before Metrics -- a Graded set's, read as its conversion --
   // has no readings of its own: its one metric's reading is the score's.
   ruleOf: (score, key, t) => (score.metrics ? score.metrics[Number(key.slice(1))]?.pass ?? null
     : Array.isArray(t?.metrics) && t.metrics.length === 1 ? score.pass : null),
-  // Every item, with its case's own metrics where it has a case.
-  read: async (t, kase, res, more) => (await readMetrics([...(t.metrics || []), ...((kase?.metrics                        ) || [])],
+  // Every item, with its case's own metrics where the eval names the
+  // dataset and the item has a case there.
+  read: async (t, kase, res, more) => (await readMetrics([...(t.metrics || []), ...((isRef(t.dataset) && kase?.metrics) || [])],
     metricInput(res, kase, more), graderCtx(t, more), t.mode, t.threshold)),
 };
 
 // ---- the lab's own scorers, as metrics ----------------------------------------
-// What the Graded set and the Single Test check, one metric each, so a test of
-// either converts to Metrics that read a run exactly as it did
-// (metrics-parity-check.js). Here rather than in metrics/builtin.ts because
-// each is the core's own matcher.
-
-// An item's case, as the Graded set scores it: its expectations, forbidden
-// terms and count bounds, with what it found, missed and invented kept.
-METRICS.case = {
-  label: "Matches its case",
-  description: "Scores the reply against the item's case in the dataset: the items it expects, forbids and how many.",
-  perItem: true,
-  needsTerms: true,
-  options: [],
-  defaults: () => ({}),
-  score(input) {
-    if (!input.kase) return null;
-    const s = scoreCase(input.kase, { ...(input.error ? { error: input.error } : {}), terms: input.terms });
-    return { pass: s.pass, score: s.score ?? (s.pass ? 1 : 0), reason: s.reasons.join(" · ") || "all matched",
-             found: s.found, missed: s.missed, invented: s.invented };
-  },
-};
+// What the Single Test checks, one metric each, so an eval of it converts to
+// Metrics that read a run exactly as it did (metrics-parity-check.js). Here
+// rather than in metrics/builtin.ts because each is the core's own matcher.
 
 // Items the reply holds -- all of them, or any -- matched as the lab matches
 // a term; a kind that yields none is matched in the replies' text instead,
 // ignoring case, as the Single Test did.
 METRICS["has-items"] = {
   label: "Has items",
+  family: "The reply's text",
   description: "Passes when the reply holds all, or any, of the items listed.",
   options: [{ key: "values", label: "Items", type: "textarea" },
             { key: "need", label: "Needs", type: "select", choices: [{ value: "all", label: "all" }, { value: "any", label: "any" }] }],
@@ -3277,6 +3258,7 @@ METRICS["has-items"] = {
 // A reply's length in characters, once trimmed.
 METRICS.length = {
   label: "Length",
+  family: "The reply's text",
   description: "Compares the reply's length in characters.",
   options: [{ key: "op", label: "Is", type: "select", choices: LENGTH_OPS.map(([value, label]) => ({ value, label })) },
             { key: "n", label: "Characters", type: "number" }],
@@ -3292,6 +3274,7 @@ METRICS.length = {
 // as the count says.
 METRICS["parse-count"] = {
   label: "Parses",
+  family: "The result",
   description: "Passes when the reply reads as the format chosen, holding the number of results set.",
   // Unformatted is one result a reply, as the Single Test counted it.
   options: [{ key: "parse", label: "As", type: "select", choices: [{ value: "", label: "Unformatted" }, { value: "csv", label: "csv" },
@@ -3310,12 +3293,13 @@ METRICS["parse-count"] = {
   },
 };
 
-/** What every test carries, kept across a conversion. */
+/** What every eval carries, kept across a conversion. */
 const common = (t           ) => ({ id: t.id, name: t.name ?? "", continueOnFailure: t.continueOnFailure !== false });
 
-// A Graded set is a Metrics test over the same dataset, its one metric the case.
+// A Graded set is a Metrics eval over the same dataset, with none of its own:
+// each case's metrics are what it scored (dataset-parity-check.js).
 LEGACY_TESTS.graded .toMetrics = t => ({ ...common(t), type: "metrics", mode: "all", threshold: null, grader: null,
-  dataset: t.dataset, over: "item", metrics: [{ type: "case" }] });
+  dataset: t.dataset, over: "item", metrics: [] });
 
 // A Single Test is Metrics over the whole run: its lists over the run's items
 // together, and exact, length and parse over each reply alone.
@@ -3333,7 +3317,39 @@ LEGACY_TESTS.single .toMetrics = t => {
   return { ...common(t), type: "metrics", mode: "all", threshold: null, grader: null, dataset: null, over: "run", metrics };
 };
 
-/** The grader a Metrics test names, reached through what the runner hands it. */
+/** [list] read over a whole run's replies so far, scored the way [mode]
+    says: a Metrics eval's verdict over the run, and an eval group's Whole run. */
+function wholeRunVerdict(list                   , ress                                       , kind               ,
+                         mode = "all", threshold                = null)                  {
+  const run = runInput(ress, !(kind != null && OUTPUT_KINDS[kind]?.terms));
+  const metrics = readRun(list, run);
+  const s = scoreOf(metrics, mode, threshold);
+  const off = metrics.filter(r => !r.pass);
+  return { pass: s ? s.pass : true, detail: off.length ? off.map(r => `${r.label}: ${r.reason}`).join(" · ") : "all matched",
+           ran: run.replies.length,
+           checks: metrics.map((r, x) => ({ key: `m${x}`, label: r.label, want: "", pass: r.pass, ...(r.pass ? {} : { note: r.reason }) })) };
+}
+
+/**
+ * An eval group's verdict on one item (docs/pipeline-model.md §17): its
+ * Every item metrics and [kase]'s own -- the item's case in the group, or
+ * null where it has none -- read of the reply under the group's scoring.
+ * The reading a Metrics eval naming the group as its dataset gives, which
+ * groups-check.js holds it to. Null where no metric had anything to read.
+ */
+function readGroup(group             , kase             , res                    , more           = {})                              {
+  return readMetrics([...group.every, ...(kase?.metrics ?? [])], metricInput(res, kase, more), graderCtx(group, more),
+                     group.scoring.mode, group.scoring.threshold);
+}
+
+/** An eval group's Whole run verdict, from the replies so far: its `run`
+    metrics under its scoring. Null for a group with no Whole run metrics,
+    which has nothing to say of a run. */
+function readGroupRun(group             , ress                                       , kind                = null)                         {
+  return group.run.length ? wholeRunVerdict(group.run, ress, kind, group.scoring.mode, group.scoring.threshold) : null;
+}
+
+/** The grader a Metrics eval names, reached through what the runner hands it. */
 function graderCtx(t           , more          )            {
   const ask = isRef(t.grader) ? more.grader?.(t.grader) : undefined;
   return ask ? { ask } : {};
@@ -3343,12 +3359,14 @@ function graderCtx(t           , more          )            {
     for one with none set: what History prints beside its name. */
 function metricSummary(m        )         {
   const entry = METRICS[m.type];
-  // Each option as it reads in the editor: a choice's label, a box that is
-  // ticked by its own label, text by its first line.
+  // Each option as it reads in the editor: a choice's label, a box by its
+  // own label where it is not as a new metric has it (ticked, or "off"),
+  // text by its first line.
+  const fresh = entry?.defaults() ?? {};
   const said = (entry?.options || []).map(o => {
     const v = m[o.key];
     if (o.type === "select") return o.choices.find(c => c.value === v)?.label ?? "";
-    if (o.type === "checkbox") return v ? o.label.toLowerCase() : "";
+    if (o.type === "checkbox") return !!v === !!fresh[o.key] ? "" : v ? o.label.toLowerCase() : `${o.label.toLowerCase()} off`;
     // Text that runs to lines (a schema) reads as its first words, run together.
     return v == null || isObj(v) ? "" : String(v).replace(/\s+/g, " ").trim().slice(0, 40);
   }).filter(Boolean);
@@ -3531,7 +3549,7 @@ STEP_TYPES.readAs = {
 // Response Format Validation: the kind a reply is read as.
 STEP_TYPES.readReply = {
   label: "Read Reply", slot: "responses", rank: 1,
-  description: "How the reply is read before tests and later jobs see it.",
+  description: "How the reply is read before evals and later jobs see it.",
   in: "text", out: step => step.out?.kind,
   // Read inside the stage: runPipeline parses each reply as it comes back.
   apply: "runPipeline",
@@ -3787,7 +3805,7 @@ STEP_TYPES.job = {
     if (!Array.isArray(c.steps)) return void bad.push(`${at}: steps has to be a list`);
     const steps              = c.steps;
     // A job's steps are the entries with a stage; one without (a job, the
-    // tests) or of no type the lab has is not one.
+    // evals) or of no type the lab has is not one.
     const unknown = steps.find(st => !slotOf(st));
     if (unknown) return void bad.push(`${at}: "${isObj(unknown) ? unknown.type : unknown}" is not a step this lab has`);
     if (steps.some(st => slotOf(st) === "target")) {
@@ -3808,54 +3826,55 @@ STEP_TYPES.job = {
     }
   },
 };
-// What a test carries besides its type's own fields.
-const TEST_FIELDS                    = ["id", "name", "continueOnFailure"];
+// What an eval carries besides its type's own fields.
+const EVAL_FIELDS                    = ["id", "name", "continueOnFailure"];
 
-/** Test [j]'s name, or the number it has always shown. */
-const testLabel = (doc                                                 , j        )         =>
-  (doc.tests?.[j]?.name || "").trim() || `Test ${j + 1}`;
+/** Eval [j]'s name, or the number it has always shown. */
+const evalLabel = (doc                                                 , j        )         =>
+  (doc.evals?.[j]?.name || "").trim() || `Eval ${j + 1}`;
 
-/** A test type that settles over the whole run rather than item by item. */
+/** An eval type that settles over the whole run rather than item by item. */
 const isWholeRun = (type                           , t            )          =>
   type?.wholeRun && t ? type.wholeRun(t) : !!type?.verdict && !type?.score;
 
 /**
- * A document's tests as a list, whatever version wrote it: a queue row keeps
- * the run document it was submitted with, so a run from before version 6
- * holds one test or null, read here as the upgrade reads it (testsList).
+ * A document's evals as a list, whatever version wrote it: a queue row keeps
+ * the run document it was submitted with, so a run from before version 11
+ * spells them `tests`, and one from before version 6 holds one or null, read
+ * here as the upgrade reads it (testsList, evalsKey).
  */
-function testsOf(doc                                        )         {
-  const t = doc?.tests;
+function evalsOf(doc                                                         )         {
+  const t = doc?.evals ?? doc?.tests;
   if (Array.isArray(t)) return t          ;
   return isObj(t) ? [{ id: "t1", name: "", ...t, continueOnFailure: true }        ] : [];
 }
 
-/** The dataset a document's tests grade against, where one does: a run
+/** The dataset a document's evals grade against, where one does: a run
     grades against one (validatePipeline says so), so the first names it. */
-function testsDataset(doc                     )                                      {
-  const t = testsOf(doc).find((x         ) => isObj(x) && isRef(x.dataset));
+function evalsDataset(doc                     )                                      {
+  const t = evalsOf(doc).find((x         ) => isObj(x) && isRef(x.dataset));
   return t && "dataset" in t ? t.dataset : null;
 }
 
-STEP_TYPES.tests = {
+STEP_TYPES.evals = {
   in: "results", out: "verdict",
   apply: "score",
-  validate(tests, ctx, bad, lastKind, doc){
-    if (!Array.isArray(tests)) return void bad.push("tests has to be a list, empty for an unscored run");
+  validate(evals, ctx, bad, lastKind, doc){
+    if (!Array.isArray(evals)) return void bad.push("evals has to be a list, empty for an unscored run");
     const seen = new Set        ();
-    tests.forEach((t           , j        ) => {
-      const at = testLabel(doc, j);
-      const type = isObj(t) && TEST_TYPES[t.type];
+    evals.forEach((t           , j        ) => {
+      const at = evalLabel(doc, j);
+      const type = isObj(t) && EVAL_TYPES[t.type];
       if (!type) return void bad.push(`${at} is of type "${t?.type}", which is not one this lab has`);
       const before = bad.length;
       if (!isStr(t.id) || !t.id.trim()) bad.push(`${at} has no id`);
-      else if (seen.has(t.id)) bad.push(`${at} has the id of another test`);
+      else if (seen.has(t.id)) bad.push(`${at} has the id of another eval`);
       else seen.add(t.id);
       if (t.name != null && !isStr(t.name)) bad.push(`${at}: name has to be text`);
       if (typeof t.continueOnFailure !== "boolean") bad.push(`${at}: continueOnFailure has to be true or false`);
-      onlyFields(t, at, [...type.fields, ...TEST_FIELDS], bad);
+      onlyFields(t, at, [...type.fields, ...EVAL_FIELDS], bad);
       type.validate(t, ctx, bad);
-      // Which kinds a test scores is only worth saying of a test that is whole.
+      // Which kinds an eval scores is only worth saying of an eval that is whole.
       if (bad.length > before) return;
       const accepts = typeof type.accepts === "function" ? type.accepts(t) : type.accepts;
       if (accepts && lastKind && OUTPUT_KINDS[lastKind] && !accepts.includes(lastKind)) {
@@ -3864,14 +3883,14 @@ STEP_TYPES.tests = {
       }
     });
     // A run is handed one dataset's body to grade against (server-side-runs §4).
-    const named = new Set(tests.filter((t           ) => isObj(t) && isRef(t.dataset)).map((t           ) => t.dataset.id));
-    if (named.size > 1) bad.push("the tests grade against one dataset at a time");
+    const named = new Set(evals.filter((t           ) => isObj(t) && isRef(t.dataset)).map((t           ) => t.dataset.id));
+    if (named.size > 1) bad.push("the evals grade against one dataset at a time");
   },
 };
 
 // ---- the document -------------------------------------------------------------
 
-const PIPELINE_FIELDS                    = ["version", "id", "name", "jobs", "targets", "tests"];
+const PIPELINE_FIELDS                    = ["version", "id", "name", "jobs", "targets", "evals"];
 // What resolving adds, and nothing else: the profiles it resolved to and the
 // run's own comment, which belongs to the run and never to the pipeline.
 const RUN_FIELDS = [...PIPELINE_FIELDS, "profiles", "comment", "plugins"];
@@ -3888,7 +3907,7 @@ function versionProblem(doc         )                {
 /** What an upgrade from version 2 needs from outside the document: the rules
     of the dataset a pipeline was graded against, which the retired
     version-2 list kind read every reply under. Without them a job is upgraded with no
-    rules, as a run with no graded test parsed. */
+    rules, as a run with no graded eval parsed. */
                              
                                                                 
                                                                     
@@ -3913,7 +3932,8 @@ function versionProblem(doc         )                {
  * as it was, for versionProblem to name. A copy: the caller's document is not
  * touched. From version 5, its test -- or none -- becomes a list of one (or
  * none), the test's id `t1`. From version 6, `chains` are `jobs`: the field
- * renames and every job's `type` becomes `"job"`.
+ * renames and every job's `type` becomes `"job"`. From version 10, its
+ * `tests` are `evals` (evalsKey).
  */
 /** Every id a current document must carry, minted for the ones [doc] lacks.
     An id it already has is kept. */
@@ -3949,6 +3969,9 @@ function profileRefs(doc           )       {
   }
 }
 
+/** [doc] with its profile references cut (profileRefs), for chaining. */
+const cutRefs = (doc           )            => { profileRefs(doc); return doc; };
+
 /** Whether any profile reference in a pipeline holds more than { id, name }. */
 function fatProfileRef(doc         )          {
   const fat = (r         ) => isObj(r) && Object.keys(r).some(k => k !== "id" && k !== "name");
@@ -3967,9 +3990,9 @@ function testsList(doc           )            {
   return doc;
 }
 
-/** A new test of [type] at the lab's defaults, named [name], continuing on failure. */
-function newTest(type        , name = "", fields                          = {})       {
-  const own = TEST_TYPES[type]?.defaults?.() ?? { type };
+/** A new eval of [type] at the lab's defaults, named [name], continuing on failure. */
+function newEval(type        , name = "", fields                          = {})       {
+  const own = EVAL_TYPES[type]?.defaults?.() ?? { type };
   return { ...own, ...fields, type, id: newId(), name, continueOnFailure: true }        ;
 }
 
@@ -4067,6 +4090,29 @@ function targetsFromScenarios(next           )            {
   return Object.fromEntries(Object.entries(next).map(([key, v]) => (key === "scenarios" ? ["targets", targets] : [key, v])));
 }
 
+/** Version 10 to 11: `tests` are `evals` -- the key renames in place, so an
+    upgraded document reads the same, key for key, and each eval is as it
+    was. Every version before 11 passes through here. */
+function evalsKey(next           )            {
+  next.version = PIPELINE_VERSION;
+  // `tests` is the spelling of every version before 11.
+  if (!("tests" in next)) return next;
+  return Object.fromEntries(Object.entries(next).filter(([k]) => k !== "evals")
+    .map(([k, v]) => (k === "tests" ? ["evals", v] : [k, v])));
+}
+
+/** Version 11 to 12: a Contains metric's Ignore case holds where the reply
+    is matched item by item, as it does where it is matched as text, and it
+    is kept as written. Until version 11's last hours (#199) every Contains
+    metric matched the reply's text and minded its Ignore case, so what a
+    stored metric says is what its author meant; only the item-by-item
+    matching #199 added, case-blind for a few hours, read it otherwise.
+    Every version before 12 ends here. */
+function caseAsWritten(next           )            {
+  next.version = PIPELINE_VERSION;
+  return next;
+}
+
 function upgradePipeline   (doc   , ctx             = {})    {
   // A current document is read as it is, but for a profile reference the Runs
   // tab saved whole (see profileRefs), which is cut back, and a step on a
@@ -4077,11 +4123,29 @@ function upgradePipeline   (doc   , ctx             = {})    {
       out = clone(out)             ;
       profileRefs(out);
     }
-    return localSteps(out, ctx)     ;
+    return localSteps(withoutCaseMetric(out), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9].includes(doc.version          )) return doc;
-  // Every version before 10 reads as version 9 first, then as 10.
-  return localSteps(targetsFromScenarios(nineOf(clone(doc)             , ctx)), ctx)     ;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11].includes(doc.version          )) return doc;
+  if (doc.version === 11) return localSteps(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc)             ))), ctx)     ;
+  // Every version before 10 reads as version 9 first, then as 10, then as 11
+  // and 12.
+  // A version-10 document is cut as a current one was (profileRefs); the
+  // earlier ones are cut on their way through nineOf.
+  const ten = doc.version === 10 ? cutRefs(clone(doc)             ) : targetsFromScenarios(nineOf(clone(doc)             , ctx));
+  return localSteps(withoutCaseMetric(caseAsWritten(evalsKey(ten))), ctx)     ;
+}
+
+/** [doc] with no Metrics eval holding the `case` metric: a dataset's cases
+    are metrics now (dataset body version 5), which an eval naming the
+    dataset adds for each item, so the case's own metrics carry what that
+    metric scored. Read so at every version, the current one included, as a
+    document saved before it went still holds it. The same document where
+    none does. */
+function withoutCaseMetric(doc           )            {
+  const has = (t         ) => isObj(t) && Array.isArray(t.metrics) && t.metrics.some((m           ) => isObj(m) && m.type === "case");
+  if (!Array.isArray(doc.evals) || !doc.evals.some(has)) return doc;
+  return { ...doc, evals: doc.evals.map((t           ) => (has(t)
+    ? { ...t, metrics: t.metrics.filter((m           ) => !(isObj(m) && m.type === "case")) } : t)) };
 }
 
 /** [doc] with each step asked of a profile whose type a target step stands
@@ -4165,7 +4229,7 @@ function tokenMappingsFromV1(set         )                {
 function blankPipeline(opts                                                           = {})           {
   const job = withImage(jobDefaults(opts.kind, opts.tokens), true);
   return { version: PIPELINE_VERSION, id: newId(), name: opts.name || "",
-           jobs: [job], targets: [], tests: [] };
+           jobs: [job], targets: [], evals: [] };
 }
 
 /**
@@ -4303,7 +4367,7 @@ function validatePipeline(input         , ctx          = {})           {
         : `no model on ${targetLabel(doc, i)}'s Target profile — manage profiles on the Setup tab`);
     }
   });
-  STEP_TYPES.tests .validate(doc.tests, c, bad, outOf(doc.jobs.at(-1)).kind, doc);
+  STEP_TYPES.evals .validate(doc.evals, c, bad, outOf(doc.jobs.at(-1)).kind, doc);
   if (bad.length) return bad;
 
   // Asked before anything is sent, so a misspelt token costs nothing and
@@ -4400,9 +4464,9 @@ function profileIds(doc                                                  )
       if (ref?.id && !ids.includes(ref.id)) ids.push(ref.id);
     }
   }
-  // Then the ones a test asks (a grader), so a run carries them too.
-  for (const t of testsOf(doc)) {
-    for (const ref of TEST_TYPES[t.type]?.profiles?.(t) ?? []) if (ref?.id && !ids.includes(ref.id)) ids.push(ref.id);
+  // Then the ones an eval asks (a grader), so a run carries them too.
+  for (const t of evalsOf(doc)) {
+    for (const ref of EVAL_TYPES[t.type]?.profiles?.(t) ?? []) if (ref?.id && !ids.includes(ref.id)) ids.push(ref.id);
   }
   return ids;
 }
@@ -4415,9 +4479,9 @@ function profileIds(doc                                                  )
 function resolvePipeline(doc          ,
                          ctx                                                                            = {})              {
   const run = clone(doc)             ;
-  // What the lab supplies a test -- its grader -- before the profiles it
+  // What the lab supplies an eval -- its grader -- before the profiles it
   // asks are carried.
-  run.tests = (Array.isArray(run.tests) ? run.tests : []).map((t           ) => TEST_TYPES[t?.type]?.resolve?.(t, ctx) ?? t);
+  run.evals = (Array.isArray(run.evals) ? run.evals : []).map((t           ) => EVAL_TYPES[t?.type]?.resolve?.(t, ctx) ?? t);
   run.profiles = {};
   for (const id of profileIds(run)) {
     const p = ctx.profiles?.(id);
@@ -4426,7 +4490,7 @@ function resolvePipeline(doc          ,
   const content = contentOf(run)             ;
   if (content?.type === "source") content.files = CONTENT_TYPES.source .expand (content, ctx.files);
   if (ctx.datasetVersion) {
-    for (const t of run.tests || []) if (isRef(t.dataset)) t.dataset.version = ctx.datasetVersion;
+    for (const t of run.evals || []) if (isRef(t.dataset)) t.dataset.version = ctx.datasetVersion;
   }
   if (isStr(ctx.comment)) run.comment = ctx.comment;
   return run;
@@ -4440,7 +4504,7 @@ function pipelineOfRun(run             , ctx             = {})           {
   delete doc.plugins;
   const content = contentOf(doc)             ;
   if (content) { delete content.files; delete content.revs; }
-  for (const t of doc.tests || []) if (isObj(t.dataset)) delete t.dataset.version;
+  for (const t of doc.evals || []) if (isObj(t.dataset)) delete t.dataset.version;
   return doc;
 }
 
@@ -4498,7 +4562,7 @@ function importPipeline(input         , ctx            = {})               {
       if (st?.profile) st.profile = remap(st.profile, "profile");
     }
   }
-  for (const t of Array.isArray(next.tests) ? next.tests : []) {
+  for (const t of Array.isArray(next.evals) ? next.evals : []) {
     if (isRef(t?.dataset)) t.dataset = remap(t.dataset, "dataset");
   }
   // Its references are this lab's now, so a step asking this lab's Echo
@@ -4530,7 +4594,7 @@ function mintIds(doc           )            {
   for (const t of Array.isArray(doc.targets) ? doc.targets : []) {
     if (isObj(t) && (!isStr(t.id) || !t.id)) t.id = newId();
   }
-  for (const t of Array.isArray(doc.tests) ? doc.tests : []) {
+  for (const t of Array.isArray(doc.evals) ? doc.evals : []) {
     if (isObj(t) && (!isStr(t.id) || !t.id)) t.id = newId();
   }
   return doc;
@@ -4608,14 +4672,14 @@ function modifierSummary(m          )         {
   return said.length ? said.join(", ") : "on";
 }
 
-// ---- the tests, in order (docs/pipeline-model.md §3) ---------------------------
-// Tests only read a run: none changes what a later one sees, and none stops
+// ---- the evals, in order (docs/pipeline-model.md §3) ---------------------------
+// Evals only read a run: none changes what a later one sees, and none stops
 // the model being sent the next item. What order changes is Continue on
-// failure. A per-item test that fails and does not continue stops the tests
+// failure. A per-item eval that fails and does not continue stops the evals
 // after it for that item alone -- they read Skipped there, and a whole-run
-// test after it pools the items it did not stop. A whole-run test settles
+// eval after it pools the items it did not stop. A whole-run eval settles
 // once every item is in, and one that fails then and does not continue
-// leaves every test after it Skipped.
+// leaves every eval after it Skipped.
 
 const SKIPPED          = Object.freeze({ skipped: true          });
 const isSkipped = (s         )               => isObj(s) && s.skipped === true;
@@ -4623,16 +4687,16 @@ const isSkipped = (s         )               => isObj(s) && s.skipped === true;
 const failedScore = (s                              )          => !!s && !isSkipped(s) && !s.pass;
 
 /**
- * One reply's scores under a run's per-item tests, by test id, in order: a
- * test with no case to score leaves no entry, and one after a failure that
+ * One reply's scores under a run's per-item evals, by eval id, in order: a
+ * eval with no case to score leaves no entry, and one after a failure that
  * does not continue reads Skipped. Null where nothing was scored.
  */
-function itemScores(run             , kase                         , res                    )                                             {
+function itemScores(run             , kase                         , res                    )                                               {
   if (!kase) return null;
-  const out                                      = {};
+  const out                                        = {};
   let stopped = false;
-  for (const t of testsOf(run)) {
-    const type = TEST_TYPES[t.type];
+  for (const t of evalsOf(run)) {
+    const type = EVAL_TYPES[t.type];
     if (!type?.score) continue;
     if (stopped) { out[t.id] = SKIPPED; continue; }
     const s = type.score(t, kase, res);
@@ -4650,19 +4714,19 @@ function productionOf(run                          , record         )
 }
 
 /**
- * itemScores for the runner, which can wait: a test that reads every item
+ * itemScores for the runner, which can wait: an eval that reads every item
  * (`read`: the Metrics, which may ask a grader) scores one with no case too.
  */
 async function itemScoresAsync(run             , kase                         , res                    ,
                                more           = {})                                                        {
   const out                                        = {};
   let stopped = false;
-  for (const t of testsOf(run)) {
-    const type = TEST_TYPES[t.type];
+  for (const t of evalsOf(run)) {
+    const type = EVAL_TYPES[t.type];
     if (isWholeRun(type, t) || (!type?.read && !(type?.score && kase))) continue;
     if (stopped) { out[t.id] = SKIPPED; continue; }
-    // A test with nothing to read on this item leaves no entry, as a graded
-    // test does on an item with no case.
+    // An eval with nothing to read on this item leaves no entry, as a graded
+    // eval does on an item with no case.
     const s = type.read ? await type.read(t, kase ?? null, res, { plain: !OUTPUT_KINDS[lastKind(run) ?? ""]?.terms, ...more })
       : type.score (t, kase , res);
     if (!s) continue;
@@ -4673,22 +4737,22 @@ async function itemScoresAsync(run             , kase                         ,
 }
 
 /**
- * Every test's reading of scenario [i] of a run, in the run's order, from
+ * Every eval's reading of scenario [i] of a run, in the run's order, from
  * the items so far. [settled] says every item is in: only then has a
- * whole-run test settled, so only then does its failure skip the tests
+ * whole-run eval settled, so only then does its failure skip the evals
  * after it.
  */
-function scenarioTests(run             , i        , items                                             ,
+function scenarioEvals(run             , i        , items                                             ,
                        settled = true)                {
   const cells = (items || []).map(it => (it && !it.unrun ? it.scenarios?.[i] : undefined));
-  // Which items a per-item test has stopped so far, for the tests after it.
+  // Which items a per-item eval has stopped so far, for the evals after it.
   const stopped = cells.map(() => false);
   let skipRest = false;
-  const tests = testsOf(run);
-  return tests.map((t, j) => {
-    const type = TEST_TYPES[t.type];
+  const evals = evalsOf(run);
+  return evals.map((t, j) => {
+    const type = EVAL_TYPES[t.type];
     const whole = isWholeRun(type, t);
-    const base = { id: t.id, label: testLabel({ tests }, j), whole, skipped: skipRest,
+    const base = { id: t.id, label: evalLabel({ evals }, j), whole, skipped: skipRest,
                    verdict: null                          , rules: type?.rules?.(t) ?? [], items: cells.map(() => null)                        ,
                    ran: 0, passed: 0, skippedItems: 0 };
     if (skipRest) {
@@ -4715,7 +4779,7 @@ function scenarioTests(run             , i        , items
   });
 }
 
-/** A scenario's pass or fail over every test that read it: null where none
+/** A scenario's pass or fail over every eval that read it: null where none
     has anything to say yet. */
 function scenarioPasses(outcomes                        )                 {
   let said = false;
@@ -4756,95 +4820,95 @@ function validateEvals(ev           , files
     return ["the graded set has to be a JSON object with a `cases` list"];
   }
   if (ev.cases != null && !Array.isArray(ev.cases)) bad.push("cases has to be a list");
+  if (ev.source != null && !isRef(ev.source)) bad.push("a dataset names its Source as { id, name }, or null");
+  // An eval group's own sections (version 7): what a Metrics eval held.
+  if (ev.scoring != null) {
+    const sc = ev.scoring;
+    if (!isObj(sc) || !SCORING_MODES.includes(sc.mode)) bad.push(`a group is scored ${SCORING_MODES.join(" or ")}`);
+    else if (sc.mode === "weighted" && typeof sc.threshold !== "number") bad.push("a group scored in points needs Pass at: the points an item has to reach");
+    else if (sc.threshold != null && typeof sc.threshold !== "number") bad.push("a group's Pass at has to be a number");
+  }
+  if (ev.grader != null && !isRef(ev.grader)) bad.push("a group names its grader as { id, name }, or null");
+  if (ev.every != null) metricsProblems(ev.every, "Every item", bad);
+  if (ev.run != null) {
+    metricsProblems(ev.run, "Whole run", bad);
+    // Read once over every reply: nothing that reads one item, or asks a grader.
+    for (const m of Array.isArray(ev.run) ? ev.run : []) {
+      const entry = isObj(m) ? METRICS[m.type] : undefined;
+      if (entry?.graded) bad.push(`Whole run: ${entry.label} is model-graded, so it cannot read a whole run`);
+      else if (entry?.perItem) bad.push(`Whole run: ${entry.label} reads one item at a time, so it cannot read a whole run`);
+    }
+  }
   if (bad.length) return bad;
 
   const graded              = ev.cases || [];
   // Identity first: every rule below reports which case is at fault, so a
   // case with no usable id makes the rest of the report unreadable.
-  const seen = new Map                ();
+  const seen = new Set        ();
   for (const c of graded) {
-    {
-      if (!c || typeof c !== "object" || Array.isArray(c)) {
-        bad.push("cases holds something that is not a case");
-        continue;
-      }
-      if (typeof c.id !== "string" || !c.id.trim()) bad.push("a case has no id");
-      else if (seen.has(c.id)) bad.push(`${c.id} is graded twice`);
-      else seen.set(c.id, "cases");
-      if (!caseFile(c).trim()) {
-        bad.push(`${c.id || "a case"} names no file`);
-      }
-      for (const key of ["expect", "forbid", "allow", "anyOf", "watch", "traits"]) {
-        if (c[key] != null && !Array.isArray(c[key])) bad.push(`${c.id}: ${key} has to be a list`);
-      }
-      for (const key of ["minCount", "maxCount"]) {
-        if (c[key] != null && !Number.isInteger(c[key])) bad.push(`${c.id}: ${key} has to be a whole number`);
-      }
-      if (c.discarded != null && c.discarded !== true) bad.push(`${c.id}: discarded is true or left out`);
-      // A case's own metrics, which a Metrics test adds to its own for this item.
-      if (c.metrics != null) metricsProblems(c.metrics, String(c.id), bad);
+    if (!isObj(c)) {
+      bad.push("cases holds something that is not a case");
+      continue;
     }
+    if (!isStr(c.id) || !c.id.trim()) bad.push("a case has no id");
+    else if (seen.has(c.id)) bad.push(`${c.id} is graded twice`);
+    else seen.add(c.id);
+    if (!caseItem(c).trim()) bad.push(`${c.id || "a case"} names no item`);
+    if (c.todo != null && typeof c.todo !== "boolean") bad.push(`${c.id}: todo is true or false`);
+    if (c.note != null && !isStr(c.note)) bad.push(`${c.id}: note has to be text`);
+    if (c.metrics != null) metricsProblems(c.metrics, String(c.id), bad);
   }
   if (bad.length) return bad;
 
-  // A term the scorer cannot match is a term that can never be produced, so
-  // the case is unpassable however well the model answers.
+  // What a case says, read from its metrics: a term the scorer cannot match
+  // can never be produced, so the case is unpassable however well the model
+  // answers -- and the same goes for a bound no reply can meet.
   const matchable = (term        ) => termIn([String(term).trim()], term);
-
+  const values = (m        )           => metricLines(m.type === "contains" ? m.value : m.values);
   for (const c of graded) {
     if (c.todo) continue;
     const say = (m        ) => bad.push(`${c.id}: ${m}`);
-    const expect = c.expect || [], forbid = c.forbid || [], anyOf = c.anyOf || [];
-
-    if (c.discarded === true) {
+    const scored = (c.metrics                         ?? []).filter(m => m.weight !== 0);
+    if (!scored.length) { say("graded but states nothing to expect"); continue; }
+    if (scored.some(m => m.type === "discarded" && !m.not)) {
       // What is discarded holds nothing, so there is nothing else to expect.
-      if (expect.length || anyOf.length || forbid.length || c.minCount != null || c.maxCount != null) {
-        say("expects its answer discarded, and states something the answer should hold too");
-      }
+      if (scored.length > 1) say("expects its answer discarded, and states something the answer should hold too");
       continue;
     }
-    const metrics = Array.isArray(c.metrics) ? c.metrics : [];
-    if (!expect.length && !anyOf.length && !metrics.length) say("graded but states nothing to expect");
-    for (const t of expect) if (!matchable(t)) say(`expects ${t}, which its own scorer cannot match`);
-    for (const g of anyOf) {
-      if (!Array.isArray(g)) { say("an anyOf group has to be a list of terms"); continue; }
-      if (!g.length) say("an anyOf group is empty, so nothing can satisfy it");
-      for (const t of g) if (!matchable(t)) say(`offers ${t}, which its own scorer cannot match`);
-    }
-    // An exception excuses only the forbidden terms inside it, so one that
-    // holds none of them changes nothing and reads as if it did.
-    for (const a of c.allow || []) {
-      if (!forbid.some((t        ) => termIn([a], t))) say(`allows ${a}, which holds nothing it forbids`);
+    const wants = scored.filter(m => !m.not && (m.type === "contains-all" || m.type === "contains-any"));
+    const forbids = scored.filter(m => m.not && m.type === "contains");
+    for (const m of wants) for (const t of values(m)) if (!matchable(t)) say(`expects ${t}, which its own scorer cannot match`);
+    // An exception excuses only the forbidden term inside it, so one that
+    // holds it not changes nothing and reads as if it did.
+    for (const m of forbids) {
+      for (const e of metricLines(m.except)) if (!termIn([e], String(m.value ?? ""), m.ignoreCase === true)) say(`allows ${e}, which holds nothing it forbids`);
     }
     // A term on both lists cannot be produced and cannot be withheld.
-    for (const t of [...expect, ...anyOf.flat()]) {
-      if (forbid.includes(t)) say(`${t} is both expected and forbidden`);
-    }
-    const { minCount: lo, maxCount: hi } = c;
-    if (lo != null && hi != null && lo > hi) say(`minCount ${lo} is above maxCount ${hi}`);
-    if (lo != null && lo < 1) say(`minCount ${lo} is not a bound`);
-    // The mirror of the floor: a ceiling below one says no answer is
-    // acceptable, which is an entry that can never pass rather than a
-    // strict one.
-    if (hi != null && hi < 1) say(`maxCount ${hi} leaves no answer that could pass`);
-    // A case that names more distinct things than the reply may carry, in
-    // the scorer's own counting of a thing (#486: an expect term is one, an
-    // anyOf group is one), can never pass however well the model answers.
-    const things = expect.length + anyOf.length;
-    if (hi != null && things > hi) {
-      say(`asks for ${things} things and maxCount ${hi} admits ${hi}`);
+    const forbidden = new Set(forbids.map(m => String(m.value ?? "")));
+    for (const m of wants) for (const t of values(m)) if (forbidden.has(t)) say(`${t} is both expected and forbidden`);
+    for (const m of scored.filter(m => m.type === "item-count" && !m.not)) {
+      const lo = m.min == null || m.min === "" ? null : Number(m.min), hi = m.max == null || m.max === "" ? null : Number(m.max);
+      if (lo != null && hi != null && lo > hi) say(`at least ${lo} is above at most ${hi}`);
+      // A ceiling below one says no answer is acceptable, which is a case
+      // that can never pass rather than a strict one.
+      if (hi != null && hi < 1) say(`at most ${hi} leaves no answer that could pass`);
+      // A case that names more distinct things than the reply may carry, in
+      // the scorer's own counting of a thing (#486: each term Contains all
+      // wants is one, each Contains any is one), can never pass.
+      const things = wants.reduce((n, w) => n + (w.type === "contains-all" ? values(w).length : 1), 0);
+      if (hi != null && things > hi) say(`asks for ${things} things and at most ${hi} admits ${hi}`);
     }
   }
 
-  // One file, one set of expectations. A file here twice is two sets of
-  // them, graded separately, and both would be listed.
+  // One item, one case. An item here twice is two cases of it, graded
+  // separately, and both would be listed.
   const where = new Map                 ();
   for (const c of graded) {
-    const name = caseFile(c);
+    const name = caseItem(c);
     const counted = where.get(name);
     if (counted) {
-      bad.push(`${name} is graded twice — one file, `
-             + `two sets of expectations. Grade it once.`);
+      bad.push(`${name} is graded twice — one item, `
+             + `two cases. Grade it once.`);
     } else where.set(name, true);
   }
   const gradedAt = (name        ) => where.has(name);
@@ -4865,7 +4929,7 @@ function validateEvals(ev           , files
   for (const f of shown) {
     if (!gradedAt(f)) {
       bad.push(`${f} is in the Source and this dataset does not grade it, `
-             + `so the Evals tab does not list it`);
+             + `so the Cases view does not list it`);
     }
   }
   return bad;
@@ -4887,7 +4951,9 @@ function evalsWarnings(ev           , prompt        )           {
   const want = Number(n), warn           = [];
   for (const c of ev.cases || []) {
     if (!c || c.todo) continue;
-    const things = (c.expect || []).length + (c.anyOf || []).length;
+    const things = (Array.isArray(c.metrics) ? c.metrics             : [])
+      .filter(m => !m.not && m.weight !== 0 && (m.type === "contains-all" || m.type === "contains-any"))
+      .reduce((k, m) => k + (m.type === "contains-all" ? metricLines(m.values).length : 1), 0);
     if (things > want) {
       warn.push(`${c.id}: asks for ${things} things and the prompt asks for up to ${want}`);
     }
@@ -4906,7 +4972,7 @@ function evalsWarnings(ev           , prompt        )           {
  * `evals-check.js` asserts the round trip against the real file.
  */
 function evalsJson(ev         )         {
-  return JSON.stringify(canonicalCases(ev), null, 2) + "\n";
+  return JSON.stringify(ev, null, 2) + "\n";
 }
 
 
@@ -4920,14 +4986,14 @@ registerMetrics({ registerKinds });
 export {
     TOKEN_DEFAULTS, TOKEN_TYPES, tokenMapping, tokenNames, resolvePrompt, tokenSet, textPrompt, words,
     loopReplyError, preparedSize,
-    termIn, forbiddenIn, scoreCase, gradedSetFrom, caseFile, canonicalCases, upgradeDatasetBody, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
+    termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
     SEED, REPLY_TOKENS_BEFORE, ANTHROPIC_MAX_TOKENS, asNumber, pinReplyTokens, mappingsFor,
     CONNECTION_TYPES, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
     EDGE_448, budgetLabel, IMAGE_FORMATS, encoderQuality,
     tallyPercent, runPipeline, validateEvals, evalsWarnings, evalsJson, readList,
     scoreSingle, singleRules, parseCount, lengthHolds, countHolds, COUNT_OPS, LENGTH_OPS,
     COUNT_OP_LABEL, LENGTH_OP_LABEL,
-    PIPELINE_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, TEST_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, productionOf,
+    PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, productionOf,
     SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf,
     registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, connectionOf, connectionSettings,
     connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
@@ -4935,9 +5001,9 @@ CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWord
     contentOf, withContent, replyOf,
     targetsOf, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    blankPipeline, upgradePipeline, fatProfileRef, newId, newTest, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
-    scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioTests, scenarioPasses, isSkipped, failedScore,
-    testLabel, testsOf, testsDataset, isWholeRun, TEST_FIELDS,
+    blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
+    scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, isSkipped, failedScore,
+    evalLabel, evalsOf, evalsDataset, isWholeRun, EVAL_FIELDS,
     pipelineToYaml, importPipeline, yamlToPipeline,
     pluginHost, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
 };