evals-lab 0.4.1 → 0.5.0

package/lab/evals-core.mjs

@@ -275,9 +275,41 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                         
  
 
-/** An eval, from version 9: Metrics. A Single Test or a Graded set is what an
-    older document held (upgradePipeline reads it converted). */
-                               
+/** A private group (docs/pipeline-model.md §17): an eval group's body held
+    in the pipeline rather than the Library -- what a version-12 Metrics
+    eval upgrades to when it is not a plain link. `casesFrom` is the Library
+    group whose cases, at its newest version, join this group's Every item
+    under this group's scoring; only the upgrade writes one. */
+                                                              
+                
+                             
+ 
+
+/** A Library eval group by reference; a run's copy says the version it
+    graded with: the body's fingerprint, and its number. */
+                                                              
+
+/** An eval, from version 13: a link to an eval group (§17). `group` names a
+    Library group, followed at its newest version (`pin: null`) or pinned at
+    one; a private group is `own`, with `group: null`. A run's copy carries
+    the version it graded with on the reference (`version`, the body's
+    fingerprint, and `n`), and the lab's grader where the group names none. */
+                                               
+                
+                         
+                     
+                 
+               
+ 
+
+/** An eval, from version 13: a link to an eval group. A Metrics eval
+    (version 9 to 12), a Single Test or a Graded set is what an older
+    document held (upgradePipeline reads it converted). */
+                             
+
+/** The rule a pipeline's linked groups pass by, together: every one, or at
+    least `count` of them. */
+                                                                            
 
 /** A pipeline's evals, in the order they read a run. */
                            
@@ -293,11 +325,14 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
               
                     
                
+                 
  
 
 /** A Setup profile as a run carries it: request settings, never the key. */
                              
                
+                                                                           
+                
               
                 
                                                                  
@@ -338,6 +373,7 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                           
               
                 
+                
                
                  
                 
@@ -364,11 +400,15 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                     
  
 
+/** A Target profile as an import or export matches it: its slug, where it
+    has one -- withSlugs gives the rest theirs. */
+                                                 
+
 /** What an import matches references against: { id, name } each. */
                             
-                            
+                                   
                            
-                            
+                          
                                                                                  
                                                                 
                                                                            
@@ -817,8 +857,8 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
  
 
 /** What validation looks references up in: `profiles(id)` and `sources(id)`
-    answer with what the id names, or nothing; `datasets` is the datasets the
-    lab holds, as `GET /api/datasets` lists them. Each is optional, and a
+    answer with what the id names, or nothing; `groups` is the eval groups
+    the lab holds. Each is optional, and a
     reference nothing is given to look up is not checked. */
                           
                                                                                                                 
@@ -826,7 +866,10 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                          
                       
                                                                                                    
-                                                      
+                                                                       
+                                                                          
+                                                                
+                                                                                            
  
 
 /** Lookups, and whether it is a run document being validated. */
@@ -904,6 +947,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                      
                                                                       
                                       
+                                                                        
+                                                                                 
+                                    
                                                                         
                                                             
                                                                          
@@ -917,6 +963,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                   
                                                            
                                                                            
+                                                                           
+                       
+                                                
  
 
 /** A job's stages, in the order they run (pipeline-model §16). A target's
@@ -2687,7 +2736,10 @@ function applyModifiers   (list                                 , kind        ,
 // scenario is a target, whose own step in each job is what it sends there.
 // 11: `tests` are `evals`: the key renames and nothing in an eval changes,
 // so a stored result's scores, keyed by eval id, read as they did.
-const PIPELINE_VERSION = 12         ;
+// 12: a Contains metric's Ignore case holds item by item too.
+// 13: an eval is a link to an eval group, or a group of the pipeline's own,
+// and the document has an overall pass rule (pipeline-model §17).
+const PIPELINE_VERSION = 13         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -2780,11 +2832,13 @@ OUTPUT_KINDS.text = {
 
 // ---- the connection a profile reference resolves to ------------------------
 // What a run carries of a Setup profile: everything a request needs and never
-// the key. A key follows the profile's id into $EVAL_API_KEY_<ID>, which the
-// server sets from its profiles store and the runner reads, so a name is only
-// a label and two profiles can share one. #46: the profile carries `type`,
-// and the run's resolved copy carries it too (docs/pipeline-model.md §7).
-const CONNECTION_FIELDS                    = ["name", "url", "model", "type", "temperature",
+// the key. A key follows the profile's slug into $EVALSLAB_API_KEY_<SLUG> -- or,
+// in a run document from before slugs, its id into $EVALSLAB_API_KEY_<ID> --
+// which the server sets from its profiles store and the runner reads, so a
+// name is only a label and two profiles can share one. #46: the profile
+// carries `type`, and the run's resolved copy carries it too
+// (docs/pipeline-model.md §7).
+const CONNECTION_FIELDS                    = ["name", "slug", "url", "model", "type", "temperature",
                            "px", "format", "quality", "options"];
 const SETTING_KEYS = [...new Set(Object.values(CONNECTION_TYPES).flatMap(t => t.settings.map(s => s.key)))];
 const OPTION_FIELDS = ["seed", "nPredict", ...DECODING_KEYS];
@@ -2794,9 +2848,69 @@ const IMAGE_FORMATS                    = ["image/jpeg", "image/webp", "image/png
 const PROFILE_ID = /^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?$/;
 const LOOKS_LIKE_A_KEY = /^(?:api[-_]?)?key$|^(?:authorization|bearer|secret|password|token)$/i;
 
-/** The variable a profile's key is read from: pm1x8k2q is EVAL_API_KEY_PM1X8K2Q. */
-const keyVar = (id        )         => "EVAL_API_KEY_"
-  + String(id).toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
+/** The variable a profile's key is read from: its slug's, when it has one --
+    anthropic is EVALSLAB_API_KEY_ANTHROPIC -- else its id's, as a run document
+    from before slugs has it: pm1x8k2q is EVALSLAB_API_KEY_PM1X8K2Q. */
+const keyVar = (id        , slug                )         => "EVALSLAB_API_KEY_"
+  + String(slug || id).toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
+
+// ---- a profile's slug (#253) ------------------------------------------------
+// What a profile is called outside the lab: in a pipeline file and in the
+// name of the variable CI keeps its key in. An id (pm1x9c0r) means nothing to
+// whoever sets that secret; a slug (anthropic) does, and stays put when the
+// lab it came from does not. Lowercase letters and digits, hyphens between,
+// so its variable is spelt one way and two slugs never spell the same one.
+const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+const SLUG_MAX = 32;
+
+/** [text] as a slug: lowercase, every other run of characters one hyphen,
+    cut to SLUG_MAX. "" when nothing of it is a letter or a digit. */
+function slugOf(text         )         {
+  return String(text ?? "").toLowerCase().normalize("NFKD").replace(/[\u0300-\u036f]/g, "")
+    .replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, SLUG_MAX).replace(/-+$/, "");
+}
+
+/** [base] made unique among [taken]: anthropic, then anthropic-2, -3. */
+function uniqueSlug(base        , taken                     )         {
+  const root = base || "profile";
+  if (!taken.has(root)) return root;
+  for (let n = 2; ; n++) {
+    const tail = `-${n}`;
+    const s = root.slice(0, SLUG_MAX - tail.length).replace(/-+$/, "") + tail;
+    if (!taken.has(s)) return s;
+  }
+}
+
+/**
+ * [list] with every profile holding a slug of its own: one it holds already
+ * is kept, tidied into a slug (Setup's field holds what was typed, a
+ * trailing hyphen and all), while no profile before it holds the same; any
+ * other is made from its name, made unique. The slugs held are settled
+ * first, so a slug minted for one profile never takes another's. The same
+ * array when nothing changed, so a reader can tell a store that needs
+ * writing back from one that does not.
+ */
+function withSlugs                                              (list              )      {
+  const taken = new Set        ();
+  const held = list.map(p => {
+    const s = slugOf(p?.slug);
+    if (!s || taken.has(s)) return "";
+    taken.add(s);
+    return s;
+  });
+  let changed = false;
+  const out = list.map((p, i) => {
+    const slug = held[i] || uniqueSlug(slugOf(p?.name), taken);
+    taken.add(slug);
+    if (slug === p?.slug) return p;
+    changed = true;
+    return { ...p, slug };
+  });
+  return changed ? out : list       ;
+}
+
+/** Whether [s] is a slug as a profile may hold one. */
+const isSlug = (s         )              => isStr(s) && SLUG.test(s) && s.length <= SLUG_MAX;
 
 /** A stored profile as a run carries it: its request settings, no key. */
 function connectionOf(p         )             {
@@ -2807,6 +2921,7 @@ function connectionOf(p         )             {
   // the options bag holds the type's own settings only.
   for (const k of settings) if (k !== "temperature" && String(p[k] ?? "").trim()) options[k] = p[k];
   const out             = { name: p.name || "unnamed", url: p.url || "", model: p.model || "", type };
+  if (isSlug(p.slug)) out.slug = p.slug;
   for (const k of ["px", "format", "quality"]         ) if (p[k] != null && p[k] !== "") out[k] = p[k]             ;
   if (String(p.temperature ?? "").trim()) out.temperature = p.temperature                   ;
   if (Object.keys(options).length) out.options = options;
@@ -2863,7 +2978,7 @@ function onlyFields(obj        , at        , allowed                   , bad
   for (const k of Object.keys(obj)) {
     if (allowed.includes(k)) continue;
     bad.push(LOOKS_LIKE_A_KEY.test(k)
-      ? `${at} carries "${k}", and a key never goes in a pipeline — $${keyFrom || "EVAL_API_KEY_<ID>"} supplies it`
+      ? `${at} carries "${k}", and a key never goes in a pipeline — $${keyFrom || "EVALSLAB_API_KEY_<ID>"} supplies it`
       : `${at} has "${k}", which is not a pipeline field`);
   }
 }
@@ -3026,7 +3141,7 @@ LEGACY_TESTS.graded = {
     if (!isRef(d) || (d.version != null && !isStr(d.version))) {
       return void bad.push("a graded eval has to name its dataset as { id, name }");
     }
-    if (ctx.datasets && !ctx.datasets.some(x => x.id === d.id)) bad.push(`Dataset ${d.name || d.id} not found`);
+    if (ctx.groups && !ctx.groups.some(x => x.id === d.id)) bad.push(`Eval group ${d.name || d.id} not found`);
   },
 };
 
@@ -3169,7 +3284,10 @@ function readRun(list                   , run             )                  {
   });
 }
 
-EVAL_TYPES.metrics = {
+// The eval of versions 9 to 12, read only to upgrade (LEGACY_TESTS.metrics.
+// toGroup) and kept whole as the reference groups-check.js and
+// pipeline-check.js hold the eval group to.
+LEGACY_TESTS.metrics = {
   label: "Metrics",
   description: "Checks of each reply, or of the whole run, scored all-must-pass or in weighted points.",
   fields: ["type", "metrics", "mode", "threshold", "grader", "dataset", "over"],
@@ -3187,7 +3305,7 @@ EVAL_TYPES.metrics = {
     // The dataset whose cases a case metric reads, and whose own metrics join.
     if (t.dataset != null) {
       if (!isRef(t.dataset) || (t.dataset.version != null && !isStr(t.dataset.version))) bad.push("the Metrics name their dataset as { id, name }");
-      else if (ctx.datasets && !ctx.datasets.some(x => x.id === t.dataset.id)) bad.push(`Dataset ${t.dataset.name || t.dataset.id} not found`);
+      else if (ctx.groups && !ctx.groups.some(x => x.id === t.dataset.id)) bad.push(`Eval group ${t.dataset.name || t.dataset.id} not found`);
     }
     if (!SCORING_MODES.includes(t.mode)) bad.push(`the Metrics are scored ${SCORING_MODES.join(" or ")}`);
     if (t.mode === "weighted" && typeof t.threshold !== "number") bad.push("weighted Metrics need a threshold: the points an item has to reach");
@@ -3228,6 +3346,151 @@ EVAL_TYPES.metrics = {
     metricInput(res, kase, more), graderCtx(t, more), t.mode, t.threshold)),
 };
 
+/** A version-12 Metrics eval as the version-13 eval it reads as (§17): a
+    link to its dataset, followed at the newest version, where it named one
+    and held nothing of its own -- no metrics, scored All, the lab's grader,
+    over each item -- and a private group holding its metrics, scoring and
+    grader otherwise, with its dataset's cases (`casesFrom`). An eval over the
+    whole run never read its dataset's cases, so its group takes none. The
+    id, name and Continue on failure are kept, so a stored score keyed by
+    the eval's id is the link's. */
+function groupOfMetrics(t           )            {
+  const { id, name, continueOnFailure } = t;
+  const common = { id, ...(name !== undefined ? { name } : {}), type: "group", continueOnFailure };
+  const metrics = Array.isArray(t.metrics) ? t.metrics : [];
+  const run = t.over === "run";
+  if (isRef(t.dataset) && !metrics.length && t.mode === "all" && t.grader == null && !run) {
+    return { ...common, group: { ...t.dataset }, pin: null };
+  }
+  const own = { version: DATASET_BODY_VERSION, source: null, scoring: { mode: t.mode, threshold: t.threshold ?? null },
+                grader: t.grader ?? null, every: run ? [] : metrics, run: run ? metrics : [], cases: [],
+                casesFrom: isRef(t.dataset) && !run ? { ...t.dataset } : null };
+  return { ...common, group: null, pin: null, own };
+}
+LEGACY_TESTS.metrics.toGroup = groupOfMetrics;
+
+/** The group an eval reads by: a private group's own body, or a link's
+    Library group as the runner hands it ([more].group). A link whose body
+    nothing hands over reads as a Library group upgraded from version 6 does
+    -- no metrics of its own, scored All -- which is what every one was
+    until a body could hold more. */
+function groupOf(t           , more           = {})                         {
+  if (isObj(t.own)) return t.own            ;
+  const body = isRef(t.group) ? more.group?.(t.group) : undefined;
+  return body ?? { version: DATASET_BODY_VERSION, source: null, scoring: { mode: "all", threshold: null }, grader: null,
+                   every: [], run: [], cases: [] };
+}
+
+/** The Library group whose cases an eval reads, where it reads one: a
+    link's group, or a private group's `casesFrom`. */
+function casesRef(t         )                  {
+  if (!isObj(t) || t.type !== "group") return null;
+  if (isRef(t.group)) return t.group            ;
+  return isObj(t.own) && isRef(t.own.casesFrom) ? t.own.casesFrom             : null;
+}
+
+/** The metrics a private group shows as its rules: its Whole run where it
+    reads the run, its Every item otherwise. */
+const ownList = (t           )           => (isObj(t.own) ? (wholeRunGroup(t) ? t.own.run : t.own.every) ?? [] : []);
+
+/** A private group read over the whole run: Whole run metrics, and nothing
+    read item by item. A group holding both is read item by item until the
+    runner reports a group's Whole run beside its items (#233). */
+const wholeRunGroup = (t           )          => isObj(t.own) && Array.isArray(t.own.run) && t.own.run.length > 0
+  && !(Array.isArray(t.own.every) && t.own.every.length) && !(Array.isArray(t.own.cases) && t.own.cases.length) && !t.own.casesFrom;
+
+// A link to an eval group (docs/pipeline-model.md §17): a Library group by
+// reference, or a private group held here (`own`). It reads each item as its
+// group does (readGroup), or -- a private group of Whole run metrics alone --
+// the whole run (readGroupRun).
+EVAL_TYPES.group = {
+  label: "Eval group",
+  description: "Checks of each reply, or of the whole run, from an eval group: linked from the Library, or this pipeline's own.",
+  fields: ["type", "group", "pin", "own", "grader"],
+  // A metric that reads terms (a case) reads only a kind that yields them.
+  accepts: t => ([...ownList(t)].some((m           ) => METRICS[m?.type]?.needsTerms)
+    ? Object.keys(OUTPUT_KINDS).filter(k => OUTPUT_KINDS[k] .terms) : null),
+  defaults: () => ({ type: "group", group: null, pin: null,
+    own: { version: DATASET_BODY_VERSION, source: null, scoring: { mode: "all", threshold: null }, grader: null,
+           every: [], run: [], cases: [], casesFrom: null } }),
+  validate(t, ctx, bad){
+    const g = t.group, own = t.own;
+    const named = (ref           , what        ) => {
+      const n = isObj(ref) ? ref.n : undefined;
+      if (!isRef(ref) || (ref.version != null && !isStr(ref.version)) || (n != null && !Number.isInteger(n))) {
+        return void bad.push(`${what} names its eval group as { id, name }`);
+      }
+      onlyFields(ref, what, ["id", "name", "version", "n"], bad);
+      const held = ctx.groups?.find(x => x.id === ref.id);
+      if (ctx.groups && !held) bad.push(`Eval group ${ref.name || ref.id} not found`);
+      return held;
+    };
+    if (t.pin != null && !(Number.isInteger(t.pin) && t.pin > 0)) bad.push("a pin is a group's version: a whole number from 1");
+    if (t.grader != null && !isRef(t.grader)) bad.push("the link names its grader as { id, name }");
+    if (g != null) {
+      if (own != null) return void bad.push("an eval links a Library group or holds its own, not both");
+      const held = named(g, "the link");
+      if (held && t.pin != null && typeof held.versions === "number" && t.pin > held.versions) {
+        bad.push(`${g.name || g.id} has no version ${t.pin}`);
+      }
+      return;
+    }
+    if (!isObj(own)) return void bad.push("an eval links a Library group, or holds its own");
+    if (t.pin != null) bad.push("a group of this pipeline's own versions with it, so it is never pinned");
+    if (own.version !== DATASET_BODY_VERSION) bad.push(`the group's own body is version ${DATASET_BODY_VERSION}`);
+    for (const key of ["scoring", "every", "run", "cases"]) if (own[key] == null) bad.push(`the group's own body has no ${key}`);
+    onlyFields(own, "the group's own body", ["version", "source", "scoring", "grader", "every", "run", "cases", "casesFrom"], bad);
+    if (bad.length) return;
+    bad.push(...validateEvals(own));
+    if (own.casesFrom != null) named(own.casesFrom, "Cases from");
+    // Read item by item or over the run: one section at a time, until a
+    // group's Whole run is reported beside its items (#233).
+    if (own.run.length && !wholeRunGroup(t)) bad.push("the group's own body reads each item or the whole run, not both");
+    // The group's own grader, or the lab's.
+    const grader = isRef(own.grader) ? own.grader : isRef(ctx.grader) ? ctx.grader : null;
+    if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a Target profile the lab's grader");
+    // A grader is asked words, and needs a model to ask.
+    const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
+    if (grader && ctx.profiles && !p) bad.push(`Target profile ${grader.name || grader.id} not found`);
+    else if (p) {
+      const type = CONNECTION_TYPES[typeOf(p            )];
+      if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
+      else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's Target profile");
+    }
+  },
+  rules: t => ownList(t).map((m        , x        ) => ({
+    key: `m${x}`, label: (isStr(m.metric) && m.metric) || METRICS[m.type]?.label || m.type,
+    want: [m.not ? "not" : "", metricSummary(m), typeof m.weight === "number" && m.weight !== 1 ? `×${m.weight}` : ""]
+      .filter(Boolean).join(" ") })),
+  profiles: t => (isObj(t.own) && isRef(t.own.grader) ? [t.own.grader] : isRef(t.grader) ? [t.grader] : []),
+  // A run carries the lab's grader where the group names none and may ask
+  // one: a model-graded metric of its own, or a case's. A link's group is
+  // the Library's, so the run's copy of the link carries it.
+  resolve(t, ctx){
+    if (!isRef(ctx.grader)) return t;
+    const grader = { id: ctx.grader.id, name: ctx.grader.name };
+    if (isObj(t.own)) {
+      return !isRef(t.own.grader) && (gradedIn(t.own.every) || isRef(t.own.casesFrom)) ? { ...t, own: { ...t.own, grader } } : t;
+    }
+    return isRef(t.group) && !isRef(t.grader) ? { ...t, grader } : t;
+  },
+  wholeRun: wholeRunGroup,
+  verdict: (t, ress, kind) => {
+    const own = t.own            ;
+    return wholeRunVerdict(own.run, ress, kind, own.scoring.mode, own.scoring.threshold);
+  },
+  // Rule m{x} is the group's own metric x, read in that place on every item.
+  ruleOf: (score, key, t) => (score.metrics ? score.metrics[Number(key.slice(1))]?.pass ?? null
+    : ownList(t).length === 1 ? score.pass : null),
+  // Every item, with its case where the group reads the Library group's
+  // cases and the item has one there.
+  read: async (t, kase, res, more) => {
+    const group = groupOf(t, more);
+    const grader = isRef(group.grader) ? group.grader : isRef(t.grader) ? t.grader : null;
+    return readGroup({ ...group, grader }               , casesRef(t) ? kase : null, res, more);
+  },
+};
+
 // ---- the lab's own scorers, as metrics ----------------------------------------
 // What the Single Test checks, one metric each, so an eval of it converts to
 // Metrics that read a run exactly as it did (metrics-parity-check.js). Here
@@ -3849,11 +4112,16 @@ function evalsOf(doc                                                         )
   return isObj(t) ? [{ id: "t1", name: "", ...t, continueOnFailure: true }        ] : [];
 }
 
-/** The dataset a document's evals grade against, where one does: a run
-    grades against one (validatePipeline says so), so the first names it. */
-function evalsDataset(doc                     )                                      {
-  const t = evalsOf(doc).find((x         ) => isObj(x) && isRef(x.dataset));
-  return t && "dataset" in t ? t.dataset : null;
+/** The Library group whose cases a document's evals read, where one does:
+    a link's group, or a private group's Cases from -- or, in a stored run
+    from before version 13, a Metrics eval's dataset. A run grades against
+    one (validatePipeline says so), so the first names it. */
+function evalsDataset(doc                     )                  {
+  for (const t of evalsOf(doc)             ) {
+    const ref = casesRef(t) ?? (isObj(t) && isRef(t.dataset) ? t.dataset             : null);
+    if (ref) return ref;
+  }
+  return null;
 }
 
 STEP_TYPES.evals = {
@@ -3882,15 +4150,17 @@ STEP_TYPES.evals = {
                + `and the last job answers with ${OUTPUT_KINDS[lastKind] .noun || lastKind}`);
       }
     });
-    // A run is handed one dataset's body to grade against (server-side-runs §4).
-    const named = new Set(evals.filter((t           ) => isObj(t) && isRef(t.dataset)).map((t           ) => t.dataset.id));
-    if (named.size > 1) bad.push("the evals grade against one dataset at a time");
+    // The worker is handed one Library group's body to grade against
+    // (server-side-runs §4) until it reads the body the queue keeps per
+    // group (#233).
+    const named = new Set(evals.map(casesRef).filter(Boolean).map(r => r .id));
+    if (named.size > 1) bad.push("the evals grade against one Library eval group at a time");
   },
 };
 
 // ---- the document -------------------------------------------------------------
 
-const PIPELINE_FIELDS                    = ["version", "id", "name", "jobs", "targets", "evals"];
+const PIPELINE_FIELDS                    = ["version", "id", "name", "jobs", "targets", "evals", "pass"];
 // What resolving adds, and nothing else: the profiles it resolved to and the
 // run's own comment, which belongs to the run and never to the pipeline.
 const RUN_FIELDS = [...PIPELINE_FIELDS, "profiles", "comment", "plugins"];
@@ -3999,6 +4269,12 @@ function newEval(type        , name = "", fields                          = {})
 /** Version 6 to 7: chains are jobs. The field renames in place -- so an
     upgraded document reads the same, key for key -- and every job's `type`
     becomes `"job"`, so an older document reads as one of today's everywhere. */
+/** A new eval linking the Library eval group [group], followed at its
+    newest version. */
+function newLink(group     , name = "")            {
+  return { type: "group", id: newId(), name, continueOnFailure: true, group: { id: group.id, name: group.name }, pin: null };
+}
+
 function jobsOf(next           )            {
   next.version = PIPELINE_VERSION;
   if (Array.isArray(next.chains)) {
@@ -4113,6 +4389,26 @@ function caseAsWritten(next           )            {
   return next;
 }
 
+/** Version 12 to 13: each Metrics eval is a link to an eval group or a
+    group of the pipeline's own (groupOfMetrics), and the document passes
+    when every linked group does. Pure: it needs nothing but the document,
+    so every reader reads the same result. Every version before 13 ends
+    here. */
+function groupsOf(next           )            {
+  next.version = PIPELINE_VERSION;
+  if (Array.isArray(next.evals)) {
+    next.evals = next.evals.map((t           ) => (isObj(t) && t.type === "metrics" ? groupOfMetrics(t) : t));
+  }
+  if (!("pass" in next)) {
+    // After the evals, where a reader of the document looks for it.
+    const at = Object.keys(next).indexOf("evals");
+    const entries = Object.entries(next);
+    entries.splice(at < 0 ? entries.length : at + 1, 0, ["pass", { mode: "all" }]);
+    return Object.fromEntries(entries);
+  }
+  return next;
+}
+
 function upgradePipeline   (doc   , ctx             = {})    {
   // A current document is read as it is, but for a profile reference the Runs
   // tab saved whole (see profileRefs), which is cut back, and a step on a
@@ -4123,24 +4419,26 @@ function upgradePipeline   (doc   , ctx             = {})    {
       out = clone(out)             ;
       profileRefs(out);
     }
-    return localSteps(withoutCaseMetric(out), ctx)     ;
+    return localSteps(out, ctx)     ;
+  }
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12].includes(doc.version          )) return doc;
+  if (doc.version === 11 || doc.version === 12) {
+    return localSteps(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc)             )))), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11].includes(doc.version          )) return doc;
-  if (doc.version === 11) return localSteps(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc)             ))), ctx)     ;
-  // Every version before 10 reads as version 9 first, then as 10, then as 11
-  // and 12.
+  // Every version before 10 reads as version 9 first, then as 10, then as 11,
+  // 12 and 13.
   // A version-10 document is cut as a current one was (profileRefs); the
   // earlier ones are cut on their way through nineOf.
   const ten = doc.version === 10 ? cutRefs(clone(doc)             ) : targetsFromScenarios(nineOf(clone(doc)             , ctx));
-  return localSteps(withoutCaseMetric(caseAsWritten(evalsKey(ten))), ctx)     ;
+  return localSteps(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten)))), ctx)     ;
 }
 
 /** [doc] with no Metrics eval holding the `case` metric: a dataset's cases
     are metrics now (dataset body version 5), which an eval naming the
     dataset adds for each item, so the case's own metrics carry what that
-    metric scored. Read so at every version, the current one included, as a
-    document saved before it went still holds it. The same document where
-    none does. */
+    metric scored. Read so at every version up to 12, as a document saved
+    before it went still holds it; version 13 holds no Metrics eval. The
+    same document where none does. */
 function withoutCaseMetric(doc           )            {
   const has = (t         ) => isObj(t) && Array.isArray(t.metrics) && t.metrics.some((m           ) => isObj(m) && m.type === "case");
   if (!Array.isArray(doc.evals) || !doc.evals.some(has)) return doc;
@@ -4229,7 +4527,7 @@ function tokenMappingsFromV1(set         )                {
 function blankPipeline(opts                                                           = {})           {
   const job = withImage(jobDefaults(opts.kind, opts.tokens), true);
   return { version: PIPELINE_VERSION, id: newId(), name: opts.name || "",
-           jobs: [job], targets: [], evals: [] };
+           jobs: [job], targets: [], evals: [], pass: { mode: "all" } };
 }
 
 /**
@@ -4368,6 +4666,7 @@ function validatePipeline(input         , ctx          = {})           {
     }
   });
   STEP_TYPES.evals .validate(doc.evals, c, bad, outOf(doc.jobs.at(-1)).kind, doc);
+  passProblems(doc.pass, Array.isArray(doc.evals) ? doc.evals.length : 0, bad);
   if (bad.length) return bad;
 
   // Asked before anything is sent, so a misspelt token costs nothing and
@@ -4382,16 +4681,45 @@ function validatePipeline(input         , ctx          = {})           {
   return bad;
 }
 
+/** The overall pass rule: every linked group, or at least [count] of the
+    [n] the pipeline has. */
+function passProblems(pass         , n        , bad          )       {
+  if (!isObj(pass) || (pass.mode !== "all" && pass.mode !== "atLeast")) {
+    return void bad.push("pass has to be { mode: all } or { mode: atLeast, count }");
+  }
+  onlyFields(pass, "pass", pass.mode === "all" ? ["mode"] : ["mode", "count"], bad);
+  if (pass.mode === "atLeast" && !(Number.isInteger(pass.count) && pass.count > 0)) bad.push("pass at least has to count a whole number of groups from 1");
+  else if (pass.mode === "atLeast" && pass.count > n) bad.push(`pass asks for ${pass.count} groups, and the pipeline has ${n}`);
+}
+
+/**
+ * What [doc] may still be right about, as sentences: a link to a group made
+ * for another Source than the pipeline's content (§17, decision 2). The run
+ * grades the cases whose items it holds, and the rest read Missing.
+ */
+function pipelineWarnings(doc         , ctx          = {})           {
+  if (!isObj(doc) || !Array.isArray(doc.evals) || !ctx.groups) return [];
+  const content = contentOf(doc             )                    ;
+  const here = content?.type === "source" && isRef(content.ref) ? content.ref.id : null;
+  const warn           = [];
+  doc.evals.forEach((t           , j        ) => {
+    const ref = casesRef(t);
+    const src = ref && ctx.groups .find(x => x.id === ref.id)?.source;
+    if (src && src.id !== here) warn.push(`${evalLabel(doc             , j)}: ${ref .name || ref .id} grades ${src.name || src.id}`);
+  });
+  return warn;
+}
+
 /** A run document's profiles table: id → connection, keyless, spellable. */
 function profilesProblems(profiles         , bad          )       {
   if (!isObj(profiles)) return void bad.push("profiles has to be an object of id → connection");
   const spelt = new Map                ();
   for (const [id, conn] of Object.entries(profiles)) {
     if (!PROFILE_ID.test(id)) {
-      bad.push(`profile id "${id}" has to be letters and digits, with at most one hyphen: its key comes from EVAL_API_KEY_<ID>`);
+      bad.push(`profile id "${id}" has to be letters and digits, with at most one hyphen: its key comes from EVALSLAB_API_KEY_<ID>`);
       continue;
     }
-    const from = keyVar(id);
+    const from = keyVar(id, isObj(conn) && isStr(conn.slug) ? conn.slug : null);
     if (spelt.has(from)) bad.push(`profiles ${spelt.get(from)} and ${id} both take their key from $${from}`);
     spelt.set(from, id);
     const at = `profile ${id}`;
@@ -4408,6 +4736,9 @@ function connectionProblems(conn                           , at        , from
     bad.push(`${at}: type has to be one of ${Object.keys(CONNECTION_TYPES).join(", ")}`);
   }
   if (!isStr(conn.name)) bad.push(`${at}: name has to be text`);
+  if (conn.slug != null && !isSlug(conn.slug)) {
+    bad.push(`${at}: slug has to be lowercase letters and digits, with hyphens between, at most ${SLUG_MAX}`);
+  }
   const url = conn.url ?? "";
   if (!isStr(url)) bad.push(`${at}: url has to be text, blank for the lab's Ollama`);
   else if (url.trim()) {
@@ -4477,7 +4808,8 @@ function profileIds(doc                                                  )
  * the Source's file list as it reads now, `ctx.comment` is the run's own.
  */
 function resolvePipeline(doc          ,
-                         ctx                                                                            = {})              {
+                         ctx                                                 
+                                                                                                                                     = {})              {
   const run = clone(doc)             ;
   // What the lab supplies an eval -- its grader -- before the profiles it
   // asks are carried.
@@ -4489,8 +4821,18 @@ function resolvePipeline(doc          ,
   }
   const content = contentOf(run)             ;
   if (content?.type === "source") content.files = CONTENT_TYPES.source .expand (content, ctx.files);
-  if (ctx.datasetVersion) {
-    for (const t of run.evals || []) if (isRef(t.dataset)) t.dataset.version = ctx.datasetVersion;
+  // Each Library group the evals read, as the version it grades with: the
+  // body's fingerprint and, where the lab numbers them, `n` -- a pinned
+  // link's pin.
+  if (ctx.groupVersion) {
+    for (const t of run.evals || []) {
+      const ref = casesRef(t);
+      const at = ref && ctx.groupVersion(ref.id);
+      if (!ref || !at) continue;
+      ref.version = at.version;
+      const n = isObj(t) && Number.isInteger(t.pin) ? t.pin : at.n;
+      if (n != null) ref.n = n;
+    }
   }
   if (isStr(ctx.comment)) run.comment = ctx.comment;
   return run;
@@ -4504,7 +4846,12 @@ function pipelineOfRun(run             , ctx             = {})           {
   delete doc.plugins;
   const content = contentOf(doc)             ;
   if (content) { delete content.files; delete content.revs; }
-  for (const t of doc.evals || []) if (isObj(t.dataset)) delete t.dataset.version;
+  // What the run recorded of each group it read is the run's. The grader
+  // it carried stays, as a Metrics eval's did: a bundle writes it down.
+  for (const t of doc.evals || []) {
+    const ref = casesRef(t);
+    if (ref) { delete ref.version; delete ref.n; }
+  }
   return doc;
 }
 
@@ -4517,9 +4864,35 @@ function pipelineOfRun(run             , ctx             = {})           {
 
 
 /** [doc] as YAML text, the pipeline's file form -- references only, so an
-    export carries no key, no URL, no file list and no file bytes. */
-function pipelineToYaml(doc          )         {
-  return yaml.dump(doc, { lineWidth: -1, noRefs: true });
+    export carries no key, no URL, no file list and no file bytes. A Target
+    profile [ctx] holds is named by its slug, `{ slug, name }`, rather than
+    by this lab's id (#253): the slug is what another lab, or CI's
+    $EVALSLAB_API_KEY_<SLUG>, knows it by. */
+function pipelineToYaml(doc          , ctx                                       = {})         {
+  const profiles = withSlugs(ctx.profiles ?? []);
+  if (!profiles.length) return yaml.dump(doc, { lineWidth: -1, noRefs: true });
+  const out = clone(doc)             ;
+  const bySlug = (ref           )            => {
+    const p = isRef(ref) ? profiles.find(x => x.id === ref.id) : null;
+    return p ? { slug: p.slug, name: ref.name ?? p.name } : ref;
+  };
+  eachProfileRef(out, bySlug);
+  return yaml.dump(out, { lineWidth: -1, noRefs: true });
+}
+
+/** Every Target profile reference in [doc] -- each target's, each of its
+    steps', each eval's grader -- replaced, in place, by [f] of it. */
+function eachProfileRef(doc           , f                               )       {
+  for (const t of targetsOf(doc)) {
+    if (t.profile) t.profile = f(t.profile);
+    for (const st of Array.isArray(t.steps) ? t.steps : []) {
+      if (st?.profile) st.profile = f(st.profile);
+    }
+  }
+  for (const t of Array.isArray(doc.evals) ? doc.evals : []) {
+    if (isObj(t) && t.grader) t.grader = f(t.grader);
+    if (isObj(t?.own) && t.own.grader) t.own.grader = f(t.own.grader);
+  }
 }
 
 /**
@@ -4527,7 +4900,7 @@ function pipelineToYaml(doc          )         {
  * name (pipeline-model §5), so a pipeline written in another lab -- ids that
  * mean nothing here -- finds its Source and Setup profiles by the names they
  * were exported under. [ctx]'s lists are what the lab holds:
- * `{ profiles, sources, datasets }`, each `{ id, name }[]`. Returns
+ * `{ profiles, sources, groups }`, each `{ id, name }[]`. Returns
  * `{ doc, missing }`, where `missing` is one sentence per reference neither
  * id nor name matched, or `{ error }` when the document is refused: not a
  * version the lab reads, or not something the lab can edit and run as it
@@ -4538,14 +4911,14 @@ function importPipeline(input         , ctx            = {})               {
   const doc = upgradePipeline(input, ctx);
   const v = versionProblem(doc);
   if (v) return { error: v };
-  const profiles = ctx.profiles ?? [], sources = ctx.sources ?? [], datasets = ctx.datasets ?? [];
+  const profiles = ctx.profiles ?? [], sources = ctx.sources ?? [], groups = ctx.groups ?? [];
   const next = clone(doc)             ;
   const missing           = [], seen = new Set        ();
-  const lists = { profile: profiles, source: sources, dataset: datasets };
+  const lists = { profile: profiles, source: sources, group: groups };
   const words = {
-    profile: (ref     ) => `Target profile ${ref.name || ref.id} not found`,
+    profile: (ref                         ) => `Target profile ${ref.slug || ref.name || ref.id} not found`,
     source: (ref     ) => `Source ${ref.name || ref.id} not found`,
-    dataset: (ref     ) => `Dataset ${ref.name || ref.id} not found`,
+    group: (ref     ) => `Eval group ${ref.name || ref.id} not found`,
   };
   const remap = (ref     , kind                    )      => {
     const hit = lists[kind].find(x => x.id === ref.id) || lists[kind].find(x => x.name === ref.name);
@@ -4554,16 +4927,34 @@ function importPipeline(input         , ctx            = {})               {
     if (!seen.has(sentence)) { seen.add(sentence); missing.push(sentence); }
     return ref;
   };
+  // A profile is named by its slug in a file this lab writes (#253), and by
+  // its id in one written before slugs: a slug is matched by slug, then by
+  // name. One this lab holds under neither is kept as a reference the run
+  // is blocked on, the slug standing in for an id.
+  const slugged = withSlugs(profiles);
+  const remapProfile = (ref           )            => {
+    if (!isObj(ref) || !isStr(ref.slug) || isRef(ref)) return isRef(ref) ? remap(ref, "profile") : ref;
+    const hit = slugged.find(x => x.slug === ref.slug) || slugged.find(x => isStr(ref.name) && x.name === ref.name);
+    if (hit) return { id: hit.id, name: hit.name };
+    const sentence = words.profile(ref                          );
+    if (!seen.has(sentence)) { seen.add(sentence); missing.push(sentence); }
+    return { id: ref.slug, name: isStr(ref.name) && ref.name ? ref.name : ref.slug };
+  };
   const content = contentOf(next)             ;
   if (content?.type === "source") content.ref = remap(content.ref, "source");
   for (const t of targetsOf(next)) {
-    if (t.profile) t.profile = remap(t.profile, "profile");
+    if (t.profile) t.profile = remapProfile(t.profile);
     for (const st of Array.isArray(t.steps) ? t.steps : []) {
-      if (st?.profile) st.profile = remap(st.profile, "profile");
+      if (st?.profile) st.profile = remapProfile(st.profile);
     }
   }
   for (const t of Array.isArray(next.evals) ? next.evals : []) {
-    if (isRef(t?.dataset)) t.dataset = remap(t.dataset, "dataset");
+    if (isRef(t?.group)) t.group = remap(t.group, "group");
+    else if (isObj(t?.own) && isRef(t.own.casesFrom)) t.own.casesFrom = remap(t.own.casesFrom, "group");
+    // The grader is a profile like any other, matched the same way: a
+    // private group's own, or the one a run's link carries.
+    if (isObj(t?.own) && t.own.grader) t.own.grader = remapProfile(t.own.grader);
+    if (isObj(t) && t.grader) t.grader = remapProfile(t.grader);
   }
   // Its references are this lab's now, so a step asking this lab's Echo
   // profile is read as an Echo step (upgradePipeline did it for ids that
@@ -4597,6 +4988,8 @@ function mintIds(doc           )            {
   for (const t of Array.isArray(doc.evals) ? doc.evals : []) {
     if (isObj(t) && (!isStr(t.id) || !t.id)) t.id = newId();
   }
+  // Nor need it say how its linked groups pass together: all of them.
+  if (!("pass" in doc)) doc.pass = { mode: "all" };
   return doc;
 }
 
@@ -4610,6 +5003,224 @@ function yamlToPipeline(text        , ctx            )               {
   return importPipeline(doc, ctx);
 }
 
+// ---- JUnit (#251) --------------------------------------------------------------
+//
+// What a CI test panel reads, from the worker's report and from a lab's
+// stored run alike (`evals-lab run --lab`, #256), so both write one format.
+// A suite is one target's eval, a case one item: a failure carries the reason
+// the item failed, an item that never ran is an error, and one the eval had
+// nothing to read of is skipped.
+
+/** One JUnit test suite: a target's eval, and a case per item it read. */
+                             
+               
+                                                                                
+ 
+
+// XML 1.0 has no way to write most control characters, escaped or not, and a
+// model's reply can hold any of them.
+const xmlText = (v         )         => String(v).replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\uFFFE\uFFFF]/g, "")
+  .replace(/[&<>"']/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&apos;" })[c] );
+
+/** [suites] as JUnit XML, under the name of what ran them. */
+function junitXml(suites                       , runner        )         {
+  const count = (cases                     , k                                 ) => cases.filter(c => c[k] != null).length;
+  const body = suites.map(st => {
+    const cases = st.cases.map(c => `    <testcase classname="${xmlText(st.name)}" name="${xmlText(c.name)}">`
+      + (c.failure != null ? `<failure message="${xmlText(c.failure)}"/>`
+        : c.error != null ? `<error message="${xmlText(c.error)}"/>`
+        : c.skipped != null ? `<skipped message="${xmlText(c.skipped)}"/>` : "")
+      + "</testcase>");
+    return `  <testsuite name="${xmlText(st.name)}" tests="${st.cases.length}" failures="${count(st.cases, "failure")}" `
+      + `errors="${count(st.cases, "error")}" skipped="${count(st.cases, "skipped")}">\n`
+      + cases.map(c => c + "\n").join("") + "  </testsuite>\n";
+  });
+  const all = suites.flatMap(st => st.cases);
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="${xmlText(runner)}" tests="${all.length}" `
+    + `failures="${count(all, "failure")}" errors="${count(all, "error")}" skipped="${count(all, "skipped")}">\n`
+    + body.join("") + "</testsuites>\n";
+}
+
+// ---- a pipeline as a CI bundle (#254) --------------------------------------
+//
+// Export for CI writes a pipeline as files a repository keeps and
+// `evals-lab run` runs with no lab around it (docs/pipeline-yaml.md, "Export
+// for CI"): the pipeline's YAML, its Target profiles by slug, and each
+// dataset it grades against as the Library's export of it. The server adds
+// the plugins and, when asked, the Source's items, and zips the lot. Read
+// back, the files are the run document the page would have submitted, with
+// each profile keyed by its slug rather than this lab's id -- the slug is
+// what the key's $EVALSLAB_API_KEY_<SLUG> is spelt from either way.
+
+/** The text files of a bundle, by path inside it. */
+                                                 
+
+/** What a bundle is read back as: the run document, and each dataset it
+    grades against, by the id its evals name it by. */
+                             
+                   
+                                        
+ 
+
+const BUNDLE_PIPELINE = "pipeline.yaml";
+const BUNDLE_PROFILES = "profiles.yaml";
+const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
+
+/**
+ * [doc] as a bundle's text files: `pipeline.yaml`, `profiles.yaml` and
+ * `datasets/<slug>.json`. [ctx.profiles] is Setup's list -- keys and all,
+ * none of which is written -- [ctx.grader] the lab's grader, written into
+ * each eval that would have asked it, and [ctx.datasets] each graded
+ * dataset by id, as `{ name, body }`. `{ error }` when a dataset or a
+ * profile the pipeline names is not given, since a bundle without it runs
+ * something else.
+ */
+function exportBundle(doc          , ctx                                                       
+                                                                                                            = {}) 
+                                               {
+  const profiles = withSlugs((ctx.profiles ?? [])                     );
+  // What the lab would supply at submit, written down: no lab supplies it later.
+  const out = clone(doc)             ;
+  out.evals = (Array.isArray(out.evals) ? out.evals : [])
+    .map((t           ) => EVAL_TYPES[t?.type]?.resolve?.(t, { grader: ctx.grader ?? null }) ?? t);
+  const table                           = {};
+  for (const id of profileIds(out)) {
+    const p = profiles.find(x => x.id === id);
+    if (!p) return { error: `Target profile ${id} not found` };
+    const { slug, ...flat } = connectionSettings(connectionOf(p)                       );
+    table[slug          ] = flat;
+  }
+  const files              = {};
+  const taken = new Set        (), written = new Set        ();
+  for (const t of evalsOf(out)               ) {
+    // A link's group, or a private group's Cases from: the reference itself.
+    const ref = casesRef(t);
+    if (!ref) continue;
+    const d = ctx.datasets?.[ref.id];
+    if (!d) return { error: `Dataset ${ref.name || ref.id} not found` };
+    // Named as the dataset is now, since the name is what a reader joins on.
+    ref.name = d.name;
+    if (written.has(ref.id)) continue;
+    const slug = uniqueSlug(slugOf(d.name) || "dataset", taken);
+    taken.add(slug);
+    written.add(ref.id);
+    // The Library's Export of it, so the file imports into any lab too. Its
+    // name is what joins it back to the eval naming it: a lab's dataset
+    // names are its own.
+    files[`datasets/${slug}.json`] = JSON.stringify({ format: "evals-lab/dataset", version: DATASET_BODY_VERSION,
+                                                       dataset: { name: d.name, body: d.body } }, null, 2) + "\n";
+  }
+  files[BUNDLE_PIPELINE] = pipelineToYaml(out, { profiles });
+  files[BUNDLE_PROFILES] = yaml.dump(table, { lineWidth: -1, noRefs: true });
+  const bad = bundleProblems(files, profiles.map(p => p.key));
+  return bad.length ? { error: bad[0]  } : { files };
+}
+
+/**
+ * Why [files] cannot leave the lab: a field named like a key anywhere in
+ * them, or any of [keys] -- Setup's own -- written anywhere at all. Empty
+ * when nothing key-shaped is in them. The server asks the same of the zip
+ * it is handed (server.py `bundle_problems`).
+ */
+function bundleProblems(files             , keys                     = [])           {
+  const bad           = [];
+  const named = (v         , at        )       => {
+    if (Array.isArray(v)) return v.forEach(x => named(x, at));
+    if (!isObj(v)) return;
+    for (const [k, x] of Object.entries(v)) {
+      if (LOOKS_LIKE_A_KEY.test(k)) bad.push(`${at} holds a field named ${k}, and a key never leaves the lab`);
+      named(x, at);
+    }
+  };
+  for (const [path, text] of Object.entries(files)) {
+    for (const key of keys) {
+      if (isStr(key) && key.trim().length >= 4 && text.includes(key.trim())) {
+        bad.push(`${path} holds a Target profile's key, and a key never leaves the lab`);
+        break;
+      }
+    }
+    let doc         ;
+    try { doc = path.endsWith(".json") ? JSON.parse(text) : yaml.load(text); } catch { continue; }
+    named(doc, path);
+  }
+  return [...new Set(bad)];
+}
+
+/**
+ * A bundle's [files] as the run they describe: `pipeline.yaml` read as an
+ * import would read it, against `profiles.yaml` and the `datasets/` it
+ * carries, then resolved as the page resolves one at submit. [ctx.items] is
+ * the Source's file list -- the names in the bundle's `items/`, or a
+ * directory CI names; with none, each case's item is the list, so an item
+ * nothing supplies is a missing item rather than an item never asked.
+ * `{ error }` names the first thing that keeps it from running: a profile
+ * or a dataset the files do not hold, a key in them, or a document the lab
+ * would refuse. The caller registers the bundle's plugins first.
+ */
+function readBundle(files             , ctx                                = {})                                 {
+  if (!isStr(files[BUNDLE_PIPELINE])) return { error: `the bundle has no ${BUNDLE_PIPELINE}` };
+  let table            = {};
+  try { table = isStr(files[BUNDLE_PROFILES]) ? yaml.load(files[BUNDLE_PROFILES] ) ?? {} : {}; }
+  catch { return { error: `${BUNDLE_PROFILES} is not YAML` }; }
+  if (!isObj(table)) return { error: `${BUNDLE_PROFILES} maps each profile's slug to its settings` };
+  const keyed = bundleProblems(files);
+  if (keyed.length) return { error: keyed[0]  };
+  const profiles                                       = [];
+  for (const [slug, p] of Object.entries(table)) {
+    if (!isSlug(slug)) return { error: `${BUNDLE_PROFILES}: ${JSON.stringify(slug)} is not a slug` };
+    if (!isObj(p)) return { error: `${BUNDLE_PROFILES}: ${slug} has to hold its settings` };
+    if (!CONNECTION_TYPES[typeOf(p           )]) {
+      return { error: `${BUNDLE_PROFILES}: ${slug} is of type ${p.type}, which this lab does not have` };
+    }
+    profiles.push({ ...p, id: slug, slug, name: isStr(p.name) && p.name ? p.name : slug });
+  }
+  // Each dataset by its name, which is what joins it to the eval naming it.
+  const datasets                                        = [];
+  for (const [path, text] of Object.entries(files)) {
+    if (!path.startsWith("datasets/")) continue;
+    if (!BUNDLE_DATASET.test(path)) return { error: `${path} is not named datasets/<slug>.json` };
+    let doc           ;
+    try { doc = JSON.parse(text); } catch { return { error: `${path} is not JSON` }; }
+    if (!isObj(doc) || doc.format !== "evals-lab/dataset" || !isObj(doc.dataset)) {
+      return { error: `${path} is not a dataset's export` };
+    }
+    if (!(Number.isInteger(doc.version) && doc.version >= 1 && doc.version <= DATASET_BODY_VERSION)) {
+      return { error: `${path} is version ${JSON.stringify(doc.version)}, and this reads versions 1 to ${DATASET_BODY_VERSION}` };
+    }
+    const body = upgradeDatasetBody(doc.dataset.body)             ;
+    if (!isObj(body) || !Array.isArray(body.cases)) return { error: `${path} holds no cases` };
+    datasets.push({ name: String(doc.dataset.name ?? ""), body: body                });
+  }
+  let raw           ;
+  try { raw = yaml.load(files[BUNDLE_PIPELINE] ); }
+  catch { return { error: `${BUNDLE_PIPELINE} is not YAML` }; }
+  // The bundle stands in for the Source: whatever it names, the items are
+  // the ones handed over.
+  const content = isObj(raw) ? contentOf(upgradePipeline(raw))              : null;
+  const refs        = [];
+  // Read as this version, whichever wrote the bundle: a link's group, or a
+  // group of its own's Cases from.
+  for (const t of isObj(raw) ? evalsOf(upgradePipeline(raw)) : []) {
+    const ref = casesRef(t);
+    if (!ref) continue;
+    if (datasets.some(d => d.name === ref.name)) refs.push({ id: ref.id, name: ref.name });
+  }
+  const imported = importPipeline(raw, { profiles, groups: refs,
+                                        sources: content?.type === "source" && isRef(content.ref) ? [content.ref] : [] });
+  if ("error" in imported) return imported;
+  if (imported.missing.length) return { error: `${imported.missing[0]}: the bundle does not hold it` };
+  const byId                              = {};
+  for (const r of refs) byId[r.id] = datasets.find(d => d.name === r.name) .body;
+  const caseItems = [...new Set(Object.values(byId).flatMap(b => gradedSetFrom(b                      ).filter(c => !c.todo).map(caseItem)).filter(Boolean))];
+  const run = resolvePipeline(imported.doc, {
+    profiles: id => profiles.find(p => p.id === id) ?? null,
+    files: ctx.items ? [...ctx.items].sort() : caseItems,
+  });
+  const bad = validatePipeline(run, { groups: refs });
+  if (bad.length) return { error: bad[0]  };
+  return { run, datasets: byId };
+}
+
 /**
  * Scenario [i] of a run document as runPipeline and a transport take it:
  * each stage's wording, kind, image flag and modifiers, the token set of
@@ -4995,15 +5606,15 @@ export {
     COUNT_OP_LABEL, LENGTH_OP_LABEL,
     PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, productionOf,
     SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf,
-    registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, connectionOf, connectionSettings,
+    registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, connectionSettings,
     connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
     targetsOf, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
+    blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, isSkipped, failedScore,
-    evalLabel, evalsOf, evalsDataset, isWholeRun, EVAL_FIELDS,
-    pipelineToYaml, importPipeline, yamlToPipeline,
+    evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, isWholeRun, EVAL_FIELDS,
+    pipelineToYaml, importPipeline, yamlToPipeline, exportBundle, readBundle, bundleProblems, junitXml,
     pluginHost, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
 };