evals-lab 0.4.1 → 0.6.0

package/lab/server.py

@@ -560,6 +560,48 @@ QUEUE_WAIT = 1.0
 WATCH_EVERY = 5.0
 
 
+# A Target profile's key is written from the page and never read back by it
+# (#258): a client holding the lab's password -- CI's runners and their logs
+# among them -- is handed no key. Where a profile holds one, what is served
+# (/api/state, the page's carried copy, a refused write's current copy) holds
+# KEY_HELD and the profile's id instead, and a write that brings that back
+# keeps the key the store holds for the id: the profile's own, or the one it
+# was cloned or restored from. KEY_HELD starts with a character no key can
+# (header_safe), so a pasted key is never taken for it.
+KEY_HELD = "\u2022held:"
+
+
+def held(key) -> bool:
+    return isinstance(key, str) and key.startswith(KEY_HELD)
+
+
+def profiles_in(name: str, body):
+    """Every profile a synced document holds: the profiles store's list, and
+    each profile version's copy."""
+    if not isinstance(body, dict):
+        return
+    if name == "promptlab.profiles":
+        for p in body.get("list") or []:
+            if isinstance(p, dict):
+                yield p
+    elif name == "promptlab.versions":
+        for kept in (body.get("profile") or {}).values() if isinstance(body.get("profile"), dict) else ():
+            for v in kept if isinstance(kept, list) else ():
+                if isinstance(v, dict) and isinstance(v.get("doc"), dict):
+                    yield v["doc"]
+
+
+def hide_keys(docs: dict) -> dict:
+    out = {}
+    for name, d in docs.items():
+        d = json.loads(json.dumps(d))
+        for p in profiles_in(name, d.get("body")):
+            if isinstance(p.get("key"), str) and p["key"] and not held(p["key"]):
+                p["key"] = KEY_HELD + str(p.get("id") or "")
+        out[name] = d
+    return out
+
+
 class Store:
     """
     Documents by name, each with a version that goes up by one per write.
@@ -579,6 +621,11 @@ class Store:
             db.execute("CREATE TABLE IF NOT EXISTS docs (name TEXT PRIMARY KEY, "
                        "version INTEGER NOT NULL, body TEXT, updated_at TEXT NOT NULL)")
             db.execute("CREATE TABLE IF NOT EXISTS runs (at TEXT PRIMARY KEY, body TEXT NOT NULL)")
+            # The key of a profile a write took out, by id, for TRASH_SECONDS:
+            # Undo puts the profile back holding KEY_HELD, and this is what
+            # it holds.
+            db.execute("CREATE TABLE IF NOT EXISTS dropped_keys (id TEXT PRIMARY KEY, "
+                       "key TEXT NOT NULL, at REAL NOT NULL)")
             # History was a document, capped at what a browser could hold. The
             # first start with the table moves what that document had into it,
             # once, and drops the document so the page stops carrying it.
@@ -600,8 +647,30 @@ class Store:
 
     def served(self) -> dict:
         """What a browser is handed: the SYNCED documents only, so a retired
-        key's rows stay in the store without reaching a page again."""
-        return {n: d for n, d in self.all().items() if n in SYNCED}
+        key's rows stay in the store without reaching a page again, and no
+        profile's key (KEY_HELD)."""
+        return hide_keys({n: d for n, d in self.all().items() if n in SYNCED})
+
+    def _keyring(self, db, now: dict) -> dict:
+        """Each profile id's key as the store holds it: its profile's, else a
+        dropped one's, else its newest version's."""
+        ring = {}
+        for p in profiles_in("promptlab.versions", now.get("promptlab.versions", {}).get("body")):
+            pid, key = str(p.get("id") or ""), p.get("key")
+            if pid not in ring and isinstance(key, str) and key and not held(key):
+                ring[pid] = key
+        db.execute("DELETE FROM dropped_keys WHERE at < ?", (time.time() - TRASH_SECONDS,))
+        ring.update(db.execute("SELECT id, key FROM dropped_keys").fetchall())
+        for p in profiles_in("promptlab.profiles", now.get("promptlab.profiles", {}).get("body")):
+            key = p.get("key")
+            if isinstance(key, str) and key and not held(key):
+                ring[str(p.get("id") or "")] = key
+        return ring
+
+    def held_key(self, key: str) -> str:
+        """The key KEY_HELD stands for, or "" when the store holds none."""
+        with self.lock, closing(sqlite3.connect(self.path)) as db, db:
+            return self._keyring(db, self._rows(db)).get(key[len(KEY_HELD):], "")
 
     def write(self, docs: dict):
         """
@@ -614,9 +683,23 @@ class Store:
             have = lambda n: now.get(n, {"version": 0, "body": None})
             stale = {n: have(n) for n, d in docs.items() if d["version"] != have(n)["version"]}
             if stale:
-                return None, stale
+                return None, hide_keys(stale)
             at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
             with db:
+                ring = self._keyring(db, now)
+                for n, d in docs.items():
+                    for p in profiles_in(n, d["body"]):
+                        if held(p.get("key")):
+                            p["key"] = ring.get(p["key"][len(KEY_HELD):], "")
+                if "promptlab.profiles" in docs:
+                    kept = {str(p.get("id") or "") for p in profiles_in(
+                        "promptlab.profiles", docs["promptlab.profiles"]["body"])}
+                    db.executemany("DELETE FROM dropped_keys WHERE id = ?", [(i,) for i in kept])
+                    db.executemany(
+                        "INSERT OR REPLACE INTO dropped_keys (id, key, at) VALUES (?, ?, ?)",
+                        [(str(p.get("id") or ""), p["key"], time.time())
+                         for p in profiles_in("promptlab.profiles", have("promptlab.profiles")["body"])
+                         if str(p.get("id") or "") not in kept and isinstance(p.get("key"), str) and p["key"]])
                 for n, d in docs.items():
                     db.execute(
                         "INSERT INTO docs (name, version, body, updated_at) VALUES (?, ?, ?, ?) "
@@ -726,7 +809,7 @@ class Sources:
         out = []
         with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
             db.row_factory = sqlite3.Row
-            for r in db.execute("SELECT id, name, system, bytes, type FROM sources "
+            for r in db.execute("SELECT id, name, system, bytes, type, created_at FROM sources "
                                 "ORDER BY system DESC, name COLLATE NOCASE"):
                 if r["system"]:
                     files = self._sample_files()
@@ -734,10 +817,13 @@ class Sources:
                                 "type": r["type"],
                                 "files": len(files), "bytes": sum(f["bytes"] for f in files)})
                 else:
-                    n = db.execute("SELECT COUNT(*) FROM source_files WHERE source = ?",
-                                   (r["id"],)).fetchone()[0]
+                    n, last = db.execute("SELECT COUNT(*), MAX(at) FROM source_files WHERE source = ?",
+                                         (r["id"],)).fetchone()
+                    # When it last changed: made, or a file added -- what a
+                    # picker orders its recent Sources by.
                     out.append({"id": r["id"], "name": r["name"], "system": False,
-                                "type": r["type"], "files": n, "bytes": r["bytes"]})
+                                "type": r["type"], "files": n, "bytes": r["bytes"],
+                                "changed": max(filter(None, (r["created_at"], last)))})
         return out
 
     def get(self, sid) -> dict:
@@ -1177,6 +1263,18 @@ class Sources:
             return None, (500, "the files could not be copied")
         return self.get(sid), None
 
+    def item_paths(self, sid):
+        """Every file of a Source, as (name, path on disk) in its own order,
+        or None when there is no such Source."""
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+            r = db.execute("SELECT system FROM sources WHERE id = ?", (sid,)).fetchone()
+            if r is None:
+                return None
+            if r[0]:
+                return [(f["name"], SAMPLES / f["name"]) for f in self._sample_files()]
+            return [(row[0], self.dir / sid / row[0]) for row in db.execute(
+                "SELECT name FROM source_files WHERE source = ? ORDER BY name", (sid,))]
+
     def zip_files(self, sid, names):
         """The bytes of a stdlib zipfile holding exactly the named files, in
         the order named, or an error. Read under the store's lock so the file
@@ -1362,15 +1460,23 @@ DATASET_FIELDS = ("version", "source", "scoring", "grader", "every", "run", "cas
 # 5 was told by its `source` alone, and earlier ones by neither.
 DATASET_BODY_VERSION = 7
 DATASET_NAME_MAX = 80
-# The file forms Export writes and Import reads. Export writes version 7;
-# Import reads it and versions 1 to 6, upgraded, and refuses anything else,
-# as a pipeline of another version is refused. Versions 1 to 3 carried a
-# prompt, which an import gives to the Prompt library.
-EXPORT_ONE = "evals-lab/dataset"
-EXPORT_ALL = "evals-lab/datasets"
+# The file forms Export writes and Import reads. Export writes an eval group
+# at version 7 (docs/pipeline-model.md §17); Import reads that, and a dataset
+# file of versions 1 to 7, upgraded, and refuses anything else, as a pipeline
+# of another version is refused. Versions 1 to 3 carried a prompt, which an
+# import gives to the Prompt library.
+EXPORT_ONE = "evals-lab/eval-group"
+EXPORT_ALL = "evals-lab/eval-groups"
+DATASET_ONE = "evals-lab/dataset"
+DATASET_ALL = "evals-lab/datasets"
 EXPORT_VERSION = 7
 IMPORT_VERSIONS = (1, 2, 3, 4, 5, 6, 7)
+# Each file form, and the key its one entry or its list sits under.
+EXPORT_KEYS = {EXPORT_ONE: "group", EXPORT_ALL: "groups", DATASET_ONE: "dataset", DATASET_ALL: "datasets"}
 SCORING_MODES = ("all", "weighted")
+# A group's newest version is edited in place by a save within this long of
+# the last, as a prompt's is (PROMPT_IDLE_SECONDS).
+GROUP_IDLE_SECONDS = float(os.environ.get("GROUP_IDLE_SECONDS", "30"))
 
 
 def blank_dataset() -> dict:
@@ -1574,6 +1680,30 @@ def fingerprint(body: dict) -> str:
     return hashlib.sha256(text.encode("utf-8")).hexdigest()[:CASE_SET_LEN]
 
 
+def pins_in(workflows) -> set:
+    """(group id, version) for every link a stored pipeline pins: the
+    promptlab.workflows body, each pipeline as its `work`. Only version 13
+    links pin, and a pipeline of an earlier version has none."""
+    out = set()
+    listed = workflows.get("list") if isinstance(workflows, dict) else None
+    for w in listed if isinstance(listed, list) else []:
+        work = w.get("work") if isinstance(w, dict) else None
+        evals = work.get("evals") if isinstance(work, dict) else None
+        for t in evals if isinstance(evals, list) else []:
+            if (isinstance(t, dict) and t.get("type") == "group" and isinstance(t.get("group"), dict)
+                    and isinstance(t["group"].get("id"), str) and type(t.get("pin")) is int):
+                out.add((t["group"]["id"], t["pin"]))
+    return out
+
+
+def group_key(ref) -> str:
+    """The key a run keeps a group's body under: `<id>@<n>`, or the id alone
+    for a reference from before the lab numbered versions."""
+    n = ref.get("n") if isinstance(ref, dict) else None
+    gid = ref.get("id") if isinstance(ref, dict) else None
+    return f"{gid}@{n}" if type(n) is int else str(gid)
+
+
 def unique_dataset_name(name: str, taken: set) -> str:
     """`Receipts` again becomes `Receipts (2)`, compared without case."""
     lower = {t.lower() for t in taken}
@@ -1627,7 +1757,7 @@ def scenario_ref(sc, i):
     what version 4's upgrade gives one, by position."""
     sid = sc.get("id") if isinstance(sc, dict) else None
     name = (sc.get("name") or "").strip() if isinstance(sc, dict) and isinstance(sc.get("name"), str) else ""
-    return (sid if isinstance(sid, str) and sid else f"s{i + 1}"), (name or f"Target {i + 1}")
+    return (sid if isinstance(sid, str) and sid else f"s{i + 1}"), (name or f"Target {chr(65 + i) if i < 26 else i + 1}")
 
 
 class Prompts:
@@ -1944,6 +2074,16 @@ class Datasets:
             # the body it was typed as is kept, as the rules were.
             db.execute("CREATE TABLE IF NOT EXISTS dataset_body_archive ("
                        "dataset_id TEXT NOT NULL, body TEXT NOT NULL, archived_at TEXT NOT NULL)")
+            # Every version of each eval group, numbered from 1, for a link to
+            # pin and a run to name (docs/pipeline-model.md §17). `ran` says a
+            # run graded with it, which freezes it for good; a pin freezes it
+            # while a stored pipeline holds the pin. Version 1 is added from
+            # the row's body at its first save, pin or run, so a group nobody
+            # touches holds what it held.
+            db.execute("CREATE TABLE IF NOT EXISTS eval_group_versions ("
+                       "group_id TEXT NOT NULL, n INTEGER NOT NULL, body TEXT NOT NULL, "
+                       "created_at TEXT NOT NULL, edited_at REAL NOT NULL, "
+                       "ran INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (group_id, n))")
             # Rows from an earlier version are converted once, in place: a
             # dataset is typed in by hand and costly to re-enter, so it is
             # upgraded rather than hidden (AGENTS.md's one exception). The
@@ -1992,16 +2132,26 @@ class Datasets:
         return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
 
     @staticmethod
-    def _doc(r, body=True):
+    def _doc(r, body=True, versions=1):
         """A row as the API answers it: a DatasetSummary, and its body with it,
-        read as today's version."""
+        read as today's version. `version` is the save counter a write is
+        arbitrated by; `versions` is how many group versions it has -- its
+        newest one's number, which a pin may name, and 1 before any is kept,
+        the row's body being version 1 in waiting."""
         parsed = upgrade_body(json.loads(r["body"]))
         out = {"id": r["id"], "name": r["name"], "cases": len(parsed.get("cases") or []),
-               "version": r["version"], "updated": r["updated_at"]}
+               "version": r["version"], "versions": versions, "updated": r["updated_at"]}
         if body:
             out["body"] = parsed
         return out
 
+    @staticmethod
+    def _counts(db) -> dict:
+        return dict(db.execute("SELECT group_id, MAX(n) FROM eval_group_versions GROUP BY group_id"))
+
+    def _row_doc(self, db, r, body=True):
+        return self._doc(r, body, self._counts(db).get(r["id"], 1))
+
     def _connect(self):
         db = sqlite3.connect(self.store.path)
         db.row_factory = sqlite3.Row
@@ -2016,13 +2166,142 @@ class Datasets:
 
     def list(self) -> list:
         with self.store.lock, self._connect() as db:
-            return [self._doc(r, body=False) for r in db.execute(
+            counts = self._counts(db)
+            return [self._doc(r, False, counts.get(r["id"], 1)) for r in db.execute(
                 "SELECT * FROM datasets WHERE trash IS NULL ORDER BY name COLLATE NOCASE, id")]
 
     def get(self, did):
         with self.store.lock, self._connect() as db:
             r = self._live(db, did)
-            return self._doc(r) if r else None
+            return self._row_doc(db, r) if r else None
+
+    # ---- a group's versions (docs/pipeline-model.md §17) -----------------
+
+    @staticmethod
+    def _head(db, did):
+        return db.execute("SELECT * FROM eval_group_versions WHERE group_id = ? "
+                          "ORDER BY n DESC LIMIT 1", (did,)).fetchone()
+
+    def _mint(self, db, r):
+        """The newest version of row [r], adding version 1 from its body
+        first if it has none: edited when the row last was, so a group made a
+        moment ago goes on being edited in place."""
+        head = self._head(db, r["id"])
+        if head is not None:
+            return head
+        try:
+            edited = calendar.timegm(time.strptime(r["updated_at"], "%Y-%m-%dT%H:%M:%SZ"))
+        except ValueError:
+            # A stamp nothing here wrote is no recent edit.
+            edited = 0
+        db.execute("INSERT INTO eval_group_versions (group_id, n, body, created_at, edited_at) "
+                   "VALUES (?, 1, ?, ?, ?)", (r["id"], r["body"], r["updated_at"], edited))
+        return self._head(db, r["id"])
+
+    @staticmethod
+    def _pins(db) -> set:
+        """Every (group, version) a stored pipeline pins, read in the caller's
+        transaction from the docs table the page writes them to."""
+        row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.workflows'").fetchone()
+        return pins_in(json.loads(row[0])) if row and row[0] else set()
+
+    def _cut(self, db, did, text):
+        """A new newest version of [did] holding [text]; returns its number."""
+        n = self._head(db, did)["n"] + 1
+        db.execute("INSERT INTO eval_group_versions (group_id, n, body, created_at, edited_at) "
+                   "VALUES (?, ?, ?, ?, ?)", (did, n, text, self._now(), time.time()))
+        return n
+
+    def _keep(self, db, r, text):
+        """[text] as row [r]'s newest version: the newest edited in place while
+        no run has graded with it, no link pins it and it was edited in the
+        last GROUP_IDLE_SECONDS, and a new version otherwise."""
+        head = self._mint(db, r)
+        if text == head["body"]:
+            return
+        fresh = time.time() - head["edited_at"] < GROUP_IDLE_SECONDS
+        if fresh and not head["ran"] and (r["id"], head["n"]) not in self._pins(db):
+            db.execute("UPDATE eval_group_versions SET body = ?, edited_at = ? WHERE group_id = ? AND n = ?",
+                       (text, time.time(), r["id"], head["n"]))
+        else:
+            self._cut(db, r["id"], text)
+
+    def versions(self, did):
+        """A group's versions, newest first, without their bodies -- version 1
+        alone, read from the row, before any is kept -- or None."""
+        with self.store.lock, self._connect() as db:
+            r = self._live(db, did)
+            if r is None:
+                return None
+            pins = self._pins(db)
+            rows = db.execute("SELECT * FROM eval_group_versions WHERE group_id = ? ORDER BY n DESC",
+                              (did,)).fetchall()
+            if not rows:
+                return [{"n": 1, "created": r["created_at"], "ran": False, "pinned": (did, 1) in pins,
+                         "fingerprint": fingerprint(json.loads(r["body"]))}]
+            return [{"n": v["n"], "created": v["created_at"], "ran": bool(v["ran"]),
+                     "pinned": (did, v["n"]) in pins, "fingerprint": fingerprint(json.loads(v["body"]))}
+                    for v in rows]
+
+    def version_body(self, did, n):
+        """Version [n]'s body, read as today's, or None."""
+        with self.store.lock, self._connect() as db:
+            r = self._live(db, did)
+            if r is None:
+                return None
+            v = db.execute("SELECT body FROM eval_group_versions WHERE group_id = ? AND n = ?",
+                           (did, n)).fetchone()
+            if v is None and n == 1 and self._head(db, did) is None:
+                v = (r["body"],)
+        return upgrade_body(json.loads(v[0])) if v else None
+
+    def restore_version(self, did, n):
+        """An older version's body as the newest version, and the row's: as a
+        prompt's Restore, nothing is rewritten, so a run or a pin naming any
+        version still reads what it named. Returns (DatasetDoc, None)."""
+        with self.store.lock, self._connect() as db, db:
+            r = self._live(db, did)
+            if r is None:
+                return None, (404, "no such eval group")
+            head = self._mint(db, r)
+            old = db.execute("SELECT body FROM eval_group_versions WHERE group_id = ? AND n = ?",
+                             (did, n)).fetchone()
+            if old is None:
+                return None, (404, "no such version")
+            if old["body"] != head["body"]:
+                self._cut(db, did, old["body"])
+                db.execute("UPDATE datasets SET body = ?, version = ?, updated_at = ? WHERE id = ?",
+                           (old["body"], r["version"] + 1, self._now(), did))
+            return self._row_doc(db, self._live(db, did)), None
+
+    def mint_pinned(self, pins):
+        """Version 1 of each pinned group that has none yet: a pin names a
+        version, so the version has to be kept from the moment it does."""
+        with self.store.lock, self._connect() as db, db:
+            for did, _ in pins:
+                r = self._live(db, did)
+                if r is not None:
+                    self._mint(db, r)
+
+    def resolve(self, did, pin=None):
+        """The version a run submitted now grades with -- [pin], or the
+        newest -- marked as graded with, in the same transaction, so no save
+        can edit it in place between this and the run keeping its body.
+        Returns (n, body), (None, why) for a pin the group has no version of,
+        or None for a group the lab does not have. A submit refused after
+        this leaves the version frozen, which costs only a new version at the
+        next save."""
+        with self.store.lock, self._connect() as db, db:
+            r = self._live(db, did) if isinstance(did, str) else None
+            if r is None:
+                return None
+            head = self._mint(db, r)
+            v = head if pin is None else db.execute(
+                "SELECT * FROM eval_group_versions WHERE group_id = ? AND n = ?", (did, pin)).fetchone()
+            if v is None:
+                return None, f"{r['name']} has no version {pin}"
+            db.execute("UPDATE eval_group_versions SET ran = 1 WHERE group_id = ? AND n = ?", (did, v["n"]))
+            return v["n"], json.loads(v["body"])
 
     def snapshot(self, did):
         """The body a run submitted now grades against, and its fingerprint;
@@ -2087,9 +2366,11 @@ class Datasets:
             if r is None:
                 return None, (404, "no such dataset")
             if r["version"] != version:
-                return None, (409, {"current": self._doc(r)})
+                return None, (409, {"current": self._row_doc(db, r)})
+            text = json.dumps(body)
+            self._keep(db, r, text)
             db.execute("UPDATE datasets SET body = ?, version = ?, updated_at = ? WHERE id = ?",
-                       (json.dumps(body), version + 1, self._now(), did))
+                       (text, version + 1, self._now(), did))
         return {"version": version + 1}, None
 
     def remove(self, did):
@@ -2115,58 +2396,69 @@ class Datasets:
             did = r["id"]
         return self.get(did), None
 
+    @staticmethod
+    def _purge(db, where, args):
+        # A group's versions go with it: the trash held them, and Undo is
+        # what would have brought them back.
+        db.execute(f"DELETE FROM eval_group_versions WHERE group_id IN (SELECT id FROM datasets WHERE {where})", args)
+        db.execute(f"DELETE FROM datasets WHERE {where}", args)
+
     def lazy_trash(self):
         """A datasets request empties what has been trashed longer than
         TRASH_SECONDS, as a Sources request does."""
         with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
-            db.execute("DELETE FROM datasets WHERE trash IS NOT NULL AND trashed_at < ?",
-                       (time.time() - TRASH_SECONDS,))
+            self._purge(db, "trash IS NOT NULL AND trashed_at < ?", (time.time() - TRASH_SECONDS,))
 
     def empty_trash(self):
         """The startup sweep: a restart has nothing to undo."""
         with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
-            db.execute("DELETE FROM datasets WHERE trash IS NOT NULL")
+            self._purge(db, "trash IS NOT NULL", ())
 
     def export(self, did):
         d = self.get(did)
         if d is None:
             return None
         return {"format": EXPORT_ONE, "version": EXPORT_VERSION,
-                "dataset": {"name": d["name"], "body": d["body"]}}
+                "group": {"name": d["name"], "body": d["body"]}}
 
     def export_all(self):
         with self.store.lock, self._connect() as db:
             rows = db.execute("SELECT * FROM datasets WHERE trash IS NULL "
                               "ORDER BY name COLLATE NOCASE, id").fetchall()
         return {"format": EXPORT_ALL, "version": EXPORT_VERSION,
-                "datasets": [{"name": r["name"], "body": upgrade_body(json.loads(r["body"]))} for r in rows]}
+                "groups": [{"name": r["name"], "body": upgrade_body(json.loads(r["body"]))} for r in rows]}
 
     def import_file(self, doc):
         """Either export's file, as new datasets: import always creates, ids
         are this lab's, and a taken name gets ` (2)`. All of it or none.
         Returns ([DatasetSummary], None), or (None, (400, one sentence))."""
         if not isinstance(doc, dict) or "format" not in doc:
-            return None, (400, "that file is not a dataset export: it has no \"format\"")
-        if doc["format"] not in (EXPORT_ONE, EXPORT_ALL):
+            return None, (400, "that file is not an eval group export: it has no \"format\"")
+        if doc["format"] not in EXPORT_KEYS:
             return None, (400, f"that file's format is {json.dumps(doc['format'])}, and the lab "
-                               f"imports {EXPORT_ONE} and {EXPORT_ALL}")
+                               f"imports {EXPORT_ONE}, {EXPORT_ALL}, {DATASET_ONE} and {DATASET_ALL}")
         version = doc.get("version")
-        # Exactly the integer: Python's True == 1.
-        if type(version) is not int or version not in IMPORT_VERSIONS:
+        # Exactly the integer: Python's True == 1. An eval group's file began
+        # at version 7; a dataset's reads back to version 1.
+        grouped = doc["format"] in (EXPORT_ONE, EXPORT_ALL)
+        if type(version) is not int or version not in ((EXPORT_VERSION,) if grouped else IMPORT_VERSIONS):
             return None, (400, f"that file is version {json.dumps(version)}, and the lab "
-                               f"reads versions 1 to {EXPORT_VERSION}")
-        if doc["format"] == EXPORT_ONE:
-            items = [doc.get("dataset")]
+                               + (f"reads version {EXPORT_VERSION}" if grouped else f"reads versions 1 to {EXPORT_VERSION}"))
+        one = doc["format"] in (EXPORT_ONE, DATASET_ONE)
+        key = EXPORT_KEYS[doc["format"]]
+        if one:
+            items = [doc.get(key)]
         else:
-            items = doc.get("datasets")
+            items = doc.get(key)
             if not isinstance(items, list):
-                return None, (400, "an export of every dataset holds them as a \"datasets\" list")
+                return None, (400, f"an export of every {'group' if grouped else 'dataset'} holds them as a \"{key}\" list")
         for k in doc:
-            if k not in ("format", "version", "dataset" if doc["format"] == EXPORT_ONE else "datasets"):
+            if k not in ("format", "version", key):
                 return None, (400, f"that file has \"{k}\", which an export does not")
         ready = []
+        what = "eval group" if grouped else "dataset"
         for i, item in enumerate(items):
-            at = "the dataset" if doc["format"] == EXPORT_ONE else f"dataset {i + 1}"
+            at = f"the {what}" if one else f"{what} {i + 1}"
             if not isinstance(item, dict) or set(item) != {"name", "body"}:
                 return None, (400, f"{at} has to be {{ \"name\", \"body\" }}")
             name, why = dataset_name(item["name"])
@@ -2339,13 +2631,17 @@ def read_pack(data: bytes):
             doc, why = load(name)
             if why:
                 return None, why
-            if (not isinstance(doc, dict) or doc.get("format") != EXPORT_ONE
-                    or doc.get("version") not in IMPORT_VERSIONS or not isinstance(doc.get("dataset"), dict)):
-                return None, f"{name} is not a dataset export the lab reads"
-            ds_name, why = dataset_name(doc["dataset"].get("name"))
+            # A pack's datasets are eval groups, in either file form: the
+            # group's (version 7) or the dataset's an older pack holds.
+            fmt = doc.get("format") if isinstance(doc, dict) else None
+            entry = doc.get(EXPORT_KEYS[fmt]) if fmt in (EXPORT_ONE, DATASET_ONE) else None
+            if (not isinstance(entry, dict) or type(doc.get("version")) is not int
+                    or doc["version"] not in ((EXPORT_VERSION,) if fmt == EXPORT_ONE else IMPORT_VERSIONS)):
+                return None, f"{name} is not an eval group export the lab reads"
+            ds_name, why = dataset_name(entry.get("name"))
             if why:
                 return None, f"{name}: {why}"
-            raw = doc["dataset"].get("body")
+            raw = entry.get("body")
             body = upgrade_body(raw) if doc["version"] < EXPORT_VERSION else raw
             why = dataset_problem(body)
             if why:
@@ -2524,11 +2820,12 @@ class Packs:
                 # the page upgrades the pipeline as it reads it.
                 evals = doc.get("evals", doc.get("tests"))
                 for t in evals if isinstance(evals, list) else [evals]:
-                    ref = t.get("dataset") if isinstance(t, dict) else None
-                    if isinstance(ref, dict):
+                    ref = eval_group_ref(t)
+                    if ref is not None:
                         did = ds_ids.get(ref.get("id")) or ds_ids.get(ref.get("name"))
                         if did:
-                            t["dataset"] = {"id": did, "name": DATASETS.get(did)["name"]}
+                            ref.clear()
+                            ref.update({"id": did, "name": DATASETS.get(did)["name"]})
                 content = content_of(doc)
                 if isinstance(content, dict) and isinstance(content.get("ref"), dict):
                     sid = src_ids.get(content["ref"].get("id")) or src_ids.get(content["ref"].get("name"))
@@ -3251,13 +3548,15 @@ class Plugins:
 # is a marker the worker checks between items -- today's semantics, stopping
 # after the item in flight, never a process kill that loses its reply.
 
-# A profile's key, as run-evals.js reads it: EVAL_API_KEY_<ID>, the id a run
-# document keys its profiles table by, in the shell-safe spelling of itself.
-# The id and not the name, so two profiles can share a name without sharing a
-# key (docs/pipeline-model.md §5). One spelling on both sides -- evals-core.ts
-# keyVar -- or the worker would never find the key the page never sent.
-def key_var(profile_id: str) -> str:
-    return "EVAL_API_KEY_" + re.sub(r"[^A-Z0-9]+", "_", str(profile_id).upper()).strip("_")
+# A profile's key, as run-evals.js reads it: EVALSLAB_API_KEY_<SLUG>, the slug the
+# run document's connection carries (#253), or EVALSLAB_API_KEY_<ID> for one from
+# before slugs -- the id it keys its profiles table by -- in the shell-safe
+# spelling of itself. Never the name, so two profiles can share a name
+# without sharing a key (docs/pipeline-model.md §5). One spelling on both
+# sides -- evals-core.ts keyVar -- or the worker would never find the key the
+# page never sent.
+def key_var(profile_id: str, slug=None) -> str:
+    return "EVALSLAB_API_KEY_" + re.sub(r"[^A-Z0-9]+", "_", str(slug or profile_id).upper()).strip("_")
 
 
 def run_items(run):
@@ -3331,7 +3630,33 @@ def brief_row(row):
         if not isinstance(it, dict) or not isinstance(it.get("scenarios"), list):
             return it
         return {**it, "scenarios": [brief_cell(c) for c in it["scenarios"]]}
-    return {**row, "results": [item(it) for it in row.get("results") or []], "brief": True}
+    # The run's verdict stays; each Target's eval by eval is the whole row's.
+    brief = {k: v for k, v in row.items() if k != "verdicts"}
+    return {**brief, "results": [item(it) for it in row.get("results") or []], "brief": True}
+
+
+# What of a worker's report a row keeps as its verdicts (#252): the run's --
+# pass, fail or incomplete, which the worker's exit code already said -- and
+# each Target's evals, by the eval's id, as `run.verdicts` names them. Only
+# these fields are carried, so nothing else the report holds lands in the row.
+# A status says whether the run finished; its verdict says whether it passed,
+# so a run that failed an eval is `done` with the verdict `fail`.
+VERDICTS = ("pass", "fail", "incomplete")
+VERDICT_FIELDS = ("name", "skipped", "verdict", "pass", "detail", "ran", "passed", "skippedItems")
+
+
+def report_verdicts(report):
+    """(verdict, verdicts as JSON) from a worker's report, or (None, None)
+    for one that carries none -- a worker from before #251."""
+    verdict = report.get("verdict") if isinstance(report, dict) else None
+    run = report.get("run") if verdict in VERDICTS else None
+    targets = run.get("verdicts") if isinstance(run, dict) else None
+    if not isinstance(targets, list):
+        return None, None
+    kept = [{eid: {k: e[k] for k in VERDICT_FIELDS if k in e}
+             for eid, e in t.items() if isinstance(e, dict)}
+            if isinstance(t, dict) else {} for t in targets]
+    return verdict, json.dumps(kept)
 
 
 class Queue:
@@ -3368,13 +3693,27 @@ class Queue:
             # one of its own.
             if "rerun_of" not in cols:
                 db.execute("ALTER TABLE queue ADD COLUMN rerun_of TEXT")
+            # The worker's verdicts (#252); a row from before them has none,
+            # and is never given one after the fact.
+            if "verdict" not in cols:
+                db.execute("ALTER TABLE queue ADD COLUMN verdict TEXT")
+            if "verdicts" not in cols:
+                db.execute("ALTER TABLE queue ADD COLUMN verdicts TEXT")
+            # The body of each eval group a run grades with, by `<id>@<n>`
+            # (docs/pipeline-model.md §17). A row from before keeps its
+            # `dataset` column, read as its one group.
+            if "groups" not in cols:
+                db.execute("ALTER TABLE queue ADD COLUMN groups TEXT")
 
     # ---- rows -----------------------------------------------------------
 
     # A row read with the submit time of the run it re-runs, if any: the
     # page names a run by that time (History's Run ID), so a "Re-run of"
-    # note reads without fetching the original.
-    SELECT = ("SELECT q.*, o.submitted_at FROM queue q "
+    # note reads without fetching the original. Its columns are named, since
+    # the order ALTER TABLE added them in is no order _row can count on.
+    SELECT = ("SELECT q.id, q.status, q.cancel, q.submitted_at, q.started_at, "
+              "q.finished_at, q.snapshot, q.results, q.progress, q.totals, q.error, "
+              "q.rerun_of, q.verdict, q.verdicts, o.submitted_at FROM queue q "
               "LEFT JOIN queue o ON o.id = q.rerun_of")
 
     @staticmethod
@@ -3387,8 +3726,9 @@ class Queue:
             "snapshot": json.loads(r[6]), "results": json.loads(r[7]),
             "progress": json.loads(r[8]), "totals": json.loads(r[9]),
             "error": r[10],
-            "rerunOf": r[12] if len(r) > 12 else None,
-            "rerunOfAt": r[13] if len(r) > 13 else None,
+            "rerunOf": r[11], "rerunOfAt": r[14],
+            "verdict": r[12],
+            "verdicts": json.loads(r[13]) if r[13] else None,
         }
 
     # A row from before run documents has no version, and nothing here can
@@ -3411,15 +3751,19 @@ class Queue:
                                        (rid,)).fetchone())
         return row if self._readable(row) else None
 
-    def list(self, limit=RUNS_PAGE, before=None, full=False):
-        """Runs, newest first, and whether more follow. `before` is a
-        `submittedAt` the page of runs stops at, so History can page through
-        them the way it pages the runs store. Each row is brief_row's unless
-        `full` asks for the whole of it."""
+    def list(self, limit=RUNS_PAGE, before=None, before_id=None, full=False):
+        """Runs, newest first, and whether more follow. `before` and
+        `before_id` are the `submittedAt` and id of the last run on the page
+        before, so History can page through them the way it pages the runs
+        store. `submittedAt` is to the second, so two runs can share one; the
+        id breaks the tie, or a page ending between them would skip the
+        second (#238). `before` alone stops at the second. Each row is
+        brief_row's unless `full` asks for the whole of it."""
         with self.lock, closing(sqlite3.connect(self.store.path)) as db:
             rows = self._all(db)
-        rows = [r for r in rows if before is None or r["submittedAt"] < before]
-        rows.sort(key=lambda r: r["submittedAt"], reverse=True)
+        key = lambda r: (r["submittedAt"], r["id"])
+        rows = [r for r in rows if before is None or key(r) < (before, before_id or "")]
+        rows.sort(key=key, reverse=True)
         page = rows[:limit]
         return (page if full else [brief_row(r) for r in page]), len(rows) > limit
 
@@ -3430,27 +3774,30 @@ class Queue:
 
     # ---- submit ---------------------------------------------------------
 
-    def submit(self, run: dict, dataset=None, rerun_of=None):
+    def submit(self, run: dict, dataset=None, rerun_of=None, groups=None):
         """
         A new queued run. `run` is the run document (docs/pipeline-model.md
         §5): the pipeline, the profiles it resolved to without their keys, its
-        content's file list in order and with its repeats, and the dataset's
-        version; `dataset` is that version's body, for a graded run;
-        `rerun_of` is the run a re-run was queued from. Returns the row. Its items are that list, or the one inline text, each through
-        every scenario -- so the total is the list's length, repeats and all,
-        the same count the runner and the page make.
+        content's file list in order and with its repeats, and each eval
+        group's version; `groups` is those versions' bodies, by `<id>@<n>`
+        (§17), and `dataset` the one body a row from before them kept, which a
+        re-run of one carries on; `rerun_of` is the run a re-run was queued
+        from. Returns the row. Its items are that list, or the one inline
+        text, each through every scenario -- so the total is the list's
+        length, repeats and all, the same count the runner and the page make.
         """
         rid = secrets.token_hex(6)
         now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
         total = len(run_items(run))
         with self.lock, closing(sqlite3.connect(self.store.path)) as db, db:
             db.execute("INSERT INTO queue (id, status, cancel, submitted_at, "
-                       "snapshot, results, progress, totals, dataset, rerun_of) "
-                       "VALUES (?, ?, 0, ?, ?, ?, ?, ?, ?, ?)",
+                       "snapshot, results, progress, totals, dataset, rerun_of, groups) "
+                       "VALUES (?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?)",
                        (rid, "queued", now, json.dumps(run), "[]",
                         json.dumps({"current": None, "n": 0, "total": total}),
                         json.dumps({"ran": 0, "passed": 0, "found": 0, "of": 0}),
-                        None if dataset is None else json.dumps(dataset), rerun_of))
+                        None if dataset is None else json.dumps(dataset), rerun_of,
+                        None if groups is None else json.dumps(groups)))
             # Its prompts' uses, in the same transaction: a run is in the
             # library the moment it is queued, or not queued at all.
             if self.prompts is not None:
@@ -3460,19 +3807,43 @@ class Queue:
             # the moment the lock is let go.
             return self._row(db.execute(self.SELECT + " WHERE q.id = ?", (rid,)).fetchone())
 
+    def groups(self, rid, raw=False):
+        """The eval group bodies a readable run grades with, by `<id>@<n>`, or
+        None for a run that is not there or kept none -- what its verdicts
+        were graded by, whatever the groups hold now. A row from before runs
+        kept a body per group answers its one `dataset` copy under its
+        group's key. A copy kept at an earlier version reads as one of
+        today's, unless [raw]: the worker is handed it as kept, since an
+        earlier run's pipeline is upgraded under the rules that copy holds."""
+        run = self.get(rid)
+        if run is None:
+            return None
+        kept = self._kept(rid)
+        if kept is None:
+            with self.lock, closing(sqlite3.connect(self.store.path)) as db:
+                old = db.execute("SELECT dataset FROM queue WHERE id = ?", (rid,)).fetchone()[0]
+            if not old:
+                return None
+            kept = {group_key(evals_dataset(run["snapshot"])): json.loads(old)}
+        return kept if raw else {k: upgrade_body(b) for k, b in kept.items()}
+
+    def _kept(self, rid):
+        """The row's own `groups` column, or None for a row from before it."""
+        with self.lock, closing(sqlite3.connect(self.store.path)) as db:
+            r = db.execute("SELECT groups FROM queue WHERE id = ?", (rid,)).fetchone()
+        return json.loads(r[0]) if r and r[0] is not None else None
+
     def dataset(self, rid, raw=False):
-        """The dataset body a readable run was submitted against, or None --
-        what its verdicts were graded by, whatever the dataset holds now. A
-        copy kept at an earlier version reads as one of today's, unless [raw]:
-        the worker is handed it as kept, since an earlier run's pipeline is
-        upgraded under the rules that copy holds."""
+        """The body of the one Library group a readable run grades its cases
+        against (evals_dataset), or None: the body the worker is handed as
+        --dataset, and what the page reads a run's cases from."""
         if self.get(rid) is None:
             return None
-        with self.lock, closing(sqlite3.connect(self.store.path)) as db:
-            r = db.execute("SELECT dataset FROM queue WHERE id = ?", (rid,)).fetchone()
-        if not (r and r[0]):
+        kept = self.groups(rid, raw=True)
+        ref = evals_dataset(self.get(rid)["snapshot"])
+        body = kept.get(group_key(ref)) if kept and ref is not None else None
+        if body is None:
             return None
-        body = json.loads(r[0])
         return body if raw else upgrade_body(body)
 
     def _plugin_args(self, run):
@@ -3487,14 +3858,18 @@ class Queue:
         return ["--plugins", str(PLUGINS.dir)], None
 
     def _dataset_args(self, run, rundir):
-        """The worker's --dataset for a graded run: the body kept with the row,
-        written beside the run document. A row queued before runs kept their
-        dataset has none, and is pinned to the dataset as it reads now, once,
-        so every later pass over it agrees. Returns (args, None) or (None, why)."""
+        """The worker's --dataset for a graded run: the body of its one
+        Library group kept with the row, written beside the run document --
+        one, until the worker reads a body per group (#233). A row queued
+        before runs kept their dataset has none, and is pinned to the dataset
+        as it reads now, once, so every later pass over it agrees. Returns
+        (args, None) or (None, why)."""
         ref = evals_dataset(run["snapshot"])
         if ref is None:
             return [], None
         body = self.dataset(run["id"], raw=True)
+        if body is None and self._kept(run["id"]) is not None:
+            return None, f"the run kept no body of the eval group {ref.get('name') or ref.get('id')!r}"
         if body is None:
             snap = DATASETS.snapshot(ref.get("id")) if DATASETS is not None else None
             if snap is None:
@@ -3510,6 +3885,27 @@ class Queue:
         (rundir / "dataset.json").write_text(json.dumps(body))
         return ["--dataset", str(rundir / "dataset.json")], None
 
+    def _grading_args(self, run, rundir):
+        """The worker's eval group bodies: --dataset for a run that grades
+        against one group, which keeps its single-body path and old-row
+        pinning, and --groups for a run that links several (#233) -- every body
+        it kept, by `<id>@<n>`, written beside the run document. Returns
+        (args, None) or (None, why)."""
+        refs = [eval_group_ref(t) for t in run["snapshot"].get("evals", []) or []]
+        keys = {group_key(r) for r in refs if r is not None}
+        if len(keys) <= 1:
+            return self._dataset_args(run, rundir)
+        kept = self.groups(run["id"], raw=True)
+        if kept is None:
+            return None, "the run kept no eval group bodies"
+        missing = sorted({(r.get("name") or r.get("id")) for r in refs
+                          if r is not None and group_key(r) not in kept})
+        if missing:
+            return None, f"the run kept no body of the eval group {', '.join(missing)}"
+        rundir.mkdir(parents=True, exist_ok=True)
+        (rundir / "groups.json").write_text(json.dumps(kept))
+        return ["--groups", str(rundir / "groups.json")], None
+
     def _behind(self, rid):
         """How many submissions stand between this one and the worker, by
         submit time -- what a waiting form names when it says what it is
@@ -3547,7 +3943,8 @@ class Queue:
         if run["status"] not in ("cancelled", "interrupted", "incomplete"):
             return None, (409, "only a cancelled, interrupted or incomplete run can be resumed")
         (self.dir / rid / "cancel").unlink(missing_ok=True)
-        self._set(rid, status="queued", cancel=0, error=None)
+        # What it reached is decided by the run it goes on to finish.
+        self._set(rid, status="queued", cancel=0, error=None, verdict=None, verdicts=None)
         return self.get(rid), None
 
     def set_comment(self, rid, comment):
@@ -3597,9 +3994,12 @@ class Queue:
         _, err = self._pinned_files(snap)
         if err:
             return None, (409, err)
+        # The group bodies the original kept: a re-run grades with exactly
+        # them, whatever the groups or the pins read now.
+        kept = self._kept(rid)
         ref = evals_dataset(snap)
         body = None
-        if ref is not None:
+        if ref is not None and kept is None:
             body = self.dataset(rid, raw=True)
             if body is None:
                 # A run that never started kept no body: the dataset's, if it
@@ -3615,7 +4015,7 @@ class Queue:
         _, err = worker_destinations(snap)
         if err:
             return None, (403, err)
-        return self.submit(snap, body, rerun_of=rid), None
+        return self.submit(snap, body, rerun_of=rid, groups=kept), None
 
     def rerun_item(self, rid, index):
         """
@@ -3644,7 +4044,7 @@ class Queue:
             return None, (403, err)
         if NODE is None:
             return None, (500, "node is not installed, so nothing can run")
-        dataset, err = self._dataset_args(run, rundir)
+        dataset, err = self._grading_args(run, rundir)
         if err:
             return None, (409, err)
         plugins, err = self._plugin_args(run)
@@ -3677,9 +4077,12 @@ class Queue:
         while len(results) <= index:
             results.append(None)
         results[index] = next((it for it in items if it and it.get("item") == index), items[0])
+        # The worker read every item to reach its verdicts, so they are the
+        # row's as it now stands.
+        verdict, verdicts = report_verdicts(report)
         with self.lock, closing(sqlite3.connect(self.store.path)) as db, db:
-            db.execute("UPDATE queue SET results = ?, error = ? WHERE id = ?",
-                       (json.dumps(results), None, run["id"]))
+            db.execute("UPDATE queue SET results = ?, error = ?, verdict = ?, verdicts = ? "
+                       "WHERE id = ?", (json.dumps(results), None, verdict, verdicts, run["id"]))
         return self.get(run["id"]), None
 
     def rescore_item(self, rid, index):
@@ -3707,7 +4110,7 @@ class Queue:
             return None, (500, "node is not installed, so nothing can run")
         rundir = self.dir / run["id"]
         rundir.mkdir(parents=True, exist_ok=True)
-        dataset, err = self._dataset_args(run, rundir)
+        dataset, err = self._grading_args(run, rundir)
         if err:
             return None, (409, err)
         plugins, err = self._plugin_args(run)
@@ -3742,9 +4145,12 @@ class Queue:
         while len(results) <= index:
             results.append(None)
         results[index] = next((it for it in items if it and it.get("item") == index), items[0])
+        # The worker read every item to reach its verdicts, so they are the
+        # row's as it now stands.
+        verdict, verdicts = report_verdicts(report)
         with self.lock, closing(sqlite3.connect(self.store.path)) as db, db:
-            db.execute("UPDATE queue SET results = ?, error = ? WHERE id = ?",
-                       (json.dumps(results), None, run["id"]))
+            db.execute("UPDATE queue SET results = ?, error = ?, verdict = ?, verdicts = ? "
+                       "WHERE id = ?", (json.dumps(results), None, verdict, verdicts, run["id"]))
         return self.get(run["id"]), None
 
     # ---- the run --------------------------------------------------------
@@ -3820,7 +4226,7 @@ class Queue:
             return self._finish(rid, "failed", error=err)
         if NODE is None:
             return self._finish(rid, "failed", error="node is not installed, so nothing can run")
-        dataset, err = self._dataset_args(run, rundir)
+        dataset, err = self._grading_args(run, rundir)
         if err:
             return self._finish(rid, "failed", error=err)
         plugins, err = self._plugin_args(run)
@@ -3855,10 +4261,11 @@ class Queue:
         report = self._read_report(rid)
         items = report["run"]["items"] if report and isinstance(report.get("run"), dict) else None
         if isinstance(items, list):
+            verdict, verdicts = report_verdicts(report)
             with self.lock, closing(sqlite3.connect(self.store.path)) as db, db:
                 if db.execute("SELECT 1 FROM queue WHERE id = ?", (rid,)).fetchone() is not None:
-                    db.execute("UPDATE queue SET results = ?, error = ? WHERE id = ?",
-                               (json.dumps(items), None, rid))
+                    db.execute("UPDATE queue SET results = ?, error = ?, verdict = ?, verdicts = ? "
+                               "WHERE id = ?", (json.dumps(items), None, verdict, verdicts, rid))
         run = self.get(rid)
         # The watchdog may have failed the run while it was on the wire; a
         # row that already left `running` is not this worker's to re-label.
@@ -3968,8 +4375,8 @@ class Queue:
         """The oldest queued run this server can read. One it cannot is never
         started: nothing here would know what it asks for."""
         with self.lock, closing(sqlite3.connect(self.store.path)) as db, db:
-            for r in db.execute("SELECT * FROM queue WHERE status = 'queued' "
-                                "ORDER BY submitted_at, rowid").fetchall():
+            for r in db.execute(self.SELECT + " WHERE q.status = 'queued' "
+                                "ORDER BY q.submitted_at, q.rowid").fetchall():
                 if not self._readable(self._row(r)):
                     continue
                 db.execute("UPDATE queue SET status = 'running', started_at = ? "
@@ -4015,10 +4422,89 @@ class Queue:
                         proc.kill()
 
 
+# The key a relayed request is sent with: the one it carries, or -- a page
+# that was handed KEY_HELD in place of a profile's key -- the one the store
+# holds for that profile.
+def relay_key(payload: dict) -> str:
+    key = str(payload.get("key") or "").strip()
+    if held(key):
+        return STORE.held_key(key).strip() if STORE is not None else ""
+    return key
+
+
 # The worker-destination rules, at submit and again at dequeue: a run's
 # profiles take their keys from the profiles store, by id, so the request
 # carries none, and the relay's rules bind where they go. Returns (env_vars,
 # None) or (None, a named refusal).
+# Export for CI (#254): a pipeline as a bundle a repository keeps and
+# `evals-lab run` runs with no lab. The page writes its text -- the core's
+# exportBundle: pipeline.yaml, profiles.yaml by slug, datasets/<slug>.json --
+# and the server adds what only it holds: every installed plugin, as a run
+# stamps them all, and, when asked, the Source's files under items/. A key is
+# refused rather than zipped: no Setup key, no token, no field named like
+# one, in any of it (the core's bundleProblems asks the same of the page's).
+BUNDLE_TEXT = re.compile(r"pipeline\.yaml|profiles\.yaml|datasets/[a-z0-9]+(?:-[a-z0-9]+)*\.json")
+BUNDLE_KEY_FIELD = re.compile(
+    r'^[\s-]*"?((?:api[-_]?)?key|authorization|bearer|secret|password|token)"?\s*:', re.I | re.M)
+
+
+def bundle_problems(files, keys=()):
+    """Why [files] -- path to text -- cannot leave the lab, or ""."""
+    for path, text in files.items():
+        for key in keys:
+            key = str(key or "").strip()
+            if len(key) >= 4 and key in text:
+                return f"{path} holds a Target profile's key, and a key never leaves the lab"
+        if TOKEN_SHAPE.search(text):
+            return f"{path} holds a token, and a key never leaves the lab"
+        m = BUNDLE_KEY_FIELD.search(text)
+        if m:
+            return f"{path} holds a field named {m.group(1)}, and a key never leaves the lab"
+    return ""
+
+
+def build_bundle(payload):
+    """The zip Export for CI downloads, from the page's [payload]:
+    { files: {path: text}, source: id or null, items: bool }. Returns
+    (bytes, None) or (None, (status, one sentence))."""
+    files = payload.get("files")
+    if not isinstance(files, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in files.items()):
+        return None, (400, "a bundle's files are text, by path")
+    for path in files:
+        if not BUNDLE_TEXT.fullmatch(path):
+            return None, (400, f"{path!r} is not a file a bundle holds")
+    if "pipeline.yaml" not in files or "profiles.yaml" not in files:
+        return None, (400, "a bundle holds pipeline.yaml and profiles.yaml")
+    stored = []
+    if STORE is not None:
+        stored = ((STORE.all().get("promptlab.profiles") or {}).get("body") or {}).get("list") or []
+    why = bundle_problems(files, [p.get("key") for p in stored if isinstance(p, dict)])
+    if why:
+        return None, (400, why)
+    items = []
+    if payload.get("items"):
+        sid = payload.get("source")
+        found = SOURCES.item_paths(sid) if isinstance(sid, str) and SOURCES is not None else None
+        if found is None:
+            return None, (404, "no such source")
+        items = found
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+        for path, text in sorted(files.items()):
+            zf.writestr(f"evals/{path}", text)
+        if PLUGINS is not None:
+            for p in PLUGINS.stamp():
+                root = PLUGINS.dir / p["id"] / p["version"]
+                for f in sorted(root.rglob("*")):
+                    if f.is_file():
+                        zf.write(f, f"evals/plugins/{p['id']}/{p['version']}/{f.relative_to(root).as_posix()}")
+        for name, path in items:
+            if not path.is_file():
+                return None, (404, f"{name!r} is not in that Source")
+            zf.write(path, f"evals/items/{name}")
+    return buf.getvalue(), None
+
+
 def worker_destinations(run: dict):
     table = run.get("profiles") if isinstance(run, dict) else None
     if not isinstance(table, dict):
@@ -4046,7 +4532,7 @@ def worker_destinations(run: dict):
             why = allowed(base, key)
             if why:
                 return None, f"Target profile {name}: {why}"
-            env[key_var(pid)] = key
+            env[key_var(pid, conn.get("slug"))] = key
     return env, None
 
 
@@ -4068,15 +4554,17 @@ def worker_destinations(run: dict):
 # step in each job is what it sends there (docs/pipeline-model.md §16).
 # 11: `tests` are `evals`; nothing in an eval changes.
 # 12: a Contains metric's Ignore case holds item by item too, kept as written.
-PIPELINE_VERSION = 12
+# 13: an eval is a link to an eval group, or a group of the pipeline's own,
+# and the document has an overall pass rule (docs/pipeline-model.md §17).
+PIPELINE_VERSION = 13
 # What a stored run may be: the current version, and the ones evals-core.ts's
 # upgradePipeline reads. A new submission is upgraded to the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)
 TARGET_CAP = 4
 # Target steps whose words the Prompt library does not record as a use: they
 # ask no model (evals-core.ts's STEP_TYPES.echo).
 UNRECORDED_STEPS = {"echo"}
-RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "evals", "profiles", "comment", "plugins")
+RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "evals", "pass", "profiles", "comment", "plugins")
 
 
 def content_of(doc):
@@ -4122,14 +4610,17 @@ BUILTIN_CONNECTION_TYPES = dict(CONNECTION_TYPES)
 BUILTIN_LOCAL = set(LOCAL_CONNECTIONS)
 BUILTIN_CHAT_PATHS = dict(CONNECTION_CHAT_PATHS)
 BUILTIN_AUTH = dict(CONNECTION_AUTH)
-CONNECTION_FIELDS = ("name", "url", "model", "type", "temperature", "px", "format",
+CONNECTION_FIELDS = ("name", "slug", "url", "model", "type", "temperature", "px", "format",
                      "quality", "options")
 LOOKS_LIKE_A_KEY = re.compile(r"^(?:api[-_]?)?key$|^(?:authorization|bearer|secret|password|token)$",
                               re.IGNORECASE)
 PROFILE_ID = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?")
+# evals-core.ts's SLUG and SLUG_MAX: what a key's variable is spelt from.
+PROFILE_SLUG = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*")
+SLUG_MAX = 32
 
 
-def _fields(obj, at, allowed_fields, bad, key_from="EVAL_API_KEY_<ID>"):
+def _fields(obj, at, allowed_fields, bad, key_from="EVALSLAB_API_KEY_<ID>"):
     for k in obj:
         if k in allowed_fields:
             continue
@@ -4200,7 +4691,12 @@ CONNECTION_LISTS = {"http": http_endpoint_test}
 
 
 def connection_problems(pid, conn, at, bad):
-    _fields(conn, at, CONNECTION_FIELDS, bad, key_var(pid))
+    slug = conn.get("slug")
+    if slug is not None and not (isinstance(slug, str) and PROFILE_SLUG.fullmatch(slug) and len(slug) <= SLUG_MAX):
+        bad.append(f"{at}: slug has to be lowercase letters and digits, with hyphens between, at most {SLUG_MAX}")
+        slug = None
+    var = key_var(pid, slug)
+    _fields(conn, at, CONNECTION_FIELDS, bad, var)
     ctype = conn.get("type")
     if not isinstance(ctype, str) or ctype not in CONNECTION_TYPES:
         bad.append(f"{at}: type has to be one of {', '.join(CONNECTION_TYPES)}")
@@ -4211,7 +4707,7 @@ def connection_problems(pid, conn, at, bad):
             # Which settings there are is the type's to say; a key among them
             # is the server's.
             allowed = list(CONNECTION_TYPES.get(ctype, ())) if ctype else list(options)
-            _fields(options, f"{at}'s options", allowed, bad, key_var(pid))
+            _fields(options, f"{at}'s options", allowed, bad, var)
         else:
             bad.append(f"{at}: options has to be an object")
     url = conn.get("url") or ""
@@ -4222,7 +4718,7 @@ def connection_problems(pid, conn, at, bad):
         parts = urllib.parse.urlsplit(url if "://" in url else "http://" + url)
         if parts.username or parts.password:
             bad.append(f"{at} has a key in its address, and a key never goes in a pipeline "
-                       f"— ${key_var(pid)} supplies it")
+                       f"— ${var} supplies it")
     for why in CONNECTION_CHECKS.get(ctype, lambda _c: [])(conn):
         bad.append(f"{at} {why}")
     # llama.cpp runs its own llama-server: a hosted address is refused, as the
@@ -4233,16 +4729,33 @@ def connection_problems(pid, conn, at, bad):
             bad.append(f"{at}: llama.cpp requires its llama-server, not a hosted model")
 
 
+def eval_group_ref(t):
+    """The Library group an eval reads the cases of, as the dict itself (so a
+    caller may stamp it in place), or None: version 13's link (`group`) or a
+    private group's `casesFrom`, or an earlier version's `dataset`
+    (evals-core.ts casesRef)."""
+    if not isinstance(t, dict):
+        return None
+    if t.get("type") == "group":
+        if isinstance(t.get("group"), dict):
+            return t["group"]
+        own = t.get("own")
+        return own["casesFrom"] if isinstance(own, dict) and isinstance(own.get("casesFrom"), dict) else None
+    return t["dataset"] if isinstance(t.get("dataset"), dict) else None
+
+
 def evals_dataset(doc):
-    """The dataset reference a document's evals grade against, or None: the
-    first eval that names one. A run grades against one dataset (the core's
+    """The Library group a document's evals read the cases of, or None: the
+    first eval that names one. A run grades against one (the core's
     validatePipeline says so). Reads a stored document of any shape --
-    version 11's `evals`, the `tests` before it, version 5's list or the one
-    test before that -- since rows keep the document they were submitted with."""
+    version 13's links, version 11's `evals`, the `tests` before it, version
+    5's list or the one test before that -- since rows keep the document they
+    were submitted with."""
     evals = doc.get("evals", doc.get("tests")) if isinstance(doc, dict) else None
     for t in evals if isinstance(evals, list) else [evals]:
-        if isinstance(t, dict) and isinstance(t.get("dataset"), dict):
-            return t["dataset"]
+        ref = eval_group_ref(t)
+        if ref is not None:
+            return ref
     return None
 
 
@@ -4325,6 +4838,7 @@ def run_problems(run):
     table = run.get("profiles")
     if not isinstance(table, dict):
         return bad + ["profiles has to be an object of id → connection"]
+    spelt = {}
     for pid, conn in table.items():
         at = f"profile {pid}"
         if not PROFILE_ID.fullmatch(str(pid)):
@@ -4333,6 +4847,11 @@ def run_problems(run):
         if not isinstance(conn, dict):
             bad.append(f"{at} has to be an object")
             continue
+        # Two profiles spelling one variable would hand one the other's key.
+        var = key_var(pid, conn.get("slug") if isinstance(conn.get("slug"), str) else None)
+        if var in spelt:
+            bad.append(f"profiles {spelt[var]} and {pid} both take their key from ${var}")
+        spelt[var] = pid
         connection_problems(pid, conn, at, bad)
     targets = run.get("targets")
     if not isinstance(targets, list) or not 1 <= len(targets) <= TARGET_CAP:
@@ -4550,9 +5069,23 @@ class Handler(BaseHTTPRequestHandler):
 
     # ---- routes ---------------------------------------------------------
 
+    # The eval groups' routes (docs/pipeline-model.md §17) are the datasets'
+    # under their new name: /api/datasets goes on answering the same rows, so
+    # dataset-diff.js and a script written against it keep working. A list
+    # asked for by the new name is `groups`.
+    GROUPS_ROUTE = "/api/eval-groups"
+    grouped = False
+
+    def _alias(self):
+        self.grouped = self.path == self.GROUPS_ROUTE or self.path.startswith((self.GROUPS_ROUTE + "/",
+                                                                                 self.GROUPS_ROUTE + "?"))
+        if self.grouped:
+            self.path = "/api/datasets" + self.path[len(self.GROUPS_ROUTE):]
+
     def do_GET(self):
         if not self._authorised():
             return
+        self._alias()
         path = self.path.split("?", 1)[0]
         # The lab is one page: a Connection and an Input make a scenario,
         # Content and Evals are shared, and one to four scenarios run over the
@@ -4599,8 +5132,16 @@ class Handler(BaseHTTPRequestHandler):
             except ValueError:
                 return self._json(400, {"error": "limit has to be a number"})
             before = (query.get("before") or [None])[0]
-            runs, more = QUEUE.list(limit, before, full=(query.get("full") or [""])[0] == "1")
+            before_id = (query.get("beforeId") or [None])[0]
+            runs, more = QUEUE.list(limit, before, before_id, full=(query.get("full") or [""])[0] == "1")
             return self._json(200, {"runs": runs, "more": more})
+        if path.startswith("/api/queue/") and path.endswith("/groups") and path.count("/") == 4:
+            # The bodies of the eval groups a run grades with, by `<id>@<n>`
+            # (§17); null for a run that kept none, as /dataset answers.
+            run_id = path.split("/")[3]
+            if QUEUE is None or QUEUE.get(run_id) is None:
+                return self._send(404, b"not found", "text/plain")
+            return self._json(200, QUEUE.groups(run_id))
         if path.startswith("/api/queue/") and path.endswith("/dataset") and path.count("/") == 4:
             # The dataset body a graded run was submitted against, which is
             # what its verdicts were graded by; the list never carries it.
@@ -4762,6 +5303,7 @@ class Handler(BaseHTTPRequestHandler):
     def do_DELETE(self):
         if not self._authorised() or not self._from_this_page():
             return
+        self._alias()
         path = self.path.split("?", 1)[0]
         if path == "/api/connections/google":
             if CONNECTIONS is None:
@@ -4829,6 +5371,7 @@ class Handler(BaseHTTPRequestHandler):
     def do_PATCH(self):
         if not self._authorised() or not self._from_this_page():
             return
+        self._alias()
         parts = self.path.split("?", 1)[0].split("/")
         if len(parts) != 4 or parts[1] != "api":
             return self._send(404, b"not found", "text/plain")
@@ -4859,6 +5402,7 @@ class Handler(BaseHTTPRequestHandler):
     def do_PUT(self):
         if not self._authorised() or not self._from_this_page():
             return
+        self._alias()
         parts = self.path.split("?", 1)[0].split("/")
         if len(parts) == 4 and parts[:3] == ["", "api", "prompts"] and PROMPTS is not None:
             return self._prompts_put(parts[3])
@@ -4921,6 +5465,7 @@ class Handler(BaseHTTPRequestHandler):
             return self._json(500, {"error": f"the relay failed: {type(e).__name__}"})
 
     def _post(self):
+        self._alias()
         path = self.path.split("?", 1)[0]
         if path == "/api/state":
             return self._state_write()
@@ -4940,6 +5485,12 @@ class Handler(BaseHTTPRequestHandler):
             return self._prompts_post(path)
         if path.startswith("/api/sources"):
             return self._sources_post(path)
+        if path == "/api/bundle":
+            data, err = build_bundle(self._payload() or {})
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._send(200, data, "application/zip",
+                              (("Content-Disposition", 'attachment; filename="evals.zip"'),))
         payload = self._payload()
         if payload is None:
             return self._json(400, {"error": "bad body"})
@@ -5012,7 +5563,7 @@ class Handler(BaseHTTPRequestHandler):
 
     def _models_list(self, payload):
         base = api_base(str(payload.get("url") or "")) or api_base(OLLAMA)
-        key = str(payload.get("key") or "").strip()
+        key = relay_key(payload)
         ctype = str(payload.get("type") or "")
         # A type that lists some other way (an HTTP endpoint: its own test
         # path, key header and headers) says where and how.
@@ -5063,7 +5614,7 @@ class Handler(BaseHTTPRequestHandler):
         # and how the key travels, per the type -- the same mirror the models
         # list uses, so a client cannot point the relay at a path of its own.
         base = api_base(str(payload.get("url") or "")) or api_base(OLLAMA)
-        key = str(payload.get("key") or "").strip()
+        key = relay_key(payload)
         if not header_safe(key):
             return self._json(400, {"error": "That key has characters that "
                                              "cannot be sent in a header, so "
@@ -5099,6 +5650,10 @@ class Handler(BaseHTTPRequestHandler):
                                        for n, d in docs.items()})
         if stale is not None:
             return self._json(409, {"stale": stale})
+        # A pin names a group's version, so the group keeps that version from
+        # the moment a pipeline pins it (§17).
+        if "promptlab.workflows" in docs and DATASETS is not None:
+            DATASETS.mint_pinned(pins_in(docs["promptlab.workflows"].get("body")))
         return self._json(200, {"versions": versions})
 
     # ---- The run queue (#530) ----------------------------------------------
@@ -5137,20 +5692,28 @@ class Handler(BaseHTTPRequestHandler):
         # The plugins it runs under, as installed now: the server's to say,
         # whatever the document claimed.
         run["plugins"] = PLUGINS.stamp() if PLUGINS is not None else []
-        # A graded run keeps the body of the dataset it names, as it reads
-        # now, and records that body's fingerprint on the reference it
-        # belongs to: the worker grades against that and nothing else.
-        ref = evals_dataset(run)
-        body = None
-        if ref is not None:
-            snap = DATASETS.snapshot(ref.get("id")) if DATASETS else None
-            if snap is None:
+        # Each Library group the evals read, resolved once (§17): a pinned
+        # link to its pin, anything else to the group's newest version. The
+        # reference records the version, `n`, and its body's fingerprint, and
+        # the body is kept with the row under `<id>@<n>`: the worker grades
+        # with that and nothing else, and the version is frozen from now on.
+        groups = {}
+        for t in run["evals"]:
+            ref = eval_group_ref(t)
+            if ref is None:
+                continue
+            pin = t.get("pin") if t.get("type") == "group" and t.get("group") is ref else None
+            got = DATASETS.resolve(ref.get("id"), pin if type(pin) is int else None) if DATASETS else None
+            if got is None:
                 return self._json(400, {"error": "the run's graded eval names no dataset this lab has"})
-            body, version = snap
-            for t in run["evals"]:
-                if isinstance(t.get("dataset"), dict) and t["dataset"].get("id") == ref.get("id"):
-                    t["dataset"]["version"] = version
-        return self._json(201, {"run": QUEUE.submit(run, body)})
+            n, body = got
+            if n is None:
+                return self._json(400, {"error": f"the run cannot be queued: {body}"})
+            ref["n"], ref["version"] = n, fingerprint(body)
+            groups[group_key(ref)] = body
+        # The worker reads a body per group now (#233), so a run may link
+        # several; each body is kept with the row under `<id>@<n>`.
+        return self._json(201, {"run": QUEUE.submit(run, groups=groups)})
 
     # ---- Datasets ----------------------------------------------------------
     # Rows in the store (Datasets above). The page reads one by id; an export
@@ -5244,9 +5807,16 @@ class Handler(BaseHTTPRequestHandler):
         DATASETS.lazy_trash()
         parts = path.split("/")
         if len(parts) == 3:
-            return self._json(200, {"datasets": DATASETS.list()})
+            return self._json(200, {"groups" if self.grouped else "datasets": DATASETS.list()})
         if len(parts) == 4 and parts[3] == "export":
-            return self._download(DATASETS.export_all(), "datasets", "all")
+            return self._download(DATASETS.export_all(), "eval-groups", "all")
+        if len(parts) == 5 and parts[4] == "versions":
+            got = DATASETS.versions(parts[3])
+            return self._json(200, {"versions": got}) if got is not None else self._send(404, b"not found", "text/plain")
+        if len(parts) == 6 and parts[4] == "versions" and parts[5].isdigit():
+            got = DATASETS.version_body(parts[3], int(parts[5]))
+            return self._json(200, {"n": int(parts[5]), "body": got}) if got is not None \
+                else self._send(404, b"not found", "text/plain")
         if len(parts) == 4 and parts[3] == "archived-rules":
             return self._json(200, {"rules": DATASETS.archived_rules()})
         if len(parts) == 4:
@@ -5256,7 +5826,7 @@ class Handler(BaseHTTPRequestHandler):
             doc = DATASETS.export(parts[3])
             if doc is None:
                 return self._send(404, b"not found", "text/plain")
-            return self._download(doc, "dataset", doc["dataset"]["name"])
+            return self._download(doc, "eval-group", doc["group"]["name"])
         return self._send(404, b"not found", "text/plain")
 
     def _download(self, doc, kind, name):
@@ -5284,6 +5854,12 @@ class Handler(BaseHTTPRequestHandler):
             if err:
                 return self._json(err[0], {"error": err[1]})
             return self._json(200, dataset)
+        if len(parts) == 7 and parts[4] == "versions" and parts[6] == "restore" and parts[5].isdigit():
+            self._payload()
+            dataset, err = DATASETS.restore_version(parts[3], int(parts[5]))
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(200, dataset)
         if len(parts) == 4 and parts[3] == "import":
             doc, err = self._dataset_payload(DATASET_IMPORT_CAP)
             if err: