evals-lab 0.15.0 → 0.16.1

package/lab/server.py

@@ -89,12 +89,6 @@ DISPLAYABLE = (".jpg", ".jpeg", ".png")
 # is what issue #270 turned out to be.
 UNDISPLAYABLE = (".heic", ".heif", ".dng", ".tif", ".tiff")
 
-# The llama.cpp presets the Setup tab offers as Apply preset…: measured
-# decoding settings, Setup's data and no dataset's. Read by the page (served
-# at /api/presets) and by evals-check.js's Decoding.kt comparison; absent, the
-# tab just has no presets to offer, so it is not fatal to start.
-PRESETS = HERE / "presets.json" if (HERE / "presets.json").is_file() else None
-
 # The page, built from web/ (React + Vite, #24 and #27's page PR): the image's
 # web stage writes it here, as `npm run build` does in a checkout. Required,
 # like the dataset: without it there is no page. WEB_DIST names another
@@ -3879,11 +3873,11 @@ def response_pins(db, sid, name) -> set:
         except (ValueError, TypeError, AttributeError):
             continue
         for w in listed if isinstance(listed, list) else []:
-            c = content_of(w.get("work")) if isinstance(w, dict) else None
-            pin = c.get("pin") if isinstance(c, dict) else None
-            if (isinstance(pin, dict) and isinstance(c.get("ref"), dict) and c["ref"].get("id") == sid
-                    and pin.get("name") == name and type(pin.get("n")) is int):
-                out.add(pin["n"] or 1)
+            for _step, c, _at in item_contents(w.get("work")) if isinstance(w, dict) else []:
+                pin = c.get("pin")
+                if (isinstance(pin, dict) and isinstance(c.get("ref"), dict) and c["ref"].get("id") == sid
+                        and pin.get("name") == name and type(pin.get("n")) is int):
+                    out.add(pin["n"] or 1)
     return out
 
 
@@ -3978,6 +3972,17 @@ def filter_set_links(doc) -> list:
     Runs wires the step and migrates the inline rules in #339; this is the one
     reader of where the links sit, for the store (#336) to resolve and keep."""
     out = []
+    # Version 20: filter sets live on the Filter steps of the flat step list
+    # (evals-core.ts); version 19 and older held them on a job's Responses
+    # steps.
+    steps = doc.get("steps") if isinstance(doc, dict) else None
+    if isinstance(steps, list) and any(isinstance(s, dict) and s.get("type") in ("getItems", "request") for s in steps):
+        for st in steps:
+            links = st.get("filterSets") if isinstance(st, dict) else None
+            for link in links if isinstance(links, list) else []:
+                if isinstance(link, dict):
+                    out.append(link)
+        return out
     jobs = doc.get("jobs") if isinstance(doc, dict) else None
     for job in jobs if isinstance(jobs, list) else []:
         steps = job.get("steps") if isinstance(job, dict) else None
@@ -4143,6 +4148,24 @@ class Prompts:
                                            "ORDER BY at DESC, run_id, job", (r["id"],))]
         return out
 
+    def used_by(self, run_ids) -> dict:
+        """Each run's prompt uses, by run id: the target, the job, and the
+        prompt and version its words were recorded as. A trashed prompt's
+        uses are left out with it."""
+        out = {}
+        ids = list(run_ids)
+        with self.store.lock, self._connect() as db:
+            for k in range(0, len(ids), 500):
+                chunk = ids[k:k + 500]
+                for u in db.execute(
+                        "SELECT u.run_id, u.scenario_id, u.job, u.version, p.id, p.name FROM prompt_uses u "
+                        "JOIN prompts p ON p.id = u.prompt_id WHERE p.trash IS NULL AND p.workspace = ? "
+                        f"AND u.run_id IN ({', '.join('?' * len(chunk))}) ORDER BY u.job",
+                        (workspace_of(self.store), *chunk)):
+                    out.setdefault(u["run_id"], []).append(
+                        {"target": u["scenario_id"], "job": u["job"], "id": u["id"], "name": u["name"], "version": u["version"]})
+        return out
+
     def list(self) -> list:
         with self.store.lock, self._connect() as db:
             return [self._summary(db, r) for r in db.execute(
@@ -5616,23 +5639,38 @@ def pack_connection(ws, documents=False):
 
 
 def repoint_targets(doc: dict, ws, documents=False) -> None:
-    """A pack pipeline's targets that name a connection this lab does not have,
-    pointed at the one pack_connection picks, its profile dropped with it."""
-    targets = doc.get("targets")
-    if not isinstance(targets, list) or STORE is None:
+    """A pack pipeline's Variants or Request cells that name a connection this
+    lab does not have, pointed at the one pack_connection picks, its profile
+    dropped with it. Version 20 keeps the connection on each Variant and on the
+    Request steps' cells (and a Variant's overrides), so each is repointed."""
+    if not isinstance(doc, dict) or STORE is None:
         return
     have = {c.get("id") for c in ((STORE.all().get("promptlab.connections") or {}).get("body") or {}).get("list") or []
             if isinstance(c, dict)}
-    pick = None
-    for t in targets:
-        ref = t.get("connection") if isinstance(t, dict) else None
-        if not isinstance(ref, dict) or ref.get("id") in have:
-            continue
-        pick = pick or pack_connection(ws, documents)
-        if pick is None:
+    state = {"pick": None, "none": False}
+
+    def repoint(o):
+        ref = o.get("connection") if isinstance(o, dict) else None
+        if not isinstance(ref, dict) or ref.get("id") in have or state["none"]:
             return
-        t["connection"] = {"id": pick["id"], "name": pick.get("name") or pick["id"]}
-        t["profile"] = None
+        if state["pick"] is None:
+            state["pick"] = pack_connection(ws, documents)
+            if state["pick"] is None:
+                state["none"] = True
+                return
+        pick = state["pick"]
+        o["connection"] = {"id": pick["id"], "name": pick.get("name") or pick["id"]}
+        if "profile" in o:
+            o["profile"] = None
+
+    for v in doc.get("variants") or []:
+        if isinstance(v, dict):
+            repoint(v)
+            for ov in (v.get("overrides") or {}).values():
+                repoint(ov)
+    for s in doc.get("steps") or []:
+        if isinstance(s, dict) and s.get("type") == "request":
+            repoint(s.get("cell"))
 
 
 def pack_requires_problem(manifest: dict, plugins: set) -> str:
@@ -5728,7 +5766,8 @@ def read_pack(data: bytes):
             if why:
                 return None, why
             if (not isinstance(doc, dict) or doc.get("version") not in READABLE_VERSIONS
-                    or not (isinstance(doc.get("jobs"), list) or isinstance(doc.get("chains"), list))):
+                    or not (isinstance(doc.get("steps"), list) or isinstance(doc.get("jobs"), list)
+                            or isinstance(doc.get("chains"), list))):
                 return None, f"{name} is not a pipeline document the lab reads"
             pack["pipelines"].append((parts[1][:-5], doc))
         elif len(parts) == 3 and parts[0] == "sources" and parts[2] == PACK_SOURCE_META:
@@ -5892,11 +5931,11 @@ class Packs:
         return out
 
     def presets(self) -> list:
-        """Every installed pack's Setup presets, each marked with its pack."""
+        """Every installed pack's Setup presets, each marked with its pack's id and name."""
         with self.store.lock, self._connect() as db:
-            rows = db.execute("SELECT id, presets FROM packs WHERE workspace = ? ORDER BY name COLLATE NOCASE",
+            rows = db.execute("SELECT id, name, presets FROM packs WHERE workspace = ? ORDER BY name COLLATE NOCASE",
                               (workspace_of(self.store),)).fetchall()
-        return [{**p, "pack": r["id"]} for r in rows for p in json.loads(r["presets"])]
+        return [{**p, "pack": r["id"], "packName": r["name"]} for r in rows for p in json.loads(r["presets"])]
 
     # The page's documents are the server's to write here too, through the
     # docs table's own arbitration: a browser holding an older copy is told
@@ -6804,12 +6843,20 @@ def key_var(profile_id: str, slug=None) -> str:
 
 
 def run_items(run):
-    """A run's items: its content's file list, in order and with its repeats,
-    or the one inline text."""
-    content = content_of(run) or {}
-    if content.get("type") in ("text", "prompt"):
-        return [None]
-    return list(content.get("files") or [])
+    """A run's items (evals-core.ts runItemsOf): each Get items step's in turn,
+    a Source's file list in order and with its repeats or one inline text; a
+    run where none adds one goes over the bare item, its prompt alone."""
+    out, bare = [], False
+    for _step, content, _at in item_contents(run):
+        kind = content.get("type")
+        if kind == "source":
+            out.extend(content.get("files") or [])
+        elif kind == "text":
+            if isinstance(content.get("text"), str) and content["text"].strip():
+                out.append(None)
+        elif kind == "prompt":
+            bare = True
+    return out or ([None] if bare else [])
 
 
 def worker_refusal(code, stderr, env):
@@ -7024,18 +7071,23 @@ class Queue:
             row = self._row(db.execute(sql, (rid, ws) if ws else (rid,)).fetchone())
         return row if self._readable(row) else None
 
-    def list(self, limit=RUNS_PAGE, before=None, before_id=None, full=False):
+    def list(self, limit=RUNS_PAGE, before=None, before_id=None, full=False, pipeline=None):
         """Runs, newest first, and whether more follow. `before` and
         `before_id` are the `submittedAt` and id of the last run on the page
         before, so History can page through them the way it pages the runs
         store. `submittedAt` is to the second, so two runs can share one; the
         id breaks the tie, or a page ending between them would skip the
-        second (#238). `before` alone stops at the second. Each row is
+        second (#238). `before` alone stops at the second. `pipeline` keeps
+        the runs of one pipeline, by its document's id: History's Overview
+        (#642) reads a window of one pipeline's runs, which paging every run
+        would find slowly where that pipeline runs rarely. Each row is
         brief_row's unless `full` asks for the whole of it."""
         with self.lock, closing(sqlite3.connect(self.store.path)) as db:
             rows = self._all(db, workspace_of(self.store))
         key = lambda r: (r["submittedAt"], r["id"])
         rows = [r for r in rows if before is None or key(r) < (before, before_id or "")]
+        if pipeline is not None:
+            rows = [r for r in rows if (r["snapshot"] or {}).get("id") == pipeline]
         rows.sort(key=key, reverse=True)
         page = rows[:limit]
         return (page if full else [brief_row(r) for r in page]), len(rows) > limit
@@ -7324,9 +7376,10 @@ class Queue:
         if run["status"] in ("queued", "running"):
             return None, (409, "a run still in progress cannot be re-run")
         snap = run["snapshot"]
-        _, err = self._pinned_files(snap)
-        if err:
-            return None, (409, err)
+        for _step, content, _at in item_contents(snap):
+            _, err = self._pinned_files(content)
+            if err:
+                return None, (409, err)
         # The group bodies the original kept: a re-run grades with exactly
         # them, whatever the groups or the pins read now; its filter-set
         # bodies travel the same way, so it cleans and gates as the original did.
@@ -7517,29 +7570,34 @@ class Queue:
         files_dir = rundir / "files"
         shutil.rmtree(rundir, ignore_errors=True)
         files_dir.mkdir(parents=True)
-        if live is not None:
-            for name in dict.fromkeys(content_of(snap).get("files") or []):
-                text = live.get(name)
-                if text is None:
-                    return None, f"{name!r} was not produced for this run"
-                (files_dir / name).write_text(text, encoding="utf-8")
-        else:
-            files, err = self._pinned_files(snap)
+        # Each Get items step's files under its own folder (item_contents): a
+        # content produced at submit from the bytes kept with the row, any
+        # other from its Source.
+        for _step, content, at in item_contents(snap):
+            folder = files_dir / at if at else files_dir
+            folder.mkdir(exist_ok=True)
+            if live is not None and (content.get("live") is True or isinstance(content.get("pin"), dict)):
+                for name in dict.fromkeys(content.get("files") or []):
+                    text = live.get(at + "/" + name if at else name)
+                    if text is None:
+                        return None, f"{name!r} was not produced for this run"
+                    (folder / name).write_text(text, encoding="utf-8")
+                continue
+            files, err = self._pinned_files(content)
             if err:
                 return None, err
             for name, path in files:
-                shutil.copy2(path, files_dir / name)
+                shutil.copy2(path, folder / name)
         (rundir / "run.json").write_text(json.dumps(snap))
         return rundir, None
 
     @staticmethod
-    def _pinned_files(snap):
-        """Each file a run document reads, by name, and the path holding the
-        bytes it pinned at submit -- or (None, why), naming the Source that
-        has gone, or the files gone from it or changed since. A document
-        with no Source reads no files."""
-        content = content_of(snap) or {}
-        if content.get("type") != "source":
+    def _pinned_files(content):
+        """Each file one Get items content reads, by name, and the path holding
+        the bytes it pinned at submit -- or (None, why), naming the Source
+        that has gone, or the files gone from it or changed since. A content
+        that is not a Source's reads no files."""
+        if not isinstance(content, dict) or content.get("type") != "source":
             return [], None
         ref = content.get("ref") or {}
         label = ref.get("name") or ref.get("id")
@@ -7992,24 +8050,57 @@ def worker_destinations(run: dict):
 # {name, n}`, which a fetch, an edit or a Restore never moves (#596).
 # 19: a job's filters are one list around its Response Format, and filters are
 # seven verbs (#615).
-PIPELINE_VERSION = 19
+PIPELINE_VERSION = 20
 # What a stored run may be: the current version, and the ones evals-core.ts's
 # upgradePipeline reads. A new submission is upgraded to the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19)
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)
 TARGET_CAP = 4
 # Target steps whose words the Prompt library does not record as a use: they
 # ask no model (evals-core.ts's STEP_TYPES.echo).
 UNRECORDED_STEPS = {"echo"}
-RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "evals", "pass", "profiles", "comment", "plugins")
+# The version-20 document is a flat step list and Variants (pipeline-model
+# §20): `steps` and `variants` in the place version 19 held `jobs` and
+# `targets`.
+RUN_FIELDS = ("version", "id", "name", "steps", "variants", "evals", "pass", "profiles", "comment", "plugins")
+
+
+def request_steps(doc):
+    """A version-20 document's Request steps, in order (evals-core.ts): one per
+    job, holding the settings every Variant shares."""
+    steps = doc.get("steps") if isinstance(doc, dict) else None
+    return [s for s in (steps if isinstance(steps, list) else [])
+            if isinstance(s, dict) and s.get("type") == "request"]
+
+
+def merge_cell(base, ov):
+    """A Variant's cell over a Request's base (evals-core.ts mergeCell): the
+    base where there is no override, the override whole where it carries its
+    own `type` (a different Answered by, or a different shape), else the base
+    with the override's fields."""
+    base = base if isinstance(base, dict) else {}
+    if not isinstance(ov, dict) or not ov:
+        return dict(base)
+    if "type" in ov:
+        return dict(ov)
+    merged = dict(base)
+    merged.update(ov)
+    return merged
 
 
 def content_of(doc):
-    """A pipeline's content, where its version keeps it: job 1's Attach
-    Content step from version 8 (evals-core.ts contentOf), the document's own
-    `content` before. The dict itself, so a caller may stamp it in place.
-    None for none."""
+    """A pipeline's content, where its version keeps it: version 20's first Get
+    items step, version 8's job-1 Attach Content step (evals-core.ts
+    contentOf), the document's own `content` before. The dict itself, so a
+    caller may stamp it in place. None for none."""
     if not isinstance(doc, dict):
         return None
+    steps = doc.get("steps")
+    if isinstance(steps, list) and any(isinstance(s, dict) and s.get("type") in ("getItems", "request") for s in steps):
+        for step in steps:
+            if isinstance(step, dict) and step.get("type") == "getItems":
+                c = step.get("content")
+                return c if isinstance(c, dict) else None
+        return None
     jobs = doc.get("jobs") if isinstance(doc.get("jobs"), list) else doc.get("chains")
     first = jobs[0] if isinstance(jobs, list) and jobs and isinstance(jobs[0], dict) else None
     if first is not None and isinstance(first.get("steps"), list):
@@ -8020,6 +8111,25 @@ def content_of(doc):
         return None
     c = doc.get("content")
     return c if isinstance(c, dict) else None
+def item_contents(doc):
+    """Every Get items step's content, in list order, as (step id, content,
+    folder) -- evals-core.ts itemContents: the first's files at the top of a
+    run's files, each later one's under `@N`, N its place among them, a name
+    no item can have (NAME_RESTRICT). A document before version 20 has its
+    one content."""
+    if not isinstance(doc, dict):
+        return []
+    steps = doc.get("steps")
+    if isinstance(steps, list) and any(isinstance(s, dict) and s.get("type") in ("getItems", "request") for s in steps):
+        out = []
+        for k, step in enumerate(s for s in steps if isinstance(s, dict) and s.get("type") == "getItems"):
+            if isinstance(step.get("content"), dict):
+                out.append((step.get("id") or "", step["content"], f"@{k + 1}" if k else ""))
+        return out
+    c = content_of(doc)
+    return [("", c, "")] if c is not None else []
+
+
 # The connection-type registry, mirrored from evals-core.ts (#46) because the
 # server is stdlib-only and cannot load it: which types ship, and the settings
 # each one's options may carry. The server only needs the field allow-list and
@@ -8298,12 +8408,23 @@ def upgrade_run(run):
 
 
 def cells_of(run):
-    """Every target's step in every job of a run, as (i, target, k, step):
-    version 10's targets and their steps, or an earlier version's scenarios
-    and their cells -- what the Prompt library records a use from, whichever
-    version a stored run is."""
+    """Every Variant's cell in every Request of a run, as (i, variant, k, cell):
+    version 20's Variants over the Request steps (evals-core.ts targetsOf), an
+    earlier version's targets/scenarios and their steps/cells -- what the
+    Prompt library records a use from, whichever version a stored run is. The
+    cell is the Request's base with the Variant's override over it."""
     if not isinstance(run, dict):
         return
+    reqs = request_steps(run)
+    if reqs:
+        variants = run.get("variants")
+        for i, v in enumerate(variants if isinstance(variants, list) else []):
+            if not isinstance(v, dict):
+                continue
+            overrides = v.get("overrides") if isinstance(v.get("overrides"), dict) else {}
+            for k, req in enumerate(reqs):
+                yield i, v, k, merge_cell(req.get("cell"), overrides.get(req.get("id")))
+        return
     targets = run.get("targets") if isinstance(run.get("targets"), list) else run.get("scenarios")
     for i, t in enumerate(targets if isinstance(targets, list) else []):
         if not isinstance(t, dict):
@@ -8327,22 +8448,46 @@ def run_problems(run):
     _fields(run, "the run", RUN_FIELDS, bad)
     if not isinstance(run.get("id"), str) or not run["id"].strip():
         bad.append("the run has no id")
-    jobs = run.get("jobs")
-    if not isinstance(jobs, list) or not jobs:
-        return bad + ["jobs has to be a list of at least one job"]
-    jids = [ch.get("id") for ch in jobs if isinstance(ch, dict)]
-    for k, ch in enumerate(jobs):
-        at = f"job {k + 1}"
-        if not isinstance(ch, dict):
-            bad.append(f"{at} has to be an object")
+    # The version-20 step list: Get items before Request 1, each Read Response
+    # and Filter after the Request it reads (evals-core.ts stepsProblems). The
+    # server checks the shape and order; tokens and kinds are the runner's.
+    steps = run.get("steps")
+    if not isinstance(steps, list):
+        return bad + ["steps has to be a list"]
+    req_ids = [s.get("id") for s in steps if isinstance(s, dict) and s.get("type") == "request"]
+    seen_ids = set()
+    saw_request = False
+    for pos, s in enumerate(steps):
+        if not isinstance(s, dict) or s.get("type") not in ("getItems", "request", "readResponse", "filter"):
+            bad.append(f"\"{s.get('type') if isinstance(s, dict) else s}\" is not a step this lab has")
             continue
-        # An id is a job's own, like a scenario's: it is how a stored run
-        # keeps pointing at the job that actually ran, whatever it is
-        # called now (#93).
-        if not isinstance(ch.get("id"), str) or not ch["id"].strip():
-            bad.append(f"{at} has no id")
-        elif jids.count(ch["id"]) > 1:
-            bad.append(f"{at} has the id of another job")
+        sid = s.get("id")
+        if not isinstance(sid, str) or not sid.strip():
+            bad.append(f"a {s['type']} step has no id")
+        elif sid in seen_ids:
+            bad.append(f"two steps have the id {sid}")
+        else:
+            seen_ids.add(sid)
+        if s["type"] == "getItems":
+            if saw_request:
+                bad.append("Get items comes before Request 1")
+        elif s["type"] == "request":
+            saw_request = True
+        else:
+            # A Read Response or Filter reads the response of a Request, and
+            # comes after it: the step it must come before names the Request.
+            label = "Read Response" if s["type"] == "readResponse" else "Filter"
+            rid = s.get("request")
+            if not isinstance(rid, str) or rid not in req_ids:
+                bad.append(f"a {label} reads a Request the experiment does not have")
+            else:
+                n = req_ids.index(rid) + 1
+                req_pos = next((j for j, o in enumerate(steps)
+                                if isinstance(o, dict) and o.get("type") == "request" and o.get("id") == rid), -1)
+                if pos < req_pos:
+                    bad.append(f"{label} {n} comes after Request {n}")
+    if not req_ids:
+        return bad + ["add a Request first"]
     table = run.get("profiles")
     if not isinstance(table, dict):
         return bad + ["profiles has to be an object of id → connection"]
@@ -8367,76 +8512,81 @@ def run_problems(run):
             bad.append(f"profiles {spelt[var]} and {cid} both take their key from ${var}")
         spelt[var] = cid
         connection_problems(pid, conn, at, bad)
-    targets = run.get("targets")
-    if not isinstance(targets, list) or not 1 <= len(targets) <= TARGET_CAP:
-        return bad + [f"a run needs between one and {TARGET_CAP} targets"]
-    ids = [t.get("id") for t in targets if isinstance(t, dict)]
-    for i, t in enumerate(targets):
-        at = f"target {i + 1}"
-        if not isinstance(t, dict):
+    variants = run.get("variants")
+    if not isinstance(variants, list) or not 1 <= len(variants) <= TARGET_CAP:
+        return bad + [f"a run needs between one and {TARGET_CAP} variants"]
+    vids = [v.get("id") for v in variants if isinstance(v, dict)]
+    for i, v in enumerate(variants):
+        at = f"variant {i + 1}"
+        if not isinstance(v, dict):
             bad.append(f"{at} has to be an object")
             continue
         # The id the Prompt library records a use under (version 4).
-        if not isinstance(t.get("id"), str) or not t["id"].strip():
+        if not isinstance(v.get("id"), str) or not v["id"].strip():
             bad.append(f"{at} has no id")
-        elif ids.count(t["id"]) > 1:
-            bad.append(f"{at} has the id of another target")
-        steps = t.get("steps")
-        if not isinstance(steps, list) or len(steps) != len(jobs):
-            bad.append(f"{at} has to have one step per job")
+        elif vids.count(v["id"]) > 1:
+            bad.append(f"{at} has the id of another variant")
+        overrides = v.get("overrides")
+        if overrides is not None and (not isinstance(overrides, dict) or any(k not in req_ids for k in overrides)):
+            bad.append(f"{at}: overrides are keyed by Request ids")
+    # Each Variant's cell in each Request -- the Request's base with the
+    # Variant's override over it (cells_of) -- what it sends and the
+    # connection the run carries for it. A Variant with no connection is one
+    # whose cells ask none (Echo) or each name their own; which needs one is
+    # the core's to judge. Whether the words may be blank (Echo answering from
+    # the item) is the core's too, and the worker refuses the run in its words.
+    seen_ref = set()
+    for i, v, k, cell in cells_of(run):
+        at = f"variant {i + 1}, job {k + 1}"
+        if not isinstance(cell, dict) or not isinstance(cell.get("type"), str):
+            bad.append(f"{at} has to name what it sends")
             continue
-        # A target with no connection is one whose steps ask none (Echo) or
-        # each name their own; which steps need one is the core's to judge.
-        # Each pair named -- the target's, a step's own -- is one the run
-        # carries (evals-core.ts targetAnswerOf).
-        def pair(o, base=None):
-            c = o.get("connection") if o.get("connection") is not None else (base or {}).get("connection")
-            p = o.get("profile")
-            return c, p
-        refs = [pair(t)] if t.get("connection") is not None or t.get("profile") is not None else []
-        refs += [pair(st, t) for st in steps
-                 if isinstance(st, dict) and (st.get("connection") is not None or st.get("profile") is not None)]
-        # Whether the words may be blank -- Echo answering from the item --
-        # is the core's to judge, and the worker refuses the run in its words.
-        for k, step in enumerate(steps):
-            if not isinstance(step, dict) or not isinstance(step.get("type"), str):
-                bad.append(f"{at}, job {k + 1} has to name what it sends")
-            elif not isinstance(step.get("prompt"), str):
-                bad.append(f"{at}, job {k + 1} has no prompt")
-            elif step.get("from") is not None and not (
-                    isinstance(step["from"], dict) and isinstance(step["from"].get("id"), str)):
-                bad.append(f"{at}, job {k + 1} names the prompt it was picked from without an id")
-        for c, p in refs:
-            if (not isinstance(c, dict) or not isinstance(c.get("id"), str)
-                    or (p is not None and not (isinstance(p, dict) and isinstance(p.get("id"), str)))
-                    or run_answer(table, answer_key(c["id"], p["id"] if p else None)) is None):
-                bad.append(f"{at} names a connection the run does not carry")
-    content = content_of(run)
-    kind = content.get("type") if isinstance(content, dict) else None
-    if kind == "source":
-        ref = content.get("ref")
-        files = content.get("files")
-        if not isinstance(ref, dict) or not isinstance(ref.get("id"), str) or not ref["id"]:
-            bad.append("a Source content has to name its Source")
-        # A live dynamic run (docs/dynamic-sources.md) produces its items at
-        # submit, so its file list is the server's to fill -- empty here is
-        # fine. Every other source run names the files it froze.
-        elif content.get("live") is True:
-            if not isinstance(files, list) or not all(isinstance(f, str) for f in files):
-                bad.append("a run over a Source needs its file list")
-        elif not isinstance(files, list) or not files or not all(isinstance(f, str) for f in files):
-            bad.append("a run over a Source needs its file list")
-        pin = content.get("pin")
-        if pin is not None and not (isinstance(pin, dict) and isinstance(pin.get("name"), str) and pin["name"]
-                                    and type(pin.get("n")) is int and pin["n"] >= 0):
-            bad.append("a Source content's pin names a response as { name, n }")
-    elif kind == "text":
-        if not isinstance(content.get("text"), str) or not content["text"].strip():
-            bad.append("a run over text needs some text")
-    elif kind == "prompt":
-        pass
-    else:
-        bad.append("a run needs a Source's files, some inline text, or Prompt only")
+        if not isinstance(cell.get("prompt"), str):
+            bad.append(f"{at} has no prompt")
+        elif cell.get("from") is not None and not (
+                isinstance(cell["from"], dict) and isinstance(cell["from"].get("id"), str)):
+            bad.append(f"{at} names the prompt it was picked from without an id")
+        c = cell.get("connection") if cell.get("connection") is not None else v.get("connection")
+        p = cell.get("profile") if cell.get("profile") is not None else v.get("profile")
+        if c is None and p is None:
+            continue
+        if (not isinstance(c, dict) or not isinstance(c.get("id"), str)
+                or (p is not None and not (isinstance(p, dict) and isinstance(p.get("id"), str)))
+                or run_answer(table, answer_key(c["id"], p["id"] if p else None)) is None):
+            if i not in seen_ref:
+                seen_ref.add(i)
+                bad.append(f"variant {i + 1} names a connection the run does not carry")
+    # Each Get items step's content; a later one is named by its place, and
+    # may add no files (a run of one file of the first, say).
+    contents = item_contents(run) or [("", None, "")]
+    for k, (_step, content, _at) in enumerate(contents):
+        said = []
+        kind = content.get("type") if isinstance(content, dict) else None
+        if kind == "source":
+            ref = content.get("ref")
+            files = content.get("files")
+            if not isinstance(ref, dict) or not isinstance(ref.get("id"), str) or not ref["id"]:
+                said.append("a Source content has to name its Source")
+            # A live dynamic run (docs/dynamic-sources.md) produces its items at
+            # submit, so its file list is the server's to fill -- empty here is
+            # fine. Every other source run names the files it froze.
+            elif content.get("live") is True:
+                if not isinstance(files, list) or not all(isinstance(f, str) for f in files):
+                    said.append("a run over a Source needs its file list")
+            elif not isinstance(files, list) or not (files or k) or not all(isinstance(f, str) for f in files):
+                said.append("a run over a Source needs its file list")
+            pin = content.get("pin")
+            if pin is not None and not (isinstance(pin, dict) and isinstance(pin.get("name"), str) and pin["name"]
+                                        and type(pin.get("n")) is int and pin["n"] >= 0):
+                said.append("a Source content's pin names a response as { name, n }")
+        elif kind == "text":
+            if not isinstance(content.get("text"), str) or not content["text"].strip():
+                said.append("a run over text needs some text")
+        elif kind == "prompt":
+            pass
+        else:
+            said.append("a run needs a Source's files, some inline text, or (No content)")
+        bad.extend(f"Get items {k + 1}: {x}" if k else x for x in said)
     evals = run.get("evals")
     if not isinstance(evals, list) or not all(isinstance(t, dict) and isinstance(t.get("type"), str) for t in evals):
         bad.append("evals has to be a list of evals, each naming its type")
@@ -8612,7 +8762,8 @@ class Handler(BaseHTTPRequestHandler):
 
     # The eval groups' routes (docs/pipeline-model.md §17) are the datasets'
     # under their new name: /api/datasets goes on answering the same rows, so
-    # dataset-diff.js and a script written against it keep working. A list
+    # the app's comparison (docs/app-data.md) and a script written against it
+    # keep working: another repository's contract, so keep its shape. A list
     # asked for by the new name is `groups`.
     GROUPS_ROUTE = "/api/eval-groups"
     grouped = False
@@ -8706,8 +8857,14 @@ class Handler(BaseHTTPRequestHandler):
                 return self._json(400, {"error": "limit has to be a number"})
             before = (query.get("before") or [None])[0]
             before_id = (query.get("beforeId") or [None])[0]
-            runs, more = QUEUE.list(limit, before, before_id, full=(query.get("full") or [""])[0] == "1")
-            return self._json(200, {"runs": runs, "more": more})
+            pipeline = (query.get("pipeline") or [None])[0]
+            runs, more = QUEUE.list(limit, before, before_id, full=(query.get("full") or [""])[0] == "1",
+                                    pipeline=pipeline)
+            # Which version of a Library prompt each target step ran: the
+            # prompt library records it, not the run's document (§14), and
+            # Overview breaks a target's line where it changes.
+            used = PROMPTS.used_by([r["id"] for r in runs]) if PROMPTS else {}
+            return self._json(200, {"runs": [{**r, "prompts": used.get(r["id"], [])} for r in runs], "more": more})
         if path.startswith("/api/queue/") and path.endswith("/groups") and path.count("/") == 4:
             # The bodies of the eval groups a run grades with, by `<id>@<n>`
             # (§17); null for a run that kept none, as /dataset answers.
@@ -8748,10 +8905,8 @@ class Handler(BaseHTTPRequestHandler):
             run["behind"] = QUEUE._behind(run["id"]) or 0
             return self._json(200, run)
         if path == "/api/presets":
-            # The presets file beside server.py, if the image has one, and
-            # every installed pack's.
-            own = json.loads(PRESETS.read_text("utf-8")) if PRESETS is not None else []
-            return self._json(200, own + (PACKS.presets() if PACKS is not None else []))
+            # Every installed pack's: the lab ships none of its own (#652).
+            return self._json(200, PACKS.presets() if PACKS is not None else [])
         if path == "/api/plugins":
             if PLUGINS is None:
                 return self._send(404, b"not found", "text/plain")
@@ -9446,28 +9601,18 @@ class Handler(BaseHTTPRequestHandler):
 
     # ---- The run queue (#530) ----------------------------------------------
 
-    def _queue_submit(self):
-        """A run document, checked and queued. The page resolves it at submit
-        (evals-core.ts resolvePipeline); anything else posting here is held to
-        the same fields."""
-        if QUEUE is None:
-            return self._send(404, b"not found", "text/plain")
-        run = self._payload()
-        if run is None:
-            return self._json(400, {"error": "bad body"})
-        run, why = upgrade_run(run)
-        if why:
-            return self._json(400, {"error": "the run cannot be queued: " + why})
-        bad = run_problems(run)
-        if bad:
-            return self._json(400, {"error": "the run cannot be queued: " + "; ".join(bad)})
-        content = content_of(run)
+    def _stamp_content(self, content):
+        """One Get items step's content made ready to queue, in place: a
+        Source's files pinned at their revision, or a dynamic Source's items
+        produced or its pinned response read. Returns (produced, None) -- the
+        bytes kept with the run by item name, or None for none -- or (None,
+        (status, error))."""
         produced = None
         if content["type"] == "source":
             sid = content["ref"]["id"]
             src = SOURCES.get(sid)
             if src is None:
-                return self._json(404, {"error": "no such Source"})
+                return None, (404, "no such Source")
             # A dynamic Source (docs/dynamic-sources.md) grades either its stored
             # Calls (snapshot) or content produced fresh at submit (live). The
             # choice is the run's (content.live) over the Source's default
@@ -9477,20 +9622,19 @@ class Handler(BaseHTTPRequestHandler):
             if src.get("type") == "dynamic":
                 entry = source_entry("dynamic", src.get("config") or {})
                 if entry is None:
-                    return self._json(403, {"error": "the lab does not offer that Source's provider"})
+                    return None, (403, "the lab does not offer that Source's provider")
                 cfg = src.get("config") or {}
                 if entry.get("produce") is None:
                     # A Supplied provider hands items in (the caller's --items,
                     # or an Upload…); the server produces nothing, so there is no
                     # live for it -- a hosted run grades the items it holds. A
-                    # Source with none is refused above by run_problems, naming
+                    # Source with none is refused by run_problems, naming
                     # the Source with no files; a run that asks it to go live is
                     # refused here, pointing at the CLI (docs/ci.md).
                     if content.get("live"):
                         noun = entry.get("noun", "item")
-                        return self._json(400, {"error":
-                            f"this Source produces nothing: it grades the {noun}s handed in "
-                            f"-- add one, or run it from the CLI with evals-lab run --items"})
+                        return None, (400, f"this Source produces nothing: it grades the {noun}s handed in "
+                                            f"-- add one, or run it from the CLI with evals-lab run --items")
                     content["live"] = False
                 else:
                     live = content.get("live")
@@ -9503,14 +9647,14 @@ class Handler(BaseHTTPRequestHandler):
                         # written by hand is graded as saved.
                         produced, err = self._produce_endpoints(sid, entry, cfg, content["files"])
                         if err:
-                            return self._json(err[0], {"error": err[1]})
+                            return None, err
                         content["live"] = True
                         content["revs"] = {name: "live:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
                                            for name, text in produced.items()}
                     elif live:
                         produced, name, err = self._produce_live(sid, entry, cfg)
                         if err:
-                            return self._json(err[0], {"error": err[1]})
+                            return None, err
                         content["live"] = True
                         # The one capture, looped as a snapshot's files are
                         # (evals-core.ts expand): Run first cannot cut below
@@ -9528,8 +9672,8 @@ class Handler(BaseHTTPRequestHandler):
                     # whatever a fetch, an edit or a Restore made current since.
                     body = SOURCES.pinned_body(sid, pin["name"], pin["n"])
                     if body is None:
-                        return self._json(400, {"error": f"the response this run grades, {pin['name']}, "
-                                                         "was removed: pick another"})
+                        return None, (400, f"the response this run grades, {pin['name']}, "
+                                                "was removed: pick another")
                     text = body.decode("utf-8", errors="replace")
                     produced = {pin["name"]: text}
                     loops = content.get("loops")
@@ -9543,8 +9687,34 @@ class Handler(BaseHTTPRequestHandler):
                 for name in dict.fromkeys(content["files"]):
                     revs[name] = SOURCES.rev(sid, name)
                     if revs[name] is None:
-                        return self._json(400, {"error": f"{name!r} is not in that Source"})
+                        return None, (400, f"{name!r} is not in that Source")
                 content["revs"] = revs
+        return produced, None
+
+    def _queue_submit(self):
+        """A run document, checked and queued. The page resolves it at submit
+        (evals-core.ts resolvePipeline); anything else posting here is held to
+        the same fields."""
+        if QUEUE is None:
+            return self._send(404, b"not found", "text/plain")
+        run = self._payload()
+        if run is None:
+            return self._json(400, {"error": "bad body"})
+        run, why = upgrade_run(run)
+        if why:
+            return self._json(400, {"error": "the run cannot be queued: " + why})
+        bad = run_problems(run)
+        if bad:
+            return self._json(400, {"error": "the run cannot be queued: " + "; ".join(bad)})
+        produced = None
+        for _step, content, at in item_contents(run):
+            got, err = self._stamp_content(content)
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            if got is not None:
+                # A later Get items' items are kept under its own folder
+                # (evals-core.ts itemContents), so two Sources' names never meet.
+                produced = {**(produced or {}), **{(at + "/" + n if at else n): t for n, t in got.items()}}
         # A connection not shared with this workspace blocks the run with the
         # sentence the Run bar shows and the connections it names, so its Share
         # link can grant them at once (docs/workspaces.md).