evals-lab 0.2.0 → 0.4.0

package/lab/server.py

@@ -24,6 +24,7 @@ given, so both follow their user between browsers; see Store and Sources.
 
 import base64
 import calendar
+import gzip
 import hashlib
 import hmac
 import io
@@ -247,6 +248,35 @@ TYPES = {
     ".png": "image/png",
 }
 
+# What is gzipped for a client that asks (#225): text, which compresses five
+# to ten times, and nothing already compressed. A body under GZIP_MIN goes as
+# it is, since the header and the gzip frame would outweigh the saving.
+COMPRESSIBLE = ("text/", "application/json", "image/svg+xml")
+GZIP_MIN = 1024
+# The built page's bundles, gzipped once: a new build is new names.
+GZIPPED: dict = {}
+
+
+def accepts_gzip(header: str) -> bool:
+    """Whether an Accept-Encoding header takes gzip: named, or `*`, with a q
+    above 0. `gzip;q=0` is a refusal, not a request."""
+    star = False
+    for part in (header or "").split(","):
+        name, _, params = part.partition(";")
+        name, q = name.strip().lower(), 1.0
+        for p in params.split(";"):
+            k, _, v = p.partition("=")
+            if k.strip().lower() == "q":
+                try:
+                    q = float(v)
+                except ValueError:
+                    q = 0.0
+        if name == "gzip":
+            return q > 0
+        if name == "*":
+            star = q > 0
+    return star
+
 
 # What a Source may accept and ever be served back as -- deliberately not
 # TYPES: an upload that could come back as text/html is stored XSS, and the
@@ -469,9 +499,10 @@ SIGN_INS = {"microsoft": lambda: microsoft_config() is not None}
 # A key taken off this list is no longer served or written, and its rows stay
 # in the store: a document is not migrated or deleted because nothing reads it.
 # promptlab.cases, .rules and their .base copies went that way when a dataset
-# became a row of its own (#86).
+# became a row of its own (#86), and promptlab.mappings once a dataset named
+# the Source it grades (#199).
 SYNCED = ("promptlab.workflows", "promptlab.profiles",
-          "promptlab.versions", "promptlab.tokens", "promptlab.mappings")
+          "promptlab.versions", "promptlab.tokens")
 MAX_DOC = 8 * 1024 * 1024
 RUNS_PAGE = 25
 # A dataset request's caps, read from Content-Length before the body is, as a
@@ -1306,7 +1337,7 @@ class Sources:
 
 # ---- Datasets ----------------------------------------------------------------
 #
-# A dataset is data a graded test names by id: its cases. (The prompt a new
+# A dataset is data a graded eval names by id: its cases. (The prompt a new
 # scenario starts from is the Prompt library's Default, below; a dataset from
 # before the library held one, and gave it to the library once.) It is one row in the
 # store's SQLite, its body one JSON document with a version that goes up by one
@@ -1325,44 +1356,158 @@ class Sources:
 # A lab from before this held a one-time import's `meta` row saying it ran;
 # it is left where it is, and nothing reads it.
 
-DATASET_FIELDS = ("cases",)
+DATASET_FIELDS = ("version", "source", "scoring", "grader", "every", "run", "cases")
+# A body's own version: evals-core.ts's DATASET_BODY_VERSION. Version 7 is an
+# eval group (docs/pipeline-model.md §17), version 6 its cases alone; version
+# 5 was told by its `source` alone, and earlier ones by neither.
+DATASET_BODY_VERSION = 7
 DATASET_NAME_MAX = 80
-# The file forms Export writes and Import reads. Export writes version 4;
-# Import reads it and versions 1 to 3, upgraded, and refuses anything else,
+# The file forms Export writes and Import reads. Export writes version 7;
+# Import reads it and versions 1 to 6, upgraded, and refuses anything else,
 # as a pipeline of another version is refused. Versions 1 to 3 carried a
 # prompt, which an import gives to the Prompt library.
 EXPORT_ONE = "evals-lab/dataset"
 EXPORT_ALL = "evals-lab/datasets"
-EXPORT_VERSION = 4
-IMPORT_VERSIONS = (1, 2, 3, 4)
+EXPORT_VERSION = 7
+IMPORT_VERSIONS = (1, 2, 3, 4, 5, 6, 7)
+SCORING_MODES = ("all", "weighted")
 
 
 def blank_dataset() -> dict:
-    return {"cases": []}
+    return group_of_v6({"source": None, "cases": []})
+
+
+# vocab: the names older versions gave a case's fields
+CASE_RENAMED = {"minTags": "minCount", "maxTags": "maxCount", "textInImage": "watch", "photo": "filename"}  # vocab: as above
+CASE_V4 = ("filename", "why", "traits", "expect", "anyOf", "forbid", "allow", "minCount", "maxCount", "watch", "discarded")
+
+
+def _words(s) -> list:
+    """evals-core.ts's words: the letters and digits of [s], lowercased."""
+    return re.findall(r"[^\W_]+", str(s or "").lower())
+
+
+def _term_in(items, term) -> bool:
+    """evals-core.ts's termIn: [term]'s words in some item, in order and adjacent."""
+    t = _words(term)
+    if not t:
+        return False
+    for item in items:
+        w = _words(item)
+        if any(w[i:i + len(t)] == t for i in range(len(w) - len(t) + 1)):
+            return True
+    return False
+
+
+def case_metrics(c: dict) -> list:
+    """A version-4 case's expectations as the metrics that say the same:
+    evals-core.ts's caseMetrics, in Python, and held to it by proxy-check.py
+    through fixtures/dataset-v7.json."""
+    def strs(v):
+        return [x for x in v if isinstance(x, str)] if isinstance(v, list) else []
+    if c.get("discarded") is True:
+        return [{"type": "discarded"}]
+    out = []
+    expect, allow = strs(c.get("expect")), strs(c.get("allow"))
+    if expect:
+        out.append({"type": "contains-all", "values": "\n".join(expect)})
+    for g in c.get("anyOf") if isinstance(c.get("anyOf"), list) else []:
+        if strs(g):
+            out.append({"type": "contains-any", "values": "\n".join(strs(g))})
+    for t in strs(c.get("forbid")):
+        # An exception excuses only the forbidden term inside it.
+        except_ = [a for a in allow if _term_in([a], t)]
+        out.append({"type": "contains", "value": t, "not": True, **({"except": "\n".join(except_)} if except_ else {})})
+    whole = lambda v: v if type(v) is int else None
+    lo, hi = whole(c.get("minCount")), whole(c.get("maxCount"))
+    if lo is not None or hi is not None:
+        out.append({"type": "item-count", "min": lo, "max": hi})
+    for t in strs(c.get("watch")):
+        out.append({"type": "contains-any", "values": t, "weight": 0})
+    return out
+
+
+def case_of_v4(c):
+    """One case of any earlier version as a version-5 one: evals-core.ts's
+    caseOfV4, in Python."""
+    if not isinstance(c, dict):
+        return c
+    was = {}
+    for k, v in c.items():
+        key = CASE_RENAMED.get(k, k)
+        if key not in was or key == k:
+            was[key] = v
+    if isinstance(was.get("item"), str):
+        was.pop("filename", None)
+    out = {}
+    if "id" in was:
+        out["id"] = was["id"]
+    out["item"] = was["item"] if isinstance(was.get("item"), str) else was["filename"] if isinstance(was.get("filename"), str) else ""
+    out["todo"] = was.get("todo") is True
+    out["note"] = was["note"] if isinstance(was.get("note"), str) else was["why"] if isinstance(was.get("why"), str) else ""
+    out["metrics"] = case_metrics(was) + (was["metrics"] if isinstance(was.get("metrics"), list) else [])
+    for k, v in was.items():
+        if k not in out and k not in CASE_V4:
+            out[k] = v
+    return out
+
+
+# The metrics whose Ignore case version 6 made mean what it says for a reply
+# read as a list: evals-core.ts's CASE_FOLDING.
+CASE_FOLDING = ("contains", "contains-all", "contains-any")
+
+
+def case_of_v5(c):
+    """A version-5 case as a version-6 one: evals-core.ts's caseOfV5, in
+    Python. Each Contains metric says Ignore case, as version 5 matched a
+    list's items whatever it said."""
+    if not isinstance(c, dict) or not isinstance(c.get("metrics"), list):
+        return c
+    return {**c, "metrics": [{**m, "ignoreCase": True}
+                             if isinstance(m, dict) and m.get("type") in CASE_FOLDING and m.get("ignoreCase") is not True
+                             else m for m in c["metrics"]]}
+
+
+def group_of_v6(body: dict) -> dict:
+    """A version-6 body as a version-7 eval group: evals-core.ts's
+    groupOfV6. Scored All, the lab's grader, and no metrics of its own for
+    every item or the whole run -- what a Metrics eval naming the dataset with
+    none of its own graded."""
+    rest = {k: v for k, v in body.items() if k != "version"}
+    return {"version": DATASET_BODY_VERSION, "source": None, "scoring": {"mode": "all", "threshold": None},
+            "grader": None, "every": [], "run": [], **rest}
 
 
 def upgrade_body(body):
-    """An earlier body as today's: evals-core.ts's upgradeDatasetBody, in
-    Python. Version 1's `imageCases` are `cases`, each case's
-    `minTags`/`maxTags` its `minCount`/`maxCount`, and its `replays` and
-    `conformance` go (fixtures/replays.json holds the parser's tests).
-    Version 2's `rules` go -- they clean a job's answer, so they are the
-    job's -- and the terms a case watches for are its `watch`. Version 3's
-    `prompt` goes: the Prompt library holds prompts now. A caller that needs
-    the rules or the prompt takes them first (`body_rules`, `body_prompt`).
-    Anything else comes back as it was."""
-    if not isinstance(body, dict):
+    """An earlier body as today's (version 7): evals-core.ts's
+    upgradeDatasetBody, in Python. Version 1's `imageCases` are `cases`, and
+    its `replays` and `conformance` go (fixtures/replays.json holds the
+    parser's tests). Version 2's `rules` go -- they clean a job's answer, so
+    they are the job's. Version 3's `prompt` goes: the Prompt library holds
+    prompts now. Version 4's case named its item `filename` and said what a
+    good answer is in expectations; each becomes its metric, `why` the
+    `note`, `traits` go, and the body names no Source yet. A caller that
+    needs the rules or the prompt takes them first (`body_rules`,
+    `body_prompt`). A body naming its Source is version 5, whose Contains
+    metrics each come to say Ignore case (`case_of_v5`). Version 6 gains a
+    group's scoring, grader, Every item and Whole run (`group_of_v6`). A body
+    saying it is version 7 comes back as it was; so does anything that is
+    not a body."""
+    if not isinstance(body, dict) or body.get("version") == DATASET_BODY_VERSION:
         return body
-    if "imageCases" not in body and "rules" not in body:
-        if "prompt" not in body:
-            return body
-        return {k: v for k, v in body.items() if k != "prompt"}
-    renamed = {"minTags": "minCount", "maxTags": "maxCount", "textInImage": "watch"}  # vocab: older names
+    # A body saying any other version is one this lab does not read, and is
+    # left for dataset_problem to refuse.
+    if "version" in body:
+        return group_of_v6(body) if body["version"] == 6 else body
+    if "source" in body:
+        up = dict(body)
+        if isinstance(body.get("cases"), list):
+            up["cases"] = [case_of_v5(c) for c in body["cases"]]
+        return group_of_v6(up)
     cases = body.get("cases") if isinstance(body.get("cases"), list) else body.get("imageCases")
     if not isinstance(cases, list):
         return body
-    cases = [{renamed.get(k, k): v for k, v in c.items()} if isinstance(c, dict) else c for c in cases]
-    return {"cases": canonical_cases(cases)}
+    return group_of_v6({"source": None, "cases": [case_of_v5(case_of_v4(c)) for c in cases]})
 
 
 def body_prompt(body):
@@ -1377,21 +1522,6 @@ def body_rules(body):
     return rules if isinstance(rules, dict) and isinstance(rules.get("rules"), list) else None
 
 
-def canonical_cases(cases: list) -> list:
-    """Every case naming its file as `filename`: evals-core.ts's
-    canonicalCases, in Python. `old` is the key a set re-synced from an
-    app's own repository arrives with; the key keeps its place, so only its
-    spelling changes."""
-    old = "photo"  # vocab: the older spelling of filename
-    out = []
-    for c in cases:
-        if isinstance(c, dict) and old in c:
-            c = {("filename" if k == old else k): v for k, v in c.items()
-                 if not (k == old and "filename" in c)}
-        out.append(c)
-    return out
-
-
 def dataset_problem(body) -> str:
     """Why [body] is not a dataset's body, in one sentence, or ""."""
     if not isinstance(body, dict):
@@ -1402,8 +1532,29 @@ def dataset_problem(body) -> str:
     for k in DATASET_FIELDS:
         if k not in body:
             return f"a dataset's body has no \"{k}\""
+    if body["version"] != DATASET_BODY_VERSION:
+        return f"a dataset's body is version {DATASET_BODY_VERSION}"
     if not isinstance(body["cases"], list) or not all(isinstance(c, dict) for c in body["cases"]):
         return "cases has to be a list of cases"
+    def is_ref(v):
+        return isinstance(v, dict) and isinstance(v.get("id"), str) and isinstance(v.get("name"), str)
+    if body["source"] is not None and not is_ref(body["source"]):
+        return "a dataset names its Source as { id, name }, or null"
+    # A group's own sections: their metrics are evals-core.ts's validateEvals'
+    # to judge, as a case's are; the shape is the server's.
+    sc = body["scoring"]
+    if not isinstance(sc, dict) or sc.get("mode") not in SCORING_MODES:
+        return "a group is scored all or weighted"
+    number = lambda v: type(v) in (int, float)
+    if sc["mode"] == "weighted" and not number(sc.get("threshold")):
+        return "a group scored in points needs Pass at: the points an item has to reach"
+    if sc.get("threshold") is not None and not number(sc.get("threshold")):
+        return "a group's Pass at has to be a number"
+    if body["grader"] is not None and not is_ref(body["grader"]):
+        return "a group names its grader as { id, name }, or null"
+    for k, label in (("every", "Every item"), ("run", "Whole run")):
+        if not isinstance(body[k], list) or not all(isinstance(m, dict) for m in body[k]):
+            return f"{label} has to be a list of metrics"
     return ""
 
 
@@ -1788,6 +1939,11 @@ class Datasets:
             # so nothing is lost if a pipeline was missed.
             db.execute("CREATE TABLE IF NOT EXISTS dataset_rules_archive ("
                        "dataset_id TEXT NOT NULL, rules TEXT NOT NULL, archived_at TEXT NOT NULL)")
+            # Each row's body as it was before the conversion below rewrote
+            # it (#199): a version-4 case's expectations became metrics, and
+            # the body it was typed as is kept, as the rules were.
+            db.execute("CREATE TABLE IF NOT EXISTS dataset_body_archive ("
+                       "dataset_id TEXT NOT NULL, body TEXT NOT NULL, archived_at TEXT NOT NULL)")
             # Rows from an earlier version are converted once, in place: a
             # dataset is typed in by hand and costly to re-enter, so it is
             # upgraded rather than hidden (AGENTS.md's one exception). The
@@ -1803,9 +1959,15 @@ class Datasets:
             given = False
             for did, name, version, raw in rows:
                 body = json.loads(raw)
+                # A version-6 body is read as version 7 (`_doc`) and saved as
+                # one at its next edit, never rewritten here (§17).
+                if isinstance(body, dict) and body.get("version") == 6:
+                    continue
                 up = upgrade_body(body)
                 if up is not body:
                     self._archive(db, did, body)
+                    db.execute("INSERT INTO dataset_body_archive (dataset_id, body, archived_at) VALUES (?, ?, ?)",
+                               (did, raw, self._now()))
                     if prompts is not None:
                         given = prompts.adopt(db, body_prompt(body), name, default=not given) is not None or given
                     db.execute("UPDATE datasets SET body = ?, version = ? WHERE id = ?",
@@ -1831,8 +1993,9 @@ class Datasets:
 
     @staticmethod
     def _doc(r, body=True):
-        """A row as the API answers it: a DatasetSummary, and its body with it."""
-        parsed = json.loads(r["body"])
+        """A row as the API answers it: a DatasetSummary, and its body with it,
+        read as today's version."""
+        parsed = upgrade_body(json.loads(r["body"]))
         out = {"id": r["id"], "name": r["name"], "cases": len(parsed.get("cases") or []),
                "version": r["version"], "updated": r["updated_at"]}
         if body:
@@ -1874,7 +2037,6 @@ class Datasets:
     def _insert(self, db, name, body):
         did = secrets.token_hex(6)
         now = self._now()
-        body = {**body, "cases": canonical_cases(body["cases"])}
         db.execute("INSERT INTO datasets (id, name, version, body, created_at, updated_at) "
                    "VALUES (?, ?, 1, ?, ?, ?)", (did, name, json.dumps(body), now, now))
         return did
@@ -1885,7 +2047,7 @@ class Datasets:
         name, why = dataset_name(name)
         if why:
             return None, (400, why)
-        body = blank_dataset() if body is None else body
+        body = blank_dataset() if body is None else upgrade_body(body)
         why = dataset_problem(body)
         if why:
             return None, (400, why)
@@ -1914,10 +2076,12 @@ class Datasets:
         the current row."""
         if type(version) is not int:
             return None, (400, "a save names the version it began from")
+        # A body of an earlier version -- from a page loaded before this one --
+        # is read as today's, as an import is.
+        body = upgrade_body(body)
         why = dataset_problem(body)
         if why:
             return None, (400, why)
-        body = {**body, "cases": canonical_cases(body["cases"])}
         with self.store.lock, self._connect() as db, db:
             r = self._live(db, did)
             if r is None:
@@ -1975,7 +2139,7 @@ class Datasets:
             rows = db.execute("SELECT * FROM datasets WHERE trash IS NULL "
                               "ORDER BY name COLLATE NOCASE, id").fetchall()
         return {"format": EXPORT_ALL, "version": EXPORT_VERSION,
-                "datasets": [{"name": r["name"], "body": json.loads(r["body"])} for r in rows]}
+                "datasets": [{"name": r["name"], "body": upgrade_body(json.loads(r["body"]))} for r in rows]}
 
     def import_file(self, doc):
         """Either export's file, as new datasets: import always creates, ids
@@ -2328,6 +2492,13 @@ class Packs:
             ds_ids = {}
             cuts = []   # (kind, id, the document as it was), kept before it changes
             for key, name, body, raw in pack["datasets"]:
+                # The Source a dataset grades may be the pack's own, named by
+                # its folder: pointed at the Source the pack made here.
+                ref = body.get("source")
+                if isinstance(ref, dict):
+                    sid = src_ids.get(ref.get("id")) or src_ids.get(ref.get("name"))
+                    if sid:
+                        body = {**body, "source": {"id": sid, "name": SOURCES.get(sid)["name"]}}
                 did = owned.get(("dataset", key))
                 current = DATASETS.get(did) if did else None
                 if current:
@@ -2349,8 +2520,10 @@ class Packs:
             new_work = []
             for key, doc in pack["pipelines"]:
                 doc = json.loads(json.dumps(doc))
-                tests = doc.get("tests")
-                for t in tests if isinstance(tests, list) else [tests]:
+                # A pack written before version 11 spells its evals `tests`;
+                # the page upgrades the pipeline as it reads it.
+                evals = doc.get("evals", doc.get("tests"))
+                for t in evals if isinstance(evals, list) else [evals]:
                     ref = t.get("dataset") if isinstance(t, dict) else None
                     if isinstance(ref, dict):
                         did = ds_ids.get(ref.get("id")) or ds_ids.get(ref.get("name"))
@@ -2487,7 +2660,7 @@ class Packs:
 #
 # A plugin is code, installed like a pack (docs/packs.md): a zip of a
 # manifest and the compiled JavaScript that registers what the lab lacks -- a
-# kind of answer, a modifier, a test type, a connection type. Its code runs in
+# kind of answer, a modifier, an eval type, a connection type. Its code runs in
 # the page and in the runner, where the registries live; this server never
 # runs it. It reads the manifest's `registers` as data, so it can refuse two
 # plugins registering one id, and so a connection type's settings, chat path
@@ -2505,7 +2678,19 @@ PLUGIN_VERSIONS = (1,)
 PLUGIN_CAP = int(os.environ.get("PLUGIN_CAP", str(16 * 1024 ** 2)))
 PLUGIN_VERSION_TEXT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}")
 PLUGIN_FILE = re.compile(r"(?:[A-Za-z0-9_-][A-Za-z0-9._-]*/)*[A-Za-z0-9_-][A-Za-z0-9._-]*\.(?:js|mjs|json|map)")
-REGISTRIES = ("outputKinds", "modifiers", "testTypes", "connectionTypes")
+REGISTRIES = ("outputKinds", "modifiers", "evalTypes", "connectionTypes")
+# evalTypes as a manifest written before pipeline version 11 spells it: a
+# plugin's own file, which the lab cannot upgrade, so it is read for good.
+OLD_REGISTRIES = {"testTypes": "evalTypes"}
+
+
+def plugin_registers(manifest: dict) -> dict:
+    """A manifest's registers under today's names: testTypes read as evalTypes."""
+    reg = dict(manifest.get("registers") or {})
+    for old, now in OLD_REGISTRIES.items():
+        if old in reg:
+            reg[now] = [*reg.get(now, []), *[x for x in reg.pop(old) if x not in reg.get(now, [])]]
+    return reg
 AUTH_WAYS = ("bearer", "x-api-key", "none")
 
 
@@ -2545,9 +2730,9 @@ def read_plugin(data: bytes):
     if not isinstance(entry, str) or entry not in files or not entry.endswith((".js", ".mjs")):
         return None, "the plugin's entry names no JavaScript file it holds"
     reg = m.get("registers") or {}
-    if not isinstance(reg, dict) or any(k not in REGISTRIES for k in reg):
+    if not isinstance(reg, dict) or any(k not in REGISTRIES and k not in OLD_REGISTRIES for k in reg):
         return None, f"a plugin's registers are {', '.join(REGISTRIES)}"
-    for k in ("outputKinds", "modifiers", "testTypes"):
+    for k in ("outputKinds", "modifiers", "evalTypes", *OLD_REGISTRIES):
         if not isinstance(reg.get(k, []), list) or not all(isinstance(x, str) and x for x in reg.get(k, [])):
             return None, f"registers.{k} is a list of ids"
     conns = reg.get("connectionTypes", [])
@@ -2570,8 +2755,8 @@ def read_plugin(data: bytes):
 
 def registered_ids(manifest: dict) -> set:
     """(registry, id) for everything a plugin's manifest says it registers."""
-    reg = manifest.get("registers") or {}
-    out = {(k, x) for k in ("outputKinds", "modifiers", "testTypes") for x in reg.get(k, [])}
+    reg = plugin_registers(manifest)
+    out = {(k, x) for k in ("outputKinds", "modifiers", "evalTypes") for x in reg.get(k, [])}
     return out | {("connectionTypes", c["id"]) for c in reg.get("connectionTypes", [])}
 
 
@@ -2924,7 +3109,7 @@ class Plugins:
         return [{"id": r["id"], "version": r["version"], "sha256": r["sha256"],
                  "entry": json.loads(r["manifest"])["entry"],
                  "description": json.loads(r["manifest"]).get("description") or "",
-                 "registers": json.loads(r["manifest"]).get("registers") or {},
+                 "registers": plugin_registers(json.loads(r["manifest"])),
                  "installed": r["installed_at"]} for r in self._rows()]
 
     def stamp(self) -> list:
@@ -3100,6 +3285,55 @@ def worker_refusal(code, stderr, env):
     return text if len(text) <= 2000 else text[:2000] + "…"
 
 
+
+# A list of runs is read for its figures -- History's table, Home's recent
+# runs, Runs' progress -- and a run's results are mostly what Results alone
+# shows: every reply, every stage's request and answer, every item a score
+# found or missed. A list of 25 runs was 1.9 MB of that (#225). So a row in a
+# list carries a brief copy of its results, `brief` says so, and one run
+# (GET /api/queue/<id>) is always whole. A brief cell keeps its time and
+# whether it ran; a brief score keeps its verdict and counts in place of its
+# lists. The replies go too: an eval over the whole run reads them, and the
+# page asks for that run whole rather than every list carrying them.
+BRIEF_SCORE = ("pass", "score", "points", "skipped")
+COUNTED = ("found", "missed", "invented")
+
+
+def brief_score(score):
+    if not isinstance(score, dict):
+        return score
+    out = {k: score[k] for k in BRIEF_SCORE if k in score}
+    for k in COUNTED:
+        if isinstance(score.get(k), list):
+            out[k] = len(score[k])
+    return out
+
+
+def brief_cell(cell):
+    if not isinstance(cell, dict):
+        return cell
+    out = {k: v for k, v in cell.items() if k not in ("res", "scores", "score")}
+    if isinstance(cell.get("res"), dict):
+        out["res"] = {k: cell["res"][k] for k in ("ms", "error") if k in cell["res"]}
+    if isinstance(cell.get("scores"), dict):
+        out["scores"] = {k: brief_score(v) for k, v in cell["scores"].items()}
+    elif "scores" in cell:
+        out["scores"] = cell["scores"]
+    # A run from before version 6 kept its one test's score as `score`, which
+    # the page reads as `scores.t1`: kept under its own name for that.
+    if "score" in cell:
+        out["score"] = brief_score(cell["score"])
+    return out
+
+
+def brief_row(row):
+    def item(it):
+        if not isinstance(it, dict) or not isinstance(it.get("scenarios"), list):
+            return it
+        return {**it, "scenarios": [brief_cell(c) for c in it["scenarios"]]}
+    return {**row, "results": [item(it) for it in row.get("results") or []], "brief": True}
+
+
 class Queue:
     STATUS = ("queued", "running", "done", "incomplete", "cancelled",
               "failed", "interrupted")
@@ -3127,11 +3361,22 @@ class Queue:
                        "dataset TEXT)")
             # A store from before runs kept their dataset gains the column;
             # its rows have none, and are pinned when first they need one.
-            if "dataset" not in [c[1] for c in db.execute("PRAGMA table_info(queue)")]:
+            cols = [c[1] for c in db.execute("PRAGMA table_info(queue)")]
+            if "dataset" not in cols:
                 db.execute("ALTER TABLE queue ADD COLUMN dataset TEXT")
+            # The run a re-run was queued from (#245); every earlier row is
+            # one of its own.
+            if "rerun_of" not in cols:
+                db.execute("ALTER TABLE queue ADD COLUMN rerun_of TEXT")
 
     # ---- rows -----------------------------------------------------------
 
+    # A row read with the submit time of the run it re-runs, if any: the
+    # page names a run by that time (History's Run ID), so a "Re-run of"
+    # note reads without fetching the original.
+    SELECT = ("SELECT q.*, o.submitted_at FROM queue q "
+              "LEFT JOIN queue o ON o.id = q.rerun_of")
+
     @staticmethod
     def _row(r):
         if r is None:
@@ -3142,6 +3387,8 @@ class Queue:
             "snapshot": json.loads(r[6]), "results": json.loads(r[7]),
             "progress": json.loads(r[8]), "totals": json.loads(r[9]),
             "error": r[10],
+            "rerunOf": r[12] if len(r) > 12 else None,
+            "rerunOfAt": r[13] if len(r) > 13 else None,
         }
 
     # A row from before run documents has no version, and nothing here can
@@ -3155,24 +3402,26 @@ class Queue:
         return row is not None and (row["snapshot"] or {}).get("version") in READABLE_VERSIONS
 
     def _all(self, db):
-        return [row for row in (self._row(r) for r in db.execute("SELECT * FROM queue"))
+        return [row for row in (self._row(r) for r in db.execute(self.SELECT))
                 if self._readable(row)]
 
     def get(self, rid):
         with self.lock, closing(sqlite3.connect(self.store.path)) as db:
-            row = self._row(db.execute("SELECT * FROM queue WHERE id = ?",
+            row = self._row(db.execute(self.SELECT + " WHERE q.id = ?",
                                        (rid,)).fetchone())
         return row if self._readable(row) else None
 
-    def list(self, limit=RUNS_PAGE, before=None):
+    def list(self, limit=RUNS_PAGE, before=None, full=False):
         """Runs, newest first, and whether more follow. `before` is a
         `submittedAt` the page of runs stops at, so History can page through
-        them the way it pages the runs store."""
+        them the way it pages the runs store. Each row is brief_row's unless
+        `full` asks for the whole of it."""
         with self.lock, closing(sqlite3.connect(self.store.path)) as db:
             rows = self._all(db)
         rows = [r for r in rows if before is None or r["submittedAt"] < before]
         rows.sort(key=lambda r: r["submittedAt"], reverse=True)
-        return rows[:limit], len(rows) > limit
+        page = rows[:limit]
+        return (page if full else [brief_row(r) for r in page]), len(rows) > limit
 
     def _set(self, rid, **fields):
         sets, vals = ", ".join(f"{k} = ?" for k in fields), list(fields.values())
@@ -3181,13 +3430,13 @@ class Queue:
 
     # ---- submit ---------------------------------------------------------
 
-    def submit(self, run: dict, dataset=None):
+    def submit(self, run: dict, dataset=None, rerun_of=None):
         """
         A new queued run. `run` is the run document (docs/pipeline-model.md
         §5): the pipeline, the profiles it resolved to without their keys, its
         content's file list in order and with its repeats, and the dataset's
-        version; `dataset` is that version's body, for a graded run. Returns
-        the row. Its items are that list, or the one inline text, each through
+        version; `dataset` is that version's body, for a graded run;
+        `rerun_of` is the run a re-run was queued from. Returns the row. Its items are that list, or the one inline text, each through
         every scenario -- so the total is the list's length, repeats and all,
         the same count the runner and the page make.
         """
@@ -3196,11 +3445,12 @@ class Queue:
         total = len(run_items(run))
         with self.lock, closing(sqlite3.connect(self.store.path)) as db, db:
             db.execute("INSERT INTO queue (id, status, cancel, submitted_at, "
-                       "snapshot, results, progress, totals, dataset) VALUES (?, ?, 0, ?, ?, ?, ?, ?, ?)",
+                       "snapshot, results, progress, totals, dataset, rerun_of) "
+                       "VALUES (?, ?, 0, ?, ?, ?, ?, ?, ?, ?)",
                        (rid, "queued", now, json.dumps(run), "[]",
                         json.dumps({"current": None, "n": 0, "total": total}),
                         json.dumps({"ran": 0, "passed": 0, "found": 0, "of": 0}),
-                        None if dataset is None else json.dumps(dataset)))
+                        None if dataset is None else json.dumps(dataset), rerun_of))
             # Its prompts' uses, in the same transaction: a run is in the
             # library the moment it is queued, or not queued at all.
             if self.prompts is not None:
@@ -3208,7 +3458,7 @@ class Queue:
             # Read back under the lock the worker dequeues under, so the
             # answer is the run as it was queued: an idle worker can take it
             # the moment the lock is let go.
-            return self._row(db.execute("SELECT * FROM queue WHERE id = ?", (rid,)).fetchone())
+            return self._row(db.execute(self.SELECT + " WHERE q.id = ?", (rid,)).fetchone())
 
     def dataset(self, rid, raw=False):
         """The dataset body a readable run was submitted against, or None --
@@ -3241,7 +3491,7 @@ class Queue:
         written beside the run document. A row queued before runs kept their
         dataset has none, and is pinned to the dataset as it reads now, once,
         so every later pass over it agrees. Returns (args, None) or (None, why)."""
-        ref = tests_dataset(run["snapshot"])
+        ref = evals_dataset(run["snapshot"])
         if ref is None:
             return [], None
         body = self.dataset(run["id"], raw=True)
@@ -3327,6 +3577,46 @@ class Queue:
                 shutil.rmtree(d, ignore_errors=True)
         return n
 
+    def rerun(self, rid):
+        """
+        A new run from a finished one's document (#245): a new id and submit
+        time, the same document -- the file revisions and the plugins it
+        pinned -- and the dataset body it was graded by, so a re-run measures
+        the model again rather than changed data. The original is left as it
+        was; the new row names it in `rerunOf`. Refused, naming what is
+        missing, when a pinned file or dataset version is no longer kept. A
+        Target's key is read from its profile now, at dequeue, as for any
+        run: none is ever stored in one.
+        """
+        run = self.get(rid)
+        if run is None:
+            return None, (404, "no such run")
+        if run["status"] in ("queued", "running"):
+            return None, (409, "a run still in progress cannot be re-run")
+        snap = run["snapshot"]
+        _, err = self._pinned_files(snap)
+        if err:
+            return None, (409, err)
+        ref = evals_dataset(snap)
+        body = None
+        if ref is not None:
+            body = self.dataset(rid, raw=True)
+            if body is None:
+                # A run that never started kept no body: the dataset's, if it
+                # still reads as the version the run was submitted against.
+                now = DATASETS.snapshot(ref.get("id")) if DATASETS is not None else None
+                if now is None or not ref.get("version") or now[1] != ref.get("version"):
+                    return None, (409, f"the version of the dataset {ref.get('name') or ref.get('id')!r} "
+                                       "this run was submitted against is no longer kept")
+                body = now[0]
+        _, err = self._plugin_args(run)
+        if err:
+            return None, (409, err)
+        _, err = worker_destinations(snap)
+        if err:
+            return None, (403, err)
+        return self.submit(snap, body, rerun_of=rid), None
+
     def rerun_item(self, rid, index):
         """
         One item against the run's snapshot, by its index, updating the row in
@@ -3473,34 +3763,48 @@ class Queue:
         has gone fails the run with the reason stated. Returns the run dir,
         or (None, error)."""
         snap = run["snapshot"]
-        content = content_of(snap) or {}
+        files, err = self._pinned_files(snap)
+        if err:
+            return None, err
         rundir = self.dir / run["id"]
         files_dir = rundir / "files"
         shutil.rmtree(rundir, ignore_errors=True)
         files_dir.mkdir(parents=True)
-        if content.get("type") == "source":
-            ref = content.get("ref") or {}
-            label = ref.get("name") or ref.get("id")
-            if SOURCES.get(ref.get("id")) is None:
-                return None, f"the source {label!r} is gone"
-            revs = content.get("revs") if isinstance(content.get("revs"), dict) else None
-            changed = []
-            for name in dict.fromkeys(content.get("files") or []):
-                path = SOURCES.file_path(ref["id"], name)
-                if path is None or not path.is_file():
-                    return None, f"{name!r} is gone from the source {label!r}"
-                if revs is not None:
-                    path = SOURCES.rev_path(ref["id"], name, revs.get(name))
-                    if path is None:
-                        changed.append(repr(name))
-                        continue
-                shutil.copy2(path, files_dir / name)
-            if changed:
-                return None, (f"{', '.join(changed)} {'has' if len(changed) == 1 else 'have'} "
-                              f"changed in the source {label!r} since this run was queued")
+        for name, path in files:
+            shutil.copy2(path, files_dir / name)
         (rundir / "run.json").write_text(json.dumps(snap))
         return rundir, None
 
+    @staticmethod
+    def _pinned_files(snap):
+        """Each file a run document reads, by name, and the path holding the
+        bytes it pinned at submit -- or (None, why), naming the Source that
+        has gone, or the files gone from it or changed since. A document
+        with no Source reads no files."""
+        content = content_of(snap) or {}
+        if content.get("type") != "source":
+            return [], None
+        ref = content.get("ref") or {}
+        label = ref.get("name") or ref.get("id")
+        if SOURCES.get(ref.get("id")) is None:
+            return None, f"the source {label!r} is gone"
+        revs = content.get("revs") if isinstance(content.get("revs"), dict) else None
+        files, changed = [], []
+        for name in dict.fromkeys(content.get("files") or []):
+            path = SOURCES.file_path(ref["id"], name)
+            if path is None or not path.is_file():
+                return None, f"{name!r} is gone from the source {label!r}"
+            if revs is not None:
+                path = SOURCES.rev_path(ref["id"], name, revs.get(name))
+                if path is None:
+                    changed.append(repr(name))
+                    continue
+            files.append((name, path))
+        if changed:
+            return None, (f"{', '.join(changed)} {'has' if len(changed) == 1 else 'have'} "
+                          f"changed in the source {label!r} since this run was queued")
+        return files, None
+
     def _execute(self, run):
         """One run, FIFO. Never more than one of these at a time: the worker
         is a single thread, so the queue is single-flight by construction."""
@@ -3748,7 +4052,7 @@ def worker_destinations(run: dict):
 
 # A run document's fields, checked before it is accepted -- the rules
 # docs/pipeline-model.md §6 gives the server, in Python because the server is
-# stdlib-only and cannot load evals-core.ts. What a kind, a modifier or a test
+# stdlib-only and cannot load evals-core.ts. What a kind, a modifier or an eval
 # means is the runner's to judge, and it fails the run with a sentence if it
 # cannot; what is here is what the server itself depends on: a version it
 # reads, its own cap, content it can count and copy, and connections that
@@ -3756,21 +4060,23 @@ def worker_destinations(run: dict):
 # 5: the pipeline and every chain have an id; version 4's scenarios keep
 # theirs, and an older run's chains are read as they are, by position.
 # 6: tests are an ordered list; a stored run's one test (or null) is read as
-# a list of one (tests_dataset).
+# a list of one (evals_dataset).
 # 7: chains are jobs: the field is `jobs` and each job's type is "job".
 # 8: a job is its steps; the content is job 1's Attach Content step.
-# 9: a test is Metrics; a Single Test or a Graded set is read converted.
+# 9: an eval is Metrics; a Single Test or a Graded set is read converted.
 # 10: a job's steps are its stages, and each scenario is a target whose own
 # step in each job is what it sends there (docs/pipeline-model.md §16).
-PIPELINE_VERSION = 10
+# 11: `tests` are `evals`; nothing in an eval changes.
+# 12: a Contains metric's Ignore case holds item by item too, kept as written.
+PIPELINE_VERSION = 12
 # What a stored run may be: the current version, and the ones evals-core.ts's
 # upgradePipeline reads. A new submission is upgraded to the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
 TARGET_CAP = 4
 # Target steps whose words the Prompt library does not record as a use: they
 # ask no model (evals-core.ts's STEP_TYPES.echo).
 UNRECORDED_STEPS = {"echo"}
-RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "tests", "profiles", "comment", "plugins")
+RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "evals", "profiles", "comment", "plugins")
 
 
 def content_of(doc):
@@ -3927,14 +4233,14 @@ def connection_problems(pid, conn, at, bad):
             bad.append(f"{at}: llama.cpp requires its llama-server, not a hosted model")
 
 
-def tests_dataset(doc):
-    """The dataset reference a document's tests grade against, or None: the
-    first test that names one. A run grades against one dataset (the core's
-    validatePipeline says so). Reads a stored document of either shape --
-    version 5's list, or the one test before it -- since rows keep the
-    document they were submitted with."""
-    tests = doc.get("tests") if isinstance(doc, dict) else None
-    for t in tests if isinstance(tests, list) else [tests]:
+def evals_dataset(doc):
+    """The dataset reference a document's evals grade against, or None: the
+    first eval that names one. A run grades against one dataset (the core's
+    validatePipeline says so). Reads a stored document of any shape --
+    version 11's `evals`, the `tests` before it, version 5's list or the one
+    test before that -- since rows keep the document they were submitted with."""
+    evals = doc.get("evals", doc.get("tests")) if isinstance(doc, dict) else None
+    for t in evals if isinstance(evals, list) else [evals]:
         if isinstance(t, dict) and isinstance(t.get("dataset"), dict):
             return t["dataset"]
     return None
@@ -4079,9 +4385,9 @@ def run_problems(run):
         pass
     else:
         bad.append("a run needs a Source's files, some inline text, or Prompt only")
-    tests = run.get("tests")
-    if not isinstance(tests, list) or not all(isinstance(t, dict) and isinstance(t.get("type"), str) for t in tests):
-        bad.append("tests has to be a list of tests, each naming its type")
+    evals = run.get("evals")
+    if not isinstance(evals, list) or not all(isinstance(t, dict) and isinstance(t.get("type"), str) for t in evals):
+        bad.append("evals has to be a list of evals, each naming its type")
     if run.get("comment") is not None and not isinstance(run["comment"], str):
         bad.append("comment has to be text")
     return bad
@@ -4146,11 +4452,25 @@ class Handler(BaseHTTPRequestHandler):
 
     # ---- helpers --------------------------------------------------------
 
-    def _send(self, code, body: bytes, ctype="application/json", headers=()):
+    def _send(self, code, body: bytes, ctype="application/json", headers=(), packed=None):
+        """`packed` keys a body that never changes under it -- a hashed
+        bundle -- so it is gzipped once and kept, not on every request."""
         self.send_response(code)
         self.send_header("Content-Type", ctype)
         for k, v in headers:
             self.send_header(k, v)
+        if ctype.startswith(COMPRESSIBLE):
+            # Said whether or not this answer is compressed, so a cache
+            # between here and the browser keys on it either way.
+            self.send_header("Vary", "Accept-Encoding")
+            if len(body) >= GZIP_MIN and accepts_gzip(self.headers.get("Accept-Encoding", "")):
+                if packed is None:
+                    body = gzip.compress(body, 6, mtime=0)
+                else:
+                    if packed not in GZIPPED:
+                        GZIPPED[packed] = gzip.compress(body, 9, mtime=0)
+                    body = GZIPPED[packed]
+                self.send_header("Content-Encoding", "gzip")
         self.send_header("Content-Length", str(len(body)))
         self.end_headers()
         self.wfile.write(body)
@@ -4235,7 +4555,7 @@ class Handler(BaseHTTPRequestHandler):
             return
         path = self.path.split("?", 1)[0]
         # The lab is one page: a Connection and an Input make a scenario,
-        # Content and Tests are shared, and one to four scenarios run over the
+        # Content and Evals are shared, and one to four scenarios run over the
         # content. It replaced the A/B page it was prototyped beside once it
         # carried everything that page did -- the graded set, the
         # Configuration group, the history -- rather than being left to rot
@@ -4255,7 +4575,7 @@ class Handler(BaseHTTPRequestHandler):
             if not name or TYPES.get(p.suffix.lower()) is None or not p.is_file():
                 return self._send(404, b"not found", "text/plain")
             return self._send(200, p.read_bytes(), TYPES[p.suffix.lower()],
-                              (("Cache-Control", "public, max-age=31536000, immutable"),))
+                              (("Cache-Control", "public, max-age=31536000, immutable"),), packed=name)
         if path == "/api/config":
             # The lab's own limits, for the page to disable rather than
             # hard-code: how many scenarios a run may have.
@@ -4279,14 +4599,14 @@ class Handler(BaseHTTPRequestHandler):
             except ValueError:
                 return self._json(400, {"error": "limit has to be a number"})
             before = (query.get("before") or [None])[0]
-            runs, more = QUEUE.list(limit, before)
+            runs, more = QUEUE.list(limit, before, full=(query.get("full") or [""])[0] == "1")
             return self._json(200, {"runs": runs, "more": more})
         if path.startswith("/api/queue/") and path.endswith("/dataset") and path.count("/") == 4:
             # The dataset body a graded run was submitted against, which is
             # what its verdicts were graded by; the list never carries it.
             #
             # A run that is there but kept no copy -- one submitted before the
-            # lab kept them, or one with no graded test -- answers null, not
+            # lab kept them, or one with no graded eval -- answers null, not
             # 404. It is not an error: the page reads it as "use the dataset
             # as it is now", and a 404 put a red line in the console every
             # time such a run was opened, which is the console people are
@@ -4820,14 +5140,14 @@ class Handler(BaseHTTPRequestHandler):
         # A graded run keeps the body of the dataset it names, as it reads
         # now, and records that body's fingerprint on the reference it
         # belongs to: the worker grades against that and nothing else.
-        ref = tests_dataset(run)
+        ref = evals_dataset(run)
         body = None
         if ref is not None:
             snap = DATASETS.snapshot(ref.get("id")) if DATASETS else None
             if snap is None:
-                return self._json(400, {"error": "the run's graded test names no dataset this lab has"})
+                return self._json(400, {"error": "the run's graded eval names no dataset this lab has"})
             body, version = snap
-            for t in run["tests"]:
+            for t in run["evals"]:
                 if isinstance(t.get("dataset"), dict) and t["dataset"].get("id") == ref.get("id"):
                     t["dataset"]["version"] = version
         return self._json(201, {"run": QUEUE.submit(run, body)})
@@ -4985,6 +5305,9 @@ class Handler(BaseHTTPRequestHandler):
         return self._send(404, b"not found", "text/plain")
 
     def _queue_action(self, path):
+        # Drained, so a keep-alive connection is not left holding the
+        # page's `{}` in front of its next request.
+        self._payload()
         parts = path.split("/")
         rid = parts[3]
         action = parts[4] if len(parts) > 4 else ""
@@ -4992,6 +5315,10 @@ class Handler(BaseHTTPRequestHandler):
             run, err = QUEUE.cancel(rid)
         elif action == "resume":
             run, err = QUEUE.resume(rid)
+        elif action == "rerun" and len(parts) == 5:
+            run, err = QUEUE.rerun(rid)
+            if not err:
+                return self._json(201, {"run": run})
         elif action == "items" and len(parts) == 6:
             run, err = QUEUE.rerun_item(rid, urllib.parse.unquote(parts[5]))
         elif action == "rescore" and len(parts) == 6: