evals-lab 0.2.0 → 0.3.0

package/lab/server.py

@@ -24,6 +24,7 @@ given, so both follow their user between browsers; see Store and Sources.
 
 import base64
 import calendar
+import gzip
 import hashlib
 import hmac
 import io
@@ -247,6 +248,35 @@ TYPES = {
     ".png": "image/png",
 }
 
+# What is gzipped for a client that asks (#225): text, which compresses five
+# to ten times, and nothing already compressed. A body under GZIP_MIN goes as
+# it is, since the header and the gzip frame would outweigh the saving.
+COMPRESSIBLE = ("text/", "application/json", "image/svg+xml")
+GZIP_MIN = 1024
+# The built page's bundles, gzipped once: a new build is new names.
+GZIPPED: dict = {}
+
+
+def accepts_gzip(header: str) -> bool:
+    """Whether an Accept-Encoding header takes gzip: named, or `*`, with a q
+    above 0. `gzip;q=0` is a refusal, not a request."""
+    star = False
+    for part in (header or "").split(","):
+        name, _, params = part.partition(";")
+        name, q = name.strip().lower(), 1.0
+        for p in params.split(";"):
+            k, _, v = p.partition("=")
+            if k.strip().lower() == "q":
+                try:
+                    q = float(v)
+                except ValueError:
+                    q = 0.0
+        if name == "gzip":
+            return q > 0
+        if name == "*":
+            star = q > 0
+    return star
+
 
 # What a Source may accept and ever be served back as -- deliberately not
 # TYPES: an upload that could come back as text/html is stored XSS, and the
@@ -469,9 +499,10 @@ SIGN_INS = {"microsoft": lambda: microsoft_config() is not None}
 # A key taken off this list is no longer served or written, and its rows stay
 # in the store: a document is not migrated or deleted because nothing reads it.
 # promptlab.cases, .rules and their .base copies went that way when a dataset
-# became a row of its own (#86).
+# became a row of its own (#86), and promptlab.mappings once a dataset named
+# the Source it grades (#199).
 SYNCED = ("promptlab.workflows", "promptlab.profiles",
-          "promptlab.versions", "promptlab.tokens", "promptlab.mappings")
+          "promptlab.versions", "promptlab.tokens")
 MAX_DOC = 8 * 1024 * 1024
 RUNS_PAGE = 25
 # A dataset request's caps, read from Content-Length before the body is, as a
@@ -1306,7 +1337,7 @@ class Sources:
 
 # ---- Datasets ----------------------------------------------------------------
 #
-# A dataset is data a graded test names by id: its cases. (The prompt a new
+# A dataset is data a graded eval names by id: its cases. (The prompt a new
 # scenario starts from is the Prompt library's Default, below; a dataset from
 # before the library held one, and gave it to the library once.) It is one row in the
 # store's SQLite, its body one JSON document with a version that goes up by one
@@ -1325,44 +1356,140 @@ class Sources:
 # A lab from before this held a one-time import's `meta` row saying it ran;
 # it is left where it is, and nothing reads it.
 
-DATASET_FIELDS = ("cases",)
+DATASET_FIELDS = ("version", "source", "cases")
+# A body's own version: evals-core.ts's DATASET_BODY_VERSION. Version 5 was
+# told by its `source` alone, and earlier ones by neither.
+DATASET_BODY_VERSION = 6
 DATASET_NAME_MAX = 80
-# The file forms Export writes and Import reads. Export writes version 4;
-# Import reads it and versions 1 to 3, upgraded, and refuses anything else,
+# The file forms Export writes and Import reads. Export writes version 6;
+# Import reads it and versions 1 to 5, upgraded, and refuses anything else,
 # as a pipeline of another version is refused. Versions 1 to 3 carried a
 # prompt, which an import gives to the Prompt library.
 EXPORT_ONE = "evals-lab/dataset"
 EXPORT_ALL = "evals-lab/datasets"
-EXPORT_VERSION = 4
-IMPORT_VERSIONS = (1, 2, 3, 4)
+EXPORT_VERSION = 6
+IMPORT_VERSIONS = (1, 2, 3, 4, 5, 6)
 
 
 def blank_dataset() -> dict:
-    return {"cases": []}
+    return {"version": DATASET_BODY_VERSION, "source": None, "cases": []}
+
+
+# vocab: the names older versions gave a case's fields
+CASE_RENAMED = {"minTags": "minCount", "maxTags": "maxCount", "textInImage": "watch", "photo": "filename"}  # vocab: as above
+CASE_V4 = ("filename", "why", "traits", "expect", "anyOf", "forbid", "allow", "minCount", "maxCount", "watch", "discarded")
+
+
+def _words(s) -> list:
+    """evals-core.ts's words: the letters and digits of [s], lowercased."""
+    return re.findall(r"[^\W_]+", str(s or "").lower())
+
+
+def _term_in(items, term) -> bool:
+    """evals-core.ts's termIn: [term]'s words in some item, in order and adjacent."""
+    t = _words(term)
+    if not t:
+        return False
+    for item in items:
+        w = _words(item)
+        if any(w[i:i + len(t)] == t for i in range(len(w) - len(t) + 1)):
+            return True
+    return False
+
+
+def case_metrics(c: dict) -> list:
+    """A version-4 case's expectations as the metrics that say the same:
+    evals-core.ts's caseMetrics, in Python, and held to it by proxy-check.py
+    through fixtures/dataset-v6.json."""
+    def strs(v):
+        return [x for x in v if isinstance(x, str)] if isinstance(v, list) else []
+    if c.get("discarded") is True:
+        return [{"type": "discarded"}]
+    out = []
+    expect, allow = strs(c.get("expect")), strs(c.get("allow"))
+    if expect:
+        out.append({"type": "contains-all", "values": "\n".join(expect)})
+    for g in c.get("anyOf") if isinstance(c.get("anyOf"), list) else []:
+        if strs(g):
+            out.append({"type": "contains-any", "values": "\n".join(strs(g))})
+    for t in strs(c.get("forbid")):
+        # An exception excuses only the forbidden term inside it.
+        except_ = [a for a in allow if _term_in([a], t)]
+        out.append({"type": "contains", "value": t, "not": True, **({"except": "\n".join(except_)} if except_ else {})})
+    whole = lambda v: v if type(v) is int else None
+    lo, hi = whole(c.get("minCount")), whole(c.get("maxCount"))
+    if lo is not None or hi is not None:
+        out.append({"type": "item-count", "min": lo, "max": hi})
+    for t in strs(c.get("watch")):
+        out.append({"type": "contains-any", "values": t, "weight": 0})
+    return out
+
+
+def case_of_v4(c):
+    """One case of any earlier version as a version-5 one: evals-core.ts's
+    caseOfV4, in Python."""
+    if not isinstance(c, dict):
+        return c
+    was = {}
+    for k, v in c.items():
+        key = CASE_RENAMED.get(k, k)
+        if key not in was or key == k:
+            was[key] = v
+    if isinstance(was.get("item"), str):
+        was.pop("filename", None)
+    out = {}
+    if "id" in was:
+        out["id"] = was["id"]
+    out["item"] = was["item"] if isinstance(was.get("item"), str) else was["filename"] if isinstance(was.get("filename"), str) else ""
+    out["todo"] = was.get("todo") is True
+    out["note"] = was["note"] if isinstance(was.get("note"), str) else was["why"] if isinstance(was.get("why"), str) else ""
+    out["metrics"] = case_metrics(was) + (was["metrics"] if isinstance(was.get("metrics"), list) else [])
+    for k, v in was.items():
+        if k not in out and k not in CASE_V4:
+            out[k] = v
+    return out
+
+
+# The metrics whose Ignore case version 6 made mean what it says for a reply
+# read as a list: evals-core.ts's CASE_FOLDING.
+CASE_FOLDING = ("contains", "contains-all", "contains-any")
+
+
+def case_of_v5(c):
+    """A version-5 case as a version-6 one: evals-core.ts's caseOfV5, in
+    Python. Each Contains metric says Ignore case, as version 5 matched a
+    list's items whatever it said."""
+    if not isinstance(c, dict) or not isinstance(c.get("metrics"), list):
+        return c
+    return {**c, "metrics": [{**m, "ignoreCase": True}
+                             if isinstance(m, dict) and m.get("type") in CASE_FOLDING and m.get("ignoreCase") is not True
+                             else m for m in c["metrics"]]}
 
 
 def upgrade_body(body):
-    """An earlier body as today's: evals-core.ts's upgradeDatasetBody, in
-    Python. Version 1's `imageCases` are `cases`, each case's
-    `minTags`/`maxTags` its `minCount`/`maxCount`, and its `replays` and
-    `conformance` go (fixtures/replays.json holds the parser's tests).
-    Version 2's `rules` go -- they clean a job's answer, so they are the
-    job's -- and the terms a case watches for are its `watch`. Version 3's
-    `prompt` goes: the Prompt library holds prompts now. A caller that needs
-    the rules or the prompt takes them first (`body_rules`, `body_prompt`).
-    Anything else comes back as it was."""
-    if not isinstance(body, dict):
+    """An earlier body as today's (version 6): evals-core.ts's
+    upgradeDatasetBody, in Python. Version 1's `imageCases` are `cases`, and
+    its `replays` and `conformance` go (fixtures/replays.json holds the
+    parser's tests). Version 2's `rules` go -- they clean a job's answer, so
+    they are the job's. Version 3's `prompt` goes: the Prompt library holds
+    prompts now. Version 4's case named its item `filename` and said what a
+    good answer is in expectations; each becomes its metric, `why` the
+    `note`, `traits` go, and the body names no Source yet. A caller that
+    needs the rules or the prompt takes them first (`body_rules`,
+    `body_prompt`). A body naming its Source is version 5, whose Contains
+    metrics each come to say Ignore case (`case_of_v5`). A body saying it is
+    version 6 comes back as it was; so does anything that is not a body."""
+    if not isinstance(body, dict) or body.get("version") == DATASET_BODY_VERSION:
         return body
-    if "imageCases" not in body and "rules" not in body:
-        if "prompt" not in body:
-            return body
-        return {k: v for k, v in body.items() if k != "prompt"}
-    renamed = {"minTags": "minCount", "maxTags": "maxCount", "textInImage": "watch"}  # vocab: older names
+    if "source" in body:
+        up = {"version": DATASET_BODY_VERSION, **body}
+        if isinstance(body.get("cases"), list):
+            up["cases"] = [case_of_v5(c) for c in body["cases"]]
+        return up
     cases = body.get("cases") if isinstance(body.get("cases"), list) else body.get("imageCases")
     if not isinstance(cases, list):
         return body
-    cases = [{renamed.get(k, k): v for k, v in c.items()} if isinstance(c, dict) else c for c in cases]
-    return {"cases": canonical_cases(cases)}
+    return {"version": DATASET_BODY_VERSION, "source": None, "cases": [case_of_v5(case_of_v4(c)) for c in cases]}
 
 
 def body_prompt(body):
@@ -1377,21 +1504,6 @@ def body_rules(body):
     return rules if isinstance(rules, dict) and isinstance(rules.get("rules"), list) else None
 
 
-def canonical_cases(cases: list) -> list:
-    """Every case naming its file as `filename`: evals-core.ts's
-    canonicalCases, in Python. `old` is the key a set re-synced from an
-    app's own repository arrives with; the key keeps its place, so only its
-    spelling changes."""
-    old = "photo"  # vocab: the older spelling of filename
-    out = []
-    for c in cases:
-        if isinstance(c, dict) and old in c:
-            c = {("filename" if k == old else k): v for k, v in c.items()
-                 if not (k == old and "filename" in c)}
-        out.append(c)
-    return out
-
-
 def dataset_problem(body) -> str:
     """Why [body] is not a dataset's body, in one sentence, or ""."""
     if not isinstance(body, dict):
@@ -1402,8 +1514,14 @@ def dataset_problem(body) -> str:
     for k in DATASET_FIELDS:
         if k not in body:
             return f"a dataset's body has no \"{k}\""
+    if body["version"] != DATASET_BODY_VERSION:
+        return f"a dataset's body is version {DATASET_BODY_VERSION}"
     if not isinstance(body["cases"], list) or not all(isinstance(c, dict) for c in body["cases"]):
         return "cases has to be a list of cases"
+    src = body["source"]
+    if src is not None and not (isinstance(src, dict) and isinstance(src.get("id"), str)
+                                and isinstance(src.get("name"), str)):
+        return "a dataset names its Source as { id, name }, or null"
     return ""
 
 
@@ -1788,6 +1906,11 @@ class Datasets:
             # so nothing is lost if a pipeline was missed.
             db.execute("CREATE TABLE IF NOT EXISTS dataset_rules_archive ("
                        "dataset_id TEXT NOT NULL, rules TEXT NOT NULL, archived_at TEXT NOT NULL)")
+            # Each row's body as it was before the conversion below rewrote
+            # it (#199): a version-4 case's expectations became metrics, and
+            # the body it was typed as is kept, as the rules were.
+            db.execute("CREATE TABLE IF NOT EXISTS dataset_body_archive ("
+                       "dataset_id TEXT NOT NULL, body TEXT NOT NULL, archived_at TEXT NOT NULL)")
             # Rows from an earlier version are converted once, in place: a
             # dataset is typed in by hand and costly to re-enter, so it is
             # upgraded rather than hidden (AGENTS.md's one exception). The
@@ -1806,6 +1929,8 @@ class Datasets:
                 up = upgrade_body(body)
                 if up is not body:
                     self._archive(db, did, body)
+                    db.execute("INSERT INTO dataset_body_archive (dataset_id, body, archived_at) VALUES (?, ?, ?)",
+                               (did, raw, self._now()))
                     if prompts is not None:
                         given = prompts.adopt(db, body_prompt(body), name, default=not given) is not None or given
                     db.execute("UPDATE datasets SET body = ?, version = ? WHERE id = ?",
@@ -1874,7 +1999,6 @@ class Datasets:
     def _insert(self, db, name, body):
         did = secrets.token_hex(6)
         now = self._now()
-        body = {**body, "cases": canonical_cases(body["cases"])}
         db.execute("INSERT INTO datasets (id, name, version, body, created_at, updated_at) "
                    "VALUES (?, ?, 1, ?, ?, ?)", (did, name, json.dumps(body), now, now))
         return did
@@ -1885,7 +2009,7 @@ class Datasets:
         name, why = dataset_name(name)
         if why:
             return None, (400, why)
-        body = blank_dataset() if body is None else body
+        body = blank_dataset() if body is None else upgrade_body(body)
         why = dataset_problem(body)
         if why:
             return None, (400, why)
@@ -1914,10 +2038,12 @@ class Datasets:
         the current row."""
         if type(version) is not int:
             return None, (400, "a save names the version it began from")
+        # A body of an earlier version -- from a page loaded before this one --
+        # is read as today's, as an import is.
+        body = upgrade_body(body)
         why = dataset_problem(body)
         if why:
             return None, (400, why)
-        body = {**body, "cases": canonical_cases(body["cases"])}
         with self.store.lock, self._connect() as db, db:
             r = self._live(db, did)
             if r is None:
@@ -2328,6 +2454,13 @@ class Packs:
             ds_ids = {}
             cuts = []   # (kind, id, the document as it was), kept before it changes
             for key, name, body, raw in pack["datasets"]:
+                # The Source a dataset grades may be the pack's own, named by
+                # its folder: pointed at the Source the pack made here.
+                ref = body.get("source")
+                if isinstance(ref, dict):
+                    sid = src_ids.get(ref.get("id")) or src_ids.get(ref.get("name"))
+                    if sid:
+                        body = {**body, "source": {"id": sid, "name": SOURCES.get(sid)["name"]}}
                 did = owned.get(("dataset", key))
                 current = DATASETS.get(did) if did else None
                 if current:
@@ -2349,8 +2482,10 @@ class Packs:
             new_work = []
             for key, doc in pack["pipelines"]:
                 doc = json.loads(json.dumps(doc))
-                tests = doc.get("tests")
-                for t in tests if isinstance(tests, list) else [tests]:
+                # A pack written before version 11 spells its evals `tests`;
+                # the page upgrades the pipeline as it reads it.
+                evals = doc.get("evals", doc.get("tests"))
+                for t in evals if isinstance(evals, list) else [evals]:
                     ref = t.get("dataset") if isinstance(t, dict) else None
                     if isinstance(ref, dict):
                         did = ds_ids.get(ref.get("id")) or ds_ids.get(ref.get("name"))
@@ -2487,7 +2622,7 @@ class Packs:
 #
 # A plugin is code, installed like a pack (docs/packs.md): a zip of a
 # manifest and the compiled JavaScript that registers what the lab lacks -- a
-# kind of answer, a modifier, a test type, a connection type. Its code runs in
+# kind of answer, a modifier, an eval type, a connection type. Its code runs in
 # the page and in the runner, where the registries live; this server never
 # runs it. It reads the manifest's `registers` as data, so it can refuse two
 # plugins registering one id, and so a connection type's settings, chat path
@@ -2505,7 +2640,19 @@ PLUGIN_VERSIONS = (1,)
 PLUGIN_CAP = int(os.environ.get("PLUGIN_CAP", str(16 * 1024 ** 2)))
 PLUGIN_VERSION_TEXT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}")
 PLUGIN_FILE = re.compile(r"(?:[A-Za-z0-9_-][A-Za-z0-9._-]*/)*[A-Za-z0-9_-][A-Za-z0-9._-]*\.(?:js|mjs|json|map)")
-REGISTRIES = ("outputKinds", "modifiers", "testTypes", "connectionTypes")
+REGISTRIES = ("outputKinds", "modifiers", "evalTypes", "connectionTypes")
+# evalTypes as a manifest written before pipeline version 11 spells it: a
+# plugin's own file, which the lab cannot upgrade, so it is read for good.
+OLD_REGISTRIES = {"testTypes": "evalTypes"}
+
+
+def plugin_registers(manifest: dict) -> dict:
+    """A manifest's registers under today's names: testTypes read as evalTypes."""
+    reg = dict(manifest.get("registers") or {})
+    for old, now in OLD_REGISTRIES.items():
+        if old in reg:
+            reg[now] = [*reg.get(now, []), *[x for x in reg.pop(old) if x not in reg.get(now, [])]]
+    return reg
 AUTH_WAYS = ("bearer", "x-api-key", "none")
 
 
@@ -2545,9 +2692,9 @@ def read_plugin(data: bytes):
     if not isinstance(entry, str) or entry not in files or not entry.endswith((".js", ".mjs")):
         return None, "the plugin's entry names no JavaScript file it holds"
     reg = m.get("registers") or {}
-    if not isinstance(reg, dict) or any(k not in REGISTRIES for k in reg):
+    if not isinstance(reg, dict) or any(k not in REGISTRIES and k not in OLD_REGISTRIES for k in reg):
         return None, f"a plugin's registers are {', '.join(REGISTRIES)}"
-    for k in ("outputKinds", "modifiers", "testTypes"):
+    for k in ("outputKinds", "modifiers", "evalTypes", *OLD_REGISTRIES):
         if not isinstance(reg.get(k, []), list) or not all(isinstance(x, str) and x for x in reg.get(k, [])):
             return None, f"registers.{k} is a list of ids"
     conns = reg.get("connectionTypes", [])
@@ -2570,8 +2717,8 @@ def read_plugin(data: bytes):
 
 def registered_ids(manifest: dict) -> set:
     """(registry, id) for everything a plugin's manifest says it registers."""
-    reg = manifest.get("registers") or {}
-    out = {(k, x) for k in ("outputKinds", "modifiers", "testTypes") for x in reg.get(k, [])}
+    reg = plugin_registers(manifest)
+    out = {(k, x) for k in ("outputKinds", "modifiers", "evalTypes") for x in reg.get(k, [])}
     return out | {("connectionTypes", c["id"]) for c in reg.get("connectionTypes", [])}
 
 
@@ -2924,7 +3071,7 @@ class Plugins:
         return [{"id": r["id"], "version": r["version"], "sha256": r["sha256"],
                  "entry": json.loads(r["manifest"])["entry"],
                  "description": json.loads(r["manifest"]).get("description") or "",
-                 "registers": json.loads(r["manifest"]).get("registers") or {},
+                 "registers": plugin_registers(json.loads(r["manifest"])),
                  "installed": r["installed_at"]} for r in self._rows()]
 
     def stamp(self) -> list:
@@ -3100,6 +3247,55 @@ def worker_refusal(code, stderr, env):
     return text if len(text) <= 2000 else text[:2000] + "…"
 
 
+
+# A list of runs is read for its figures -- History's table, Home's recent
+# runs, Runs' progress -- and a run's results are mostly what Results alone
+# shows: every reply, every stage's request and answer, every item a score
+# found or missed. A list of 25 runs was 1.9 MB of that (#225). So a row in a
+# list carries a brief copy of its results, `brief` says so, and one run
+# (GET /api/queue/<id>) is always whole. A brief cell keeps its time and
+# whether it ran; a brief score keeps its verdict and counts in place of its
+# lists. The replies go too: an eval over the whole run reads them, and the
+# page asks for that run whole rather than every list carrying them.
+BRIEF_SCORE = ("pass", "score", "points", "skipped")
+COUNTED = ("found", "missed", "invented")
+
+
+def brief_score(score):
+    if not isinstance(score, dict):
+        return score
+    out = {k: score[k] for k in BRIEF_SCORE if k in score}
+    for k in COUNTED:
+        if isinstance(score.get(k), list):
+            out[k] = len(score[k])
+    return out
+
+
+def brief_cell(cell):
+    if not isinstance(cell, dict):
+        return cell
+    out = {k: v for k, v in cell.items() if k not in ("res", "scores", "score")}
+    if isinstance(cell.get("res"), dict):
+        out["res"] = {k: cell["res"][k] for k in ("ms", "error") if k in cell["res"]}
+    if isinstance(cell.get("scores"), dict):
+        out["scores"] = {k: brief_score(v) for k, v in cell["scores"].items()}
+    elif "scores" in cell:
+        out["scores"] = cell["scores"]
+    # A run from before version 6 kept its one test's score as `score`, which
+    # the page reads as `scores.t1`: kept under its own name for that.
+    if "score" in cell:
+        out["score"] = brief_score(cell["score"])
+    return out
+
+
+def brief_row(row):
+    def item(it):
+        if not isinstance(it, dict) or not isinstance(it.get("scenarios"), list):
+            return it
+        return {**it, "scenarios": [brief_cell(c) for c in it["scenarios"]]}
+    return {**row, "results": [item(it) for it in row.get("results") or []], "brief": True}
+
+
 class Queue:
     STATUS = ("queued", "running", "done", "incomplete", "cancelled",
               "failed", "interrupted")
@@ -3164,15 +3360,17 @@ class Queue:
                                        (rid,)).fetchone())
         return row if self._readable(row) else None
 
-    def list(self, limit=RUNS_PAGE, before=None):
+    def list(self, limit=RUNS_PAGE, before=None, full=False):
         """Runs, newest first, and whether more follow. `before` is a
         `submittedAt` the page of runs stops at, so History can page through
-        them the way it pages the runs store."""
+        them the way it pages the runs store. Each row is brief_row's unless
+        `full` asks for the whole of it."""
         with self.lock, closing(sqlite3.connect(self.store.path)) as db:
             rows = self._all(db)
         rows = [r for r in rows if before is None or r["submittedAt"] < before]
         rows.sort(key=lambda r: r["submittedAt"], reverse=True)
-        return rows[:limit], len(rows) > limit
+        page = rows[:limit]
+        return (page if full else [brief_row(r) for r in page]), len(rows) > limit
 
     def _set(self, rid, **fields):
         sets, vals = ", ".join(f"{k} = ?" for k in fields), list(fields.values())
@@ -3241,7 +3439,7 @@ class Queue:
         written beside the run document. A row queued before runs kept their
         dataset has none, and is pinned to the dataset as it reads now, once,
         so every later pass over it agrees. Returns (args, None) or (None, why)."""
-        ref = tests_dataset(run["snapshot"])
+        ref = evals_dataset(run["snapshot"])
         if ref is None:
             return [], None
         body = self.dataset(run["id"], raw=True)
@@ -3748,7 +3946,7 @@ def worker_destinations(run: dict):
 
 # A run document's fields, checked before it is accepted -- the rules
 # docs/pipeline-model.md §6 gives the server, in Python because the server is
-# stdlib-only and cannot load evals-core.ts. What a kind, a modifier or a test
+# stdlib-only and cannot load evals-core.ts. What a kind, a modifier or an eval
 # means is the runner's to judge, and it fails the run with a sentence if it
 # cannot; what is here is what the server itself depends on: a version it
 # reads, its own cap, content it can count and copy, and connections that
@@ -3756,21 +3954,23 @@ def worker_destinations(run: dict):
 # 5: the pipeline and every chain have an id; version 4's scenarios keep
 # theirs, and an older run's chains are read as they are, by position.
 # 6: tests are an ordered list; a stored run's one test (or null) is read as
-# a list of one (tests_dataset).
+# a list of one (evals_dataset).
 # 7: chains are jobs: the field is `jobs` and each job's type is "job".
 # 8: a job is its steps; the content is job 1's Attach Content step.
-# 9: a test is Metrics; a Single Test or a Graded set is read converted.
+# 9: an eval is Metrics; a Single Test or a Graded set is read converted.
 # 10: a job's steps are its stages, and each scenario is a target whose own
 # step in each job is what it sends there (docs/pipeline-model.md §16).
-PIPELINE_VERSION = 10
+# 11: `tests` are `evals`; nothing in an eval changes.
+# 12: a Contains metric's Ignore case holds item by item too, kept as written.
+PIPELINE_VERSION = 12
 # What a stored run may be: the current version, and the ones evals-core.ts's
 # upgradePipeline reads. A new submission is upgraded to the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
 TARGET_CAP = 4
 # Target steps whose words the Prompt library does not record as a use: they
 # ask no model (evals-core.ts's STEP_TYPES.echo).
 UNRECORDED_STEPS = {"echo"}
-RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "tests", "profiles", "comment", "plugins")
+RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "evals", "profiles", "comment", "plugins")
 
 
 def content_of(doc):
@@ -3927,14 +4127,14 @@ def connection_problems(pid, conn, at, bad):
             bad.append(f"{at}: llama.cpp requires its llama-server, not a hosted model")
 
 
-def tests_dataset(doc):
-    """The dataset reference a document's tests grade against, or None: the
-    first test that names one. A run grades against one dataset (the core's
-    validatePipeline says so). Reads a stored document of either shape --
-    version 5's list, or the one test before it -- since rows keep the
-    document they were submitted with."""
-    tests = doc.get("tests") if isinstance(doc, dict) else None
-    for t in tests if isinstance(tests, list) else [tests]:
+def evals_dataset(doc):
+    """The dataset reference a document's evals grade against, or None: the
+    first eval that names one. A run grades against one dataset (the core's
+    validatePipeline says so). Reads a stored document of any shape --
+    version 11's `evals`, the `tests` before it, version 5's list or the one
+    test before that -- since rows keep the document they were submitted with."""
+    evals = doc.get("evals", doc.get("tests")) if isinstance(doc, dict) else None
+    for t in evals if isinstance(evals, list) else [evals]:
         if isinstance(t, dict) and isinstance(t.get("dataset"), dict):
             return t["dataset"]
     return None
@@ -4079,9 +4279,9 @@ def run_problems(run):
         pass
     else:
         bad.append("a run needs a Source's files, some inline text, or Prompt only")
-    tests = run.get("tests")
-    if not isinstance(tests, list) or not all(isinstance(t, dict) and isinstance(t.get("type"), str) for t in tests):
-        bad.append("tests has to be a list of tests, each naming its type")
+    evals = run.get("evals")
+    if not isinstance(evals, list) or not all(isinstance(t, dict) and isinstance(t.get("type"), str) for t in evals):
+        bad.append("evals has to be a list of evals, each naming its type")
     if run.get("comment") is not None and not isinstance(run["comment"], str):
         bad.append("comment has to be text")
     return bad
@@ -4146,11 +4346,25 @@ class Handler(BaseHTTPRequestHandler):
 
     # ---- helpers --------------------------------------------------------
 
-    def _send(self, code, body: bytes, ctype="application/json", headers=()):
+    def _send(self, code, body: bytes, ctype="application/json", headers=(), packed=None):
+        """`packed` keys a body that never changes under it -- a hashed
+        bundle -- so it is gzipped once and kept, not on every request."""
         self.send_response(code)
         self.send_header("Content-Type", ctype)
         for k, v in headers:
             self.send_header(k, v)
+        if ctype.startswith(COMPRESSIBLE):
+            # Said whether or not this answer is compressed, so a cache
+            # between here and the browser keys on it either way.
+            self.send_header("Vary", "Accept-Encoding")
+            if len(body) >= GZIP_MIN and accepts_gzip(self.headers.get("Accept-Encoding", "")):
+                if packed is None:
+                    body = gzip.compress(body, 6, mtime=0)
+                else:
+                    if packed not in GZIPPED:
+                        GZIPPED[packed] = gzip.compress(body, 9, mtime=0)
+                    body = GZIPPED[packed]
+                self.send_header("Content-Encoding", "gzip")
         self.send_header("Content-Length", str(len(body)))
         self.end_headers()
         self.wfile.write(body)
@@ -4235,7 +4449,7 @@ class Handler(BaseHTTPRequestHandler):
             return
         path = self.path.split("?", 1)[0]
         # The lab is one page: a Connection and an Input make a scenario,
-        # Content and Tests are shared, and one to four scenarios run over the
+        # Content and Evals are shared, and one to four scenarios run over the
         # content. It replaced the A/B page it was prototyped beside once it
         # carried everything that page did -- the graded set, the
         # Configuration group, the history -- rather than being left to rot
@@ -4255,7 +4469,7 @@ class Handler(BaseHTTPRequestHandler):
             if not name or TYPES.get(p.suffix.lower()) is None or not p.is_file():
                 return self._send(404, b"not found", "text/plain")
             return self._send(200, p.read_bytes(), TYPES[p.suffix.lower()],
-                              (("Cache-Control", "public, max-age=31536000, immutable"),))
+                              (("Cache-Control", "public, max-age=31536000, immutable"),), packed=name)
         if path == "/api/config":
             # The lab's own limits, for the page to disable rather than
             # hard-code: how many scenarios a run may have.
@@ -4279,14 +4493,14 @@ class Handler(BaseHTTPRequestHandler):
             except ValueError:
                 return self._json(400, {"error": "limit has to be a number"})
             before = (query.get("before") or [None])[0]
-            runs, more = QUEUE.list(limit, before)
+            runs, more = QUEUE.list(limit, before, full=(query.get("full") or [""])[0] == "1")
             return self._json(200, {"runs": runs, "more": more})
         if path.startswith("/api/queue/") and path.endswith("/dataset") and path.count("/") == 4:
             # The dataset body a graded run was submitted against, which is
             # what its verdicts were graded by; the list never carries it.
             #
             # A run that is there but kept no copy -- one submitted before the
-            # lab kept them, or one with no graded test -- answers null, not
+            # lab kept them, or one with no graded eval -- answers null, not
             # 404. It is not an error: the page reads it as "use the dataset
             # as it is now", and a 404 put a red line in the console every
             # time such a run was opened, which is the console people are
@@ -4820,14 +5034,14 @@ class Handler(BaseHTTPRequestHandler):
         # A graded run keeps the body of the dataset it names, as it reads
         # now, and records that body's fingerprint on the reference it
         # belongs to: the worker grades against that and nothing else.
-        ref = tests_dataset(run)
+        ref = evals_dataset(run)
         body = None
         if ref is not None:
             snap = DATASETS.snapshot(ref.get("id")) if DATASETS else None
             if snap is None:
-                return self._json(400, {"error": "the run's graded test names no dataset this lab has"})
+                return self._json(400, {"error": "the run's graded eval names no dataset this lab has"})
             body, version = snap
-            for t in run["tests"]:
+            for t in run["evals"]:
                 if isinstance(t.get("dataset"), dict) and t["dataset"].get("id") == ref.get("id"):
                     t["dataset"]["version"] = version
         return self._json(201, {"run": QUEUE.submit(run, body)})

package/lab/web/dist/assets/{gallery-o7c4lfpn.css → gallery-DFeJkfUw.css}

@@ -1 +1 @@
-.gallery-index,.gallery-one{max-width:720px;padding:var(--s4);min-width:0;margin:0 auto}.gallery-index h1{font-size:var(--t-title);font-weight:600}.gallery-index ul{gap:var(--s2);margin:0;padding:0;list-style:none;display:grid}.gallery-index li{gap:var(--s2);flex-wrap:wrap;align-items:baseline;min-height:44px;display:flex}.gallery-index a{color:var(--amber)}.gallery-index code{font:var(--t-note) var(--mono);color:var(--faint)}.gallery-nav{gap:var(--s2);margin-bottom:var(--s3);font-size:var(--t-note);color:var(--dim);align-items:center;min-height:44px;display:flex}.gallery-nav a{color:var(--amber)}.note{color:var(--dim);font-size:var(--t-label);margin:0}
+.gallery-index,.gallery-one{max-width:720px;padding:var(--s4);min-width:0;margin:0 auto}.gallery-index h1{font-size:var(--t-title);font-weight:600}.gallery-index ul{gap:var(--s2);margin:0;padding:0;list-style:none;display:grid}.gallery-index li{gap:var(--s2);flex-wrap:wrap;align-items:baseline;min-height:44px;display:flex}.gallery-index a{color:var(--amber)}.gallery-index code{font:var(--t-note) var(--mono);color:var(--faint)}.gallery-nav{gap:var(--s2);margin-bottom:var(--s3);font-size:var(--t-note);color:var(--dim);align-items:center;min-height:44px;display:flex}.gallery-nav a{color:var(--amber)}.note{color:var(--dim);font-size:var(--t-label);margin:0}.gallery-row{gap:var(--s2);flex-wrap:wrap;display:flex}