evals-lab 0.8.0 → 0.9.0

package/lab/server.py

@@ -1229,6 +1229,9 @@ class Workspaces:
             db.execute("DELETE FROM dataset_body_archive WHERE dataset_id IN "
                        "(SELECT id FROM datasets WHERE workspace = ?)", (ws,))
             db.execute("DELETE FROM datasets WHERE workspace = ?", (ws,))
+            db.execute("DELETE FROM filter_set_versions WHERE set_id IN "
+                       "(SELECT id FROM filter_sets WHERE workspace = ?)", (ws,))
+            db.execute("DELETE FROM filter_sets WHERE workspace = ?", (ws,))
             db.execute("DELETE FROM prompt_uses WHERE prompt_id IN "
                        "(SELECT id FROM prompts WHERE workspace = ?)", (ws,))
             db.execute("DELETE FROM prompt_versions WHERE prompt_id IN "
@@ -2266,6 +2269,22 @@ def upgrade_body(body):
     return recorded_ids_v7(group_of_v6({"source": None, "cases": [case_of_v5(case_of_v4(c)) for c in cases]}))
 
 
+# The filter-set body's version: evals-core.ts's FILTER_SET_VERSION. Version 1
+# is the first -- the { version, filters } body #334 introduces. A filter set
+# is a Library row the lab stores and versions like an eval group (#336,
+# FilterSets below); this and its twin are the body it keeps, held to
+# evals-core.ts by proxy-check.py.
+FILTER_SET_BODY_VERSION = 1
+
+
+def upgrade_filter_set_body(body):
+    """An earlier filter-set body as today's (version 1): evals-core.ts's
+    upgradeFilterSetBody, in Python. There is no earlier version yet, so a
+    version-1 body comes back as it was, and so does anything that is not a
+    body. Pure."""
+    return body
+
+
 def body_prompt(body):
     """The prompt an earlier body held, or "": what the library is given."""
     prompt = body.get("prompt") if isinstance(body, dict) else None
@@ -2365,6 +2384,90 @@ def unique_dataset_name(name: str, taken: set) -> str:
     return f"{name} ({n})"
 
 
+# ---- Filter sets (the store's, FilterSets below) ---------------------------
+#
+# A filter set is a Library row like an eval group: a versioned { version,
+# filters } body (evals-core.ts FilterSetBody), kept so a pipeline can link it
+# and a run grade against the body it was submitted with (#336). The fields
+# are the body's; the per-filter shape is the core's to judge, as a case's
+# metrics are (dataset_problem). The suffixing is the generic one.
+FILTER_SET_FIELDS = ("version", "filters")
+FILTER_SET_NAME_MAX = 80
+unique_filter_set_name = unique_dataset_name
+
+
+def filter_set_problem(body) -> str:
+    """Why [body] is not a filter set's body, in one sentence, or "":
+    evals-core.ts's filterSetBodyProblems, to the depth the server checks."""
+    if not isinstance(body, dict):
+        return "a filter set's body is a JSON object"
+    for k in body:
+        if k not in FILTER_SET_FIELDS:
+            return f"a filter set's body has \"{k}\", which is not a filter-set field"
+    for k in FILTER_SET_FIELDS:
+        if k not in body:
+            return f"a filter set's body has no \"{k}\""
+    if body["version"] != FILTER_SET_BODY_VERSION:
+        return f"a filter set's body is version {FILTER_SET_BODY_VERSION}"
+    if not isinstance(body["filters"], list) or not all(isinstance(f, dict) for f in body["filters"]):
+        return "filters has to be a list of filters"
+    return ""
+
+
+def filter_set_name(raw):
+    """A name, trimmed, or (None, why)."""
+    if not isinstance(raw, str) or not raw.strip():
+        return None, "a filter set needs a name"
+    name = raw.strip()
+    if len(name) > FILTER_SET_NAME_MAX:
+        return None, "that name is too long"
+    return name, None
+
+
+def filter_set_links(doc) -> list:
+    """Every filter-set link a run or pipeline document carries: a job's
+    Responses steps list them under `filterSets` (evals-core.ts FilterSetLink).
+    A link names a Library set (`set`, followed latest or pinned at `pin`) or
+    holds a private one inline (`own`); only a Library link reads the store.
+    Runs wires the step and migrates the inline rules in #339; this is the one
+    reader of where the links sit, for the store (#336) to resolve and keep."""
+    out = []
+    jobs = doc.get("jobs") if isinstance(doc, dict) else None
+    for job in jobs if isinstance(jobs, list) else []:
+        steps = job.get("steps") if isinstance(job, dict) else None
+        for st in steps if isinstance(steps, list) else []:
+            links = st.get("filterSets") if isinstance(st, dict) else None
+            for link in links if isinstance(links, list) else []:
+                if isinstance(link, dict):
+                    out.append(link)
+    return out
+
+
+def filter_set_ref(link):
+    """The Library filter set a link reads, as the dict itself (so a caller may
+    stamp it in place), or None for a private (`own`) link."""
+    if not isinstance(link, dict):
+        return None
+    s = link.get("set")
+    return s if isinstance(s, dict) and isinstance(s.get("id"), str) else None
+
+
+def filter_set_pins_in(workflows) -> set:
+    """(filter-set id, version) for every filter-set link a stored pipeline
+    pins: the promptlab.workflows body, each pipeline as its `work`, as
+    pins_in reads eval-group pins. A Library link with an integer `pin`
+    freezes that version from the moment a pipeline pins it (§17)."""
+    out = set()
+    listed = workflows.get("list") if isinstance(workflows, dict) else None
+    for w in listed if isinstance(listed, list) else []:
+        work = w.get("work") if isinstance(w, dict) else None
+        for link in filter_set_links(work):
+            ref = filter_set_ref(link)
+            if ref is not None and type(link.get("pin")) is int:
+                out.add((ref["id"], link["pin"]))
+    return out
+
+
 # ---- The Prompt library -----------------------------------------------------
 #
 # Every prompt the lab has, run or not, each keeping every version of its text,
@@ -3164,6 +3267,319 @@ class Datasets:
         return [{k: v for k, v in self.get(did).items() if k != "body"} for did in created], None
 
 
+class FilterSets:
+    """Filter sets as a Library row, versioned like eval groups (#336,
+    docs/pipeline-model.md §18). The shape mirrors Datasets exactly -- a row
+    per set, every version kept in `filter_set_versions` for a link to pin and
+    a run to name -- without a dataset's prompt, scoring or import/export: a
+    filter set's body is just { version, filters }. A run keeps the body of
+    each set it links (Queue.submit's `filter_sets`), so it grades against what
+    it was submitted with whatever the set holds later."""
+
+    def __init__(self, store: Store):
+        self.store = store
+        with store.lock, closing(sqlite3.connect(store.path)) as db, db:
+            db.execute("CREATE TABLE IF NOT EXISTS filter_sets ("
+                       "id TEXT PRIMARY KEY, name TEXT NOT NULL, "
+                       "version INTEGER NOT NULL, body TEXT NOT NULL, "
+                       "created_at TEXT NOT NULL, updated_at TEXT NOT NULL, "
+                       "trash TEXT, trashed_at REAL, workspace TEXT)")
+            # Every version of each set, numbered from 1, as eval_group_versions
+            # keeps a group's (§17): `ran` freezes it for good once a run grades
+            # with it, a pin freezes it while a pipeline holds the pin, and
+            # version 1 is minted from the row's body at its first save, pin or
+            # run. A set from before workspaces backfills to the default.
+            db.execute("CREATE TABLE IF NOT EXISTS filter_set_versions ("
+                       "set_id TEXT NOT NULL, n INTEGER NOT NULL, body TEXT NOT NULL, "
+                       "created_at TEXT NOT NULL, edited_at REAL NOT NULL, "
+                       "ran INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (set_id, n))")
+            db.execute("UPDATE filter_sets SET workspace = ? WHERE workspace IS NULL", (store.default_ws,))
+
+    @staticmethod
+    def _now():
+        return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+    @staticmethod
+    def _doc(r, body=True, versions=1):
+        """A row as the API answers it: its summary, and its body with it, read
+        as today's version. `version` is the save counter a write is arbitrated
+        by; `versions` is its newest kept version's number, which a pin may
+        name, 1 before any is kept."""
+        parsed = upgrade_filter_set_body(json.loads(r["body"]))
+        out = {"id": r["id"], "name": r["name"],
+               "filters": len(parsed.get("filters") or []),
+               "version": r["version"], "versions": versions, "updated": r["updated_at"]}
+        if body:
+            out["body"] = parsed
+        return out
+
+    @staticmethod
+    def _counts(db) -> dict:
+        return dict(db.execute("SELECT set_id, MAX(n) FROM filter_set_versions GROUP BY set_id"))
+
+    def _row_doc(self, db, r, body=True):
+        return self._doc(r, body, self._counts(db).get(r["id"], 1))
+
+    def _connect(self):
+        db = sqlite3.connect(self.store.path)
+        db.row_factory = sqlite3.Row
+        return closing(db)
+
+    def _live(self, db, fid):
+        return db.execute("SELECT * FROM filter_sets WHERE id = ? AND trash IS NULL AND workspace = ?",
+                          (fid, workspace_of(self.store))).fetchone()
+
+    def _names(self, db, but=None):
+        return {r[0] for r in db.execute(
+            "SELECT name FROM filter_sets WHERE trash IS NULL AND id IS NOT ? AND workspace = ?",
+            (but, workspace_of(self.store)))}
+
+    def list(self) -> list:
+        with self.store.lock, self._connect() as db:
+            counts = self._counts(db)
+            return [self._doc(r, False, counts.get(r["id"], 1)) for r in db.execute(
+                "SELECT * FROM filter_sets WHERE trash IS NULL AND workspace = ? ORDER BY name COLLATE NOCASE, id",
+                (workspace_of(self.store),))]
+
+    def get(self, fid):
+        with self.store.lock, self._connect() as db:
+            r = self._live(db, fid)
+            return self._row_doc(db, r) if r else None
+
+    # ---- a set's versions (docs/pipeline-model.md §17, §18) --------------
+
+    @staticmethod
+    def _head(db, fid):
+        return db.execute("SELECT * FROM filter_set_versions WHERE set_id = ? "
+                          "ORDER BY n DESC LIMIT 1", (fid,)).fetchone()
+
+    def _mint(self, db, r):
+        """The newest version of row [r], adding version 1 from its body first
+        if it has none, edited when the row last was -- as a group's _mint."""
+        head = self._head(db, r["id"])
+        if head is not None:
+            return head
+        try:
+            edited = calendar.timegm(time.strptime(r["updated_at"], "%Y-%m-%dT%H:%M:%SZ"))
+        except ValueError:
+            edited = 0
+        db.execute("INSERT INTO filter_set_versions (set_id, n, body, created_at, edited_at) "
+                   "VALUES (?, 1, ?, ?, ?)", (r["id"], r["body"], r["updated_at"], edited))
+        return self._head(db, r["id"])
+
+    def _pins(self, db) -> set:
+        """Every (set, version) a stored pipeline pins, read in the caller's
+        transaction from this workspace's workflows document."""
+        row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.workflows' AND workspace = ?",
+                         (workspace_of(self.store),)).fetchone()
+        return filter_set_pins_in(json.loads(row[0])) if row and row[0] else set()
+
+    def _cut(self, db, fid, text):
+        n = self._head(db, fid)["n"] + 1
+        db.execute("INSERT INTO filter_set_versions (set_id, n, body, created_at, edited_at) "
+                   "VALUES (?, ?, ?, ?, ?)", (fid, n, text, self._now(), time.time()))
+        return n
+
+    def _keep(self, db, r, text):
+        """[text] as row [r]'s newest version: edited in place while no run has
+        graded with it, no link pins it and it was edited in the last
+        GROUP_IDLE_SECONDS, and a new version otherwise -- a group's _keep."""
+        head = self._mint(db, r)
+        if text == head["body"]:
+            return
+        fresh = time.time() - head["edited_at"] < GROUP_IDLE_SECONDS
+        if fresh and not head["ran"] and (r["id"], head["n"]) not in self._pins(db):
+            db.execute("UPDATE filter_set_versions SET body = ?, edited_at = ? WHERE set_id = ? AND n = ?",
+                       (text, time.time(), r["id"], head["n"]))
+        else:
+            self._cut(db, r["id"], text)
+
+    def versions(self, fid):
+        """A set's versions, newest first, without their bodies -- version 1
+        alone, read from the row, before any is kept -- or None."""
+        with self.store.lock, self._connect() as db:
+            r = self._live(db, fid)
+            if r is None:
+                return None
+            pins = self._pins(db)
+            rows = db.execute("SELECT * FROM filter_set_versions WHERE set_id = ? ORDER BY n DESC",
+                              (fid,)).fetchall()
+            if not rows:
+                return [{"n": 1, "created": r["created_at"], "ran": False, "pinned": (fid, 1) in pins,
+                         "fingerprint": fingerprint(json.loads(r["body"]))}]
+            return [{"n": v["n"], "created": v["created_at"], "ran": bool(v["ran"]),
+                     "pinned": (fid, v["n"]) in pins, "fingerprint": fingerprint(json.loads(v["body"]))}
+                    for v in rows]
+
+    def version_body(self, fid, n):
+        """Version [n]'s body, read as today's, or None."""
+        with self.store.lock, self._connect() as db:
+            r = self._live(db, fid)
+            if r is None:
+                return None
+            v = db.execute("SELECT body FROM filter_set_versions WHERE set_id = ? AND n = ?",
+                           (fid, n)).fetchone()
+            if v is None and n == 1 and self._head(db, fid) is None:
+                v = (r["body"],)
+        return upgrade_filter_set_body(json.loads(v[0])) if v else None
+
+    def restore_version(self, fid, n):
+        """An older version's body as the newest version, and the row's:
+        nothing is rewritten, so a run or a pin naming any version still reads
+        what it named. Returns (doc, None)."""
+        with self.store.lock, self._connect() as db, db:
+            r = self._live(db, fid)
+            if r is None:
+                return None, (404, "no such filter set")
+            head = self._mint(db, r)
+            old = db.execute("SELECT body FROM filter_set_versions WHERE set_id = ? AND n = ?",
+                             (fid, n)).fetchone()
+            if old is None:
+                return None, (404, "no such version")
+            if old["body"] != head["body"]:
+                self._cut(db, fid, old["body"])
+                db.execute("UPDATE filter_sets SET body = ?, version = ?, updated_at = ? WHERE id = ?",
+                           (old["body"], r["version"] + 1, self._now(), fid))
+            return self._row_doc(db, self._live(db, fid)), None
+
+    def mint_pinned(self, pins):
+        """Version 1 of each pinned set that has none yet: a pin names a
+        version, so the version has to be kept from the moment it does."""
+        with self.store.lock, self._connect() as db, db:
+            for fid, _ in pins:
+                r = self._live(db, fid)
+                if r is not None:
+                    self._mint(db, r)
+
+    def resolve(self, fid, pin=None):
+        """The version a run submitted now grades with -- [pin], or the newest
+        -- marked as graded with, in the same transaction, so no save can edit
+        it in place between this and the run keeping its body. Returns
+        (n, body), (None, why) for a pin the set has no version of, or None for
+        a set the lab does not have -- as a group's resolve."""
+        with self.store.lock, self._connect() as db, db:
+            r = self._live(db, fid) if isinstance(fid, str) else None
+            if r is None:
+                return None
+            head = self._mint(db, r)
+            v = head if pin is None else db.execute(
+                "SELECT * FROM filter_set_versions WHERE set_id = ? AND n = ?", (fid, pin)).fetchone()
+            if v is None:
+                return None, f"{r['name']} has no version {pin}"
+            db.execute("UPDATE filter_set_versions SET ran = 1 WHERE set_id = ? AND n = ?", (fid, v["n"]))
+            return v["n"], json.loads(v["body"])
+
+    def snapshot(self, fid):
+        """The body a run submitted now grades against, and its fingerprint;
+        None for a set the lab does not have."""
+        with self.store.lock, self._connect() as db:
+            r = self._live(db, fid) if isinstance(fid, str) else None
+        if r is None:
+            return None
+        body = json.loads(r["body"])
+        return body, fingerprint(body)
+
+    def _insert(self, db, name, body):
+        fid = secrets.token_hex(6)
+        now = self._now()
+        db.execute("INSERT INTO filter_sets (id, name, version, body, created_at, updated_at, workspace) "
+                   "VALUES (?, ?, 1, ?, ?, ?, ?)",
+                   (fid, name, json.dumps(body), now, now, workspace_of(self.store)))
+        return fid
+
+    def create(self, name, body=None):
+        """A new filter set, blank unless [body] is given -- New, and Duplicate,
+        which sends the source's body. A name already taken gets ` (2)`."""
+        name, why = filter_set_name(name)
+        if why:
+            return None, (400, why)
+        body = {"version": FILTER_SET_BODY_VERSION, "filters": []} if body is None \
+            else upgrade_filter_set_body(body)
+        why = filter_set_problem(body)
+        if why:
+            return None, (400, why)
+        with self.store.lock, self._connect() as db, db:
+            fid = self._insert(db, unique_filter_set_name(name, self._names(db)), body)
+        return self.get(fid), None
+
+    def rename(self, fid, name):
+        """A new label. The body and its version are untouched: a rename is not
+        an edit a browser holding the body has to reload for."""
+        name, why = filter_set_name(name)
+        if why:
+            return None, (400, why)
+        with self.store.lock, self._connect() as db, db:
+            if self._live(db, fid) is None:
+                return None, (404, "no such filter set")
+            if name.lower() in {n.lower() for n in self._names(db, but=fid)}:
+                return None, (409, f"{name!r} is taken by another filter set")
+            db.execute("UPDATE filter_sets SET name = ?, updated_at = ? WHERE id = ?",
+                       (name, self._now(), fid))
+        return self.get(fid), None
+
+    def save(self, fid, version, body):
+        """The body, written at [version] -- the one it began from. Returns
+        ({version}, None), or (None, error); a stale version's error carries
+        the current row."""
+        if type(version) is not int:
+            return None, (400, "a save names the version it began from")
+        body = upgrade_filter_set_body(body)
+        why = filter_set_problem(body)
+        if why:
+            return None, (400, why)
+        with self.store.lock, self._connect() as db, db:
+            r = self._live(db, fid)
+            if r is None:
+                return None, (404, "no such filter set")
+            if r["version"] != version:
+                return None, (409, {"current": self._row_doc(db, r)})
+            text = json.dumps(body)
+            self._keep(db, r, text)
+            db.execute("UPDATE filter_sets SET body = ?, version = ?, updated_at = ? WHERE id = ?",
+                       (text, version + 1, self._now(), fid))
+        return {"version": version + 1}, None
+
+    def remove(self, fid):
+        """Into the trash, at once; Undo restores it. Returns (token, None)."""
+        token = secrets.token_hex(6)
+        with self.store.lock, self._connect() as db, db:
+            if self._live(db, fid) is None:
+                return None, (404, "no such filter set")
+            db.execute("UPDATE filter_sets SET trash = ?, trashed_at = ? WHERE id = ?",
+                       (token, time.time(), fid))
+        return token, None
+
+    def restore(self, token):
+        """A trashed set back, under a new ` (2)` name if its own has been taken
+        since. Returns (doc, None)."""
+        with self.store.lock, self._connect() as db, db:
+            r = db.execute("SELECT * FROM filter_sets WHERE trash = ? AND workspace = ?",
+                           (str(token), workspace_of(self.store))).fetchone()
+            if r is None:
+                return None, (404, "no such trash entry")
+            name = unique_filter_set_name(r["name"], self._names(db, but=r["id"]))
+            db.execute("UPDATE filter_sets SET trash = NULL, trashed_at = NULL, name = ? WHERE id = ?",
+                       (name, r["id"]))
+            fid = r["id"]
+        return self.get(fid), None
+
+    @staticmethod
+    def _purge(db, where, args):
+        db.execute(f"DELETE FROM filter_set_versions WHERE set_id IN (SELECT id FROM filter_sets WHERE {where})", args)
+        db.execute(f"DELETE FROM filter_sets WHERE {where}", args)
+
+    def lazy_trash(self):
+        """A filter-sets request empties what has been trashed longer than
+        TRASH_SECONDS, as a datasets request does."""
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
+            self._purge(db, "trash IS NOT NULL AND trashed_at < ?", (time.time() - TRASH_SECONDS,))
+
+    def empty_trash(self):
+        """The startup sweep: a restart has nothing to undo."""
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
+            self._purge(db, "trash IS NOT NULL", ())
+
+
 def make_thumbs(sid, name):
     """JPEGs a browser can render, beside a stored file: a tile-size one
     for every image type, and a full-size render for the formats a browser
@@ -4457,6 +4873,12 @@ class Queue:
             # `dataset` column, read as its one group.
             if "groups" not in cols:
                 db.execute("ALTER TABLE queue ADD COLUMN groups TEXT")
+            # The body of each filter set a run links, by `<id>@<n>` (#336,
+            # docs/pipeline-model.md §18): a run cleans and gates its replies
+            # against these, never the set as it reads later. A row from before
+            # them has none, which is a run that links none.
+            if "filter_sets" not in cols:
+                db.execute("ALTER TABLE queue ADD COLUMN filter_sets TEXT")
             # The workspace a run belongs to (docs/workspaces.md): History is
             # per-workspace, so the list and every id-keyed read filter by it.
             # The worker loop is the one global reader -- it grades every
@@ -4542,30 +4964,35 @@ class Queue:
 
     # ---- submit ---------------------------------------------------------
 
-    def submit(self, run: dict, dataset=None, rerun_of=None, groups=None):
+    def submit(self, run: dict, dataset=None, rerun_of=None, groups=None, filter_sets=None):
         """
         A new queued run. `run` is the run document (docs/pipeline-model.md
         §5): the pipeline, the profiles it resolved to without their keys, its
         content's file list in order and with its repeats, and each eval
         group's version; `groups` is those versions' bodies, by `<id>@<n>`
         (§17), and `dataset` the one body a row from before them kept, which a
-        re-run of one carries on; `rerun_of` is the run a re-run was queued
-        from. Returns the row. Its items are that list, or the one inline
-        text, each through every scenario -- so the total is the list's
-        length, repeats and all, the same count the runner and the page make.
+        re-run of one carries on; `filter_sets` is the body of each Library
+        filter set the run links, by `<id>@<n>` (§18), so a run cleans and
+        gates its replies against what it was submitted with; `rerun_of` is the
+        run a re-run was queued from. Returns the row. Its items are that list,
+        or the one inline text, each through every scenario -- so the total is
+        the list's length, repeats and all, the same count the runner and the
+        page make.
         """
         rid = secrets.token_hex(6)
         now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
         total = len(run_items(run))
         with self.lock, closing(sqlite3.connect(self.store.path)) as db, db:
             db.execute("INSERT INTO queue (id, status, cancel, submitted_at, "
-                       "snapshot, results, progress, totals, dataset, rerun_of, groups, workspace) "
-                       "VALUES (?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                       "snapshot, results, progress, totals, dataset, rerun_of, groups, filter_sets, workspace) "
+                       "VALUES (?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                        (rid, "queued", now, json.dumps(run), "[]",
                         json.dumps({"current": None, "n": 0, "total": total}),
                         json.dumps({"ran": 0, "passed": 0, "found": 0, "of": 0}),
                         None if dataset is None else json.dumps(dataset), rerun_of,
-                        None if groups is None else json.dumps(groups), workspace_of(self.store)))
+                        None if groups is None else json.dumps(groups),
+                        None if filter_sets is None else json.dumps(filter_sets),
+                        workspace_of(self.store)))
             # Its prompts' uses, in the same transaction: a run is in the
             # library the moment it is queued, or not queued at all.
             if self.prompts is not None:
@@ -4614,6 +5041,21 @@ class Queue:
             return None
         return body if raw else upgrade_body(body)
 
+    def filter_sets(self, rid, raw=False):
+        """The bodies of the Library filter sets a readable run links, by
+        `<id>@<n>` (§18), or None for a run that is not there or links none --
+        what its replies were cleaned and gated against, whatever the sets hold
+        now. A body kept at an earlier version reads as one of today's, unless
+        [raw]."""
+        if self.get(rid) is None:
+            return None
+        with self.lock, closing(sqlite3.connect(self.store.path)) as db:
+            r = db.execute("SELECT filter_sets FROM queue WHERE id = ?", (rid,)).fetchone()
+        kept = json.loads(r[0]) if r and r[0] is not None else None
+        if kept is None:
+            return None
+        return kept if raw else {k: upgrade_filter_set_body(b) for k, b in kept.items()}
+
     def _plugin_args(self, run):
         """The worker's --plugins, when the run recorded any; a run whose
         plugins have changed since is refused, naming them."""
@@ -4674,6 +5116,28 @@ class Queue:
         (rundir / "groups.json").write_text(json.dumps(kept))
         return ["--groups", str(rundir / "groups.json")], None
 
+    def _filter_args(self, run, rundir):
+        """The worker's --filter-sets for a run that links a Library filter set
+        in Responses: the body it kept for each, by `<id>@<n>`, written beside
+        the run document, so it cleans and gates its replies against what it
+        was submitted with (§18). A private (`own`) set carries its body in the
+        document, so it needs nothing here, and a run that links no Library set
+        passes no --filter-sets. Returns (args, None) or (None, why)."""
+        refs = [filter_set_ref(link) for link in filter_set_links(run["snapshot"])]
+        keys = {group_key(r) for r in refs if r is not None}
+        if not keys:
+            return [], None
+        kept = self.filter_sets(run["id"], raw=True)
+        if kept is None:
+            return None, "the run kept no filter-set bodies"
+        missing = sorted({(r.get("name") or r.get("id")) for r in refs
+                          if r is not None and group_key(r) not in kept})
+        if missing:
+            return None, f"the run kept no body of the filter set {', '.join(missing)}"
+        rundir.mkdir(parents=True, exist_ok=True)
+        (rundir / "filter-sets.json").write_text(json.dumps(kept))
+        return ["--filter-sets", str(rundir / "filter-sets.json")], None
+
     def _behind(self, rid):
         """How many submissions stand between this one and the worker, by
         submit time -- what a waiting form names when it says what it is
@@ -4766,8 +5230,10 @@ class Queue:
         if err:
             return None, (409, err)
         # The group bodies the original kept: a re-run grades with exactly
-        # them, whatever the groups or the pins read now.
+        # them, whatever the groups or the pins read now; its filter-set
+        # bodies travel the same way, so it cleans and gates as the original did.
         kept = self._kept(rid)
+        kept_filters = self.filter_sets(rid, raw=True)
         ref = evals_dataset(snap)
         body = None
         if ref is not None and kept is None:
@@ -4786,7 +5252,7 @@ class Queue:
         _, err = worker_destinations(snap)
         if err:
             return None, (403, err)
-        return self.submit(snap, body, rerun_of=rid, groups=kept), None
+        return self.submit(snap, body, rerun_of=rid, groups=kept, filter_sets=kept_filters), None
 
     def rerun_item(self, rid, index):
         """
@@ -4821,7 +5287,10 @@ class Queue:
         plugins, err = self._plugin_args(run)
         if err:
             return None, (409, err)
-        dataset = [*dataset, *plugins]
+        filters, err = self._filter_args(run, rundir)
+        if err:
+            return None, (409, err)
+        dataset = [*dataset, *plugins, *filters]
         results = [r for r in run["results"] if r is not None]
         (rundir / "results.json").write_text(json.dumps(results))
         args = [NODE, str(HERE / "run-evals.js"), "--run", str(rundir / "run.json"),
@@ -5003,7 +5472,10 @@ class Queue:
         plugins, err = self._plugin_args(run)
         if err:
             return self._finish(rid, "failed", error=err)
-        dataset = [*dataset, *plugins]
+        filters, err = self._filter_args(run, rundir)
+        if err:
+            return self._finish(rid, "failed", error=err)
+        dataset = [*dataset, *plugins, *filters]
         results = [r for r in run["results"] if r is not None]
         (rundir / "results.json").write_text(json.dumps(results))
         progress_path = rundir / "progress.jsonl"
@@ -5364,10 +5836,12 @@ def worker_destinations(run: dict):
 # 12: a Contains metric's Ignore case holds item by item too, kept as written.
 # 13: an eval is a link to an eval group, or a group of the pipeline's own,
 # and the document has an overall pass rule (docs/pipeline-model.md §17).
-PIPELINE_VERSION = 13
+# 14: a job's Responses stage links filter sets on a `filters` step, and the
+# inline Drop items / Reject rules migrate into a private one (§18, #339).
+PIPELINE_VERSION = 14
 # What a stored run may be: the current version, and the ones evals-core.ts's
 # upgradePipeline reads. A new submission is upgraded to the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)
 TARGET_CAP = 4
 # Target steps whose words the Prompt library does not record as a use: they
 # ask no model (evals-core.ts's STEP_TYPES.echo).
@@ -5732,6 +6206,7 @@ if DATA_DIR:
     SOURCES = Sources(STORE)
     PROMPTS = Prompts(STORE)
     DATASETS = Datasets(STORE, PROMPTS)
+    FILTER_SETS = FilterSets(STORE)
     PACKS = Packs(STORE)
     PLUGINS = Plugins(STORE)
     CONNECTIONS = Connections(STORE)
@@ -5740,7 +6215,8 @@ if DATA_DIR:
     QUEUE = Queue(STORE)
     QUEUE.prompts = PROMPTS
 else:
-    STORE = WORKSPACES = SOURCES = PROMPTS = DATASETS = PACKS = PLUGINS = CONNECTIONS = MICROSOFT = GOOGLE = QUEUE = None
+    STORE = WORKSPACES = SOURCES = PROMPTS = DATASETS = FILTER_SETS = PACKS = PLUGINS = \
+        CONNECTIONS = MICROSOFT = GOOGLE = QUEUE = None
 
 
 class NoRedirects(urllib.request.HTTPRedirectHandler):
@@ -5995,6 +6471,14 @@ class Handler(BaseHTTPRequestHandler):
             if QUEUE is None or QUEUE.get(run_id, self.ws) is None:
                 return self._send(404, b"not found", "text/plain")
             return self._json(200, QUEUE.dataset(run_id))
+        if path.startswith("/api/queue/") and path.endswith("/filter-sets") and path.count("/") == 4:
+            # The bodies of the filter sets a run cleaned and gated its replies
+            # against, by `<id>@<n>` (§18); null for a run that linked none, as
+            # /dataset and /groups answer.
+            run_id = path.split("/")[3]
+            if QUEUE is None or QUEUE.get(run_id, self.ws) is None:
+                return self._send(404, b"not found", "text/plain")
+            return self._json(200, QUEUE.filter_sets(run_id))
         if path.startswith("/api/queue/"):
             if QUEUE is None:
                 return self._send(404, b"not found", "text/plain")
@@ -6093,6 +6577,8 @@ class Handler(BaseHTTPRequestHandler):
             })
         if path.startswith("/api/datasets"):
             return self._datasets_get(path)
+        if path.startswith("/api/filter-sets"):
+            return self._filter_sets_get(path)
         if path == "/api/prompts" or path.startswith("/api/prompts/"):
             return self._prompts_get(path)
         if path.startswith("/api/sources"):
@@ -6194,6 +6680,14 @@ class Handler(BaseHTTPRequestHandler):
             if err:
                 return self._json(err[0], {"error": err[1]})
             return self._json(200, {"trash": token})
+        if path.startswith("/api/filter-sets/"):
+            parts = path.split("/")
+            if FILTER_SETS is None or len(parts) != 4:
+                return self._send(404, b"not found", "text/plain")
+            token, err = FILTER_SETS.remove(parts[3])
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(200, {"trash": token})
         if path.startswith("/api/prompts/"):
             parts = path.split("/")
             if PROMPTS is None or len(parts) != 4:
@@ -6244,6 +6738,14 @@ class Handler(BaseHTTPRequestHandler):
             if err:
                 return self._json(err[0], {"error": err[1]})
             return self._json(200, dataset)
+        if parts[2] == "filter-sets" and FILTER_SETS is not None:
+            payload, err = self._dataset_payload(DATASET_CAP, "a filter set")
+            if err:
+                return err()
+            fs, err = FILTER_SETS.rename(parts[3], payload.get("name") if isinstance(payload, dict) else None)
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(200, fs)
         if parts[2] == "sources" and SOURCES is not None:
             payload = self._payload() or {}
             source, err = SOURCES.rename(parts[3], payload.get("name"))
@@ -6318,6 +6820,17 @@ class Handler(BaseHTTPRequestHandler):
                 code, said = err
                 return self._json(code, said if isinstance(said, dict) else {"error": said})
             return self._json(200, saved)
+        if len(parts) == 4 and parts[:3] == ["", "api", "filter-sets"] and FILTER_SETS is not None:
+            payload, err = self._dataset_payload(DATASET_CAP, "a filter set")
+            if err:
+                return err()
+            if not isinstance(payload, dict):
+                return self._json(400, {"error": "a save is { version, body }"})
+            saved, err = FILTER_SETS.save(parts[3], payload.get("version"), payload.get("body"))
+            if err:
+                code, said = err
+                return self._json(code, said if isinstance(said, dict) else {"error": said})
+            return self._json(200, saved)
         if len(parts) != 4 or parts[:3] != ["", "api", "datasets"] or DATASETS is None:
             return self._send(404, b"not found", "text/plain")
         payload, err = self._dataset_payload(DATASET_CAP)
@@ -6356,6 +6869,8 @@ class Handler(BaseHTTPRequestHandler):
             return self._queue_action(path)
         if path.startswith("/api/datasets"):
             return self._datasets_post(path)
+        if path.startswith("/api/filter-sets"):
+            return self._filter_sets_post(path)
         if path == "/api/packs" or path.startswith("/api/packs/"):
             return self._packs_post(path)
         if path == "/api/workspaces" or path.startswith("/api/workspaces/"):
@@ -6629,10 +7144,14 @@ class Handler(BaseHTTPRequestHandler):
                                        for n, d in docs.items()}, self.ws)
         if stale is not None:
             return self._json(409, {"stale": stale})
-        # A pin names a group's version, so the group keeps that version from
-        # the moment a pipeline pins it (§17).
-        if "promptlab.workflows" in docs and DATASETS is not None:
-            DATASETS.mint_pinned(pins_in(docs["promptlab.workflows"].get("body")))
+        # A pin names a group's or filter set's version, so it keeps that
+        # version from the moment a pipeline pins it (§17, §18).
+        if "promptlab.workflows" in docs:
+            body = docs["promptlab.workflows"].get("body")
+            if DATASETS is not None:
+                DATASETS.mint_pinned(pins_in(body))
+            if FILTER_SETS is not None:
+                FILTER_SETS.mint_pinned(filter_set_pins_in(body))
         return self._json(200, {"versions": versions})
 
     # ---- The run queue (#530) ----------------------------------------------
@@ -6698,9 +7217,29 @@ class Handler(BaseHTTPRequestHandler):
                 return self._json(400, {"error": f"the run cannot be queued: {body}"})
             ref["n"], ref["version"] = n, fingerprint(body)
             groups[group_key(ref)] = body
+        # Each Library filter set the run links, resolved the same way (§18): a
+        # pinned link to its pin, anything else to the set's newest version. The
+        # reference records the version and its fingerprint, and the body is
+        # kept with the row under `<id>@<n>` -- the run cleans and gates its
+        # replies against that, and the version is frozen from now on. A private
+        # (`own`) link carries its body in the document, so it reads no store.
+        filter_sets = {}
+        for link in filter_set_links(run):
+            ref = filter_set_ref(link)
+            if ref is None:
+                continue
+            pin = link.get("pin")
+            got = FILTER_SETS.resolve(ref.get("id"), pin if type(pin) is int else None) if FILTER_SETS else None
+            if got is None:
+                return self._json(400, {"error": "the run links no filter set this lab has"})
+            n, body = got
+            if n is None:
+                return self._json(400, {"error": f"the run cannot be queued: {body}"})
+            ref["n"], ref["version"] = n, fingerprint(body)
+            filter_sets[group_key(ref)] = body
         # The worker reads a body per group now (#233), so a run may link
         # several; each body is kept with the row under `<id>@<n>`.
-        return self._json(201, {"run": QUEUE.submit(run, groups=groups)})
+        return self._json(201, {"run": QUEUE.submit(run, groups=groups, filter_sets=filter_sets or None)})
 
     # ---- Datasets ----------------------------------------------------------
     # Rows in the store (Datasets above). The page reads one by id; an export
@@ -6867,6 +7406,58 @@ class Handler(BaseHTTPRequestHandler):
             return self._json(201, dataset)
         return self._send(404, b"not found", "text/plain")
 
+    # ---- Filter sets -------------------------------------------------------
+    # Rows in the store (FilterSets above), read and written as datasets are
+    # (docs/pipeline-model.md §18); no export or import, which datasets have for
+    # CI. The page reads one by id, lists them, and keeps its versions.
+
+    def _filter_sets_get(self, path):
+        if FILTER_SETS is None:
+            return self._send(404, b"not found", "text/plain")
+        FILTER_SETS.lazy_trash()
+        parts = path.split("/")
+        if len(parts) == 3:
+            return self._json(200, {"filterSets": FILTER_SETS.list()})
+        if len(parts) == 5 and parts[4] == "versions":
+            got = FILTER_SETS.versions(parts[3])
+            return self._json(200, {"versions": got}) if got is not None else self._send(404, b"not found", "text/plain")
+        if len(parts) == 6 and parts[4] == "versions" and parts[5].isdigit():
+            got = FILTER_SETS.version_body(parts[3], int(parts[5]))
+            return self._json(200, {"n": int(parts[5]), "body": got}) if got is not None \
+                else self._send(404, b"not found", "text/plain")
+        if len(parts) == 4:
+            d = FILTER_SETS.get(parts[3])
+            return self._json(200, d) if d else self._send(404, b"not found", "text/plain")
+        return self._send(404, b"not found", "text/plain")
+
+    def _filter_sets_post(self, path):
+        if FILTER_SETS is None:
+            return self._send(404, b"not found", "text/plain")
+        parts = path.split("/")
+        if len(parts) == 6 and parts[3] == "trash" and parts[5] == "restore":
+            self._payload()
+            fs, err = FILTER_SETS.restore(parts[4])
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(200, fs)
+        if len(parts) == 7 and parts[4] == "versions" and parts[6] == "restore" and parts[5].isdigit():
+            self._payload()
+            fs, err = FILTER_SETS.restore_version(parts[3], int(parts[5]))
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(200, fs)
+        if len(parts) == 3:
+            payload, err = self._dataset_payload(DATASET_CAP, "a filter set")
+            if err:
+                return err()
+            if not isinstance(payload, dict):
+                return self._json(400, {"error": "a new filter set is { name, body? }"})
+            fs, err = FILTER_SETS.create(payload.get("name"), payload.get("body"))
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(201, fs)
+        return self._send(404, b"not found", "text/plain")
+
     def _queue_action(self, path):
         # Drained, so a keep-alive connection is not left holding the
         # page's `{}` in front of its next request.
@@ -7164,6 +7755,8 @@ def main():
         print(f"datasets:  {len(DATASETS.list())} in the store", flush=True)
     else:
         print("datasets:  (none -- datasets need DATA_DIR)", flush=True)
+    if FILTER_SETS is not None:
+        FILTER_SETS.empty_trash()
     if PROMPTS is not None:
         PROMPTS.empty_trash()
         # The runs from before the library, read into it once.