evals-lab 0.13.0 → 0.14.0

package/lab/server.py

@@ -375,13 +375,13 @@ _CALL_UPLOADS = {".txt": "text/plain; charset=utf-8",
                  ".csv": "text/csv; charset=utf-8",
                  ".md": "text/markdown; charset=utf-8"}
 SOURCE_PROVIDERS = {
-    "http-get": {"label": "HTTP GET", "noun": "call", "tier": "pull", "flag": None,
+    "http-get": {"label": "HTTP GET", "noun": "response", "tier": "pull", "flag": None,
                  "uploads": dict(_CALL_UPLOADS)},
     # The API call provider (docs/dynamic-sources.md § 2): HTTP GET plus a verb,
     # headers and a body, keyed from a connection -- the same relay fetch, one
     # layer up. `produce` and `take` are attached below, beside http-get's.
     # `keyed`: it sends a connection's key, so a pack may not carry one (read_pack).
-    "api": {"label": "API call", "noun": "call", "tier": "pull", "flag": None, "keyed": True,
+    "api": {"label": "API call", "noun": "response", "tier": "pull", "flag": None, "keyed": True,
             "uploads": dict(_CALL_UPLOADS), "longConfig": {"body": 8000}},
     # Supplied has no `produce`: the caller hands items in (the CLI's --items, or
     # an Upload… by hand), so the server fetches and executes nothing -- it only
@@ -668,7 +668,44 @@ def take_record(name, data):
     return json.dumps(kept, indent=2).encode("utf-8"), None
 
 
+def describe_record(data):
+    """A kept record's first response, as the Calls table says it (#587):
+    one from a run was Recorded when production ran it; one written by hand
+    was Added, an uploaded one Uploaded, and an older Edited one Edited --
+    each with the reply's status, when it has one. None for bytes that are
+    not a record."""
+    try:
+        value = json.loads(data.decode("utf-8"))
+    except (UnicodeDecodeError, ValueError):
+        return None
+    if not isinstance(value, dict):
+        return None
+    origin = (value.get("origin") or {}).get("kind")
+    res = value.get("result") if isinstance(value.get("result"), dict) else {}
+    status = res.get("status") if isinstance(res.get("status"), int) else None
+    kind = {"run": "recorded", "hand": "added", "edited": "edited"}.get(origin, "uploaded")
+    return {"kind": kind, "status": status,
+            "at": value.get("at") if kind == "recorded" and isinstance(value.get("at"), str) else None}
+
+
+def record_key(data):
+    """flows/record.ts's recordKey for a call from a run: its step, run and
+    iteration, kept by an edited record too, so Import runs never brings
+    production's call back beside the edit. None for a record from no run,
+    and for bytes that are not one."""
+    try:
+        value = json.loads(data.decode("utf-8"))
+    except (UnicodeDecodeError, ValueError):
+        return None
+    origin = value.get("origin") if isinstance(value, dict) else None
+    if not isinstance(origin, dict) or not origin.get("run"):
+        return None
+    return json.dumps({"step": value.get("step"), "run": origin["run"], "iteration": origin.get("iteration") or ""})
+
+
 WORKFLOW_PLATFORMS["power-automate"]["take"] = take_record
+WORKFLOW_PLATFORMS["power-automate"]["describe"] = describe_record
+WORKFLOW_PLATFORMS["power-automate"]["key"] = record_key
 
 
 # ---- A dynamic Source's Calls ------------------------------------------------
@@ -1859,6 +1896,9 @@ class Workspaces:
                        "(SELECT id FROM sources WHERE workspace = ? AND system = 0)", (ws,))
             db.execute("DELETE FROM source_definitions WHERE source IN "
                        "(SELECT id FROM sources WHERE workspace = ? AND system = 0)", (ws,))
+            for table in ("source_calls", "source_responses"):
+                db.execute(f"DELETE FROM {table} WHERE source IN "
+                           "(SELECT id FROM sources WHERE workspace = ? AND system = 0)", (ws,))
             db.execute("DELETE FROM sources WHERE workspace = ? AND system = 0", (ws,))
             db.execute("DELETE FROM eval_group_versions WHERE group_id IN "
                        "(SELECT id FROM datasets WHERE workspace = ?)", (ws,))
@@ -2046,6 +2086,17 @@ class Sources:
                        "origin TEXT NOT NULL, at TEXT NOT NULL, "
                        "provider TEXT, request TEXT, status INTEGER, error TEXT, file TEXT, "
                        "PRIMARY KEY (source, name))")
+            # An item's responses (#587): each fetch, import or edit of a
+            # Call or a record kept with its bytes, so Restore can make an
+            # earlier one current again. The file in source_files is the
+            # current one's bytes, and what a run grades; these rows are
+            # only read back by Restore. An item that has had one response
+            # has no rows yet: its first is written when a second arrives.
+            db.execute("CREATE TABLE IF NOT EXISTS source_responses ("
+                       "source TEXT NOT NULL, name TEXT NOT NULL, n INTEGER NOT NULL, "
+                       "at TEXT NOT NULL, kind TEXT NOT NULL, status INTEGER, body BLOB NOT NULL, "
+                       "current INTEGER NOT NULL DEFAULT 0, "
+                       "PRIMARY KEY (source, name, n))")
             db.execute("INSERT OR IGNORE INTO sources (id, name, system, bytes, created_at) "
                        "VALUES (?, 'Sample library', 1, 0, ?)",
                        (self.SYSTEM_ID, time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())))
@@ -2256,10 +2307,11 @@ class Sources:
             if m.get("kind") == "source" and isinstance(m.get("source"), str):
                 held.add(m["source"])
         with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
-            for (sid,) in db.execute("SELECT DISTINCT source FROM source_definitions "
-                                     "WHERE source NOT IN (SELECT id FROM sources)").fetchall():
-                if sid not in held:
-                    db.execute("DELETE FROM source_definitions WHERE source = ?", (sid,))
+            for table in ("source_definitions", "source_calls", "source_responses"):
+                for (sid,) in db.execute(f"SELECT DISTINCT source FROM {table} "
+                                         "WHERE source NOT IN (SELECT id FROM sources)").fetchall():
+                    if sid not in held:
+                        db.execute(f"DELETE FROM {table} WHERE source = ?", (sid,))
 
     def type_of(self, sid):
         """A Source's enforcement entry -- its kind's, or its workflow
@@ -2446,6 +2498,16 @@ class Sources:
             if missing:
                 return None, (404, f"{missing[0]!r} is not in that Source")
             total = sum(rows[n][0] for n in names)
+            marks = ",".join("?" * len(names))
+            # A Call's provenance row and an item's responses go to the trash
+            # with its file, so Undo puts the item back as it was.
+            calls = [list(r) for r in db.execute(
+                "SELECT name, origin, at, provider, request, status, error, file FROM source_calls "
+                f"WHERE source = ? AND file IN ({marks})", (sid, *names))]
+            responses = [[r[0], r[1], r[2], r[3], r[4], base64.b64encode(r[5]).decode("ascii"), r[6]]
+                         for r in db.execute(
+                             "SELECT name, n, at, kind, status, body, current FROM source_responses "
+                             f"WHERE source = ? AND name IN ({marks})", (sid, *names))]
             with db:
                 for name in names:
                     db.execute("DELETE FROM source_files WHERE source = ? AND name = ?",
@@ -2454,6 +2516,8 @@ class Sources:
                     # with it (a File Library has no such rows).
                     db.execute("DELETE FROM source_calls WHERE source = ? AND file = ?",
                                (sid, name))
+                    db.execute("DELETE FROM source_responses WHERE source = ? AND name = ?",
+                               (sid, name))
                 db.execute("UPDATE sources SET bytes = bytes - ? WHERE id = ?",
                            (total, sid))
         token = secrets.token_hex(6)
@@ -2462,7 +2526,8 @@ class Sources:
         (entry / "thumbs").mkdir(parents=True)
         (entry / "manifest.json").write_text(json.dumps({
             "kind": "files", "source": sid, "at": time.time(),
-            "files": [{"name": n, "bytes": rows[n][0], "at": rows[n][1]} for n in names]}))
+            "files": [{"name": n, "bytes": rows[n][0], "at": rows[n][1]} for n in names],
+            "calls": calls, "responses": responses}))
         src = self.dir / sid
         for name in names:
             shutil.move(str(src / name), str(entry / "files" / name))
@@ -2527,6 +2592,12 @@ class Sources:
             if r["file"] and r["file"] in sizes:
                 call["bytes"] = sizes[r["file"]]
             out.append(call)
+        # When and how its current response came: the Updated column (#587).
+        held = self.responses(sid) or {}
+        for call in out:
+            now = next((x for x in held.get(call["name"], []) if x["current"]), None)
+            if now is not None:
+                call["updated"] = {"kind": now["kind"], "at": now["at"]}
         return out
 
     def _record_call(self, sid, name, origin, provider, request, status, error, file):
@@ -2540,11 +2611,18 @@ class Sources:
                         provider, json.dumps(request) if request else None, status, error, file))
 
     def snapshot(self, sid):
-        """Produce one Call now and keep it (Snapshot now): the server fetches
-        through the provider, the `take` hook refuses a token and redacts, and
-        the content is kept as a text file graded as an item. A failed fetch is
-        kept as a Call with its error, not dropped. Returns (call, None), or
-        (None, error)."""
+        """Fetch now (#587): produce the Source's Call through the provider,
+        the `take` hook refusing a token and redacting, and keep it as a
+        version of the response its name says (#596). A response already held
+        gains the version and keeps its row, whatever was done to it -- an edit
+        is a version too, so nothing forks -- unless the bytes are the current
+        version's, which adds nothing. A name not held is a new response. A
+        failed fetch leaves a response's current version as it is; one with
+        no response yet is kept as a Call with its error, not dropped.
+        Returns ({name, outcome, ...}, None) -- outcome `fetched`,
+        `unchanged`, `added` or `failed`, and for `fetched` the response it
+        replaced (`was`) and the one it added (`n`), what Undo hands back --
+        or (None, error)."""
         got, err = self._dynamic_entry(sid)
         if err:
             return None, err
@@ -2559,46 +2637,231 @@ class Sources:
         request = result.get("request")
         status = result.get("status")
         content = result.get("content")
-        if content is None:
-            # A failed production: kept as a Call with its error, no file.
-            name = self._snapshot_name(sid, dynamic_item_name(cfg, (request or {}).get("url", "")
-                                                              or secrets.token_hex(6)))
-            self._record_call(sid, name, "snapshot", provider, request, status, result.get("error"), None)
-            return {"name": name, "origin": "snapshot", "at": None, "provider": provider,
-                    "request": request, "status": status, "error": result.get("error")}, None
-        name = self._snapshot_name(sid, dynamic_item_name(cfg, content))
-        kept, take_err = entry["take"](name, content)
-        if take_err:
-            code, message = take_err
-            self._record_call(sid, name, "snapshot", provider, request, status, message, None)
-            return {"name": name, "origin": "snapshot", "at": None, "provider": provider,
-                    "request": request, "status": status, "error": message}, None
-        # The name is stable (the URL's), so a re-snapshot overwrites the item
-        # rather than adding a second: remove the old file first, then add. An
-        # Edited Call is never the one overwritten (_snapshot_name).
-        self.remove_files(sid, [name])
-        stored, add_err = self.add_file(sid, name, kept)
-        if add_err:
-            return None, add_err
-        self._record_call(sid, stored, "snapshot", provider, request, status, None, stored)
-        return {"name": stored, "origin": "snapshot", "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
-                "provider": provider, "request": request, "status": status, "bytes": len(kept)}, None
-
-    def _snapshot_name(self, sid, name):
-        """The name a snapshot is kept under: [name], unless an Edited Call
-        holds it -- a snapshot never overwrites an edit -- and then the first
-        of `<stem>-2`, `<stem>-3`... that none does. One that is not Edited is
-        overwritten, as the stable name always was."""
+        failure = result.get("error")
+        name = kept = None
+        if content is not None:
+            name = dynamic_item_name(cfg, content)
+            kept, take_err = entry["take"](name, content)
+            if take_err:
+                failure = take_err[1]
+        else:
+            name = dynamic_item_name(cfg, (request or {}).get("url", "") or secrets.token_hex(6))
+        path = self.file_path(sid, name)
+        held = path is not None and path.is_file()
+        if failure is not None:
+            if not held:
+                self._record_call(sid, name, "snapshot", provider, request, status, failure, None)
+            return {"name": name, "outcome": "failed", "provider": provider,
+                    "request": request, "status": status, "error": failure}, None
+        if not held:
+            stored, add_err = self.add_file(sid, name, kept)
+            if add_err:
+                return None, add_err
+            self._record_call(sid, stored, "snapshot", provider, request, status, None, stored)
+            return {"name": stored, "outcome": "added", "provider": provider, "request": request,
+                    "status": status, "bytes": len(kept)}, None
+        if path.read_bytes() == kept:
+            return {"name": name, "outcome": "unchanged", "status": status}, None
+        added, err = self._respond(sid, name, kept, "fetched", status)
+        if err:
+            return None, err
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
+            db.execute("UPDATE source_calls SET request = ?, provider = ?, error = NULL, file = ? "
+                       "WHERE source = ? AND name = ?",
+                       (json.dumps(request) if request else None, provider, name, sid, name))
+        return {"name": name, "outcome": "fetched", "provider": provider, "request": request,
+                "status": status, "bytes": len(kept), **added}, None
+
+    # ---- an item's responses (#587) -----------------------------------------
+    #
+    # A Call or a record keeps its responses: each fetch, import or edit is
+    # one, kept with its bytes, and the newest -- or the one Restore picks --
+    # is current: its bytes are the file a run grades. Nothing forks; a copy
+    # (Clone) is a new item with a response of its own.
+
+    # How many responses an item keeps: the oldest that is neither current nor
+    # pinned by a pipeline (#596) goes.
+    RESPONSES_KEPT = 20
+    # The kind of response a Call's provenance row says its file was.
+    ORIGIN_KINDS = {"snapshot": "fetched", "run": "fetched", "edited": "edited",
+                    "upload": "uploaded", "hand": "added", "copy": "copied"}
+    RESPONSE_KINDS = ("fetched", "recorded", "uploaded", "added", "edited", "copied")
+
+    def _described(self, sid, name, data):
+        """What an item's bytes were before it kept responses: from its Call's
+        provenance row, else from its kind's `describe` hook (a record says
+        whether production recorded it), else an upload, as of the file."""
         with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
-            edited = {r[0] for r in db.execute(
-                "SELECT name FROM source_calls WHERE source = ? AND origin = 'edited'", (sid,))}
-        if name not in edited:
-            return name
-        stem, ext = os.path.splitext(name)
-        n = 2
-        while f"{stem}-{n}{ext}" in edited:
-            n += 1
-        return f"{stem}-{n}{ext}"
+            row = db.execute("SELECT origin, at, status FROM source_calls WHERE source = ? AND name = ?",
+                             (sid, name)).fetchone()
+            f = db.execute("SELECT at FROM source_files WHERE source = ? AND name = ?",
+                           (sid, name)).fetchone()
+        if row is not None:
+            return {"kind": self.ORIGIN_KINDS.get(row[0], "uploaded"), "at": row[1], "status": row[2]}
+        describe = (self.type_of(sid) or {}).get("describe")
+        got = describe(data) if describe else None
+        out = {"kind": "uploaded", "at": f[0] if f else None, "status": None}
+        if isinstance(got, dict):
+            out.update({k: v for k, v in got.items() if v is not None})
+        return out
+
+    def _respond(self, sid, name, data, kind, status):
+        """[data] as [name]'s new current response: the file rewritten in
+        place, as an edit always was, and the row added. An item with no rows
+        yet first gains one for the bytes it held. Returns ({n, was}, None),
+        or (None, error)."""
+        path = self.file_path(sid, name)
+        if path is None or not path.is_file():
+            return None, (404, f"{name} is not in that Source")
+        old = path.read_bytes()
+        first = None
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+            seeded = db.execute("SELECT 1 FROM source_responses WHERE source = ? AND name = ?",
+                                (sid, name)).fetchone() is not None
+        if not seeded:
+            first = self._described(sid, name, old)
+        now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+        folder = self.dir / sid
+        with tempfile.NamedTemporaryFile("wb", dir=folder, delete=False) as f:
+            f.write(data)
+            tmp = f.name
+        try:
+            with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
+                total = db.execute("SELECT bytes FROM sources WHERE id = ?", (sid,)).fetchone()[0]
+                if total + len(data) - len(old) > MAX_SOURCE:
+                    raise _Refused((413, "that would take the source past its cap"))
+                n, was = self._response_rows(db, sid, name, old, first, data, kind, status, now)
+                db.execute("UPDATE source_files SET bytes = ? WHERE source = ? AND name = ?",
+                           (len(data), sid, name))
+                db.execute("UPDATE sources SET bytes = bytes + ? WHERE id = ?", (len(data) - len(old), sid))
+        except _Refused as no:
+            os.unlink(tmp)
+            return None, no.err
+        os.replace(tmp, folder / name)
+        return {"n": n, "was": was}, None
+
+    def _response_rows(self, db, sid, name, old, first, data, kind, status, now):
+        """The rows behind _respond and an edit, inside the caller's
+        transaction: the bytes [name] held as its first response when it had
+        none ([first] describes them), [data] as the new current one, the
+        oldest past RESPONSES_KEPT dropped -- never one a pipeline pins
+        (#596), which does not count against the rest. Returns (n, was)."""
+        if first is not None:
+            db.execute("INSERT OR IGNORE INTO source_responses (source, name, n, at, kind, status, "
+                       "body, current) VALUES (?, ?, 1, ?, ?, ?, ?, 1)",
+                       (sid, name, first["at"] or now, first["kind"], first["status"], old))
+        was = db.execute("SELECT n FROM source_responses WHERE source = ? AND name = ? AND current = 1",
+                         (sid, name)).fetchone()
+        n = db.execute("SELECT COALESCE(MAX(n), 0) + 1 FROM source_responses WHERE source = ? "
+                       "AND name = ?", (sid, name)).fetchone()[0]
+        db.execute("UPDATE source_responses SET current = 0 WHERE source = ? AND name = ?", (sid, name))
+        db.execute("INSERT INTO source_responses (source, name, n, at, kind, status, body, current) "
+                   "VALUES (?, ?, ?, ?, ?, ?, ?, 1)", (sid, name, n, now, kind, status, data))
+        pinned = response_pins(db, sid, name)
+        loose = [r for (r,) in db.execute(
+            "SELECT n FROM source_responses WHERE source = ? AND name = ? AND current = 0 "
+            "ORDER BY n DESC", (sid, name)).fetchall() if r not in pinned]
+        for gone in loose[self.RESPONSES_KEPT - 1:]:
+            db.execute("DELETE FROM source_responses WHERE source = ? AND name = ? AND n = ?", (sid, name, gone))
+        db.execute("UPDATE source_calls SET status = ? WHERE source = ? AND name = ?", (status, sid, name))
+        return n, (was[0] if was else None)
+
+    def pinned_body(self, sid, name, n):
+        """The bytes of [name]'s response [n] a run's pin names (#596), or
+        None when the response is gone. `n` 0 is an item that kept one
+        response and no rows: its file, or once it gained rows, the first,
+        which holds those bytes."""
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+            row = db.execute("SELECT body FROM source_responses WHERE source = ? AND name = ? AND n = ?",
+                             (sid, name, n or 1)).fetchone()
+            seeded = row is not None or db.execute(
+                "SELECT 1 FROM source_responses WHERE source = ? AND name = ?", (sid, name)).fetchone() is not None
+        if row is not None:
+            return bytes(row[0])
+        if n == 0 and not seeded:
+            path = self.file_path(sid, name)
+            return path.read_bytes() if path is not None and path.is_file() else None
+        return None
+
+    def responses(self, sid):
+        """Every item's responses, newest first, by name: {n, at, kind,
+        status, current} -- no bytes. An item that has kept one response has
+        no rows, and reads as that one, `n` 0, described as it was made. None
+        for no such Source."""
+        if self.get(sid) is None:
+            return None
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+            names = [r[0] for r in db.execute("SELECT name FROM source_files WHERE source = ?", (sid,))]
+            rows = db.execute("SELECT name, n, at, kind, status, current FROM source_responses "
+                              "WHERE source = ? ORDER BY n DESC", (sid,)).fetchall()
+        out = {}
+        for name, n, at, kind, status, current in rows:
+            out.setdefault(name, []).append({"n": n, "at": at, "kind": kind, "status": status,
+                                             "current": bool(current)})
+        for name in names:
+            if name not in out:
+                path = self.file_path(sid, name)
+                data = path.read_bytes() if path is not None and path.is_file() else b""
+                out[name] = [{"n": 0, **self._described(sid, name, data), "current": True}]
+        return out
+
+    def restore_response(self, sid, name, n, drop=None):
+        """Makes [name]'s response [n] current: its bytes the file again, the
+        response it kept, not a new one. [drop] removes a response outright as
+        well -- Undo of a fetch or an edit takes back the one it added. Returns
+        ({name, n}, None), or (None, error)."""
+        entry, err = self._calls_entry(sid)
+        if err:
+            return None, err
+        path = self.file_path(sid, name)
+        if path is None or not path.is_file():
+            return None, (404, f"{name} is not in that Source")
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+            row = db.execute("SELECT body, status FROM source_responses WHERE source = ? AND name = ? AND n = ?",
+                             (sid, name, n)).fetchone()
+        if row is None:
+            return None, (404, f"{name} has no such response")
+        data, status = bytes(row[0]), row[1]
+        old = path.read_bytes()
+        folder = self.dir / sid
+        with tempfile.NamedTemporaryFile("wb", dir=folder, delete=False) as f:
+            f.write(data)
+            tmp = f.name
+        try:
+            with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
+                total = db.execute("SELECT bytes FROM sources WHERE id = ?", (sid,)).fetchone()[0]
+                if total + len(data) - len(old) > MAX_SOURCE:
+                    raise _Refused((413, "that would take the source past its cap"))
+                db.execute("UPDATE source_responses SET current = (n = ?) WHERE source = ? AND name = ?",
+                           (n, sid, name))
+                if drop is not None and drop != n:
+                    db.execute("DELETE FROM source_responses WHERE source = ? AND name = ? AND n = ?",
+                               (sid, name, drop))
+                db.execute("UPDATE source_files SET bytes = ? WHERE source = ? AND name = ?",
+                           (len(data), sid, name))
+                db.execute("UPDATE sources SET bytes = bytes + ? WHERE id = ?", (len(data) - len(old), sid))
+                db.execute("UPDATE source_calls SET status = ? WHERE source = ? AND name = ?",
+                           (status, sid, name))
+        except _Refused as no:
+            os.unlink(tmp)
+            return None, no.err
+        os.replace(tmp, folder / name)
+        return {"name": name, "n": n}, None
+
+    def key_held(self, sid, key, value):
+        """The name of the file of [sid] whose [key] (its kind's `key` hook)
+        is [value], or None: a record from a run already held is not kept a
+        second time, however it was edited since."""
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+            names = [r[0] for r in db.execute("SELECT name FROM source_files WHERE source = ?", (sid,))]
+        for name in names:
+            path = self.file_path(sid, name)
+            try:
+                if path is not None and key(path.read_bytes()) == value:
+                    return name
+            except OSError:
+                continue
+        return None
 
     # ---- a Call opened in the Call dialog: edited, renamed, cloned (#438) ---
     #
@@ -2608,7 +2871,7 @@ class Sources:
     # the rest redacted. A File Library has no hook and no Calls.
 
     # The origins a dynamic Call's provenance row may be put back to (Undo).
-    CALL_ORIGINS = ("snapshot", "run", "upload", "hand", "edited")
+    CALL_ORIGINS = ("snapshot", "run", "upload", "hand", "edited", "copy")
 
     def _calls_entry(self, sid):
         """The enforcement entry of a user Source whose files are Calls, or
@@ -2636,11 +2899,13 @@ class Sources:
         the Source's `take` hook as an upload does. A rename keeps the
         extension and refuses a name in use, or a Call that a dataset's cases
         name ([named_by] answers which datasets do), since a case finds its
-        item by name. A dynamic Call's origin becomes `edited` when its text
-        changed -- a rename alone is no edit -- unless [origin] puts back the
-        one it had (Undo). Nothing is written unless all of it passes.
-        Returns ({name, origin, bytes}, None), or (None, (code, sentence)) --
-        (code, sentence, "name") when the Name field is what is wrong."""
+        item by name. Changed text is a new response, `edited` -- a rename
+        alone is no edit -- and the item's responses follow a rename. [origin]
+        puts back a dynamic Call's provenance (an older page's Undo). Nothing
+        is written unless all of it passes. Returns ({name, origin, bytes} and,
+        for an edit, the response it added `n` and the one it replaced `was`,
+        None), or (None, (code, sentence)) -- (code, sentence, "name") when
+        the Name field is what is wrong."""
         entry, err = self._calls_entry(sid)
         if err:
             return None, err
@@ -2677,12 +2942,26 @@ class Sources:
             if take_err:
                 return None, take_err
         changed = data != old
+        first = status = None
+        if changed:
+            with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+                seeded = db.execute("SELECT 1 FROM source_responses WHERE source = ? AND name = ?",
+                                    (sid, name)).fetchone() is not None
+                held = db.execute("SELECT status FROM source_calls WHERE source = ? AND name = ?",
+                                  (sid, name)).fetchone()
+            if not seeded:
+                first = self._described(sid, name, old)
+            # An edited reply says its own status (a record's); an edited Call
+            # keeps the status of the fetch it was.
+            described = entry["describe"](data) if entry.get("describe") else None
+            status = (described or {}).get("status") if described is not None else (held[0] if held else None)
         folder = self.dir / sid
         tmp = None
         if changed or new != name:
             with tempfile.NamedTemporaryFile("wb", dir=folder, delete=False) as f:
                 f.write(data)
                 tmp = f.name
+        added = {}
         try:
             with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
                 if new != name and new in self._names_held(db, sid):
@@ -2693,13 +2972,18 @@ class Sources:
                 db.execute("UPDATE source_files SET name = ?, bytes = ? WHERE source = ? AND name = ?",
                            (new, len(data), sid, name))
                 db.execute("UPDATE sources SET bytes = bytes + ? WHERE id = ?", (len(data) - len(old), sid))
+                db.execute("UPDATE source_responses SET name = ? WHERE source = ? AND name = ?", (new, sid, name))
                 row = db.execute("SELECT origin FROM source_calls WHERE source = ? AND name = ?",
                                  (sid, name)).fetchone()
                 now = None
                 if row is not None:
-                    now = origin or ("edited" if changed else row[0])
+                    now = origin or row[0]
                     db.execute("UPDATE source_calls SET name = ?, file = ?, origin = ? "
                                "WHERE source = ? AND name = ?", (new, new, now, sid, name))
+                if changed:
+                    n, was = self._response_rows(db, sid, new, old, first, data, "edited", status,
+                                                 time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
+                    added = {"n": n, "was": was}
         except _Refused as no:
             if tmp:
                 os.unlink(tmp)
@@ -2708,13 +2992,14 @@ class Sources:
             os.replace(tmp, folder / new)
             if new != name:
                 (folder / name).unlink(missing_ok=True)
-        return {"name": new, "origin": now, "bytes": len(data)}, None
+        return {"name": new, "origin": now, "bytes": len(data), **added}, None
 
     def clone_call(self, sid, name):
-        """A copy of one Call under `<stem>-copy` (then `-copy-2`, ...), its
-        file and, for a dynamic Call, its provenance -- origin, request,
-        status and when it was produced -- as they are. Returns ({name}, None),
-        or (None, error)."""
+        """A copy of one Call under `<stem>-copy` (then `-copy-2`, ...): its
+        current bytes as the copy's one response, `copied`, and for a dynamic
+        Call its request and provider, its origin `copy` -- so a fetch never
+        reaches it and a live run keeps it as saved (#587). Returns ({name},
+        None), or (None, error)."""
         entry, err = self._calls_entry(sid)
         if err:
             return None, err
@@ -2731,13 +3016,18 @@ class Sources:
             if copy not in held:
                 break
             n += 1
-        stored, err = self.add_file(sid, copy, path.read_bytes())
+        data = path.read_bytes()
+        stored, err = self.add_file(sid, copy, data)
         if err:
             return None, err
+        now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
         with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
             db.execute("INSERT INTO source_calls (source, name, origin, at, provider, request, status, error, file) "
-                       "SELECT source, ?, origin, at, provider, request, status, error, ? "
-                       "FROM source_calls WHERE source = ? AND name = ?", (stored, stored, sid, name))
+                       "SELECT source, ?, 'copy', ?, provider, request, status, error, ? "
+                       "FROM source_calls WHERE source = ? AND name = ?", (stored, now, stored, sid, name))
+            # A copy starts its own responses: a fetch never reaches it.
+            db.execute("INSERT INTO source_responses (source, name, n, at, kind, status, body, current) "
+                       "VALUES (?, ?, 1, ?, 'copied', NULL, ?, 1)", (sid, stored, now, data))
         return {"name": stored}, None
 
     def remove_calls(self, sid, names):
@@ -2833,6 +3123,14 @@ class Sources:
                                "VALUES (?, ?, ?, ?)",
                                (sid, f["name"], f.get("bytes", 0), f.get("at", "")))
                 db.execute("UPDATE sources SET bytes = bytes + ? WHERE id = ?", (total, sid))
+                # An entry from before calls and responses were trashed has neither.
+                for c in m.get("calls") or []:
+                    db.execute("INSERT OR REPLACE INTO source_calls (source, name, origin, at, provider, "
+                               "request, status, error, file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (sid, *c))
+                for r in m.get("responses") or []:
+                    db.execute("INSERT OR REPLACE INTO source_responses (source, name, n, at, kind, status, "
+                               "body, current) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                               (sid, r[0], r[1], r[2], r[3], r[4], base64.b64decode(r[5]), r[6]))
         th = self.dir / sid / ".thumbs"
         th.mkdir(parents=True, exist_ok=True)
         for f in files:
@@ -3498,6 +3796,26 @@ def pins_in(workflows) -> set:
     return out
 
 
+def response_pins(db, sid, name) -> set:
+    """Every response `n` of Source [sid]'s item [name] a stored pipeline pins
+    (#596), across every workspace's promptlab.workflows document, read in
+    the caller's transaction. A pin of `n` 0 -- an item before it kept rows --
+    pins the first row, which holds those bytes."""
+    out = set()
+    for (body,) in db.execute("SELECT body FROM docs WHERE name = 'promptlab.workflows'"):
+        try:
+            listed = (json.loads(body) or {}).get("list") if body else None
+        except (ValueError, TypeError, AttributeError):
+            continue
+        for w in listed if isinstance(listed, list) else []:
+            c = content_of(w.get("work")) if isinstance(w, dict) else None
+            pin = c.get("pin") if isinstance(c, dict) else None
+            if (isinstance(pin, dict) and isinstance(c.get("ref"), dict) and c["ref"].get("id") == sid
+                    and pin.get("name") == name and type(pin.get("n")) is int):
+                out.add(pin["n"] or 1)
+    return out
+
+
 def group_key(ref) -> str:
     """The key a run keeps a group's body under: `<id>@<n>`, or the id alone
     for a reference from before the lab numbered versions."""
@@ -5191,19 +5509,30 @@ def demo_pack() -> bytes:
 NO_WORDS = {"echo", "recorded", "http"}
 
 
-def pack_connection(ws):
+# The built-in connection types that read a document whole, and the suffixes
+# that are one: evals-core.ts's `documents: true` and DOCUMENT_TYPES (#558).
+DOCUMENT_CONNECTIONS = ("openai-compatible", "anthropic")
+DOCUMENT_SUFFIXES = (".pdf",)
+
+
+def pack_connection(ws, documents=False):
     """The AI Connection a pack's target answers on where it names one this
     lab lacks (#550): the lab's grader, or else the first connection that is
     asked words, either shared with workspace [ws]; None where there is none,
-    so the target keeps its placeholder and the Run bar says to add one."""
+    so the target keeps its placeholder and the Run bar says to add one. A
+    pipeline that sends [documents] looks first among the connections that
+    read one (#560), so the Run bar has nothing to refuse where the lab has
+    one."""
     body = ((STORE.all().get("promptlab.connections") or {}).get("body") or {}) if STORE is not None else {}
     words = [c for c in body.get("list") or []
              if isinstance(c, dict) and isinstance(c.get("id"), str) and c["id"]
              and c.get("type") not in NO_WORDS and STORE.shared("profile", c["id"], ws)]
+    if documents:
+        words = [c for c in words if c.get("type") in DOCUMENT_CONNECTIONS] or words
     return next((c for c in words if c["id"] == body.get("grader")), words[0] if words else None)
 
 
-def repoint_targets(doc: dict, ws) -> None:
+def repoint_targets(doc: dict, ws, documents=False) -> None:
     """A pack pipeline's targets that name a connection this lab does not have,
     pointed at the one pack_connection picks, its profile dropped with it."""
     targets = doc.get("targets")
@@ -5216,7 +5545,7 @@ def repoint_targets(doc: dict, ws) -> None:
         ref = t.get("connection") if isinstance(t, dict) else None
         if not isinstance(ref, dict) or ref.get("id") in have:
             continue
-        pick = pick or pack_connection(ws)
+        pick = pick or pack_connection(ws, documents)
         if pick is None:
             return
         t["connection"] = {"id": pick["id"], "name": pick.get("name") or pick["id"]}
@@ -5611,11 +5940,15 @@ class Packs:
                             ref.clear()
                             ref.update({"id": did, "name": DATASETS.get(did)["name"]})
                 content = content_of(doc)
+                documents = False
                 if isinstance(content, dict) and isinstance(content.get("ref"), dict):
-                    sid = src_ids.get(content["ref"].get("id")) or src_ids.get(content["ref"].get("name"))
+                    ref = content["ref"]
+                    files = pack["sources"].get(ref.get("id")) or pack["sources"].get(ref.get("name")) or []
+                    documents = any(Path(f).suffix.lower() in DOCUMENT_SUFFIXES for f, _ in files)
+                    sid = src_ids.get(ref.get("id")) or src_ids.get(ref.get("name"))
                     if sid:
                         content["ref"] = {"id": sid, "name": SOURCES.get(sid)["name"]}
-                repoint_targets(doc, ws)
+                repoint_targets(doc, ws, documents)
                 new_work.append((key, doc))
             def place(body):
                 body.setdefault("list", [])
@@ -6542,7 +6875,8 @@ class Queue:
             # a dynamic Source graded live is fetched at submit and kept here,
             # by item name, so the worker stays a pure dir-reader -- _materialise
             # writes these into the run dir rather than copying a Source file. A
-            # snapshot run keeps none and reads the Source's stored Calls.
+            # run over a pinned response (#596) keeps that response's bytes here
+            # too; a snapshot run keeps none and reads the Source's stored Calls.
             if "produced" not in cols:
                 db.execute("ALTER TABLE queue ADD COLUMN produced TEXT")
             # The workspace a run belongs to (docs/workspaces.md): History is
@@ -6726,7 +7060,8 @@ class Queue:
     def produced(self, rid):
         """A live dynamic run's produced item bytes, by item name, or None for
         a run that produced none (a snapshot run, or any non-dynamic run). The
-        values are the text the server fetched at submit, kept so a start, a
+        values are the text the server fetched at submit, or the pinned
+        response's (#596), kept so a start, a
         resume and a re-run grade the same bytes (docs/dynamic-sources.md)."""
         with self.lock, closing(sqlite3.connect(self.store.path)) as db:
             r = db.execute("SELECT produced FROM queue WHERE id = ?", (rid,)).fetchone()
@@ -7088,7 +7423,8 @@ class Queue:
         # A live dynamic run carries its items' bytes with the row, fetched at
         # submit (docs/dynamic-sources.md): they are written into the run dir
         # rather than copied from a Source, so the worker still just reads a
-        # directory of files. A snapshot run has none and reads the Source.
+        # directory of files. A pinned run (#596) carries its response's bytes
+        # the same way. A snapshot run has none and reads the Source.
         live = self.produced(run["id"])
         rundir = self.dir / run["id"]
         files_dir = rundir / "files"
@@ -7556,10 +7892,12 @@ def worker_destinations(run: dict):
 # (answer_key), each entry naming the connection whose key it takes (#445).
 # 17: an HTTP Request target has no `api` of its own; its job's flow step's
 # is the one its body is read by (#477).
-PIPELINE_VERSION = 17
+# 18: a dynamic Source's content graded as saved pins one response, `pin:
+# {name, n}`, which a fetch, an edit or a Restore never moves (#596).
+PIPELINE_VERSION = 18
 # What a stored run may be: the current version, and the ones evals-core.ts's
 # upgradePipeline reads. A new submission is upgraded to the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18)
 TARGET_CAP = 4
 # Target steps whose words the Prompt library does not record as a use: they
 # ask no model (evals-core.ts's STEP_TYPES.echo).
@@ -7990,6 +8328,10 @@ def run_problems(run):
                 bad.append("a run over a Source needs its file list")
         elif not isinstance(files, list) or not files or not all(isinstance(f, str) for f in files):
             bad.append("a run over a Source needs its file list")
+        pin = content.get("pin")
+        if pin is not None and not (isinstance(pin, dict) and isinstance(pin.get("name"), str) and pin["name"]
+                                    and type(pin.get("n")) is int and pin["n"] >= 0):
+            bad.append("a Source content's pin names a response as { name, n }")
     elif kind == "text":
         if not isinstance(content.get("text"), str) or not content["text"].strip():
             bad.append("a run over text needs some text")
@@ -9056,7 +9398,18 @@ class Handler(BaseHTTPRequestHandler):
                     live = content.get("live")
                     if live is None:
                         live = str(cfg.get("live")).lower() in ("1", "true", "yes", "on") or not src.get("files")
-                    if live:
+                    if live and content["files"]:
+                        # Live data over held items (#587): each response the
+                        # document names is fetched again under its own name,
+                        # so its case applies; a copy, an upload or an item
+                        # written by hand is graded as saved.
+                        produced, err = self._produce_endpoints(sid, entry, cfg, content["files"])
+                        if err:
+                            return self._json(err[0], {"error": err[1]})
+                        content["live"] = True
+                        content["revs"] = {name: "live:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
+                                           for name, text in produced.items()}
+                    elif live:
                         produced, name, err = self._produce_live(sid, entry, cfg)
                         if err:
                             return self._json(err[0], {"error": err[1]})
@@ -9070,6 +9423,20 @@ class Handler(BaseHTTPRequestHandler):
                             produced[name].encode("utf-8")).hexdigest()}
                     else:
                         content["live"] = False
+                pin = content.get("pin")
+                if not content["live"] and isinstance(pin, dict):
+                    # Saved response pins one (#596): the run grades that
+                    # response's bytes, kept with the row as a live run's are,
+                    # whatever a fetch, an edit or a Restore made current since.
+                    body = SOURCES.pinned_body(sid, pin["name"], pin["n"])
+                    if body is None:
+                        return self._json(400, {"error": f"the response this run grades, {pin['name']}, "
+                                                         "was removed: pick another"})
+                    text = body.decode("utf-8", errors="replace")
+                    produced = {pin["name"]: text}
+                    loops = content.get("loops")
+                    content["files"] = [pin["name"]] * (loops if type(loops) is int and loops >= 1 else 1)
+                    content["revs"] = {pin["name"]: "pin:" + hashlib.sha256(body).hexdigest()}
             if produced is None:
                 # Snapshot (or a non-dynamic Source): pin the Source's own files
                 # at their current revision, the server's to say whatever the
@@ -9139,6 +9506,43 @@ class Handler(BaseHTTPRequestHandler):
         return self._json(201, {"run": QUEUE.submit(run, groups=groups, filter_sets=filter_sets or None,
                                                      produced=produced)})
 
+    @staticmethod
+    def _produce_endpoints(sid, entry, cfg, files):
+        """Every item [files] names, for a live run over a Source that holds
+        some (#587): a response -- a Call fetched from a request -- fetched
+        now from the request it was, under its own name; any other item (a
+        copy, an upload, one by hand) its saved bytes. Returns ({name: text},
+        None), or (None, (code, sentence)) -- a fetch that fails refuses the
+        submit, naming the response."""
+        rows = {c["name"]: c for c in SOURCES.calls(sid) or []}
+        out = {}
+        for name in dict.fromkeys(files):
+            call = rows.get(name) or {}
+            request = call.get("request") if isinstance(call.get("request"), dict) else {}
+            if call.get("origin") in ("snapshot", "run", "edited") and request.get("url"):
+                # The request it was: a response whose Source has moved on
+                # since is still asked what it was asked.
+                result = entry["produce"]({**cfg, **{k: request[k] for k in ("url", "method")
+                                                      if isinstance(request.get(k), str)}})
+                if result.get("refused"):
+                    return None, result["refused"]
+                if result.get("content") is None:
+                    return None, (502, f"the live fetch of {name} produced nothing: "
+                                       + (result.get("error") or "unknown"))
+                kept, take_err = entry["take"](name, result["content"])
+                if take_err:
+                    return None, take_err
+                out[name] = kept.decode("utf-8")
+                continue
+            path = SOURCES.file_path(sid, name)
+            if path is None or not path.is_file():
+                return None, (400, f"{name!r} is not in that Source")
+            try:
+                out[name] = path.read_text("utf-8")
+            except (OSError, UnicodeDecodeError):
+                return None, (400, f"{name} is not text")
+        return out, None
+
     @staticmethod
     def _produce_live(sid, entry, cfg):
         """Produce a live dynamic Source's item now, for a run being queued:
@@ -9507,14 +9911,17 @@ class Handler(BaseHTTPRequestHandler):
         parts = path.split("/")
         if len(parts) == 3:
             sources = SOURCES.list()
-            return self._json(200, {"sources": sources, "cap": MAX_SOURCE,
-                                    "used": sum(s["bytes"] for s in sources if not s["system"])})
+            return self._json(200, {"sources": sources, "cap": MAX_SOURCE})
         if len(parts) == 4:
             source = SOURCES.get(parts[3])
             return self._json(200, source) if source else self._send(404, b"not found", "text/plain")
         if len(parts) == 5 and parts[4] == "definition":
             got = SOURCES.definition(parts[3])
             return self._json(200, got) if got else self._send(404, b"not found", "text/plain")
+        if len(parts) == 5 and parts[4] == "responses":
+            held = SOURCES.responses(parts[3])
+            return self._json(200, {"responses": held}) if held is not None \
+                else self._send(404, b"not found", "text/plain")
         if len(parts) == 5 and parts[4] == "calls":
             calls = SOURCES.calls(parts[3])
             return self._json(200, {"calls": calls}) if calls is not None \
@@ -9590,7 +9997,17 @@ class Handler(BaseHTTPRequestHandler):
             if err:
                 code, message = err
                 return self._json(code, {"error": message})
-            return self._json(201, {"call": call})
+            return self._json(201 if call["outcome"] == "added" else 200, {"call": call})
+        if len(parts) == 6 and parts[4] == "responses" and parts[5] == "restore":
+            payload = self._payload() or {}
+            name = source_name(payload.get("name"))
+            n, drop = payload.get("n"), payload.get("drop")
+            if name is None or type(n) is not int or not (drop is None or type(drop) is int):
+                return self._json(400, {"error": "a restore is { name, n, drop? }"})
+            got, err = SOURCES.restore_response(parts[3], name, n, drop)
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(200, {"call": got})
         if len(parts) == 6 and parts[4] == "calls" and parts[5] == "remove":
             names = (self._payload() or {}).get("names")
             if not isinstance(names, list) or not names:
@@ -9683,12 +10100,21 @@ class Handler(BaseHTTPRequestHandler):
         if len(body) != length:
             return self._json(400, {"error": "the body ended before Content-Length promised"})
         # A type that checks what it takes (a flow's records) has it first.
-        take = (SOURCES.type_of(sid) or {}).get("take")
+        entry = SOURCES.type_of(sid) or {}
+        take = entry.get("take")
         if take is not None:
             body, err = take(name, body)
             if err:
                 code, message = err
                 return self._json(code, {"error": message})
+        # One a kind already holds by its key (a record of the same run's call,
+        # edited or not) is not kept twice (#587).
+        key = entry.get("key")
+        value = key(body) if key is not None else None
+        if value is not None:
+            held = SOURCES.key_held(sid, key, value)
+            if held is not None:
+                return self._json(200, {"source": sid, "name": held, "unchanged": True})
         stored, err = SOURCES.add_file(sid, name, body)
         if err:
             code, message = err
@@ -9783,11 +10209,16 @@ class Handler(BaseHTTPRequestHandler):
 # key that comes is the connection's from then on, a rotated one included.
 # A connection carries its share set (`shares`, workspace slugs and the
 # sentinels; absent is All) from version 2: version 1 carried none, so an
-# import reads its connections as All.
+# import reads its connections as All. Version 3 carries (Default)'s settings
+# (`connectionProfileDefault`, #603), which the profiles store keeps beside
+# its list; an import sets them only on a lab whose (Default) sets nothing.
 
 LAB_FORMAT = "evals-lab/lab"
-LAB_VERSIONS = (1, 2)
-LAB_FIELDS = ("format", "version", "exportedAt", "connections", "connectionProfiles", "grader", "workspaces")
+LAB_VERSIONS = (1, 2, 3)
+LAB_FIELDS = ("format", "version", "exportedAt", "connections", "connectionProfiles", "connectionProfileDefault",
+              "grader", "workspaces")
+# What a Connection Profile holds that is not a setting, so (Default) holds none.
+PROFILE_META = ("id", "name", "connection", "fromProfile")
 
 
 def lab_export(only=None) -> dict:
@@ -9838,6 +10269,9 @@ def lab_export(only=None) -> dict:
     if (isinstance(body.get("grader"), str) and body["grader"] and body["grader"] not in nowhere
             and (not only or body["grader"] in kept)):
         out["grader"] = body["grader"]
+    # (Default) answers on any connection, so every workspace's file has it.
+    if isinstance(profiles.get("default"), dict) and profiles["default"]:
+        out["connectionProfileDefault"] = profiles["default"]
     return out
 
 
@@ -9894,6 +10328,19 @@ def lab_problems(doc) -> str:
         why = _keyless(p, what)
         if why:
             return why
+    if "connectionProfileDefault" in doc:
+        d = doc["connectionProfileDefault"]
+        if doc["version"] < 3:
+            return f"a lab file of version {doc['version']} holds no connectionProfileDefault"
+        if not isinstance(d, dict):
+            return "the connection profile (Default) is an object of settings"
+        bad = next((k for k, v in d.items() if k in PROFILE_META or not isinstance(v, str)), None)
+        if bad:
+            return f"the connection profile (Default)'s {bad} is not a setting"
+        # A profile's key is its connection's, and (Default) names none.
+        held = next((k for k, v in d.items() if LOOKS_LIKE_A_KEY.match(k) and v), None)
+        if held:
+            return f"the connection profile (Default) holds a {held}; a lab file holds no key"
     if "grader" in doc and not isinstance(doc["grader"], str):
         return "the grader is not a connection's id"
     seen = set()
@@ -9992,6 +10439,10 @@ def lab_import(doc, keys) -> tuple:
                 profs.append(dict(p))
                 pchanged = True
                 lines.append(f"connection profile {p['name']} added")
+        if doc.get("connectionProfileDefault") and not pbody.get("default"):
+            pbody["default"] = dict(doc["connectionProfileDefault"])
+            pchanged = True
+            lines.append("connection profile (Default) set")
         if doc.get("grader") and not body.get("grader"):
             body["grader"] = doc["grader"]
             changed = True