cortad 0.3.8 → 0.3.10

package/lib/pyhook/sitecustomize.py

@@ -118,7 +118,7 @@ def _install():
     import re
     import sys
     import time
-    from urllib.parse import urlsplit
+    from urllib.parse import quote, unquote, urlsplit, urlunsplit
 
     ctx = contextvars.ContextVar("cortad_request", default=None)
     limit = 65536
@@ -882,6 +882,31 @@ def _install():
             return False
         return bool(retrieval_host.search(parts.hostname or "")) or bool(retrieval_path.search(parts.path or ""))
 
+    # A call to one Qdrant collection, by Qdrant's own REST shape (lib/wire.cjs qdrantCall): every
+    # PUT, PATCH and DELETE writes, and a POST does unless it is one of Qdrant's reads spelled as a
+    # POST. /collections/aliases names no collection: a write of the server itself.
+    qdrant_collection = re.compile(r"^/collections/([^/]+)(/.*)?$")
+    qdrant_read_post = re.compile(r"^/points(?:/(?:search|query|recommend|discover)(?:/(?:batch|groups))?|/scroll|/count|/facet|/search/matrix/(?:pairs|offsets))?$")
+    qdrant_points = re.compile(r"^/points(?:/(?:delete|payload|payload/delete|payload/clear|vectors|vectors/delete|batch))?$")
+
+    def qdrant_call(url, method):
+        try:
+            parts = urlsplit(str(url))
+        except ValueError:
+            return None
+        if parts.scheme not in ("http", "https"):
+            return None
+        m = qdrant_collection.match(parts.path or "")
+        if not m:
+            return None
+        name, rest = m.group(1), m.group(2) or ""
+        verb = str(method or "GET").upper()
+        write = verb in ("PUT", "PATCH", "DELETE") or (verb == "POST" and not qdrant_read_post.match(rest))
+        # "points" changes what the collection holds and a copy stands in for it; "create" makes it;
+        # "schema" (an index, an alias, a snapshot, its settings or its deletion) is held, never copied.
+        kind = None if not write else "create" if rest == "" and verb == "PUT" else "points" if qdrant_points.match(rest) and verb != "DELETE" else "schema"
+        return {"collection": None if name == "aliases" else unquote(name), "rest": rest, "write": write, "kind": kind, "parts": parts}
+
     def passages_from(raw):
         out, nodes = [], [0]
 
@@ -1344,6 +1369,11 @@ def _install():
                 return False
         elif run_seen["at"] is None or time.monotonic() - run_seen["at"] > run_live_s:
             return False
+        # A read of a Qdrant collection, however its server is named, and any call to a copy this
+        # hook made (copy_of below), go out.
+        q = qdrant_call(url, method)
+        if q and (not q["write"] or (q["collection"] and our_copy.search(q["collection"]))):
+            return False
         try:
             parts = urlsplit(str(url))
             host = (parts.hostname or "").lower()
@@ -1353,6 +1383,150 @@ def _install():
         except Exception:
             return True
 
+    # A write to a Qdrant collection made inside a run goes to a copy of that collection on the same
+    # server, made at the first such write: the collection is read page by page and written under
+    # `<name>_cortad_<pid>`, and from then on every call a run makes to that collection, reads too,
+    # goes to the copy, so the app finds what it wrote. The original is never written. A copy that
+    # cannot be made (over the size a run copies, a server that refused) leaves the write held like
+    # any other. Each copy is reported as a `copy` row and the command deletes them when it ends.
+    # The person's own calls, outside a run, go where they were sent (lib/trace.cjs does the same).
+    import threading
+    our_copy = re.compile(r"_cortad_\d+$")
+    copies = {}
+    copies_lock = threading.Lock()
+    qdrant_cap = 200000
+    # What one process copies at most: collections, and points across them; past it a write is held.
+    copies_max = 8
+    points_max = 300000
+    copied = {"collections": 0, "points": 0}
+    copy_said = set()
+
+    def copy_say(host, collection, why):
+        if (host, collection) in copy_said:
+            return
+        copy_said.add((host, collection))
+        try:
+            sys.stderr.write("cortad: a request Cortad sent made this app write into the Qdrant collection %s at %s. Cortad could not copy that collection on its server (%s), so the write was kept on this machine and answered with a stand-in success.\n" % (collection, host, why))
+        except Exception:
+            pass
+
+    def in_run():
+        req = ctx.get()
+        if req:
+            return bool(req.get("turn"))
+        return run_seen["at"] is not None and time.monotonic() - run_seen["at"] <= run_live_s
+
+    # None to leave the call alone; hold for a write of the server itself; else the collection's
+    # copy when there is one, or none yet for a write, which makes it.
+    def guarded(url, method):
+        call = qdrant_call(url, method)
+        if not call:
+            return None
+        if not call["collection"]:
+            return {"hold": True} if call["write"] and in_run() else None
+        if our_copy.search(call["collection"]) or not in_run():
+            return None
+        key = "%s|%s" % (call["parts"].netloc, call["collection"])
+        copy = copies.get(key)
+        if copy:
+            return {"key": key, "call": call, "copy": copy}
+        if not call["write"]:
+            return None
+        if call["kind"] == "schema":
+            return {"hold": True}
+        if copied["collections"] >= copies_max or copied["points"] >= points_max:
+            copy_say(call["parts"].netloc, call["collection"], "this run already copied %d collections, %d points" % (copied["collections"], copied["points"]))
+            return {"hold": True}
+        return {"key": key, "call": call, "copy": None}
+
+    # Qdrant's own API over urllib, which this file leaves unpatched: never through the app's clients.
+    def qdrant_http(base, key, method, path, body=None):
+        import urllib.error
+        import urllib.request
+        data = json.dumps(body).encode("utf-8") if body is not None else None
+        headers = {"content-type": "application/json"}
+        if key:
+            headers["api-key"] = key
+        req = urllib.request.Request(base + path, data=data, method=method, headers=headers)
+        try:
+            with urllib.request.urlopen(req, timeout=30) as r:
+                return json.loads(r.read().decode("utf-8", "replace") or "null"), r.status
+        except urllib.error.HTTPError as e:
+            return None, e.code
+
+    # The copy made: "empty" when the server holds no such collection yet (the app makes it, under
+    # the copy's name), "points" once every point is across, None when none could be made.
+    def clone_qdrant(base, key, source, clone):
+        at = lambda name: "/collections/" + quote(name, safe="")
+        info, status = qdrant_http(base, key, "GET", at(source))
+        if status == 404:
+            return ("empty", 0)
+        result = (info or {}).get("result") if status == 200 else None
+        if not result:
+            return None
+        if (result.get("points_count") or 0) > qdrant_cap:
+            return None
+        params = (result.get("config") or {}).get("params") or {}
+        made = {"vectors": params.get("vectors")}
+        if params.get("sparse_vectors"):
+            made["sparse_vectors"] = params["sparse_vectors"]
+        if "on_disk_payload" in params:
+            made["on_disk_payload"] = params["on_disk_payload"]
+        if qdrant_http(base, key, "PUT", at(clone), made)[1] != 200:
+            return None
+        try:
+            for field, schema in (result.get("payload_schema") or {}).items():
+                qdrant_http(base, key, "PUT", at(clone) + "/index?wait=true", {"field_name": field, "field_schema": schema.get("params") or schema.get("data_type")})
+            offset = None
+            while True:
+                page, status = qdrant_http(base, key, "POST", at(source) + "/points/scroll", {"limit": 256, "with_payload": True, "with_vector": True, **({"offset": offset} if offset is not None else {})})
+                if status != 200:
+                    raise RuntimeError(status)
+                got = (page or {}).get("result") or {}
+                points = got.get("points") or []
+                if points and qdrant_http(base, key, "PUT", at(clone) + "/points?wait=true", {"points": [{"id": p.get("id"), "vector": p.get("vector"), "payload": p.get("payload")} for p in points]})[1] != 200:
+                    raise RuntimeError("upsert")
+                offset = got.get("next_page_offset")
+                if offset is None:
+                    return ("points", result.get("points_count") or 0)
+        except Exception:
+            qdrant_http(base, key, "DELETE", at(clone))
+            return None
+
+    # The copy's name, made now when none exists; None when none could be made. The key the app
+    # sent is the key the copy is made with, and never leaves this process.
+    def copy_of(g, header_of):
+        with copies_lock:
+            have = copies.get(g["key"])
+            if have:
+                return have
+            parts = g["call"]["parts"]
+            base = "%s://%s" % (parts.scheme, parts.netloc)
+            auth = str(header_of("authorization") or "")
+            bearer = re.match(r"^Bearer\s+(\S+)", auth, re.I)
+            key = str(header_of("api-key") or "") or (bearer.group(1) if bearer else "")
+            clone = "%s_cortad_%d" % (g["call"]["collection"][:40], os.getpid())
+            made = clone_qdrant(base, key, g["call"]["collection"], clone)
+            if not made:
+                copy_say(parts.netloc, g["call"]["collection"], "the copy did not go through")
+                return None
+            copied["collections"] += 1
+            copied["points"] += made[1]
+            copies[g["key"]] = clone
+            write({"copy": {"host": parts.hostname or "", "base": base, "collection": g["call"]["collection"], "clone": clone}})
+            return clone
+
+    def to_copy(url, clone, call):
+        parts = call["parts"]
+        return urlunsplit((parts.scheme, parts.netloc, "/collections/" + quote(clone, safe="") + call["rest"], parts.query, parts.fragment))
+
+    # The copy a call goes to, waited for off the event loop: None to hold the call instead.
+    async def copy_for(g, header_of):
+        import asyncio
+        if g.get("hold"):
+            return None
+        return g["copy"] or await asyncio.get_running_loop().run_in_executor(None, copy_of, g, header_of)
+
     # A GraphQL query reads; a mutation writes. A `query` in JSON that is not GraphQL (SQL over HTTP,
     # where a SELECT can write through a function) is held.
     # ClickHouse, Trino and Presto take every query as a POST of its text, a SELECT too, and a SELECT
@@ -2468,19 +2642,34 @@ def _install():
                 dep(request.url, response.status_code, req=req, caller=caller)
             return response
 
+        def held_answer(request, sent):
+            kind = request.headers.get("content-type", "")
+            held_row(request.url, request.method, sent, kind, ctx.get())
+            return module.Response(200, headers={"content-type": "application/json"}, content=held_reply(sent, kind), request=request)
+
         # A held call that cannot be answered fails in the app; it is never sent instead.
         def held_httpx(request, sent):
             if not held_here(request.url, request.method, lambda: keys_in(request.headers)) or reads_only(sent, request.url, request.headers.get):
                 return None
-            kind = request.headers.get("content-type", "")
-            held_row(request.url, request.method, sent, kind, ctx.get())
-            return module.Response(200, headers={"content-type": "application/json"}, content=held_reply(sent, kind), request=request)
+            return held_answer(request, sent)
+
+        # A call to a Qdrant collection goes to the run's copy of it, `clone`; with none, it is held.
+        def routed_httpx(request, g, clone, sent):
+            if clone is None:
+                return held_answer(request, sent)
+            request.url = module.URL(to_copy(request.url, clone, g["call"]))
+            return None
 
         for cls in (module.Client, module.AsyncClient):
             send = cls.send
             if cls is module.Client:
                 def sync_send(self, request, *a, _send=send, **k):
                     sent = sent_of(request)
+                    g = guarded(request.url, request.method)
+                    if g is not None:
+                        answered = routed_httpx(request, g, None if g.get("hold") else (g["copy"] or copy_of(g, request.headers.get)), sent)
+                        if answered is not None:
+                            return answered
                     held = held_httpx(request, sent)
                     if held is not None:
                         return held
@@ -2504,6 +2693,11 @@ def _install():
             else:
                 async def async_send(self, request, *a, _send=send, **k):
                     sent = sent_of(request)
+                    g = guarded(request.url, request.method)
+                    if g is not None:
+                        answered = routed_httpx(request, g, await copy_for(g, request.headers.get), sent)
+                        if answered is not None:
+                            return answered
                     held = held_httpx(request, sent)
                     if held is not None:
                         return held
@@ -2535,6 +2729,9 @@ def _install():
             raw = text(request.body, reply_max)
             if reads_only(raw, request.url, request.headers.get):
                 return None
+            return held_answer(request, raw)
+
+        def held_answer(request, raw):
             import datetime
             models, structures = sys.modules["requests.models"], sys.modules["requests.structures"]
             kind = request.headers.get("content-type", "")
@@ -2546,6 +2743,12 @@ def _install():
             return response
 
         def sent(self, request, *a, **k):
+            g = guarded(request.url, request.method)
+            if g is not None:
+                clone = None if g.get("hold") else (g["copy"] or copy_of(g, request.headers.get))
+                if clone is None:
+                    return held_answer(request, text(request.body, reply_max))
+                request.url = to_copy(request.url, clone, g["call"])
             answered = held(request)
             if answered is not None:
                 return answered
@@ -2601,7 +2804,18 @@ def _install():
         async def requested(self, method, str_or_url, *a, **k):
             body = k.get("json") if k.get("json") is not None else k.get("data")
             url = whole(self, str_or_url)
-            if held_here(url, method, lambda: keys_of(self, k)) and not reads_only(text(body, reply_max), url, lambda name: header_of(self, k, name)):
+            g = guarded(url, method)
+            hold = False
+            if g is not None:
+                clone = await copy_for(g, lambda name: header_of(self, k, name))
+                if clone is None:
+                    hold = True
+                else:
+                    parts = g["call"]["parts"]
+                    url = to_copy(url, clone, g["call"])
+                    # A session with a base address takes the path alone.
+                    str_or_url = url if getattr(self, "_base_url", None) is None else url[len("%s://%s" % (parts.scheme, parts.netloc)):]
+            if hold or (held_here(url, method, lambda: keys_of(self, k)) and not reads_only(text(body, reply_max), url, lambda name: header_of(self, k, name))):
                 port = held_port()
                 if port is None:
                     raise module.ClientConnectionError("this write was kept from leaving and could not be answered")

package/lib/qdrant.mjs

@@ -5,8 +5,9 @@ export const QDRANT_CAP = 200_000;
 const BATCH = 256;
 const at = (name) => `/collections/${encodeURIComponent(name)}`;
 
+// `conn.fetch`: the fetch to use, when the caller's global one is wrapped (the hook in the app).
 async function call(conn, method, path, body) {
-  const res = await fetch(`${conn.base}${path}`, {
+  const res = await (conn.fetch ?? fetch)(`${conn.base}${path}`, {
     method,
     headers: { "content-type": "application/json", ...(conn.apiKey ? { "api-key": conn.apiKey } : {}) },
     body: body === undefined ? undefined : JSON.stringify(body),
@@ -34,7 +35,7 @@ export async function cloneQdrant(conn, source, clone) {
       const page = await call(conn, "POST", `${at(source)}/points/scroll`, { limit: BATCH, with_payload: true, with_vector: true, ...(offset === undefined ? {} : { offset }) });
       if (page.points.length) await call(conn, "PUT", `${at(clone)}/points?wait=true`, { points: page.points.map(({ id, vector, payload }) => ({ id, vector, payload })) });
       offset = page.next_page_offset;
-      if (offset === null || offset === undefined) return { how: "points" };
+      if (offset === null || offset === undefined) return { how: "points", points: info.points_count ?? 0 };
     }
   } catch (e) {
     if (made) await dropQdrant(conn, clone);

package/lib/reach.mjs

@@ -4,7 +4,7 @@ import { plural, upper } from "./words.mjs";
 
 // One request to each endpoint status lists as not reached, from this machine to the app on its port.
 // Each body is the one Cortad read that endpoint's client sending (the server's `request`).
-// The connect command sends these by itself, one at a time, each tagged x-cortad-turn: reach:auto:<n>
+// The connect command sends these by itself, up to four at a time, each tagged x-cortad-turn: reach:auto:<n>
 // (sendOurs), signed in only as a test account this machine made. Headers are never the agent's: they
 // go from here to the app on this machine and nowhere else, and only their names are printed. These
 // are real requests with real effects.
@@ -31,7 +31,7 @@ export function notUp(runner) {
 // The rows a call sends and the ones it leaves: the named paths, else every row with a ready request
 // that is not marked by hand.
 export function picked(contact, receipts, paths = []) {
-  const rows = unreached(contact, receipts);
+  const rows = unreached(contact, receipts, { retests: true });
   const named = new Set(Array.isArray(paths) ? paths : []);
   const chosen = named.size ? rows.filter((m) => named.has(m.path) || named.has(keyOf(m))) : rows.filter((m) => !m.unsure);
   const sent = chosen.filter((m) => m.request);
@@ -72,15 +72,15 @@ export function reachText({ rows, sent, left, unknown }, answers, how) {
   ].filter(Boolean).join("\n");
 }
 
-// Cortad's own test requests, sent by the connect command: one at a time, so what the app prints
-// belongs to one of them, each with the app's own origin where one is known. An endpoint that answers
+// Cortad's own test requests, sent by the connect command, each with the app's own origin where one is known. An endpoint that answers
 // 401 or 403 is asked once more as the test account this machine signed in, when there is one
 // (`signIn(row)` gives { name, value } or null); the token never leaves this machine.
 // `stop()`: true when the rest should wait, for the agent's ask; the answers so far come back.
 // Four at a time: one each in turn left a 22-endpoint app waiting nearly two minutes behind its two
 // slowest replies (ulaim, 2026-10-06). Answers keep the rows' order; a row is not sent once stop() says so.
+// `onAnswer(answer, i)`: called as each row's answer lands, so one slow endpoint holds back no other's.
 const AT_ONCE = 4;
-export async function sendOurs(rows, { runner, origin, signIn = async () => null, next, stop = () => false, fetchImpl = fetch, atOnce = AT_ONCE }) {
+export async function sendOurs(rows, { runner, origin, signIn = async () => null, next, stop = () => false, onAnswer = () => {}, fetchImpl = fetch, atOnce = AT_ONCE }) {
   const app = appUrl(runner);
   const base = () => ({ ...(origin ? { origin, referer: `${origin}/` } : {}), [RUN_TURN]: `reach:auto:${next()}` });
   const answers = [];
@@ -101,6 +101,7 @@ export async function sendOurs(rows, { runner, origin, signIn = async () => null
       if (stopped || stop()) { stopped = true; return; }
       const i = at++;
       answers[i] = await one(rows[i]);
+      try { onAnswer(answers[i], i); } catch { /* the answer stands whatever its listener did */ }
     }
   };
   await Promise.all(Array.from({ length: Math.min(atOnce, rows.length) }, lane));
@@ -179,7 +180,13 @@ export async function sendOne(m, app, headers, fetchImpl, secrets = []) {
     res = await fetchImpl(url, { method, headers: json && !typed ? { "content-type": "application/json", ...headers } : headers,
       ...(json ? { body: JSON.stringify(body) } : {}), redirect: "manual", signal: AbortSignal.timeout(REACH_MS) });
   } catch (err) {
-    return { line: `${label}: ${failedSaid(err, app.port || 80)}`, ok: false, status: 0, down: /^(?:ECONNREFUSED|ECONNRESET)$/.test(err?.cause?.code ?? err?.code ?? "") };
+    const code = err?.cause?.code ?? err?.code ?? "";
+    // Refused, nothing reached the app, and a GET cut off changes nothing: both are sent again
+    // (`down`). A POST whose connection was reset may have been taken by the app before it went, so
+    // it is an answer nobody saw (`unknown`) and is never sent again by itself.
+    const unknown = code === "ECONNRESET" && method !== "GET";
+    return { line: `${label}: ${failedSaid(err, app.port || 80)}${unknown ? " It may have reached your app, so Cortad does not send it again by itself." : ""}`, ok: false, status: 0,
+      down: !unknown && /^(?:ECONNREFUSED|ECONNRESET)$/.test(code), ...(unknown ? { unknown: true } : {}) };
   }
   let text;
   try { text = await res.text(); } catch { return { line: `${label}: ${res.status}, and the answer had not finished after ${REACH_MS / 1000} seconds.`, ok: false, status: res.status }; }

package/lib/read-text.mjs

@@ -87,30 +87,30 @@ const listed = (names) => (names.length > 1 ? `${names.slice(0, -1).join(", ")}
 
 // What the connect command does on this machine, from the facts it keeps in the runner's record:
 // what it started and stops, the database copies, the settings its app runs with, what it loads,
-// what a run holds and puts back, and what it keeps and writes outside this repository. At most
-// eight lines, the two for the person first.
+// what a run holds and puts back, and what it keeps outside this repository. The first line is the
+// one the terminal says at connect (connectedText); the rest is read when someone asks.
 export function machineText(f) {
   if (!f) return ["No connect command is running for this repository now, so nothing Cortad started is running here."];
   const lines = [
     f.started
-      ? `For the person: Cortad started your app in ${f.dir === "." ? "this folder" : f.dir} and stops it, with everything it started for it, when you press Ctrl-C in its terminal; your code and settings files are never edited.`
-      : `For the person: Cortad uses your app already running on port ${f.port} and leaves it running when you press Ctrl-C in its terminal; your code and settings files are never edited.`,
-    f.copies === "made"
-      ? "For the person: Where Cortad could copy a database, your app runs on the copy, made from your own at every connect and deleted when this command ends, so records created in a copy are gone at the next connect; keep records every run needs in your own database. Each Data line in status says what happens to one store."
-      : "For the person: No copy of a database was made for this session, so what test conversations create stays where your app keeps its data; each Data line in status names one store.",
+      ? `Cortad started your app in ${f.dir === "." ? "this folder" : f.dir} and stops it when this command stops. Your code and settings files are never edited.`
+      : `Cortad uses your app already running on port ${f.port} and leaves it running. Your code and settings files are never edited.`,
   ];
+  if (f.copies === "made") lines.push("Test conversations save into a copy of your database, made fresh at each connect and deleted when this command stops. Your own data is not changed.");
   // Env values reach only a process the command started itself.
-  if (f.started) lines.push(f.changed?.length ? `Your env file values go into your app's process alone, with ${listed(f.changed)} set for this session only; no value is sent to Cortad.` : "Your env file values go into your app's process alone, unchanged; no value is sent to Cortad.");
-  lines.push(f.hooked ? "A hook loaded into your app records each request that reaches a model and the model calls it makes; request headers stay on this machine and are deleted when the command stops."
-    : f.proxy ? `A local proxy on port ${f.proxy} sees each request sent to it and the model calls your app makes through it; request headers stay on this machine and are deleted when the command stops.`
-      : "Nothing is loaded into your app, since Cortad did not start it, so its model calls are not seen.");
+  if (f.started) lines.push(`Your env values stay on this machine${f.changed?.length ? `, with ${listed(f.changed)} set for this session only` : ""}. None is sent to Cortad.`);
+  lines.push(f.hooked ? "Cortad sees each request that reaches your model and the model calls it makes. Request headers stay on this machine and are deleted when the command stops."
+    : f.proxy ? `A local proxy on port ${f.proxy} sees each request sent to it and the model calls your app makes through it. Request headers stay on this machine and are deleted when the command stops.`
+      : "Cortad did not start your app, so it does not see the calls your app makes to its model.");
   // Only the hook holds a write or records a file: through the proxy, or with nothing loaded, both go as the app does them.
-  lines.push(f.hooked ? "During a run, a write your app sends to a service off this machine, such as a payment or an email, is answered inside your app and never sent, unless it carries a test key, goes to a provider's sandbox, or goes to a host you said yes to on the card; a call named as a read where every call is a POST, such as customers.get, goes out."
-    : "During a run, a write your app sends to a service off this machine, such as a payment or an email, is sent for real.");
-  lines.push(f.hooked ? "Files your app writes in its own folder during a run are put back when run_status reads the run's end; its log files are left as they are." : "Files your app writes in its own folder stay as it writes them.");
-  lines.push("Kept between connects in ~/.cortad on this machine: this repository's key for later runs, the key the test accounts' passwords are made from, the files to put back, a list of what to undo, and your app's output from the last session with the values from your env files masked, which the next connect starts over.");
+  lines.push(f.hooked ? "During a run, anything your app would send to an outside service, such as a payment or an email, is answered inside your app and never sent, unless it carries a test key, goes to a provider's sandbox, or goes to a host you allowed on the card. A call that only reads goes out."
+    : "During a run, what your app sends to an outside service, such as a payment or an email, is sent for real.");
+  lines.push(f.hooked ? "Files your app writes in its own folder during a run are put back when the run ends. Its log files are left as they are." : "Files your app writes in its own folder stay as it writes them.");
+  lines.push("Kept in ~/.cortad on this machine between connects: this repository's key, the key the test accounts' passwords are made from, what to put back after a run, and your app's output from the last session with your env values masked.");
   return lines;
 }
+// What the terminal says once the app answers: the one line the person needs.
+export const connectedText = (f) => machineText(f).slice(0, 1);
 
 export function showText(d, show, page = 1) {
   if (!d.read || (d.read.complete === false && show !== "reviews")) return "Cortad is still reading this repository.";

package/lib/replay.mjs

@@ -37,6 +37,8 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
   let newestTable = null;
   // Each server a hooked process opened a connection to, by host and port.
   const conns = new Map();
+  // The Qdrant collections the hook copied for a run's writes, by server and name (lib/stores.mjs drops them).
+  const copies = new Map();
   // The model's last words and status for each of a run's turns on an endpoint the app answers
   // later, by turn (lib/wire.cjs answeredLater).
   const answers = new Map();
@@ -48,7 +50,7 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
   // requires, to one without.
   const bunHook = /\s/.test(HOOK) ? (() => {
     const dir = mkdtempSync(join(tmpdir(), "cortad-"));
-    for (const name of ["trace.cjs", "wire.cjs", "tied.cjs"]) copyFileSync(join(HERE, name), join(dir, name));
+    for (const name of ["trace.cjs", "wire.cjs", "tied.cjs", "qdrant.mjs"]) copyFileSync(join(HERE, name), join(dir, name));
     return join(dir, "trace.cjs");
   })() : HOOK;
   // The hooks are handed to whatever is started: a Node app loads the first, a Bun app the same file
@@ -96,6 +98,7 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
       if (Number.isInteger(row.listen)) { ports.add(row.listen); continue; }
       if (row.routes) { const t = routeTable(row.routes); if (t) { tables.set(t.port, t); newestTable = t; } continue; }
       if (row.conn) { const { host, port } = row.conn; if (typeof host === "string" && Number.isInteger(port)) conns.set(`${host}:${port}`, { host, port }); continue; }
+      if (row.copy) { const { base, collection, clone } = row.copy; if ([base, collection, clone].every((v) => typeof v === "string" && v)) copies.set(`${base}|${collection}`, { base, collection, clone }); continue; }
       if (row.answer) {
         const { turn, status, reply } = row.answer;
         if (TURN.test(turn ?? "")) answers.set(turn, { status: Number.isInteger(status) ? status : 0, reply: typeof reply === "string" ? reply : "" });
@@ -158,6 +161,8 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
     registry: (port) => { poll(); return tables.get(port) ?? newestTable ?? {}; },
     // The servers the app has connected to so far (lib/stores.mjs names the ones no setting does).
     connections: () => { poll(); return [...conns.values()]; },
+    // The collections the hook copied on a Qdrant server for a run's writes, to be deleted at the end.
+    copies: () => { poll(); return [...copies.values()]; },
     // The model's last words for a run's turn on an endpoint the app answers later, once the hook
     // has written them; null at `until`, or once `signal` aborts. `forget` drops a turn sent again.
     answered: async (turn, until, signal) => {

package/lib/spec.mjs

@@ -11,10 +11,20 @@ export const VERSION = JSON.parse(readFileSync(new URL("../package.json", import
 export const SPEC = "(?:cortad(?:@[A-Za-z0-9._-]+)?|/[A-Za-z0-9._/-]+)";
 const SAFE = new RegExp(`^${SPEC}$`);
 
+// A pin older than this very version loses to it: a connect run as cortad@0.3.8 wrote a skill and an
+// MCP server pinned to 0.3.7, and the agent asked whether the two were meant to mix.
 export function cliSpec({ env = process.env, version = VERSION } = {}) {
-  if (env.CORTAD_CLI_SPEC && SAFE.test(env.CORTAD_CLI_SPEC)) return env.CORTAD_CLI_SPEC;
-  return `cortad@${version}`;
+  const pinned = env.CORTAD_CLI_SPEC && SAFE.test(env.CORTAD_CLI_SPEC) ? env.CORTAD_CLI_SPEC : null;
+  if (!pinned) return `cortad@${version}`;
+  const released = /^cortad@(\d+\.\d+\.\d+)$/.exec(pinned)?.[1];
+  return released && newer(version, released) ? `cortad@${version}` : pinned;
 }
+const newer = (a, b) => {
+  if (!/^\d+\.\d+\.\d+$/.test(a)) return false;
+  const [x, y] = [a, b].map((v) => v.split(".").map(Number));
+  for (let i = 0; i < 3; i += 1) if (x[i] !== y[i]) return x[i] > y[i];
+  return false;
+};
 
 // npx needs -y to fetch a package without asking; a local folder is not fetched.
 export const npxArgs = (spec) => (isAbsolute(spec) ? [spec] : ["-y", spec]);

package/lib/stores.mjs

@@ -232,7 +232,7 @@ const HELD = {
   "neon-key": (s) => ["there is no NEON_API_KEY in its settings to make a branch with", `Set NEON_API_KEY, or point ${s.names[0]} at a database on this machine`],
   neon: (s) => ["Neon did not make a branch of it", `Check NEON_API_KEY, or point ${s.names[0]} at a database on this machine`],
   supabase: (s) => ["a local Supabase needs supabase/config.toml, Docker and the supabase command", `Add them, or point ${s.names[0]} at a local Supabase`],
-  engine: (s) => [`Cortad cannot copy a hosted ${ENGINE[s.engine]}`, `Point ${s.names[0]} at a ${ENGINE[s.engine]} on this machine`],
+  engine: (s) => [`no copy of a hosted ${ENGINE[s.engine]} is made`, `Point ${s.names[0]} at a ${ENGINE[s.engine]} on this machine`],
   parts: (s) => ["its settings name it in parts, which cannot be pointed at a copy", `Put its whole address in one setting, or point ${s.names[0]} at a database on this machine`],
   "no-migrations": (s) => ["no migrations were found to build a copy on this machine from", `Point ${s.names[0]} at a database on this machine`],
   "no-local": (s) => ["no Postgres on this machine let us make a database to build a copy in", "Start Postgres on this machine"],
@@ -247,8 +247,14 @@ const HELD = {
 // card told the owner, on the card, that "the owner can say yes to it on the card in the browser".
 const heldWhy = (s, why, tool) => {
   const [reason, fix] = (HELD[why] ?? HELD.other)(s, tool);
-  return `${listed(s.names)} ${s.names.length === 1 ? "points" : "point"} at ${s.engine === "redis" || s.db ? "the" : "a"} ${whatOf(s)} off this machine, and ${reason}, so the run would write into it as it is. ${fix}, then run the connect command again.`;
+  return `${listed(s.names)} ${s.names.length === 1 ? "points" : "point"} at ${s.engine === "redis" || s.db ? "the" : "a"} ${whatOf(s)} that is not on this machine, and ${reason}, so test conversations would save into it. ${fix}, then run the connect command again.`;
 };
+// A Qdrant server the app writes into is kept by the hook in the app instead of by a copy made here:
+// the first write a run makes into a collection copies that collection on the same server and the
+// app's later calls go to the copy (lib/trace.cjs, lib/pyhook/sitecustomize.py). So a hosted Qdrant
+// holds nothing, and a local one whose collection no setting names is not a risk either.
+export const GUARDED_ENGINES = new Set(["qdrant"]);
+const guardedKept = (s) => `Your app's ${ENGINE[s.engine]} collections at ${listed(s.names)} are not written into by a run: the first time a run writes into one, that collection is copied on the same server and the app's calls go to the copy, which is deleted when this command ends.`;
 const hostedKept = (s, got) => (got.how === "neon"
   ? `Your app runs against a Neon branch of its ${whatOf(s)}, made for this session through ${listed(s.names)}; the original is not touched, and the branch is deleted when this command ends and made again from the original at the next connect.`
   : got.how === "supabase"
@@ -259,7 +265,7 @@ export const storeId = (s) => createHash("sha256").update(s.key).digest("hex").s
 // A store off this machine the app was pointed at a copy of and reached anyway (dotenv's override):
 // trials write into the original, so it holds Run. `blocked`: the rows already holding it.
 export const bypassedOf = (connections, stores, blocked) => stores
-  .filter((s) => !onThisMachine(s.host) && !blocked.some((b) => b.id === storeId(s)) && connections.some((c) => hostKey(c.host) === hostKey(s.host)))
+  .filter((s) => !onThisMachine(s.host) && !GUARDED_ENGINES.has(s.engine) && !blocked.some((b) => b.id === storeId(s)) && connections.some((c) => hostKey(c.host) === hostKey(s.host)))
   .map((s) => ({ id: storeId(s), names: s.names, why: heldWhy(s, "bypassed") }));
 
 // A copy that throws is one that did not go through, said as such; it never stops the app starting.
@@ -292,6 +298,12 @@ export function storeKeeper({ ledgerFile, values = {}, dirs = [], fetchFn, local
       const pass = [];
       for (const s of stores) {
         if (done.has(s.key)) continue;
+        if (GUARDED_ENGINES.has(s.engine) && (!onThisMachine(s.host) || !s.points.length || !s.db)) {
+          if (await down(s)) continue;
+          done.add(s.key);
+          copies.push(guardedKept(s));
+          continue;
+        }
         if (!onThisMachine(s.host)) {
           done.add(s.key);
           const got = started ? await hosted.keep(s).catch(() => ({ why: ["other"] })) : { why: ["started"] };
@@ -329,6 +341,21 @@ export function storeKeeper({ ledgerFile, values = {}, dirs = [], fetchFn, local
       for (const { s, row } of made.splice(0)) if (await drop(s, row.made, row.marker)) book.remove(row);
       await hosted.drop();
     },
+    // The collections the hook in the app copied for a run's writes (lib/trace.cjs copyOf), deleted
+    // with the key the settings hold for that server. Each is in the ledger from the moment it is
+    // seen, so one a killed session left is deleted by the next.
+    // ponytail: a server no setting names a key for is asked without one; a copy that refuses stays
+    // under its _cortad_ name for the person to see. The key the app sent never leaves the app.
+    async dropCopies(copies, stores) {
+      const rows = copies.map((c) => ({ kind: "qdrant-copy", key: c.base, made: c.clone }));
+      for (const row of rows) book.add(row);
+      const hostOf = (base) => { try { return new URL(base).hostname; } catch { return base; } };
+      for (const row of [...rows, ...book.orphans("qdrant-copy")]) {
+        // The hook names the server with its port and QDRANT_URL may not: the key is the host's.
+        const conn = { base: row.key, apiKey: stores.find((s) => s.engine === "qdrant" && s.conn?.base && hostOf(s.conn.base) === hostOf(row.key))?.conn.apiKey };
+        if (await dropQdrant(conn, row.made)) book.remove(row);
+      }
+    },
   };
 }
 
@@ -341,7 +368,7 @@ export function unnamedHeld(connections, stores, blocked) {
   for (const { host, port } of connections) {
     const local = onThisMachine(host);
     const engine = local ? ENGINE_AT[port] : HOSTED.find(([, re]) => re.test(String(host).toLowerCase()))?.[0] ?? ENGINE_AT[port];
-    if (!engine || stores.some((s) => (local ? `${hostKey(s.host)}:${s.port}` === `localhost:${port}` : s.engine === engine && !onThisMachine(s.host)))) continue;
+    if (!engine || GUARDED_ENGINES.has(engine) || stores.some((s) => (local ? `${hostKey(s.host)}:${s.port}` === `localhost:${port}` : s.engine === engine && !onThisMachine(s.host)))) continue;
     const id = createHash("sha256").update(`${engine}|${local ? "localhost" : String(host).toLowerCase()}|${port}`).digest("hex").slice(0, 16);
     if ([...blocked, ...rows].some((b) => b.id === id)) continue;
     // Named by its engine where a setting's name would stand: the API takes a name, never an address.