cortad 0.3.0-rc.3 → 0.3.0-rc.5

package/lib/pyhook/sitecustomize.py

@@ -7,6 +7,7 @@
 import os
 
 _FILE = os.environ.get("CORTAD_TRACE_FILE")
+_RULES = os.environ.get("CORTAD_RULES_FILE")
 
 
 def _install():
@@ -25,6 +26,115 @@ def _install():
     model_host = re.compile(r"(?:^|\.)(?:openai\.com|anthropic\.com|fireworks\.ai|openrouter\.ai|groq\.com|mistral\.ai|together\.xyz|together\.ai|deepseek\.com|cohere\.ai|cohere\.com|perplexity\.ai|x\.ai|openai\.azure\.com|cognitiveservices\.azure\.com|replicate\.com|huggingface\.co|cerebras\.ai|deepinfra\.com|novita\.ai|moonshot\.cn|dashscope\.aliyuncs\.com|bigmodel\.cn|ai-gateway\.vercel\.sh|gateway\.ai\.cloudflare\.com|helicone\.ai|portkey\.ai)$", re.I)
     model_path = re.compile(r"/(?:chat/completions|completions|responses|messages|embeddings)$|:(?:generateContent|streamGenerateContent)|/invoke(?:-with-response-stream)?$|/api/(?:chat|generate)$", re.I)
 
+    # The turn a message came in under, when the run tagged it (one opaque id per request), so a
+    # model call and its prompt can be pinned to the reply they produced while turns overlap.
+    turn_ok = re.compile(r"^[A-Za-z0-9:_.-]{1,80}$")
+
+    def turn_now():
+        req = ctx.get()
+        return req.get("turn") if req else None
+
+    # The customer's own rule sentences, written beside the trace by the run, so each model call can
+    # say which of them its prompt carried. Absent file, nothing is claimed. Reloaded on change.
+    rules_at = [-1.0]
+    rules = [None]
+    slot = re.compile(r"\{[^}]*\}|\$\{[^}]*\}|%[sd]|<[^>]{1,40}>")
+
+    def norm(s):
+        s = str(s or "")
+        s = re.sub(r"\\u([0-9a-fA-F]{4})", lambda m: chr(int(m.group(1), 16)), s)
+        s = re.sub(r"\\[nrt]", " ", s).replace('\\"', '"').replace("\\\\", "\\")
+        return re.sub(r"\s+", " ", s.lower()).strip()
+
+    def rules_now():
+        if not _RULES:
+            return None
+        try:
+            at = os.stat(_RULES).st_mtime
+            if at != rules_at[0]:
+                rules_at[0] = at
+                with open(_RULES, encoding="utf-8") as f:
+                    raw = json.load(f)
+                out = []
+                for r in raw if isinstance(raw, list) else []:
+                    if not isinstance(r, dict) or not isinstance(r.get("id"), str) or not isinstance(r.get("text"), str):
+                        continue
+                    parts = [p for p in (norm(x) for x in slot.split(r["text"])) if len(p) >= 12]
+                    if parts:
+                        out.append((r["id"], parts))
+                rules[0] = out
+        except Exception:
+            pass
+        return rules[0]
+
+    # What the app's tools answered, as the prompt of the next model call carries them (chat tool
+    # messages, responses function outputs, Anthropic tool_result blocks, Gemini functionResponse
+    # parts): the material a reply's facts rest on. Bounded per call.
+    tool_text, tools_max = 3000, 12
+
+    def text_of(v):
+        if isinstance(v, str):
+            return v
+        if isinstance(v, list):
+            return "\n".join(p for p in ((x.get("text") or x.get("content") or "") if isinstance(x, dict) else str(x or "") for x in v) if p)
+        if isinstance(v, dict):
+            try:
+                return json.dumps(v, ensure_ascii=False)
+            except (TypeError, ValueError):
+                return ""
+        return ""
+
+    def tools_in(sent):
+        body = parsed(sent) if isinstance(sent, str) else None
+        if not isinstance(body, dict):
+            return None
+        out, names = [], {}
+
+        def add(name, text):
+            t = text_of(text)[:tool_text]
+            if t.strip() and len(out) < tools_max and all(o["text"] != t for o in out):
+                out.append({"name": str(name or "")[:80], "text": t})
+
+        messages = body.get("messages") if isinstance(body.get("messages"), list) else []
+        for m in messages:
+            if not isinstance(m, dict):
+                continue
+            for c in m.get("tool_calls") or []:
+                if isinstance(c, dict) and c.get("id") and isinstance(c.get("function"), dict):
+                    names[c["id"]] = c["function"].get("name")
+            if isinstance(m.get("content"), list):
+                for c in m["content"]:
+                    if isinstance(c, dict) and c.get("type") == "tool_use" and c.get("id"):
+                        names[c["id"]] = c.get("name")
+        for m in messages:
+            if not isinstance(m, dict):
+                continue
+            if m.get("role") in ("tool", "function"):
+                add(m.get("name") or names.get(m.get("tool_call_id")), m.get("content"))
+            if isinstance(m.get("content"), list):
+                for c in m["content"]:
+                    if isinstance(c, dict) and c.get("type") == "tool_result":
+                        add(names.get(c.get("tool_use_id")), c.get("content"))
+        items = body.get("input") if isinstance(body.get("input"), list) else []
+        for it in items:
+            if isinstance(it, dict) and it.get("type") == "function_call" and it.get("call_id"):
+                names[it["call_id"]] = it.get("name")
+        for it in items:
+            if isinstance(it, dict) and it.get("type") == "function_call_output":
+                add(names.get(it.get("call_id")), it.get("output"))
+        for c in body.get("contents") if isinstance(body.get("contents"), list) else []:
+            for p in c.get("parts") if isinstance(c, dict) and isinstance(c.get("parts"), list) else []:
+                if isinstance(p, dict) and isinstance(p.get("functionResponse"), dict):
+                    add(p["functionResponse"].get("name"), p["functionResponse"].get("response"))
+        return out or None
+
+    def rules_in(sent):
+        found = rules_now()
+        if found is None:
+            return None
+        body = norm(sent)
+        return [rid for rid, parts in found if all(p in body for p in parts)]
+
     def write(row):
         try:
             fd = os.open(_FILE, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
@@ -33,6 +143,17 @@ def _install():
         except OSError:
             pass
 
+    # Every other outbound call: the host and what it answered, never a byte of it. A search
+    # provider over its limit for a whole run was invisible until this row existed.
+    def dep(url, status, code=None):
+        try:
+            host = (urlsplit(str(url)).hostname or "").lower()
+            if not host or host in ("localhost", "127.0.0.1", "::1"):
+                return
+            turn = turn_now()
+            write({"dep": {"at": int(time.time() * 1000), "host": host[:253], "status": int(status or 0), **({"code": str(code)[:40]} if code else {}), **({"turn": turn} if turn else {})}})
+        except Exception:
+            pass
     def is_model_call(url):
         try:
             parts = urlsplit(str(url))
@@ -141,19 +262,31 @@ def _install():
         except Exception:
             return ""
 
-    def meter(url, sent, status, kind, raw):
+    def meter(url, sent, status, kind, raw, turn=None):
         try:
             tokens, model = read_reply(kind, (raw or "")[:reply_max])
             parts = urlsplit(str(url))
             row = {"at": int(time.time() * 1000), "host": parts.netloc.replace(":443", ""), "model": str(asked_for(url, sent) or model or "")[:160],
                    "status": int(status or 0), "usage": tokens is not None}
             row.update(tokens or {"promptTokens": 0, "cachedTokens": 0, "completionTokens": 0})
+            turn = turn or turn_now()
+            if turn:
+                row["turn"] = turn
+            as_text = sent if isinstance(sent, str) else text(sent)
+            found = rules_in(as_text)
+            if found is not None:
+                row["rules"] = found
+            tools = tools_in(as_text)
+            if tools:
+                row["tools"] = tools
             write({"call": row})
         except Exception:
             pass
 
     def started(method, path, headers):
-        return {"method": method, "path": path, "headers": headers, "chunks": [], "size": 0, "noted": False}
+        turn = headers.pop("x-cortad-turn", None)
+        return {"method": method, "path": path, "headers": headers, "chunks": [], "size": 0, "noted": False,
+                "turn": turn if isinstance(turn, str) and turn_ok.match(turn) else None}
 
     def keep(req, chunk):
         if chunk and req["size"] < limit:
@@ -330,6 +463,7 @@ def _install():
 
         def watch(request, response, sent):
             if not is_model_call(request.url):
+                dep(request.url, response.status_code)
                 return response
             kind = response.headers.get("content-type", "")
             if getattr(response, "_content", None) is not None:
@@ -356,9 +490,11 @@ def _install():
                         pass
                     try:
                         response = _send(self, request, *a, **k)
-                    except Exception:
+                    except Exception as err:
                         if is_model_call(request.url):
                             meter(request.url, sent, 0, "", "")
+                        else:
+                            dep(request.url, 0, type(err).__name__)
                         raise
                     try:
                         return watch(request, response, sent)
@@ -374,9 +510,11 @@ def _install():
                         pass
                     try:
                         response = await _send(self, request, *a, **k)
-                    except Exception:
+                    except Exception as err:
                         if is_model_call(request.url):
                             meter(request.url, sent, 0, "", "")
+                        else:
+                            dep(request.url, 0, type(err).__name__)
                         raise
                     try:
                         return watch(request, response, sent)
@@ -392,8 +530,15 @@ def _install():
                 note(request.url, request.body)
             except Exception:
                 pass
-            response = send(self, request, *a, **k)
             try:
+                response = send(self, request, *a, **k)
+            except Exception as err:
+                if not is_model_call(request.url):
+                    dep(request.url, 0, type(err).__name__)
+                raise
+            try:
+                if not is_model_call(request.url):
+                    dep(request.url, response.status_code)
                 if is_model_call(request.url):
                     # A streamed reply is counted as a call whose counts were not read.
                     raw = response.content.decode("utf-8", "replace") if getattr(response, "_content_consumed", False) else ""
@@ -413,9 +558,16 @@ def _install():
                 note(str_or_url, body)
             except Exception:
                 pass
-            response = await request(self, method, str_or_url, *a, **k)
+            try:
+                response = await request(self, method, str_or_url, *a, **k)
+            except Exception as err:
+                if not is_model_call(str_or_url):
+                    dep(str_or_url, 0, type(err).__name__)
+                raise
             if is_model_call(str_or_url):
-                response._cortad = (str_or_url, text(body))
+                response._cortad = (str_or_url, text(body), turn_now())
+            else:
+                dep(str_or_url, response.status)
             return response
 
         # Metered when your app reads the reply, or, for one it streams, when the reply is let go.
@@ -426,14 +578,14 @@ def _install():
             call = getattr(self, "_cortad", None)
             if call:
                 self._cortad = None
-                meter(call[0], call[1], self.status, self.headers.get("content-type", ""), (raw or b"").decode("utf-8", "replace"))
+                meter(call[0], call[1], self.status, self.headers.get("content-type", ""), (raw or b"").decode("utf-8", "replace"), call[2])
             return raw
 
         def release_kept(self, *a, **k):
             call = getattr(self, "_cortad", None)
             if call:
                 self._cortad = None
-                meter(call[0], call[1], self.status, self.headers.get("content-type", ""), "")
+                meter(call[0], call[1], self.status, self.headers.get("content-type", ""), "", call[2])
             return release(self, *a, **k)
 
         module.ClientSession._request = requested

package/lib/replay.mjs

@@ -11,7 +11,7 @@ const HOOK = join(HERE, "trace.cjs");
 const PYHOOK = join(HERE, "pyhook");
 export const CAPTURED = "captured";
 // Never replayed: they describe one connection, not the caller.
-const HOP = /^(?:host|content-length|connection|keep-alive|transfer-encoding|upgrade|expect|te|trailer|accept-encoding|x-cortad-as)$/i;
+const HOP = /^(?:host|content-length|connection|keep-alive|transfer-encoding|upgrade|expect|te|trailer|accept-encoding|x-cortad-as|x-cortad-turn)$/i;
 
 export function makeCapture({ work, keepSecret, onDoor }) {
   const file = join(work, "trace.jsonl");
@@ -29,6 +29,9 @@ export function makeCapture({ work, keepSecret, onDoor }) {
   // ponytail: Node, Bun and Python. Go, Ruby, Java and PHP apps are asked for their route on the screen.
   const env = (base) => ({
     CORTAD_TRACE_FILE: file,
+    // The run writes the customer's rule sentences here (the engine's /tmp/rules.json lands in this
+    // folder), and the hook reads which of them each model call's prompt carried.
+    CORTAD_RULES_FILE: join(work, "rules.json"),
     NODE_OPTIONS: `${base.NODE_OPTIONS ?? ""} --require ${JSON.stringify(HOOK)}`.trim(),
     BUN_OPTIONS: `${base.BUN_OPTIONS ?? ""} --preload=${bunHook}`.trim(),
     PYTHONPATH: [PYHOOK, base.PYTHONPATH].filter(Boolean).join(":"),
@@ -72,6 +75,15 @@ export function makeCapture({ work, keepSecret, onDoor }) {
 
 // Every model call the hook wrote down, kept here: totals per host and model, and the newest rows.
 const ROWS = 5000;
+// The turn a row was pinned to by the run's own tag, and the rule ids the call's prompt carried.
+const TURN = /^[A-Za-z0-9:_.-]{1,80}$/;
+const RULE_ID = /^[\w:-]{1,64}$/;
+const turnOf = (v) => (typeof v === "string" && TURN.test(v) ? { turn: v } : {});
+const rulesOf = (v) => (Array.isArray(v) ? { rules: v.filter((id) => typeof id === "string" && RULE_ID.test(id)).slice(0, 300) } : {});
+// What the app's tools answered behind a call, as the hook read them off the next prompt.
+const toolsOf = (v) => (Array.isArray(v) && v.length
+  ? { tools: v.filter((t) => t && typeof t.text === "string" && t.text.trim()).slice(0, 12).map((t) => ({ name: String(t.name ?? "").slice(0, 80), text: t.text.slice(0, 3000) })) }
+  : {});
 const meter = (() => {
   const rows = [];
   const deps = [];
@@ -82,14 +94,15 @@ const meter = (() => {
       rows.push({
         at: count(call.at), host: call.host.slice(0, 253), model: String(call.model ?? "").slice(0, 160), status: count(call.status),
         promptTokens: count(call.promptTokens), cachedTokens: count(call.cachedTokens), completionTokens: count(call.completionTokens),
-        usage: call.usage === true,
+        usage: call.usage === true, ...turnOf(call.turn), ...rulesOf(call.rules), ...toolsOf(call.tools),
       });
       if (rows.length > ROWS) rows.splice(0, rows.length - ROWS);
     },
     // A service their settings name: only its setting, host and status travel, never a byte of it.
     dep(d) {
-      if (!d || typeof d.env !== "string" || !/^[A-Z_][A-Z0-9_]*$/.test(d.env) || typeof d.host !== "string") return;
-      deps.push({ at: count(d.at), env: d.env, host: d.host.slice(0, 253), status: count(d.status), ...(typeof d.code === "string" ? { code: d.code.slice(0, 40) } : {}) });
+      if (!d || typeof d.host !== "string" || !d.host) return;
+      const env = typeof d.env === "string" && /^[A-Z_][A-Z0-9_]*$/.test(d.env) ? d.env : undefined;
+      deps.push({ at: count(d.at), ...(env ? { env } : {}), host: d.host.slice(0, 253), status: count(d.status), ...(typeof d.code === "string" ? { code: d.code.slice(0, 40) } : {}), ...turnOf(d.turn) });
       if (deps.length > ROWS) deps.splice(0, deps.length - ROWS);
     },
     report() {
@@ -107,7 +120,7 @@ const meter = (() => {
       }
       return {
         totals: [...totals.values()].sort((a, b) => b.calls - a.calls),
-        rows: rows.slice(-200).reverse().map(({ at, host, model, status }) => ({ at, host, model, status })),
+        rows: rows.slice(-400).reverse().map(({ at, host, model, status, turn, rules, tools }) => ({ at, host, model, status, ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}) })),
         deps: deps.slice(-400).reverse(),
       };
     },

package/lib/start.mjs

@@ -55,6 +55,7 @@ function pythonStart(dir, onPath) {
   const py = venv ? JSON.stringify(venv) : existsSync(join(dir, "uv.lock")) && onPath("uv") ? "uv run python" : existsSync(join(dir, "poetry.lock")) && onPath("poetry") ? "poetry run python" : "python3";
   if (existsSync(join(dir, "manage.py"))) return { cmd: `${py} manage.py runserver`, serves: true };
   const named = [...new Set([...ENTRIES, ...entriesIn(dir)])].map((entry) => [entry, read(join(dir, entry))]).filter(([, text]) => text);
+  const manifest = `${read(join(dir, "requirements.txt"))}\n${read(join(dir, "pyproject.toml"))}\n${read(join(dir, "Pipfile"))}`;
   // The file that serves before the file that merely runs: src/run_agent.py runs a conversation in
   // the terminal and src/run_service.py is the app, and they sit in one folder.
   for (const [entry, text] of [...named.filter(([, text]) => SERVES.test(text)), ...named]) {
@@ -65,10 +66,19 @@ function pythonStart(dir, onPath) {
     if (fast) return { cmd: `${py} -m uvicorn ${module}:${fast[1]} --host 127.0.0.1 --port 8000`, serves: true };
     const flask = /^(\w+)\s*=\s*Flask\(/m.exec(text);
     if (flask) return { cmd: `${py} -m flask --app ${module} run --port 5000`, serves: true };
+    // The app object built elsewhere and only bound or re-exported here: resumeforge's
+    // backend/app/main.py is `from .application import (app, ...)`, and the server the manifest
+    // names says what serves it.
+    const bound = BOUND_APP.exec(text)?.[1] ?? IMPORTED_APP.exec(text)?.slice(1).find(Boolean);
+    if (bound && /\b(?:fastapi|starlette|litestar|quart)\b/i.test(manifest)) return { cmd: `${py} -m uvicorn ${module}:${bound} --host 127.0.0.1 --port 8000`, serves: true };
+    if (bound && /\bflask\b/i.test(manifest)) return { cmd: `${py} -m flask --app ${module}:${bound} run --port 5000`, serves: true };
   }
   return null;
 }
 
+// `app = create_app()` at the top of the file, or `from .application import app`.
+const BOUND_APP = /^(app|application)\s*=\s*[\w.]+\(/m;
+const IMPORTED_APP = /^from\s+[\w.]+\s+import\s+(?:\(\s*[^)]*?\b(app|application)\b[^)]*\)|[^\n(]*\b(app|application)\b)/m;
 const KNOWN = ["package.json", "pyproject.toml", "requirements.txt", "manage.py", "Pipfile", "uv.lock"];
 
 const startOf = (dir, onPath) => nodeStart(dir, onPath) ?? pythonStart(dir, onPath);

package/lib/trace.cjs

@@ -22,6 +22,73 @@ if (FILE) {
     // own launcher included, so "loaded" is not "watching your app": only a listener on the app's port is.
     const listening = (port) => { if (Number.isInteger(port) && port > 0) row({ listen: port, pid: process.pid }); };
     const MAX = 65536;
+    // The turn a message came in under, when the run tagged it: one opaque id per request, so a
+    // model call and its prompt can be pinned to the reply they produced even while five turns are
+    // in flight. It is read here and never shown to your app's own code path beyond the header.
+    const TURN = /^[A-Za-z0-9:_.-]{1,80}$/;
+    const turnOf = (headers) => {
+      const t = headers && (typeof headers.get === "function" ? headers.get("x-cortad-turn") : headers["x-cortad-turn"]);
+      return typeof t === "string" && TURN.test(t) ? t : undefined;
+    };
+    // The customer's own rule sentences, written beside the trace by the run, so each model call can
+    // say which of them its prompt carried: a rule is then asked only of a reply whose call was told
+    // it. Absent file, nothing is claimed either way. Reloaded when the file changes.
+    const RULES = process.env.CORTAD_RULES_FILE;
+    let rules = null, rulesAt = -1;
+    const norm = (s) => String(s || "")
+      .replace(/\\u([0-9a-fA-F]{4})/g, (_, h) => String.fromCharCode(parseInt(h, 16)))
+      .replace(/\\[nrt]/g, " ").replace(/\\"/g, '"').replace(/\\\\/g, "\\")
+      .toLowerCase().replace(/\s+/g, " ").trim();
+    // A sentence with a slot in it ("answer in {language}") is matched by its literal parts.
+    const partsOf = (text) => text.split(/\{[^}]*\}|\$\{[^}]*\}|%[sd]|<[^>]{1,40}>/).map(norm).filter((p) => p.length >= 12);
+    const rulesNow = () => {
+      if (!RULES) return null;
+      try {
+        const at = fs.statSync(RULES).mtimeMs;
+        if (at !== rulesAt) {
+          rulesAt = at;
+          const list = JSON.parse(fs.readFileSync(RULES, "utf8"));
+          rules = Array.isArray(list)
+            ? list.filter((r) => r && typeof r.id === "string" && typeof r.text === "string").map((r) => ({ id: r.id, parts: partsOf(r.text) })).filter((r) => r.parts.length)
+            : [];
+        }
+      } catch { /* not written yet, or gone */ }
+      return rules;
+    };
+    // What the app's tools answered, as the prompt of the next model call carries them: the tool
+    // messages of a chat body, the function outputs of a responses body, the tool_result blocks of
+    // an Anthropic one, the functionResponse parts of a Gemini one. They are the material a reply's
+    // facts rest on, and without them a real ticket id read as invented. Bounded per call.
+    const TOOL_TEXT = 3000, TOOLS_MAX = 12;
+    const textOf = (v) => (typeof v === "string" ? v : Array.isArray(v) ? v.map((p) => (p && typeof p === "object" ? (p.text || p.content || "") : String(p || ""))).filter(Boolean).join("\n") : v && typeof v === "object" ? JSON.stringify(v) : "");
+    const toolsIn = (sent) => {
+      let body; try { body = JSON.parse(sent); } catch { return undefined; }
+      if (!body || typeof body !== "object") return undefined;
+      const out = [];
+      const names = new Map();
+      const add = (name, text) => { const t = textOf(text).slice(0, TOOL_TEXT); if (t.trim() && out.length < TOOLS_MAX && !out.some((o) => o.text === t)) out.push({ name: String(name || "").slice(0, 80), text: t }); };
+      for (const m of Array.isArray(body.messages) ? body.messages : []) {
+        if (!m || typeof m !== "object") continue;
+        for (const c of Array.isArray(m.tool_calls) ? m.tool_calls : []) if (c && c.id && c.function) names.set(c.id, c.function.name);
+        if (Array.isArray(m.content)) for (const c of m.content) if (c && c.type === "tool_use" && c.id) names.set(c.id, c.name);
+      }
+      for (const m of Array.isArray(body.messages) ? body.messages : []) {
+        if (!m || typeof m !== "object") continue;
+        if (m.role === "tool" || m.role === "function") add(m.name || names.get(m.tool_call_id), m.content);
+        if (Array.isArray(m.content)) for (const c of m.content) if (c && c.type === "tool_result") add(names.get(c.tool_use_id), c.content);
+      }
+      const items = Array.isArray(body.input) ? body.input : [];
+      for (const it of items) if (it && it.type === "function_call" && it.call_id) names.set(it.call_id, it.name);
+      for (const it of items) if (it && it.type === "function_call_output") add(names.get(it.call_id), it.output);
+      for (const c of Array.isArray(body.contents) ? body.contents : []) for (const p of c && Array.isArray(c.parts) ? c.parts : []) if (p && p.functionResponse) add(p.functionResponse.name, p.functionResponse.response);
+      return out.length ? out : undefined;
+    };
+    const rulesIn = (sent) => {
+      const list = rulesNow();
+      if (!list) return undefined;
+      const body = norm(sent);
+      return list.filter((r) => r.parts.every((p) => body.includes(p))).map((r) => r.id);
+    };
     // Where models are served, by host, and the paths every OpenAI-shaped or vendor endpoint ends with.
     const MODEL_HOST = /(?:^|\.)(?:openai\.com|anthropic\.com|fireworks\.ai|openrouter\.ai|groq\.com|mistral\.ai|together\.xyz|together\.ai|deepseek\.com|cohere\.ai|cohere\.com|perplexity\.ai|x\.ai|googleapis\.com|openai\.azure\.com|cognitiveservices\.azure\.com|amazonaws\.com|replicate\.com|huggingface\.co|cerebras\.ai|deepinfra\.com|novita\.ai|moonshot\.cn|dashscope\.aliyuncs\.com|bigmodel\.cn|ai-gateway\.vercel\.sh|gateway\.ai\.cloudflare\.com|helicone\.ai|portkey\.ai)$/i;
     const MODEL_PATH = /\/(?:chat\/completions|completions|responses|messages|embeddings)$|:(?:generateContent|streamGenerateContent)|\/invoke(?:-with-response-stream)?$|\/api\/(?:chat|generate)$/i;
@@ -50,7 +117,8 @@ if (FILE) {
     http.Server.prototype.emit = function (type, req, ...rest) {
       if (type === "listening") { try { listening(this.address()?.port); } catch { /* not a TCP server */ } }
       if (type !== "request" || !req || !req.method || /^(?:GET|HEAD|OPTIONS)$/.test(req.method)) return emit.call(this, type, req, ...rest);
-      const ctx = { method: req.method, path: req.url || "/", headers: { ...req.headers }, chunks: [], size: 0, noted: false };
+      const { "x-cortad-turn": _turn, ...headers } = req.headers;
+      const ctx = { method: req.method, path: req.url || "/", headers, chunks: [], size: 0, noted: false, turn: turnOf(req.headers) };
       const push = req.push;
       req.push = function (chunk, encoding) {
         if (chunk && ctx.size < MAX) { const b = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding); ctx.chunks.push(b); ctx.size += b.length; }
@@ -85,7 +153,7 @@ if (FILE) {
         if (!req || /^(?:GET|HEAD|OPTIONS)$/.test(req.method)) return handler.call(self, req, server);
         let path = "/";
         try { const u = new URL(req.url); path = u.pathname + u.search; } catch { /* keep "/" */ }
-        const ctx = { method: req.method, path, headers: Object.fromEntries(req.headers), chunks: [], size: 0, noted: false };
+        const ctx = { method: req.method, path, headers: Object.fromEntries([...req.headers].filter(([k]) => k !== "x-cortad-turn")), chunks: [], size: 0, noted: false, turn: turnOf(req.headers) };
         // ponytail: a body is copied only when it is small or says it is text; an upload is left alone.
         const size = Number(req.headers.get("content-length") || 0);
         if (size <= MAX || /json|text|form/i.test(req.headers.get("content-type") || "")) {
@@ -166,10 +234,12 @@ if (FILE) {
       const m = /\/models\/([^/:]+):|\/model\/([^/]+)\/(?:invoke|converse)/.exec(path || "");
       return m ? decodeURIComponent(m[1] || m[2]) : "";
     };
-    const meter = ({ host, path, sent, status, type, body }) => {
+    const meter = ({ host, path, sent, status, type, body, turn }) => {
       try {
         const reply = readReply(type, String(body || "").slice(0, REPLY_MAX));
-        const row = { at: Date.now(), host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens) };
+        const rules = rulesIn(sent);
+        const tools = toolsIn(sent);
+        const row = { at: Date.now(), host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens), ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}) };
         fs.appendFileSync(FILE, JSON.stringify({ call: row }) + "\n", { mode: 0o600 });
       } catch { /* the command is gone */ }
     };
@@ -196,7 +266,8 @@ if (FILE) {
       } catch { return null; }
     };
     const depRow = (dep, status, code) => {
-      try { fs.appendFileSync(FILE, JSON.stringify({ dep: { at: Date.now(), env: dep.env, host: dep.host, status, code: code ? String(code).slice(0, 40) : undefined } }) + "\n", { mode: 0o600 }); } catch { /* the command is gone */ }
+      const turn = (als.getStore() || {}).turn;
+      try { fs.appendFileSync(FILE, JSON.stringify({ dep: { at: Date.now(), ...(dep.env ? { env: dep.env } : {}), host: dep.host, status, code: code ? String(code).slice(0, 40) : undefined, ...(turn ? { turn } : {}) } }) + "\n", { mode: 0o600 }); } catch { /* the command is gone */ }
     };
 
     // Outbound: the model call your app makes while it handles that request.
@@ -210,7 +281,10 @@ if (FILE) {
           if (isModelCall(u.host, u.pathname)) url = u;
         } catch { /* not a URL we can read */ }
         if (!url) {
-          const dep = depOf(input);
+          // Every outbound host, named by the setting that points at it when one does, and by the
+          // host alone otherwise: the search provider behind TAVILY_API_KEY has no URL setting.
+          let dep = depOf(input);
+          if (!dep) { try { const u = new URL(typeof input === "string" ? input : input && input.url ? input.url : String(input)); if (/^https?:$/.test(u.protocol) && !/^(localhost|127\.0\.0\.1|\[::1\])$/.test(u.hostname)) dep = { host: u.hostname }; } catch { /* not a URL */ } }
           if (!dep) return realFetch.apply(this, arguments);
           return realFetch.apply(this, arguments).then(
             (res) => { depRow(dep, res.status); return res; },
@@ -219,7 +293,7 @@ if (FILE) {
         }
         const sent = bodyText(init && init.body);
         note(als.getStore(), sent);
-        const call = { host: url.host, path: url.pathname, sent };
+        const call = { host: url.host, path: url.pathname, sent, turn: (als.getStore() || {}).turn };
         return realFetch.apply(this, arguments).then((res) => {
           try {
             // A clone is a tee: your app's branch gets every byte as fast as it reads, this one is read to the end.
@@ -248,7 +322,7 @@ if (FILE) {
             const end = req.end;
             req.write = function (chunk, ...r) { if (chunk && typeof chunk !== "function") parts.push(Buffer.from(chunk)); return write.call(this, chunk, ...r); };
             req.end = function (chunk, ...r) { if (chunk && typeof chunk !== "function") parts.push(Buffer.from(chunk)); sent = Buffer.concat(parts).toString("utf8"); note(ctx, sent); return end.call(this, chunk, ...r); };
-            const call = { host: String(host || ""), path: String(path || "").split("?")[0] };
+            const call = { host: String(host || ""), path: String(path || "").split("?")[0], turn: ctx && ctx.turn };
             let done = false;
             const once = (row) => { if (!done) { done = true; meter({ ...call, sent, ...row }); } };
             req.once("error", () => once({ status: 0, type: "", body: "" }));

package/lib/verbs.mjs

@@ -127,7 +127,13 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
       const more = await call("GET", path(page, d.runId ?? jobId));
       if (!more.ok) return more;
       d.findings = [...(d.findings ?? []), ...(more.data.findings ?? [])];
-      if (more.data.byLine) d.byLine = [...(d.byLine ?? []), ...more.data.byLine];
+      // Every page carries the whole byLine; merged by line with the ids joined, or the first
+      // page's groups print twice.
+      for (const g of more.data.byLine ?? []) {
+        const held = (d.byLine ?? []).find((h) => h.path === g.path && h.line === g.line);
+        if (held) held.findingIds = [...new Set([...held.findingIds, ...g.findingIds])];
+        else d.byLine = [...(d.byLine ?? []), g];
+      }
     }
     return first;
   };

package/local.mjs

@@ -434,7 +434,8 @@ async function verb(job) {
     case "lock": return { locked: Array.isArray(b.hosts) ? b.hosts.length : 0 };
     // What your app spent on its providers, from the hook inside it. An app this command did not
     // start has no hook, and the empty answer says the meter is absent rather than that nothing was spent.
-    case "usage": return capture?.usage() ?? {};
+    // Masked like every other reply: a tool's answer can carry a value from their env files.
+    case "usage": return capture ? JSON.parse(mask(JSON.stringify(capture.usage() ?? {}))) : {};
     // A world is ended from this terminal, never from the cloud.
     case "destroy": return { ok: true };
     default: return { ok: true };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.0-rc.3",
+  "version": "0.3.0-rc.5",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"

package/skill/SKILL.md

@@ -50,7 +50,7 @@ Take the person through them worst first, each with its file and line. A long li
 ## Fixing one finding
 
 1. One change, in the file and near the line the finding names.
-2. `verify <findingId>`. It answers within a second, and `run_status` follows it.
+2. `verify <findingId>`. It answers within a second, and `run_status` follows it. Cortad starts the app again itself before the replay, so the saved edit is what plays.
 3. Read the move on the `Visible trials:` and `Held-out trials:` lines:
    - `improved` without `inside the noise`, with the held-out line improved too or `no pair`: the behavior moved. The change stays.
    - `inside the noise`, `no change` or `unsettled`: the trials cannot tell the change from chance. The file goes back to how it was.