cortad 0.3.1-rc.1 → 0.3.1-rc.2

package/lib/pyhook/sitecustomize.py

@@ -84,6 +84,24 @@ def _install():
                 return ""
         return ""
 
+    def items(v):
+        return [x for x in v if isinstance(x, dict)] if isinstance(v, list) else []
+
+    # Only what came after the person's latest message belongs to this turn: a thread the app
+    # resends whole carries every earlier turn's tool answers too.
+    def since(seq, is_person):
+        at = -1
+        for i, m in enumerate(seq):
+            if is_person(m):
+                at = i
+        return seq[at + 1:]
+
+    def person_said(m):
+        return m.get("role") == "user" and not any(isinstance(c, dict) and c.get("type") == "tool_result" for c in (m.get("content") if isinstance(m.get("content"), list) else []))
+
+    def person_asked(c):
+        return c.get("role") == "user" and not any(p.get("functionResponse") for p in items(c.get("parts")))
+
     def tools_in(sent):
         body = parsed(sent) if isinstance(sent, str) else None
         if not isinstance(body, dict):
@@ -95,39 +113,258 @@ def _install():
             if t.strip() and len(out) < tools_max and all(o["text"] != t for o in out):
                 out.append({"name": str(name or "")[:80], "text": t})
 
-        messages = body.get("messages") if isinstance(body.get("messages"), list) else []
+        messages = items(body.get("messages"))
         for m in messages:
-            if not isinstance(m, dict):
-                continue
-            for c in m.get("tool_calls") or []:
-                if isinstance(c, dict) and c.get("id") and isinstance(c.get("function"), dict):
+            for c in items(m.get("tool_calls")):
+                if c.get("id") and isinstance(c.get("function"), dict):
                     names[c["id"]] = c["function"].get("name")
-            if isinstance(m.get("content"), list):
-                for c in m["content"]:
-                    if isinstance(c, dict) and c.get("type") == "tool_use" and c.get("id"):
-                        names[c["id"]] = c.get("name")
-        for m in messages:
-            if not isinstance(m, dict):
-                continue
+            for c in items(m.get("content")):
+                if c.get("type") == "tool_use" and c.get("id"):
+                    names[c["id"]] = c.get("name")
+        for m in since(messages, person_said):
             if m.get("role") in ("tool", "function"):
                 add(m.get("name") or names.get(m.get("tool_call_id")), m.get("content"))
-            if isinstance(m.get("content"), list):
-                for c in m["content"]:
-                    if isinstance(c, dict) and c.get("type") == "tool_result":
-                        add(names.get(c.get("tool_use_id")), c.get("content"))
-        items = body.get("input") if isinstance(body.get("input"), list) else []
-        for it in items:
-            if isinstance(it, dict) and it.get("type") == "function_call" and it.get("call_id"):
+            for c in items(m.get("content")):
+                if c.get("type") == "tool_result":
+                    add(names.get(c.get("tool_use_id")), c.get("content"))
+        inputs = items(body.get("input"))
+        for it in inputs:
+            if it.get("type") == "function_call" and it.get("call_id"):
                 names[it["call_id"]] = it.get("name")
-        for it in items:
-            if isinstance(it, dict) and it.get("type") == "function_call_output":
+        for it in since(inputs, lambda x: x.get("role") == "user"):
+            if it.get("type") == "function_call_output":
                 add(names.get(it.get("call_id")), it.get("output"))
-        for c in body.get("contents") if isinstance(body.get("contents"), list) else []:
-            for p in c.get("parts") if isinstance(c, dict) and isinstance(c.get("parts"), list) else []:
-                if isinstance(p, dict) and isinstance(p.get("functionResponse"), dict):
+        for c in since(items(body.get("contents")), person_asked):
+            for p in items(c.get("parts")):
+                if isinstance(p.get("functionResponse"), dict):
                     add(p["functionResponse"].get("name"), p["functionResponse"].get("response"))
         return out or None
 
+    # What the model asked the app to run: chat tool_calls (a stream's pieces joined by position), a
+    # legacy function_call, a responses function_call item, an Anthropic or Bedrock tool use, a
+    # Gemini functionCall. Read off the model's reply, and off this turn's earlier calls as the
+    # prompt resends them. Names and arguments only: every value clipped, a secret never written.
+    calls_max, value_max, args_max = 12, 200, 1200
+    secret_key = re.compile(r"(?:^|_)(?:pass(?:word|phrase)?|secret|token|api_?key|authorization|cookie|session(?:_id)?|credentials?|private_key)$")
+    secret_value = re.compile(r"^(?:Bearer\s|Basic\s|sk-|pk_|rk_|ghp_|gho_|github_pat_|xox[abpr]-|AKIA|AIza|eyJ[\w-]{10,}\.)")
+
+    def clipped(v, depth=0):
+        if isinstance(v, str):
+            return "[secret]" if secret_value.match(v) else (v[:value_max] + "…" if len(v) > value_max else v)
+        if isinstance(v, list):
+            return [] if depth > 4 else [clipped(x, depth + 1) for x in v[:20]]
+        if isinstance(v, dict):
+            if depth > 4:
+                return {}
+            return {k: ("[secret]" if secret_key.search(re.sub(r"([a-z])([A-Z])", r"\1_\2", str(k)).lower()) else clipped(x, depth + 1)) for k, x in list(v.items())[:40]}
+        return v
+
+    def args_text(raw):
+        v = raw
+        if isinstance(raw, str):
+            v = parsed(raw) if raw.strip() else {}
+            if v is None:
+                v = raw
+        return json.dumps(clipped({} if v is None else v), ensure_ascii=False, separators=(",", ":"))[:args_max]
+
+    def calls_of(events):
+        whole, parts, n = {}, {}, [0]
+
+        def put(key, name, args):
+            if name:
+                if not key:
+                    n[0] += 1
+                    key = "w%d" % n[0]
+                whole[key] = (name, args)
+
+        def piece(key, name, args):
+            p = parts.setdefault(key, ["", ""])
+            if name and not p[0]:
+                p[0] = name
+            if isinstance(args, str):
+                p[1] += args
+
+        def blocks(content):
+            for b in items(content):
+                if b.get("type") == "tool_use":
+                    put(b.get("id"), b.get("name"), b.get("input"))
+                if isinstance(b.get("toolUse"), dict):
+                    put(b["toolUse"].get("toolUseId"), b["toolUse"].get("name"), b["toolUse"].get("input"))
+
+        for e in events:
+            if not isinstance(e, dict):
+                continue
+            for c in items(e.get("choices")):
+                m = c.get("message") if isinstance(c.get("message"), dict) else None
+                if m:
+                    for t in items(m.get("tool_calls")):
+                        f = t.get("function") if isinstance(t.get("function"), dict) else None
+                        if f:
+                            put(t.get("id"), f.get("name"), f.get("arguments"))
+                    if isinstance(m.get("function_call"), dict):
+                        put(None, m["function_call"].get("name"), m["function_call"].get("arguments"))
+                d = c.get("delta") if isinstance(c.get("delta"), dict) else None
+                if d:
+                    for t in items(d.get("tool_calls")):
+                        f = t.get("function") if isinstance(t.get("function"), dict) else {}
+                        piece("c%s.%s" % (c.get("index") or 0, t.get("index", t.get("id"))), f.get("name"), f.get("arguments"))
+                    if isinstance(d.get("function_call"), dict):
+                        piece("f%s" % (c.get("index") or 0), d["function_call"].get("name"), d["function_call"].get("arguments"))
+            resp = e.get("response") if isinstance(e.get("response"), dict) else {}
+            for it in items(e.get("output")) + items([e.get("item")]) + items(resp.get("output")):
+                if it.get("type") == "function_call":
+                    put(it.get("call_id") or it.get("id"), it.get("name"), it.get("arguments"))
+            blocks(e.get("content"))
+            if isinstance(e.get("message"), dict):
+                blocks(e["message"].get("content"))
+            if isinstance(e.get("output"), dict) and isinstance(e["output"].get("message"), dict):
+                blocks(e["output"]["message"].get("content"))
+            cb = e.get("content_block") if isinstance(e.get("content_block"), dict) else {}
+            if e.get("type") == "content_block_start" and cb.get("type") == "tool_use":
+                piece("a%s" % e.get("index"), cb.get("name"), "")
+            delta = e.get("delta") if isinstance(e.get("delta"), dict) else {}
+            if e.get("type") == "content_block_delta" and delta.get("type") == "input_json_delta":
+                piece("a%s" % e.get("index"), "", delta.get("partial_json"))
+            for c in items(e.get("candidates")):
+                for p in items((c.get("content") or {}).get("parts") if isinstance(c.get("content"), dict) else None):
+                    if isinstance(p.get("functionCall"), dict):
+                        put(None, p["functionCall"].get("name"), p["functionCall"].get("args"))
+        found = list(whole.values()) + [tuple(p) for p in parts.values()]
+        return [{"name": str(name)[:80], "arguments": args_text(args)} for name, args in found if name]
+
+    # This turn's earlier calls, as the prompt resends them after the person's latest message.
+    def called_before(body):
+        events = []
+        for m in since(items(body.get("messages")), person_said):
+            if m.get("role") == "assistant":
+                events.append({"choices": [{"message": m}], "content": m.get("content")})
+        events.append({"output": since(items(body.get("input")), lambda x: x.get("role") == "user")})
+        for c in since(items(body.get("contents")), person_asked):
+            if c.get("role") == "model":
+                events.append({"candidates": [{"content": c}]})
+        return calls_of(events)
+
+    def called_in(sent, events):
+        body = parsed(sent) if isinstance(sent, str) else None
+        out = []
+        for c in (called_before(body) if isinstance(body, dict) else []) + calls_of(events):
+            if len(out) < calls_max and not any(o["name"] == c["name"] and o["arguments"] == c["arguments"] for o in out):
+                out.append(c)
+        return out or None
+
+    # The passages the prompt carried as retrieved context: a block the app itself labels as
+    # context, documents, knowledge, sources or search results, in its system prompt or from the
+    # person's latest message on, and Anthropic document and search_result blocks. The app's own
+    # instructions are not passages: only what sits under such a label is.
+    passage_text, passages_max = 3000, 6
+    material = re.compile(r"\b(?:retriev\w*|context|knowledge|documents?|sources?|search[ _-]?results?|references?|passages?|excerpts?|snippets?|chunks?|background|faq)\b|检索|知识|参考资料|资料|上下文|文档|背景", re.I)
+    heading = re.compile(r"^\s*(?:#{1,6}\s+[^\n]{1,80}|[^\n]{1,80}[:：]\s*(?:\([^\n)]*\))?|\[[^\n\]]{1,80}\]|={2,}\s*[^\n]{1,80}?\s*={2,})\s*$")
+    # An inline label is a noun phrase that ends in the material word ("context:", "background:",
+    # "retrieved passages:"): "4. Knowledge:" in a numbered instruction and a JSON key are not.
+    inline = re.compile(r"^\s*(?:[-*]\s+)?([A-Za-z\u4e00-\u9fff][A-Za-z\u4e00-\u9fff '’_-]{0,29})[:：]\s*\S")
+    named = re.compile(r"(?:" + material.pattern + r")\s*$", re.I)
+    tagged = re.compile(r"<([A-Za-z][\w-]*)[^>]*>([\s\S]*?)</\1>")
+
+    def labelled(text, add):
+        def lift(m):
+            if material.search(m.group(1).replace("_", " ")):
+                add(m.group(1), m.group(2))
+                return ""
+            return m.group(0)
+        paras = re.split(r"\n\s*\n", tagged.sub(lift, str(text or "")))
+        i = 0
+        while i < len(paras):
+            lines = paras[i].split("\n")
+            head = lines[0]
+            if heading.match(head) and material.search(head):
+                body = "\n".join(lines[1:])
+                # A heading alone on its paragraph labels what follows, up to the next heading.
+                if not body.strip():
+                    while i + 1 < len(paras) and len(body) < passage_text and not heading.match(paras[i + 1].split("\n")[0]):
+                        i += 1
+                        body += ("\n\n" if body else "") + paras[i]
+                add(re.sub(r"[#:：\[\]=]", "", head).strip(), body)
+            else:
+                label = next((m for m in (inline.match(l) for l in lines) if m and named.search(m.group(1))), None)
+                if label:
+                    add(label.group(1).strip(), paras[i])
+            i += 1
+
+    def passages_in(sent):
+        body = parsed(sent) if isinstance(sent, str) else None
+        if not isinstance(body, dict):
+            return None
+        out = []
+
+        def add(name, text):
+            t = str(text or "").strip()[:passage_text]
+            if len(t) >= 20 and len(out) < passages_max and all(o["text"] != t for o in out):
+                out.append({"name": str(name or "")[:80], "text": t})
+
+        def scan(content):
+            if isinstance(content, str):
+                return labelled(content, add)
+            for b in items(content):
+                if b.get("type") == "document":
+                    src = b.get("source") if isinstance(b.get("source"), dict) else None
+                    add(b.get("title") or "document", text_of(src.get("data") if src and src.get("data") is not None else (src or {}).get("content")) if src else text_of(b.get("content")))
+                elif b.get("type") == "search_result":
+                    add(b.get("title") or b.get("source") or "search result", text_of(b.get("content")))
+                elif isinstance(b.get("text"), str):
+                    labelled(b["text"], add)
+
+        messages = items(body.get("messages"))
+        for m in messages:
+            if m.get("role") in ("system", "developer"):
+                scan(m.get("content"))
+        last = max([i for i, m in enumerate(messages) if person_said(m)] or [0])
+        for m in messages[last:]:
+            if m.get("role") == "user":
+                scan(m.get("content"))
+        scan(body.get("system"))
+        scan(body.get("instructions"))
+        if isinstance(body.get("systemInstruction"), dict):
+            scan(body["systemInstruction"].get("parts"))
+        inputs = [{"role": "user", "content": body["input"]}] if isinstance(body.get("input"), str) else items(body.get("input"))
+        last = max([i for i, it in enumerate(inputs) if it.get("role") == "user"] or [0])
+        for it in inputs[last:]:
+            if it.get("role") in ("user", "system", "developer"):
+                scan(it.get("content"))
+        return out or None
+
+    # The passages a retrieval call answered with: a vector store, a search index or a web search,
+    # read off its reply when it is JSON. Text fields only, bounded like the rest.
+    retrieval_host = re.compile(r"pinecone\.io|qdrant|weaviate|chroma|zilliz|milvus|turbopuffer|upstash\.io|algolia|typesense|meilisearch|elastic|opensearch|vespa|tavily\.com|exa\.ai|serper\.dev|serpapi\.com|search\.brave\.com|bing\.microsoft\.com|jina\.ai", re.I)
+    retrieval_path = re.compile(r"/(?:query|search|_search|retrieve|similarity_search|rerank|hybrid)(?:/|$)|/points/(?:search|query)|/rpc/match\w*", re.I)
+    passage_key = re.compile(r"^(?:text|content|page_?content|pageContent|chunk|snippet|passage|body|document|documents|description|answer|raw_content|highlights?|excerpt)$", re.I)
+
+    def is_retrieval(url):
+        try:
+            parts = urlsplit(str(url))
+        except ValueError:
+            return False
+        return bool(retrieval_host.search(parts.hostname or "")) or bool(retrieval_path.search(parts.path or ""))
+
+    def passages_from(raw):
+        out, nodes = [], [0]
+
+        def walk(v, key, depth):
+            nodes[0] += 1
+            if len(out) >= passages_max or depth > 7 or nodes[0] > 4000:
+                return
+            if isinstance(v, str):
+                t = v.strip()[:passage_text]
+                if passage_key.match(key) and len(t) >= 20 and all(o["text"] != t for o in out):
+                    out.append({"name": key, "text": t})
+            elif isinstance(v, list):
+                for x in v:
+                    walk(x, key, depth + 1)
+            elif isinstance(v, dict):
+                for k, x in v.items():
+                    walk(x, str(k), depth + 1)
+        walk(parsed(raw) if isinstance(raw, str) else None, "", 0)
+        return out or None
+
     # A framework that folds its tools' answers into the next prompt as plain text sends no tool
     # message to find: within one turn, what a later call's prompt carries that the call before did
     # not, outside the model's own words, is that material. The first call holds the customer's
@@ -189,13 +426,14 @@ def _install():
 
     # Every other outbound call: the host and what it answered, never a byte of it. A search
     # provider over its limit for a whole run was invisible until this row existed.
-    def dep(url, status, code=None):
+    # A store on this machine (a local Qdrant or Chroma) is said too when it answered with passages.
+    def dep(url, status, code=None, passages=None, turn=None):
         try:
             host = (urlsplit(str(url)).hostname or "").lower()
-            if not host or host in ("localhost", "127.0.0.1", "::1"):
+            if not host or (host in ("localhost", "127.0.0.1", "::1") and not passages):
                 return
-            turn = turn_now()
-            write({"dep": {"at": int(time.time() * 1000), "host": host[:253], "status": int(status or 0), **({"code": str(code)[:40]} if code else {}), **({"turn": turn} if turn else {})}})
+            turn = turn or turn_now()
+            write({"dep": {"at": int(time.time() * 1000), "host": host[:253], "status": int(status or 0), **({"code": str(code)[:40]} if code else {}), **({"turn": turn} if turn else {}), **({"passages": passages} if passages else {})}})
         except Exception:
             pass
     def is_model_call(url):
@@ -283,7 +521,7 @@ def _install():
             if isinstance(part, dict):
                 usage.update(part)
             model = e.get("model") or msg.get("model") or resp.get("model") or e.get("modelVersion") or model
-        return tokens_of(usage or None), model
+        return tokens_of(usage or None), model, events
 
     def asked_for(url, sent):
         body = parsed(sent) if isinstance(sent, str) else None
@@ -308,7 +546,7 @@ def _install():
 
     def meter(url, sent, status, kind, raw, turn=None):
         try:
-            tokens, model = read_reply(kind, (raw or "")[:reply_max])
+            tokens, model, events = read_reply(kind, (raw or "")[:reply_max])
             parts = urlsplit(str(url))
             row = {"at": int(time.time() * 1000), "host": parts.netloc.replace(":443", ""), "model": str(asked_for(url, sent) or model or "")[:160],
                    "status": int(status or 0), "usage": tokens is not None}
@@ -323,6 +561,12 @@ def _install():
             tools = tools_in(as_text) or material_in(as_text, turn)
             if tools:
                 row["tools"] = tools
+            called = called_in(as_text, events)
+            if called:
+                row["called"] = called
+            passages = passages_in(as_text)
+            if passages:
+                row["passages"] = passages
             write({"call": row})
         except Exception:
             pass
@@ -693,21 +937,31 @@ def _install():
                     self.finish()
 
         def watch(request, response, sent):
-            if not is_model_call(request.url):
+            model = is_model_call(request.url)
+            if not model and not is_retrieval(request.url):
                 dep(request.url, response.status_code)
                 return response
             kind = response.headers.get("content-type", "")
             if getattr(response, "_content", None) is not None:
-                meter(request.url, sent, response.status_code, kind, response.content.decode("utf-8", "replace"))
+                if model:
+                    meter(request.url, sent, response.status_code, kind, response.content.decode("utf-8", "replace"))
+                else:
+                    dep(request.url, response.status_code, passages=passages_from(response.content.decode("utf-8", "replace")))
                 return response
-            done = lambda raw: meter(request.url, sent, response.status_code, kind, unpacked(raw, response.headers.get("content-encoding")))
+            turn = turn_now()
+            if model:
+                done = lambda raw: meter(request.url, sent, response.status_code, kind, unpacked(raw, response.headers.get("content-encoding")), turn)
+            else:
+                done = lambda raw: dep(request.url, response.status_code, passages=passages_from(unpacked(raw, response.headers.get("content-encoding"))), turn=turn)
             stream = response.stream
             if hasattr(stream, "__aiter__") and hasattr(stream, "aclose"):
                 response.stream = AsyncKept(stream, done)
             elif hasattr(stream, "__iter__"):
                 response.stream = SyncKept(stream, done)
-            else:
+            elif model:
                 meter(request.url, sent, response.status_code, kind, "")
+            else:
+                dep(request.url, response.status_code)
             return response
 
         for cls in (module.Client, module.AsyncClient):
@@ -769,7 +1023,8 @@ def _install():
                 raise
             try:
                 if not is_model_call(request.url):
-                    dep(request.url, response.status_code)
+                    read = is_retrieval(request.url) and not k.get("stream")
+                    dep(request.url, response.status_code, passages=passages_from(response.content.decode("utf-8", "replace")) if read else None)
                 if is_model_call(request.url):
                     # A streamed reply is counted as a call whose counts were not read.
                     raw = response.content.decode("utf-8", "replace") if getattr(response, "_content_consumed", False) else ""
@@ -797,6 +1052,8 @@ def _install():
                 raise
             if is_model_call(str_or_url):
                 response._cortad = (str_or_url, text(body), turn_now())
+            elif is_retrieval(str_or_url):
+                response._cortad_dep = (str_or_url, turn_now())
             else:
                 dep(str_or_url, response.status)
             return response
@@ -806,6 +1063,10 @@ def _install():
 
         async def read_kept(self, *a, **k):
             raw = await read(self, *a, **k)
+            store = getattr(self, "_cortad_dep", None)
+            if store:
+                self._cortad_dep = None
+                dep(store[0], self.status, passages=passages_from((raw or b"").decode("utf-8", "replace")), turn=store[1])
             call = getattr(self, "_cortad", None)
             if call:
                 self._cortad = None
@@ -813,6 +1074,10 @@ def _install():
             return raw
 
         def release_kept(self, *a, **k):
+            store = getattr(self, "_cortad_dep", None)
+            if store:
+                self._cortad_dep = None
+                dep(store[0], self.status, turn=store[1])
             call = getattr(self, "_cortad", None)
             if call:
                 self._cortad = None

package/lib/read-text.mjs

@@ -36,6 +36,14 @@ const named = (label, group, max = 12) => {
 // "Journeys (6):" when every one is here, "Journeys (6), 4 listed:" when not.
 const counted = (label, count, n) => `${label} (${num(count ?? n)})${has(count) && n < count ? `, ${num(n)} listed` : ""}:`;
 
+// The records the app already holds, read from its own data files, that a trial may name: an agent
+// that sees a trial ask about SO20260810001 can tell it is a real order, not an invented one.
+const recordsLine = (records) => {
+  const collections = (records?.collections ?? []).filter((c) => c?.name && c.examples?.length);
+  if (!collections.length) return null;
+  return `Records the trials can name: ${collections.map((c) => `${c.name} (${c.examples.slice(0, 3).map((e) => clip(e, 40)).join(", ")})`).join(", ")}.`;
+};
+
 // The walkthrough, in the order the person is walked through it.
 export function readLines(r) {
   const lines = ["What Cortad read:"];
@@ -43,7 +51,7 @@ export function readLines(r) {
     const examples = (r.rules.examples ?? []).slice(0, 3);
     lines.push(`${rulesHead(r.rules)}${examples.length ? ` ${examples.length} of them:` : ""}`, ...examples.map(ruleLine));
   }
-  for (const line of [named("Journeys", r.journeys), named("Simulated users", r.profiles), named("Endpoints", r.doors)]) if (line) lines.push(line);
+  for (const line of [named("Journeys", r.journeys), named("Simulated users", r.profiles), named("Endpoints", r.doors), recordsLine(r.records)]) if (line) lines.push(line);
   const m = r.machine;
   if (m) {
     const misses = m.misses ?? [];

package/lib/replay.mjs

@@ -106,9 +106,15 @@ const RULE_ID = /^[\w:-]{1,64}$/;
 const turnOf = (v) => (typeof v === "string" && TURN.test(v) ? { turn: v } : {});
 const rulesOf = (v) => (Array.isArray(v) ? { rules: v.filter((id) => typeof id === "string" && RULE_ID.test(id)).slice(0, 300) } : {});
 // What the app's tools answered behind a call, as the hook read them off the next prompt.
-const toolsOf = (v) => (Array.isArray(v) && v.length
-  ? { tools: v.filter((t) => t && typeof t.text === "string" && t.text.trim()).slice(0, 12).map((t) => ({ name: String(t.name ?? "").slice(0, 80), text: t.text.slice(0, 3000) })) }
-  : {});
+const texts = (v) => (Array.isArray(v) ? v.filter((t) => t && typeof t.text === "string" && t.text.trim()).slice(0, 12).map((t) => ({ name: String(t.name ?? "").slice(0, 80), text: t.text.slice(0, 3000) })) : []);
+const toolsOf = (v) => (texts(v).length ? { tools: texts(v) } : {});
+// The tools the model asked for behind a call, names and clipped arguments, and the passages its
+// prompt was handed as retrieved context, or a retrieval call answered with.
+const calledOf = (v) => {
+  const called = Array.isArray(v) ? v.filter((c) => c && typeof c.name === "string" && c.name && typeof c.arguments === "string").slice(0, 12).map((c) => ({ name: c.name.slice(0, 80), arguments: c.arguments.slice(0, 1200) })) : [];
+  return called.length ? { called } : {};
+};
+const passagesOf = (v) => (texts(v).length ? { passages: texts(v).slice(0, 6) } : {});
 const meter = (() => {
   const rows = [];
   const deps = [];
@@ -119,7 +125,7 @@ const meter = (() => {
       rows.push({
         at: count(call.at), host: call.host.slice(0, 253), model: String(call.model ?? "").slice(0, 160), status: count(call.status),
         promptTokens: count(call.promptTokens), cachedTokens: count(call.cachedTokens), completionTokens: count(call.completionTokens),
-        usage: call.usage === true, ...turnOf(call.turn), ...rulesOf(call.rules), ...toolsOf(call.tools),
+        usage: call.usage === true, ...turnOf(call.turn), ...rulesOf(call.rules), ...toolsOf(call.tools), ...calledOf(call.called), ...passagesOf(call.passages),
       });
       if (rows.length > ROWS) rows.splice(0, rows.length - ROWS);
     },
@@ -127,7 +133,7 @@ const meter = (() => {
     dep(d) {
       if (!d || typeof d.host !== "string" || !d.host) return;
       const env = typeof d.env === "string" && /^[A-Z_][A-Z0-9_]*$/.test(d.env) ? d.env : undefined;
-      deps.push({ at: count(d.at), ...(env ? { env } : {}), host: d.host.slice(0, 253), status: count(d.status), ...(typeof d.code === "string" ? { code: d.code.slice(0, 40) } : {}), ...turnOf(d.turn) });
+      deps.push({ at: count(d.at), ...(env ? { env } : {}), host: d.host.slice(0, 253), status: count(d.status), ...(typeof d.code === "string" ? { code: d.code.slice(0, 40) } : {}), ...turnOf(d.turn), ...passagesOf(d.passages) });
       if (deps.length > ROWS) deps.splice(0, deps.length - ROWS);
     },
     report() {
@@ -145,7 +151,7 @@ const meter = (() => {
       }
       return {
         totals: [...totals.values()].sort((a, b) => b.calls - a.calls),
-        rows: rows.slice(-400).reverse().map(({ at, host, model, status, turn, rules, tools }) => ({ at, host, model, status, ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}) })),
+        rows: rows.slice(-400).reverse().map(({ at, host, model, status, turn, rules, tools, called, passages }) => ({ at, host, model, status, ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages ? { passages } : {}) })),
         deps: deps.slice(-400).reverse(),
       };
     },

package/lib/text.mjs

@@ -57,7 +57,10 @@ function scoreLine(d) {
   // The number never stands without its denominator: "93 of 100" read as broad assurance to four
   // agents whose runs had measured a quarter of the checks.
   const on = has(d.questionsAsked) && has(d.questionsOf) && d.questionsOf > 0 ? ` on the ${num(d.questionsAsked)} of ${plural(d.questionsOf, "check")} measured` : "";
-  return `${floor ? "Provisional score" : "Score"} ${num(d.score)} of 100${on}${interval(d.ci)}${floor ? `, from ${plural(d.decided, "decided reading")}: under the 22-reading floor, so not a verdict yet` : ""}.`;
+  // The replies that failed stand beside the number: a 100 over the replies that answered, with 62 of
+  // 89 error replies unsaid, read as a clean bill.
+  const crashed = d.replies && d.replies.crashed > 0 ? `; ${num(d.replies.crashed)} of ${plural(d.replies.total, "reply", "replies")} were failures and are not in it` : "";
+  return `${floor ? "Provisional score" : "Score"} ${num(d.score)} of 100${on}${interval(d.ci)}${floor ? `, from ${plural(d.decided, "decided reading")}: under the 22-reading floor, so not a verdict yet` : ""}${crashed}.`;
 }
 // A row the run held back names a door and why, or, when no door could take it, only the reason.
 const heldBackLine = (h) => h.why === "not testable here"

package/lib/trace.cjs

@@ -3,8 +3,10 @@
 // your AI's door, with the exact body it takes and the sign-in it carried, learned from a message you
 // sent yourself rather than guessed from code. Second, every model call your app makes: the host, the
 // model it asked for, the status and the token counts the provider sent back, so a run can say which
-// model answered and what it cost you. Reply text is never written down. Both go to a file only you
-// can read, in the command's own folder on this machine. Nothing here talks to a network.
+// model answered and what it cost you, which tools it asked for (names and clipped arguments, never a
+// secret) and the passages its prompt was handed as retrieved context. The words of a reply are never
+// written down. Both go to a file only you can read, in the command's own folder on this machine.
+// Nothing here talks to a network.
 "use strict";
 const FILE = process.env.CORTAD_TRACE_FILE;
 if (FILE) {
@@ -243,26 +245,181 @@ if (FILE) {
     // facts rest on, and without them a real ticket id read as invented. Bounded per call.
     const TOOL_TEXT = 3000, TOOLS_MAX = 12;
     const textOf = (v) => (typeof v === "string" ? v : Array.isArray(v) ? v.map((p) => (p && typeof p === "object" ? (p.text || p.content || "") : String(p || ""))).filter(Boolean).join("\n") : v && typeof v === "object" ? JSON.stringify(v) : "");
+    // Only what came after the person's latest message belongs to this turn: a thread the app
+    // resends whole carries every earlier turn's tool answers too.
+    const list = (v) => (Array.isArray(v) ? v.filter((x) => x && typeof x === "object") : []);
+    const since = (items, isPerson) => { let at = -1; items.forEach((m, i) => { if (isPerson(m)) at = i; }); return items.slice(at + 1); };
+    const personSaid = (m) => m.role === "user" && !(Array.isArray(m.content) && m.content.some((c) => c && c.type === "tool_result"));
+    const personAsked = (c) => c.role === "user" && !list(c.parts).some((p) => p.functionResponse);
     const toolsIn = (sent) => {
       let body; try { body = JSON.parse(sent); } catch { return undefined; }
       if (!body || typeof body !== "object") return undefined;
       const out = [];
       const names = new Map();
       const add = (name, text) => { const t = textOf(text).slice(0, TOOL_TEXT); if (t.trim() && out.length < TOOLS_MAX && !out.some((o) => o.text === t)) out.push({ name: String(name || "").slice(0, 80), text: t }); };
-      for (const m of Array.isArray(body.messages) ? body.messages : []) {
-        if (!m || typeof m !== "object") continue;
-        for (const c of Array.isArray(m.tool_calls) ? m.tool_calls : []) if (c && c.id && c.function) names.set(c.id, c.function.name);
-        if (Array.isArray(m.content)) for (const c of m.content) if (c && c.type === "tool_use" && c.id) names.set(c.id, c.name);
+      for (const m of list(body.messages)) {
+        for (const c of list(m.tool_calls)) if (c.id && c.function) names.set(c.id, c.function.name);
+        for (const c of list(m.content)) if (c.type === "tool_use" && c.id) names.set(c.id, c.name);
       }
-      for (const m of Array.isArray(body.messages) ? body.messages : []) {
-        if (!m || typeof m !== "object") continue;
+      for (const m of since(list(body.messages), personSaid)) {
         if (m.role === "tool" || m.role === "function") add(m.name || names.get(m.tool_call_id), m.content);
-        if (Array.isArray(m.content)) for (const c of m.content) if (c && c.type === "tool_result") add(names.get(c.tool_use_id), c.content);
+        for (const c of list(m.content)) if (c.type === "tool_result") add(names.get(c.tool_use_id), c.content);
       }
-      const items = Array.isArray(body.input) ? body.input : [];
-      for (const it of items) if (it && it.type === "function_call" && it.call_id) names.set(it.call_id, it.name);
-      for (const it of items) if (it && it.type === "function_call_output") add(names.get(it.call_id), it.output);
-      for (const c of Array.isArray(body.contents) ? body.contents : []) for (const p of c && Array.isArray(c.parts) ? c.parts : []) if (p && p.functionResponse) add(p.functionResponse.name, p.functionResponse.response);
+      const items = list(body.input);
+      for (const it of items) if (it.type === "function_call" && it.call_id) names.set(it.call_id, it.name);
+      for (const it of since(items, (x) => x.role === "user")) if (it.type === "function_call_output") add(names.get(it.call_id), it.output);
+      for (const c of since(list(body.contents), personAsked)) for (const p of list(c.parts)) if (p.functionResponse) add(p.functionResponse.name, p.functionResponse.response);
+      return out.length ? out : undefined;
+    };
+
+    // What the model asked the app to run: chat tool_calls (a stream's pieces joined by position), a
+    // legacy function_call, a responses function_call item, an Anthropic or Bedrock tool use, a
+    // Gemini functionCall. Read off the model's reply, and off this turn's earlier calls as the
+    // prompt resends them. Names and arguments only: every value clipped, a secret never written.
+    const CALLS_MAX = 12, VALUE_MAX = 200, ARGS_MAX = 1200;
+    const SECRET_KEY = /(?:^|_)(?:pass(?:word|phrase)?|secret|token|api_?key|authorization|cookie|session(?:_id)?|credentials?|private_key)$/;
+    const SECRET_VALUE = /^(?:Bearer\s|Basic\s|sk-|pk_|rk_|ghp_|gho_|github_pat_|xox[abpr]-|AKIA|AIza|eyJ[\w-]{10,}\.)/;
+    const clipped = (v, depth) => {
+      if (typeof v === "string") return SECRET_VALUE.test(v) ? "[secret]" : v.length > VALUE_MAX ? v.slice(0, VALUE_MAX) + "…" : v;
+      if (Array.isArray(v)) return depth > 4 ? [] : v.slice(0, 20).map((x) => clipped(x, depth + 1));
+      if (!v || typeof v !== "object") return v;
+      const o = {};
+      if (depth > 4) return o;
+      for (const [k, x] of Object.entries(v).slice(0, 40)) o[k] = SECRET_KEY.test(k.replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase()) ? "[secret]" : clipped(x, depth + 1);
+      return o;
+    };
+    const argsText = (raw) => {
+      let v = raw;
+      if (typeof raw === "string") { try { v = raw.trim() ? JSON.parse(raw) : {}; } catch { v = raw; } }
+      return JSON.stringify(clipped(v === undefined || v === null ? {} : v, 0)).slice(0, ARGS_MAX);
+    };
+    const callsOf = (events) => {
+      const whole = new Map();
+      const parts = new Map();
+      let n = 0;
+      const put = (key, name, args) => { if (name) whole.set(key || `w${n++}`, { name, args }); };
+      const piece = (key, name, args) => {
+        const p = parts.get(key) || { name: "", args: "" };
+        if (name && !p.name) p.name = name;
+        if (typeof args === "string") p.args += args;
+        parts.set(key, p);
+      };
+      const blocks = (content) => {
+        for (const b of list(content)) {
+          if (b.type === "tool_use") put(b.id, b.name, b.input);
+          if (b.toolUse) put(b.toolUse.toolUseId, b.toolUse.name, b.toolUse.input);
+        }
+      };
+      for (const e of events) {
+        if (!e || typeof e !== "object") continue;
+        for (const c of list(e.choices)) {
+          const m = c.message;
+          if (m) {
+            for (const t of list(m.tool_calls)) if (t.function) put(t.id, t.function.name, t.function.arguments);
+            if (m.function_call) put(null, m.function_call.name, m.function_call.arguments);
+          }
+          const d = c.delta;
+          if (d) {
+            for (const t of list(d.tool_calls)) piece(`c${c.index || 0}.${t.index ?? t.id}`, t.function && t.function.name, t.function && t.function.arguments);
+            if (d.function_call) piece(`f${c.index || 0}`, d.function_call.name, d.function_call.arguments);
+          }
+        }
+        for (const it of [...list(e.output), ...list(e.item ? [e.item] : []), ...list(e.response && e.response.output)]) {
+          if (it.type === "function_call") put(it.call_id || it.id, it.name, it.arguments);
+        }
+        blocks(e.content);
+        blocks(e.message && e.message.content);
+        blocks(e.output && e.output.message && e.output.message.content);
+        if (e.type === "content_block_start" && e.content_block && e.content_block.type === "tool_use") piece(`a${e.index}`, e.content_block.name, "");
+        if (e.type === "content_block_delta" && e.delta && e.delta.type === "input_json_delta") piece(`a${e.index}`, "", e.delta.partial_json);
+        for (const c of list(e.candidates)) for (const p of list(c.content && c.content.parts)) if (p.functionCall) put(null, p.functionCall.name, p.functionCall.args);
+      }
+      return [...whole.values(), ...parts.values()].filter((p) => p.name).map((p) => ({ name: String(p.name).slice(0, 80), arguments: argsText(p.args) }));
+    };
+    // This turn's earlier calls, as the prompt resends them after the person's latest message.
+    const calledBefore = (body) => callsOf([
+      ...since(list(body.messages), personSaid).filter((m) => m.role === "assistant").map((m) => ({ choices: [{ message: m }], content: m.content })),
+      { output: since(list(body.input), (x) => x.role === "user") },
+      ...since(list(body.contents), personAsked).filter((c) => c.role === "model").map((c) => ({ candidates: [{ content: c }] })),
+    ]);
+    const calledIn = (sent, events) => {
+      const body = parse(sent);
+      const out = [];
+      for (const c of [...(body && typeof body === "object" ? calledBefore(body) : []), ...callsOf(events)]) {
+        if (out.length < CALLS_MAX && !out.some((o) => o.name === c.name && o.arguments === c.arguments)) out.push(c);
+      }
+      return out.length ? out : undefined;
+    };
+
+    // The passages the prompt carried as retrieved context: a block the app itself labels as
+    // context, documents, knowledge, sources or search results, in its system prompt or from the
+    // person's latest message on, and Anthropic document and search_result blocks. The app's own
+    // instructions are not passages: only what sits under such a label is.
+    const PASSAGE_TEXT = 3000, PASSAGES_MAX = 6;
+    const MATERIAL = /\b(?:retriev\w*|context|knowledge|documents?|sources?|search[ _-]?results?|references?|passages?|excerpts?|snippets?|chunks?|background|faq)\b|检索|知识|参考资料|资料|上下文|文档|背景/i;
+    const HEADING = /^\s*(?:#{1,6}\s+[^\n]{1,80}|[^\n]{1,80}[:：]\s*(?:\([^\n)]*\))?|\[[^\n\]]{1,80}\]|={2,}\s*[^\n]{1,80}?\s*={2,})\s*$/;
+    // An inline label is a noun phrase that ends in the material word ("context:", "background:",
+    // "retrieved passages:"): "4. Knowledge:" in a numbered instruction and a JSON key are not.
+    const INLINE = /^\s*(?:[-*]\s+)?([A-Za-z\u4e00-\u9fff][A-Za-z\u4e00-\u9fff '’_-]{0,29})[:：]\s*\S/;
+    const NAMED = new RegExp(`(?:${MATERIAL.source})\\s*$`, "i");
+    const TAGGED = /<([A-Za-z][\w-]*)[^>]*>([\s\S]*?)<\/\1>/g;
+    const labelled = (text, add) => {
+      const rest = String(text || "").replace(TAGGED, (all, tag, inner) => { if (MATERIAL.test(tag.replace(/_/g, " "))) { add(tag, inner); return ""; } return all; });
+      const paras = rest.split(/\n\s*\n/);
+      for (let i = 0; i < paras.length; i++) {
+        const lines = paras[i].split("\n");
+        const head = lines[0];
+        if (HEADING.test(head) && MATERIAL.test(head)) {
+          let body = lines.slice(1).join("\n");
+          // A heading alone on its paragraph labels what follows, up to the next heading.
+          if (!body.trim()) while (i + 1 < paras.length && body.length < PASSAGE_TEXT && !HEADING.test(paras[i + 1].split("\n")[0])) body += (body ? "\n\n" : "") + paras[++i];
+          add(head.replace(/[#:：[\]=]/g, "").trim(), body);
+          continue;
+        }
+        const inline = lines.map((l) => INLINE.exec(l)).find((m) => m && NAMED.test(m[1]));
+        if (inline) add(inline[1].trim(), paras[i]);
+      }
+    };
+    const passagesIn = (sent) => {
+      const body = parse(sent);
+      if (!body || typeof body !== "object") return undefined;
+      const out = [];
+      const add = (name, text) => { const t = String(text || "").trim().slice(0, PASSAGE_TEXT); if (t.length >= 20 && out.length < PASSAGES_MAX && !out.some((o) => o.text === t)) out.push({ name: String(name || "").slice(0, 80), text: t }); };
+      const scan = (content) => {
+        if (typeof content === "string") return labelled(content, add);
+        for (const b of list(content)) {
+          if (b.type === "document") add(b.title || "document", b.source ? textOf(b.source.data ?? b.source.content) : textOf(b.content));
+          else if (b.type === "search_result") add(b.title || b.source || "search result", textOf(b.content));
+          else if (typeof b.text === "string") labelled(b.text, add);
+        }
+      };
+      const messages = list(body.messages);
+      for (const m of messages) if (m.role === "system" || m.role === "developer") scan(m.content);
+      for (const m of messages.slice(Math.max(0, messages.findLastIndex(personSaid)))) if (m.role === "user") scan(m.content);
+      scan(body.system);
+      scan(body.instructions);
+      scan(body.systemInstruction && body.systemInstruction.parts);
+      const input = typeof body.input === "string" ? [{ role: "user", content: body.input }] : list(body.input);
+      for (const it of input.slice(Math.max(0, input.findLastIndex((x) => x.role === "user")))) if (it.role === "user" || it.role === "system" || it.role === "developer") scan(it.content);
+      return out.length ? out : undefined;
+    };
+    // The passages a retrieval call answered with: a vector store, a search index or a web search,
+    // read off its reply when it is JSON. Text fields only, bounded like the rest.
+    const RETRIEVAL_HOST = /pinecone\.io|qdrant|weaviate|chroma|zilliz|milvus|turbopuffer|upstash\.io|algolia|typesense|meilisearch|elastic|opensearch|vespa|tavily\.com|exa\.ai|serper\.dev|serpapi\.com|search\.brave\.com|bing\.microsoft\.com|jina\.ai/i;
+    const RETRIEVAL_PATH = /\/(?:query|search|_search|retrieve|similarity_search|rerank|hybrid)(?:\/|$)|\/points\/(?:search|query)|\/rpc\/match\w*/i;
+    const isRetrieval = (u) => RETRIEVAL_HOST.test(u.hostname) || RETRIEVAL_PATH.test(u.pathname);
+    const PASSAGE_KEY = /^(?:text|content|page_?content|pageContent|chunk|snippet|passage|body|document|documents|description|answer|raw_content|highlights?|excerpt)$/i;
+    const passagesFrom = (raw) => {
+      const out = [];
+      let nodes = 0;
+      const walk = (v, key, depth) => {
+        if (out.length >= PASSAGES_MAX || depth > 7 || ++nodes > 4000) return;
+        const t = typeof v === "string" ? v.trim().slice(0, PASSAGE_TEXT) : "";
+        if (t) { if (PASSAGE_KEY.test(key) && t.length >= 20 && !out.some((o) => o.text === t)) out.push({ name: key, text: t }); return; }
+        if (Array.isArray(v)) { for (const x of v) walk(x, key, depth + 1); return; }
+        if (v && typeof v === "object") for (const [k, x] of Object.entries(v)) walk(x, k, depth + 1);
+      };
+      walk(parse(raw), "", 0);
       return out.length ? out : undefined;
     };
     // A framework that folds its tools' answers into the next prompt as plain text (an agent loop
@@ -445,7 +602,7 @@ if (FILE) {
         if (part && typeof part === "object") Object.assign(usage, part);
         model = e.model || (e.message && e.message.model) || (e.response && e.response.model) || e.modelVersion || model;
       }
-      return { tokens: tokensOf(Object.keys(usage).length ? usage : null), model };
+      return { tokens: tokensOf(Object.keys(usage).length ? usage : null), model, events };
     };
     // The model asked for, from what was sent; Gemini and Bedrock put it in the path instead.
     const askedFor = (path, sent) => {
@@ -459,7 +616,9 @@ if (FILE) {
         const reply = readReply(type, String(body || "").slice(0, REPLY_MAX));
         const rules = rulesIn(sent);
         const tools = toolsIn(sent) ?? materialIn(sent, turn);
-        const row = { at: Date.now(), host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens), ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}) };
+        const called = calledIn(sent, reply.events);
+        const passages = passagesIn(sent);
+        const row = { at: Date.now(), host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens), ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages ? { passages } : {}) };
         fs.appendFileSync(FILE, JSON.stringify({ call: row }) + "\n", { mode: 0o600 });
       } catch { /* the command is gone */ }
     };
@@ -485,9 +644,8 @@ if (FILE) {
         return name ? { env: name, host: u.hostname } : null;
       } catch { return null; }
     };
-    const depRow = (dep, status, code) => {
-      const turn = (als.getStore() || {}).turn;
-      try { fs.appendFileSync(FILE, JSON.stringify({ dep: { at: Date.now(), ...(dep.env ? { env: dep.env } : {}), host: dep.host, status, code: code ? String(code).slice(0, 40) : undefined, ...(turn ? { turn } : {}) } }) + "\n", { mode: 0o600 }); } catch { /* the command is gone */ }
+    const depRow = (dep, status, code, passages, turn = (als.getStore() || {}).turn) => {
+      try { fs.appendFileSync(FILE, JSON.stringify({ dep: { at: Date.now(), ...(dep.env ? { env: dep.env } : {}), host: dep.host, status, code: code ? String(code).slice(0, 40) : undefined, ...(turn ? { turn } : {}), ...(passages ? { passages } : {}) } }) + "\n", { mode: 0o600 }); } catch { /* the command is gone */ }
     };
 
     // Outbound: the model call your app makes while it handles that request.
@@ -504,10 +662,21 @@ if (FILE) {
           // Every outbound host, named by the setting that points at it when one does, and by the
           // host alone otherwise: the search provider behind TAVILY_API_KEY has no URL setting.
           let dep = depOf(input);
-          if (!dep) { try { const u = new URL(typeof input === "string" ? input : input && input.url ? input.url : String(input)); if (/^https?:$/.test(u.protocol) && !/^(localhost|127\.0\.0\.1|\[::1\])$/.test(u.hostname)) dep = { host: u.hostname }; } catch { /* not a URL */ } }
+          let retrieval = false;
+          try {
+            const u = new URL(typeof input === "string" ? input : input && input.url ? input.url : String(input));
+            // A store on this machine (a local Qdrant or Chroma) is still where the passages came from.
+            retrieval = /^https?:$/.test(u.protocol) && isRetrieval(u);
+            if (!dep && /^https?:$/.test(u.protocol) && (retrieval || !/^(localhost|127\.0\.0\.1|\[::1\])$/.test(u.hostname))) dep = { host: u.hostname };
+          } catch { /* not a URL */ }
           if (!dep) return realFetch.apply(this, arguments);
+          const turn = (als.getStore() || {}).turn;
           return realFetch.apply(this, arguments).then(
-            (res) => { depRow(dep, res.status); return res; },
+            (res) => {
+              if (!retrieval) { depRow(dep, res.status, undefined, undefined, turn); return res; }
+              try { res.clone().text().then((t) => depRow(dep, res.status, undefined, passagesFrom(t), turn), () => depRow(dep, res.status, undefined, undefined, turn)); } catch { depRow(dep, res.status, undefined, undefined, turn); }
+              return res;
+            },
             (err) => { depRow(dep, 0, err && err.cause && err.cause.code); throw err; },
           );
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.1-rc.1",
+  "version": "0.3.1-rc.2",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"