cortad 0.3.1-rc.3 → 0.3.1-rc.4

package/lib/pyhook/sitecustomize.py

@@ -96,8 +96,12 @@ def _install():
                 at = i
         return seq[at + 1:]
 
+    # A user message that only hands a tool's answer back ("Observation: ...", <tool_response>) is
+    # the agent loop talking, not the person.
+    handed_back = re.compile(r"^\s*(?:Observation\s*:|<(?:tool_response|tool_result|function_results?)>)", re.I)
+
     def person_said(m):
-        return m.get("role") == "user" and not any(isinstance(c, dict) and c.get("type") == "tool_result" for c in (m.get("content") if isinstance(m.get("content"), list) else []))
+        return m.get("role") == "user" and not any(isinstance(c, dict) and c.get("type") == "tool_result" for c in (m.get("content") if isinstance(m.get("content"), list) else [])) and not handed_back.match(text_of(m.get("content")))
 
     def person_asked(c):
         return c.get("role") == "user" and not any(p.get("functionResponse") for p in items(c.get("parts")))
@@ -138,6 +142,9 @@ def _install():
             for p in items(c.get("parts")):
                 if isinstance(p.get("functionResponse"), dict):
                     add(p["functionResponse"].get("name"), p["functionResponse"].get("response"))
+        declared = declared_in(body)
+        for name, said in observed_in(turn_texts(body, True, declared), declared):
+            add(name, said)
         return out or None
 
     # What the model asked the app to run: chat tool_calls (a stream's pieces joined by position), a
@@ -148,15 +155,15 @@ def _install():
     secret_key = re.compile(r"(?:^|_)(?:pass(?:word|phrase)?|secret|token|api_?key|authorization|cookie|session(?:_id)?|credentials?|private_key)$")
     secret_value = re.compile(r"^(?:Bearer\s|Basic\s|sk-|pk_|rk_|ghp_|gho_|github_pat_|xox[abpr]-|AKIA|AIza|eyJ[\w-]{10,}\.)")
 
-    def clipped(v, depth=0):
+    def clipped(v, depth=0, most=value_max):
         if isinstance(v, str):
-            return "[secret]" if secret_value.match(v) else (v[:value_max] + "…" if len(v) > value_max else v)
+            return "[secret]" if secret_value.match(v) else (v[:most] + "…" if len(v) > most else v)
         if isinstance(v, list):
-            return [] if depth > 4 else [clipped(x, depth + 1) for x in v[:20]]
+            return [] if depth > 4 else [clipped(x, depth + 1, most) for x in v[:20]]
         if isinstance(v, dict):
             if depth > 4:
                 return {}
-            return {k: ("[secret]" if secret_key.search(re.sub(r"([a-z])([A-Z])", r"\1_\2", str(k)).lower()) else clipped(x, depth + 1)) for k, x in list(v.items())[:40]}
+            return {k: ("[secret]" if secret_key.search(re.sub(r"([a-z])([A-Z])", r"\1_\2", str(k)).lower()) else clipped(x, depth + 1, most)) for k, x in list(v.items())[:40]}
         return v
 
     def args_text(raw):
@@ -244,10 +251,230 @@ def _install():
                 events.append({"candidates": [{"content": c}]})
         return calls_of(events)
 
+    # Tool use a model writes in its words instead of as a structured call. A ReAct agent (CrewAI,
+    # LangChain) writes "Action: name" then "Action Input: {...}" and is handed "Observation: ..."
+    # back in its next prompt; others write <tool_call>{...}</tool_call>, <function=name>,
+    # <invoke name="..."> or a JSON object that names the tool. A name counts only when the request
+    # declares that tool, in its tools field or in the tool list its prompt carries, so a thought, a
+    # "Final Answer" or prose that says Action is never a call.
+    final = re.compile(r"final[\s_-]*answer", re.I)
+    tool_name = re.compile(r"[A-Za-z_][\w.-]{0,79}", re.A)
+
+    def prompt_text(body):
+        inp = body.get("input")
+        return "\n".join(
+            [text_of(m.get("content")) for m in items(body.get("messages")) if m.get("role") != "assistant"]
+            + [text_of(body.get("system")), text_of(body.get("instructions")), body.get("prompt") if isinstance(body.get("prompt"), str) else "", inp if isinstance(inp, str) else ""]
+            + [text_of(x.get("content")) for x in items(inp) if x.get("role") and x.get("role") != "assistant"]
+            + [text_of((body.get("systemInstruction") or {}).get("parts") if isinstance(body.get("systemInstruction"), dict) else None)]
+            + [text_of(c.get("parts")) for c in items(body.get("contents")) if c.get("role") != "model"])
+
+    def declared_in(body):
+        out = set()
+
+        def add(n):
+            v = n.strip().strip("\"'`") if isinstance(n, str) else ""
+            if tool_name.fullmatch(v) and not final.fullmatch(v):
+                out.add(v)
+
+        def get(d, k):
+            return d.get(k) if isinstance(d, dict) else None
+
+        for t in items(body.get("tools")):
+            add(t.get("name"))
+            add(get(t.get("function"), "name"))
+            add(get(t.get("toolSpec"), "name"))
+            for d in items(t.get("functionDeclarations") or t.get("function_declarations")):
+                add(d.get("name"))
+        for f in items(body.get("functions")):
+            add(f.get("name"))
+        for t in items(get(body.get("toolConfig"), "tools")):
+            add(get(t.get("toolSpec"), "name"))
+        text = prompt_text(body)[:200000]
+        for m in re.finditer(r"^[ \t]*Tool Name:[ \t]*([^\n]+)", text, re.I | re.M):
+            add(m.group(1))
+        for m in re.finditer(r"\b(?:one of|name of|names? from)[ \t]*\[([^\]\n]{1,2000})\]", text, re.I):
+            for x in m.group(1).split(","):
+                add(x)
+        for m in re.finditer(r"valid \"?action\"? values?:?[ \t]*([^\n]{1,2000})", text, re.I):
+            for x in re.split(r",|\bor\b", m.group(1)):
+                add(x)
+        for m in re.finditer(r"<(tools|functions)>([\s\S]*?)</\1>", text, re.I):
+            for k in re.finditer(r"\"name\"\s*:\s*\"([^\"]+)\"", m.group(2)):
+                add(k.group(1))
+        for m in re.finditer(r"\"name\"\s*:\s*\"([^\"]+)\"\s*,\s*\"(?:description|parameters|input_schema)\"", text):
+            add(m.group(1))
+        for m in re.finditer(r"<(?:tool|function)\s+name\s*=\s*[\"']([^\"']+)[\"']", text, re.I):
+            add(m.group(1))
+        return out
+
+    react = re.compile(r"(?:^|\n)[ \t>*_#]*Action[ \t]*\d*[ \t*_]*:[ \t*_`]*([^\n`*]*?)[ \t*_`]*\n+[ \t>*_#]*Action[ \t]*\d*[ \t_]*Input[ \t*_]*:[ \t*_]*([\s\S]*?)(?=\n[ \t>*_#]*(?:Observation|Thought|Final[ \t]*Answer|Action)\b|\x00|\Z)", re.I)
+    tag_call = re.compile(r"<(tool_call|function_call|tool_use)>([\s\S]*?)</\1>", re.I)
+    fn_tag = re.compile(r"<function=([\w.-]+)>([\s\S]*?)</function>", re.I)
+    invoke = re.compile(r"<invoke\s+name\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</invoke>", re.I)
+    observed = re.compile(r"(?:^|\n)[ \t>*_]*Observation[ \t*_]*:[ \t]*([\s\S]*?)(?=\n[ \t>*_#]*(?:Thought|Action|Final[ \t]*Answer)\b|\x00|\Z)|<(tool_response|tool_result|function_results?|observation)>([\s\S]*?)</\2>", re.I)
+
+    def params_of(s):
+        return {m.group(1): m.group(2).strip() for m in re.finditer(r"<parameter(?:=|\s+name\s*=\s*[\"'])([\w.-]+)[\"']?\s*>([\s\S]*?)</parameter>", s, re.I)}
+
+    # Each balanced JSON object in the text, outermost first; its insides are not read again.
+    def objects_in(text):
+        out, i = [], text.find("{")
+        while i != -1 and len(out) < 20:
+            depth, in_str, esc, end = 0, False, False, -1
+            for j in range(i, min(len(text), i + 20000)):
+                ch = text[j]
+                if in_str:
+                    if esc:
+                        esc = False
+                    elif ch == "\\":
+                        esc = True
+                    elif ch == '"':
+                        in_str = False
+                    continue
+                if ch == '"':
+                    in_str = True
+                elif ch == "{":
+                    depth += 1
+                elif ch == "}":
+                    depth -= 1
+                    if depth == 0:
+                        end = j
+                        break
+            v = parsed(text[i:end + 1]) if end != -1 else None
+            if isinstance(v, dict):
+                out.append((i, v))
+                i = end
+            i = text.find("{", i + 1)
+        return out
+
+    # An Action Input that opens with a JSON object is that object: a model that runs on past it
+    # ("Observ: ...") does not put its own invention into the arguments.
+    def leading_json(s):
+        t = s.strip()
+        o = objects_in(t)[:1] if t.startswith("{") else []
+        return o[0][1] if o and o[0][0] == 0 else t
+
+    # A tool's schema carries a description; a call does not.
+    def json_call(v):
+        f = v["function"] if isinstance(v.get("function"), dict) else v
+        if "description" in f:
+            return None
+        name = next((x for x in (f.get("name"), v.get("tool"), v.get("tool_name"), v.get("action"), v.get("function") if isinstance(v.get("function"), str) else None) if isinstance(x, str)), None)
+        if not name:
+            return None
+        args = next((x for x in (f.get("arguments"), f.get("args"), f.get("parameters"), v.get("action_input"), v.get("tool_input"), v.get("input"), v.get("args"), v.get("arguments"), v.get("parameters")) if x is not None), None)
+        return name, args
+
+    def written_calls(text, declared):
+        found = []
+        if not text or not declared:
+            return found
+        s = text[:50000]
+
+        def take(at, name, args):
+            n = str(name or "").strip()
+            if n in declared:
+                found.append((at, n, args.strip() if isinstance(args, str) else args))
+
+        for m in react.finditer(s):
+            take(m.start(), m.group(1), leading_json(m.group(2)))
+        for m in tag_call.finditer(s):
+            v = parsed(m.group(2).strip())
+            c = json_call(v) if isinstance(v, dict) else None
+            if c:
+                take(m.start(), *c)
+        for m in fn_tag.finditer(s):
+            v = parsed(m.group(2).strip())
+            take(m.start(), m.group(1), v if isinstance(v, (dict, list)) else params_of(m.group(2)))
+        for m in invoke.finditer(s):
+            take(m.start(), m.group(1), params_of(m.group(2)))
+        for at, v in objects_in(s):
+            c = json_call(v)
+            if c:
+                take(at, *c)
+        return sorted(found, key=lambda c: c[0])
+
+    # What a tool answered, named by the call written just before it.
+    def observed_in(text, declared):
+        s = str(text or "")[:50000]
+        calls = written_calls(s, declared)
+        out = []
+        if calls:
+            for m in observed.finditer(s):
+                before = [c for c in calls if c[0] < m.start()]
+                if before:
+                    out.append((before[-1][1], (m.group(1) if m.group(1) is not None else m.group(3)).strip()))
+        return out
+
+    # The model's own words: the text it answered, whole or streamed, in every provider's shape.
+    def reply_text(events):
+        s = []
+        for e in events:
+            if not isinstance(e, dict):
+                continue
+            for c in items(e.get("choices")):
+                if isinstance(c.get("message"), dict):
+                    s.append(text_of(c["message"].get("content")))
+                if isinstance(c.get("delta"), dict) and isinstance(c["delta"].get("content"), str):
+                    s.append(c["delta"]["content"])
+                if isinstance(c.get("text"), str):
+                    s.append(c["text"])
+            if e.get("type") == "response.output_text.delta" and isinstance(e.get("delta"), str):
+                s.append(e["delta"])
+            resp = e.get("response") if isinstance(e.get("response"), dict) else {}
+            for it in items(e.get("output")) + items(resp.get("output")):
+                if it.get("type") == "message":
+                    s.extend(p["text"] for p in items(it.get("content")) if isinstance(p.get("text"), str))
+            delta = e.get("delta") if isinstance(e.get("delta"), dict) else {}
+            if e.get("type") == "content_block_delta" and isinstance(delta.get("text"), str):
+                s.append(delta["text"])
+            msg = e.get("message") if isinstance(e.get("message"), dict) else {}
+            out_msg = e["output"].get("message") if isinstance(e.get("output"), dict) and isinstance(e["output"].get("message"), dict) else {}
+            for b in items(e.get("content")) + items(msg.get("content")) + items(out_msg.get("content")):
+                if isinstance(b.get("text"), str):
+                    s.append(b["text"])
+            if isinstance(msg.get("content"), str) and not e.get("choices"):
+                s.append(msg["content"])
+            for c in items(e.get("candidates")):
+                for p in items((c.get("content") or {}).get("parts") if isinstance(c.get("content"), dict) else None):
+                    if isinstance(p.get("text"), str) and not p.get("thought"):
+                        s.append(p["text"])
+            if isinstance(e.get("response"), str):
+                s.append(e["response"])
+        return "".join(s)
+
+    # This turn as the prompt carries it: the model's earlier words after the person's latest
+    # message, and everything from that message on, where a single-prompt agent keeps its scratchpad.
+    # A user message right after the model wrote a tool call is the agent loop's nudge ("Analyze the
+    # tool result"), never the person: after a call, only the loop speaks.
+    def turn_texts(body, with_person, declared):
+        def since_person(seq, is_person):
+            at = -1
+            for i, m in enumerate(seq):
+                if is_person(m):
+                    at = i
+            return seq[max(0, at):] if with_person else seq[at + 1:]
+
+        def ours(role):
+            return role not in ("system", "developer") if with_person else role in ("assistant", "model")
+
+        inp = body.get("input")
+        ms = items(body.get("messages"))
+        after_call = {i for i in range(1, len(ms)) if ms[i - 1].get("role") == "assistant" and written_calls(text_of(ms[i - 1].get("content")), declared)}
+        out = [text_of(m.get("content")) for m in since_person([dict(m, _i=i) for i, m in enumerate(ms)], lambda m: person_said(m) and m["_i"] not in after_call) if ours(m.get("role"))]
+        out += [text_of(x.get("content") if x.get("content") is not None else x.get("output")) for x in since_person(items(inp), lambda x: x.get("role") == "user") if ours(x.get("role"))]
+        out += [text_of(c.get("parts")) for c in since_person(items(body.get("contents")), person_asked) if ours(c.get("role"))]
+        if with_person:
+            out += [body.get("prompt") if isinstance(body.get("prompt"), str) else "", inp if isinstance(inp, str) else ""]
+        return "\x00\n".join(out)
+
     def called_in(sent, events):
         body = parsed(sent) if isinstance(sent, str) else None
         out = []
-        for c in (called_before(body) if isinstance(body, dict) else []) + calls_of(events):
+        declared = declared_in(body) if isinstance(body, dict) else set()
+        written = [{"name": n, "arguments": args_text(a)} for _, n, a in written_calls(turn_texts(body, False, declared) if isinstance(body, dict) else "", declared) + written_calls(reply_text(events), declared)]
+        for c in (called_before(body) if isinstance(body, dict) else []) + calls_of(events) + written:
             if len(out) < calls_max and not any(o["name"] == c["name"] and o["arguments"] == c["arguments"] for o in out):
                 out.append(c)
         return out or None
@@ -365,50 +592,6 @@ def _install():
         walk(parsed(raw) if isinstance(raw, str) else None, "", 0)
         return out or None
 
-    # A framework that folds its tools' answers into the next prompt as plain text sends no tool
-    # message to find: within one turn, what a later call's prompt carries that the call before did
-    # not, outside the model's own words, is that material. The first call holds the customer's
-    # message and the system prompt, present in every later call, so neither is ever counted.
-    seen_by_turn = {}
-
-    def texts_of(body):
-        out = set()
-
-        def take(v):
-            t = text_of(v).strip()
-            if t:
-                out.add(t[:tool_text])
-        for m in body.get("messages") if isinstance(body.get("messages"), list) else []:
-            if isinstance(m, dict) and m.get("role") != "assistant":
-                take(m.get("content"))
-        for it in body.get("input") if isinstance(body.get("input"), list) else []:
-            if isinstance(it, dict) and it.get("role") != "assistant" and it.get("type") != "function_call":
-                take(it.get("content") if it.get("content") is not None else it.get("output"))
-        if isinstance(body.get("system"), str):
-            take(body["system"])
-        for c in body.get("contents") if isinstance(body.get("contents"), list) else []:
-            if isinstance(c, dict) and c.get("role") != "model":
-                for p in c.get("parts") if isinstance(c.get("parts"), list) else []:
-                    if isinstance(p, dict) and isinstance(p.get("text"), str):
-                        take(p["text"])
-        return out
-
-    def material_in(sent, turn):
-        if not turn:
-            return None
-        body = parsed(sent) if isinstance(sent, str) else None
-        if not isinstance(body, dict):
-            return None
-        now = texts_of(body)
-        before = seen_by_turn.get(turn)
-        seen_by_turn[turn] = now
-        if len(seen_by_turn) > 200:
-            seen_by_turn.pop(next(iter(seen_by_turn)))
-        if before is None:
-            return None
-        fresh = [{"name": "", "text": t} for t in now if t not in before][:tools_max]
-        return fresh or None
-
     def rules_in(sent):
         found = rules_now()
         if found is None:
@@ -416,12 +599,45 @@ def _install():
         body = norm(sent)
         return [rid for rid, parts in found if all(p in body for p in parts)]
 
+    # Every value this file writes passes one mask, whichever path wrote it: a key inside a value
+    # ("sk-...", a bearer token, a JWT) and what anyone wrote after "password is" or "token:". The
+    # door row's sign-in headers are kept on purpose: the command replays them from this machine.
+    secret_text = [
+        re.compile(r"""((?:pass(?:word|phrase|wd)|pwd|密码|口令)["']?\s*(?:is\b|was\b|[:=：]|是|为)\s*["']?)([^\s"'\\,;}，。；、]+)""", re.I),
+        re.compile(r"""((?:secret(?:[ _-]?key)?|api[ _-]?key|apikey|access[ _-]?key|private[ _-]?key|client[ _-]?secret|(?:auth|access|refresh)[ _-]?token|token)["']?\s*[:=：]\s*["']?)([^\s"'\\,;}，。；、]+)""", re.I),
+    ]
+    secret_word = re.compile(r"\b(?:Bearer\s+(?=[\w.~+/=-]*\d)[\w.~+/=-]{16,}|[spr]k[-_](?=[\w-]*\d)[\w-]{8,}|gh[po]_\w{16,}|github_pat_\w{16,}|xox[abpr]-[\w-]{8,}|AKIA[0-9A-Z]{12,}|AIza[\w-]{20,}|eyJ[\w-]{10,}\.[\w-]{4,}\.[\w-]*)", re.A)
+
+    def mask_text(t):
+        for pattern in secret_text:
+            t = pattern.sub(r"\1[secret]", t)
+        return secret_word.sub("[secret]", t)
+
+    def scrub(v, depth=0):
+        if isinstance(v, str):
+            return mask_text(v)
+        if isinstance(v, (list, tuple)):
+            return [scrub(x, depth + 1) for x in v]
+        if isinstance(v, dict) and depth < 12:
+            return {k: (x if k == "headers" and depth == 0 else scrub(x, depth + 1)) for k, x in v.items()}
+        return v
+
+    # A door's body goes up to the run as JSON, so it is masked as JSON: its string values, never its shape.
+    def body_scrubbed(raw):
+        b = parsed(raw)
+        return json.dumps(scrub(b), ensure_ascii=False) if isinstance(b, (dict, list)) else mask_text(raw)
+
     def write(row):
         try:
+            if not (row.get("hello") or row.get("listen") or row.get("routes")):
+                out = scrub(row)
+                if isinstance(row.get("body"), str):
+                    out["body"] = body_scrubbed(row["body"])
+                row = out
             fd = os.open(_FILE, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
             with os.fdopen(fd, "a", encoding="utf-8") as f:
                 f.write(json.dumps(row) + "\n")
-        except OSError:
+        except Exception:
             pass
 
     # Every other outbound call: the host and what it answered, never a byte of it. A search
@@ -558,7 +774,7 @@ def _install():
             found = rules_in(as_text)
             if found is not None:
                 row["rules"] = found
-            tools = tools_in(as_text) or material_in(as_text, turn)
+            tools = tools_in(as_text)
             if tools:
                 row["tools"] = tools
             called = called_in(as_text, events)
@@ -571,7 +787,185 @@ def _install():
         except Exception:
             pass
 
+    # The app's own functions the read names as its tools. An app whose code picks the tool itself
+    # (a classifier answers {"intent":"order"} and the code calls query_order()) never names one on
+    # the wire, so the function is wrapped where it is defined and wherever it was imported by name:
+    # each call writes its name, its clipped arguments and what it returned, on the turn it ran in.
+    # The run writes the list beside the rules, after the app has loaded, so it is applied to the
+    # modules already loaded and looked at again when more load.
+    tools_file = os.environ.get("CORTAD_TOOLS_FILE")
+    ident = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+    targets, swept, wrappers = [[]], [None], {}
+    unsaid = set()
+
+    def unwatched(t, why):
+        if (t["file"], t["fn"]) in unsaid:
+            return
+        unsaid.add((t["file"], t["fn"]))
+        try:
+            sys.stderr.write("cortad: calls to %s in %s are not recorded: %s.\n" % (t["name"], t["file"], why))
+        except Exception:
+            pass
+
+    def plain_of(v, depth=0):
+        if v is None or isinstance(v, (str, bool, int, float)):
+            return v
+        if depth > 4:
+            return "[%s]" % type(v).__name__
+        if isinstance(v, (list, tuple, set)):
+            return [plain_of(x, depth + 1) for x in list(v)[:20]]
+        if isinstance(v, dict):
+            return {str(k): plain_of(x, depth + 1) for k, x in list(v.items())[:40]}
+        dump = getattr(v, "model_dump", None)
+        if callable(dump):
+            try:
+                return plain_of(dump(), depth + 1)
+            except Exception:
+                pass
+        return "[%s]" % type(v).__name__
+
+    # The returned value's shape and a clipped text: a string as it is, anything else named by kind.
+    def returned(v):
+        if v is None:
+            return "returned nothing"
+        if isinstance(v, str):
+            return v
+        kind = "list of %d" % len(v) if isinstance(v, (list, tuple)) else "object" if isinstance(v, dict) or hasattr(v, "model_dump") else "value"
+        return "%s: %s" % (kind, json.dumps(clipped(plain_of(v), 0, tool_text), ensure_ascii=False, separators=(",", ":")))
+
+    def tool_row(name, sig, a, k, value, error, turn):
+        try:
+            try:
+                named = dict(sig.bind_partial(*a, **k).arguments) if sig else None
+            except TypeError:
+                named = None
+            if named is None:
+                named = {**{str(i): x for i, x in enumerate(a)}, **k}
+            text = "raised %s: %s" % (type(error).__name__, error) if error is not None else returned(value)
+            row = {"at": int(time.time() * 1000), "host": "in-app", "status": 200,
+                   "called": [{"name": name, "arguments": args_text(plain_of(named))}],
+                   "tools": [{"name": name, "text": str(text)[:tool_text]}]}
+            if turn:
+                row["turn"] = turn
+            write({"dep": row})
+        except Exception:
+            pass
+
+    def wrap_tool(fn, name):
+        import functools
+        import inspect
+        try:
+            sig = inspect.signature(fn)
+        except (TypeError, ValueError):
+            sig = None
+        if inspect.iscoroutinefunction(fn):
+            async def wrapped(*a, **k):
+                turn = turn_now()
+                try:
+                    out = await fn(*a, **k)
+                except Exception as e:
+                    tool_row(name, sig, a, k, None, e, turn)
+                    raise
+                tool_row(name, sig, a, k, out, None, turn)
+                return out
+        else:
+            def wrapped(*a, **k):
+                turn = turn_now()
+                try:
+                    out = fn(*a, **k)
+                except Exception as e:
+                    tool_row(name, sig, a, k, None, e, turn)
+                    raise
+                tool_row(name, sig, a, k, out, None, turn)
+                return out
+        functools.update_wrapper(wrapped, fn)
+        wrapped.__cortad_tool__ = fn
+        return wrapped
+
+    def targets_now():
+        if not tools_file:
+            return []
+        try:
+            with open(tools_file, encoding="utf-8") as f:
+                raw = json.load(f)
+        except Exception:
+            return []
+        out = []
+        for t in raw if isinstance(raw, list) else []:
+            if isinstance(t, dict) and isinstance(t.get("name"), str) and t["name"] and isinstance(t.get("file"), str) and t["file"].endswith(".py") and ident.match(str(t.get("function"))):
+                out.append({"name": t["name"][:80], "file": re.sub(r"^\.?/+", "", t["file"]), "fn": t["function"]})
+        # One function named twice (by its own name, and as "query_order (fallback)" where a router
+        # imported it) is recorded under its own name.
+        return sorted(out, key=lambda t: t["name"] != t["fn"])
+
+    def sweep_tools():
+        if not tools_file:
+            return
+        try:
+            at = os.stat(tools_file).st_mtime
+        except OSError:
+            return
+        mark = (at, len(sys.modules))
+        if swept[0] == mark:
+            return
+        if swept[0] is None or swept[0][0] != at:
+            targets[0] = targets_now()
+        swept[0] = mark
+        if not targets[0]:
+            return
+        import inspect
+        loaded = list(sys.modules.values())
+        by_file = {}
+        for m in loaded:
+            f = getattr(m, "__file__", None)
+            if isinstance(f, str) and f.endswith(".py"):
+                by_file.setdefault(f.replace(os.sep, "/"), m)
+        fresh = {}
+        for t in targets[0]:
+            m = next((mod for f, mod in by_file.items() if f == t["file"] or f.endswith("/" + t["file"])), None)
+            if m is None:
+                continue
+            fn = getattr(m, t["fn"], None)
+            if getattr(fn, "__cortad_tool__", None) is not None:
+                continue
+            if not (inspect.isfunction(fn) or inspect.ismethod(fn)):
+                unwatched(t, "the file has no function named %s" % t["fn"] if fn is None else "%s there is not a function this hook can wrap" % t["fn"])
+                continue
+            held = wrappers.get(id(fn))
+            w = held[1] if held is not None and held[0] is fn else wrap_tool(fn, t["name"])
+            wrappers[id(fn)] = (fn, w)
+            try:
+                setattr(m, t["fn"], w)
+            except Exception:
+                unwatched(t, "its module does not let %s be replaced" % t["fn"])
+                continue
+            fresh[id(fn)] = (fn, w)
+        if not fresh:
+            return
+        # Wherever the app imported it by name (from tools import query_order) or put it in a table
+        # of its own (ROUTES = {"order": query_order}), the same function is swapped for the wrapper.
+        here = os.getcwd().replace(os.sep, "/") + "/"
+        for m in loaded:
+            d = getattr(m, "__dict__", None)
+            if not isinstance(d, dict):
+                continue
+            f = str(getattr(m, "__file__", "") or "").replace(os.sep, "/")
+            ours = f.startswith(here) and "/site-packages/" not in f and "/dist-packages/" not in f
+            for k, v in list(d.items()):
+                hit = fresh.get(id(v))
+                if hit is not None and hit[0] is v:
+                    d[k] = hit[1]
+                elif ours and isinstance(v, dict):
+                    for kk, vv in list(v.items()):
+                        hit = fresh.get(id(vv))
+                        if hit is not None and hit[0] is vv:
+                            v[kk] = hit[1]
+
     def started(method, path, headers):
+        try:
+            sweep_tools()
+        except Exception:
+            pass
         turn = headers.pop("x-cortad-turn", None)
         return {"method": method, "path": path, "headers": headers, "chunks": [], "size": 0, "noted": False,
                 "turn": turn if isinstance(turn, str) and turn_ok.match(turn) else None}

package/lib/replay.mjs

@@ -35,6 +35,9 @@ export function makeCapture({ work, keepSecret, onDoor }) {
     // The run writes the customer's rule sentences here (the engine's /tmp/rules.json lands in this
     // folder), and the hook reads which of them each model call's prompt carried.
     CORTAD_RULES_FILE: join(work, "rules.json"),
+    // The read's tools by name, file and function, written by the run the same way: the hook wraps
+    // those functions so a tool your own code runs is seen even when no model named it.
+    CORTAD_TOOLS_FILE: join(work, "tools.json"),
     NODE_OPTIONS: `${base.NODE_OPTIONS ?? ""} --require ${JSON.stringify(HOOK)}`.trim(),
     BUN_OPTIONS: `${base.BUN_OPTIONS ?? ""} --preload=${bunHook}`.trim(),
     PYTHONPATH: [PYHOOK, base.PYTHONPATH].filter(Boolean).join(":"),
@@ -130,10 +133,12 @@ const meter = (() => {
       if (rows.length > ROWS) rows.splice(0, rows.length - ROWS);
     },
     // A service their settings name: only its setting, host and status travel, never a byte of it.
+    // A tool function of the app's own ("in-app") carries its call and what it returned, as a
+    // model call's row does.
     dep(d) {
       if (!d || typeof d.host !== "string" || !d.host) return;
       const env = typeof d.env === "string" && /^[A-Z_][A-Z0-9_]*$/.test(d.env) ? d.env : undefined;
-      deps.push({ at: count(d.at), ...(env ? { env } : {}), host: d.host.slice(0, 253), status: count(d.status), ...(typeof d.code === "string" ? { code: d.code.slice(0, 40) } : {}), ...turnOf(d.turn), ...passagesOf(d.passages) });
+      deps.push({ at: count(d.at), ...(env ? { env } : {}), host: d.host.slice(0, 253), status: count(d.status), ...(typeof d.code === "string" ? { code: d.code.slice(0, 40) } : {}), ...turnOf(d.turn), ...passagesOf(d.passages), ...toolsOf(d.tools), ...calledOf(d.called) });
       if (deps.length > ROWS) deps.splice(0, deps.length - ROWS);
     },
     report() {

package/lib/text.mjs

@@ -138,10 +138,13 @@ function stoppedLine(d) {
 
 const moveText = (m) => `held ${kn(m.before)} readings before, ${kn(m.after)} after; ${m.moved}, ${signed(m.point)} points${interval(m)}${m.insideNoise ? ", inside the noise" : ""}.`;
 
+// Over zero pairs there is no move to print: the line says verification is unavailable and why,
+// never "no change, 0 points" measured on nothing. The visible side's reason is the said line.
 function verifyLines(v) {
   const lines = [`Verify of ${v.findingId}${v.path ? ` at ${at(v.path, v.line)}` : ""}.`];
-  if (v.visible) lines.push(`Visible trials: ${moveText(v.visible)}`);
-  if (v.holdout) lines.push(`Held-out trials: ${moveText(v.holdout)}`);
+  if (v.visible && !v.visible.unavailable) lines.push(`Visible trials: ${moveText(v.visible)}`);
+  if (v.holdout?.unavailable) lines.push(`Held-out trials: verification unavailable: ${v.holdout.unavailable}.`);
+  else if (v.holdout) lines.push(`Held-out trials: ${moveText(v.holdout)}`);
   else if (v.holdout === null) lines.push("Held-out trials: no pair for this question.");
   if (v.overfit) lines.push("Overfit: the visible trials moved and the held-out trials did not.");
   if (v.said) lines.push(v.said);
@@ -190,7 +193,8 @@ function ledgerLines(l) {
   }
   if (l.fixed) {
     lines.push(`${plural(l.fixed.length, "fix", "fixes")} verified since that run${l.fixed.length ? ":" : "."}`);
-    for (const f of l.fixed) lines.push(`  ${at(f.path, f.line)} (${f.findingId}): held ${kn(f.before)} readings before, ${kn(f.after)} after; ${f.moved}, ${signed(f.move)} points${interval(f)}.`);
+    // A fix on a finding with no file (a crash, a standard) is named by its id, never "undefined".
+    for (const f of l.fixed) lines.push(`  ${f.path ? `${at(f.path, f.line)} (${f.findingId})` : f.findingId}: held ${kn(f.before)} readings before, ${kn(f.after)} after; ${f.moved}, ${signed(f.move)} points${interval(f)}.`);
   }
   const n = l.nextRun;
   if (n) lines.push(`The next run would play ${[plural(n.trials, "trial"), has(n.heldOut) && `${num(n.heldOut)} held out`, has(n.newFromChanges) && `${num(n.newFromChanges)} new from the changes`].filter(Boolean).join(", ")}.`);

package/lib/trace.cjs

@@ -16,9 +16,30 @@ if (FILE) {
     const http = require("node:http");
     const https = require("node:https");
     const als = new AsyncLocalStorage();
-    // Said once, so the command knows a message sent to this app can be seen arriving.
-    const row = (value) => { try { fs.appendFileSync(FILE, JSON.stringify(value) + "\n", { mode: 0o600 }); } catch { /* the command is gone */ } };
+    // Every value this file writes passes one mask, whichever path wrote it: a key inside a value
+    // ("sk-...", a bearer token, a JWT) and what anyone wrote after "password is" or "token:". The
+    // door row's sign-in headers are kept on purpose: the command replays them from this machine.
+    const SECRET_TEXT = [
+      /((?:pass(?:word|phrase|wd)|pwd|密码|口令)["']?\s*(?:is\b|was\b|[:=：]|是|为)\s*["']?)([^\s"'\\,;}，。；、]+)/gi,
+      /((?:secret(?:[ _-]?key)?|api[ _-]?key|apikey|access[ _-]?key|private[ _-]?key|client[ _-]?secret|(?:auth|access|refresh)[ _-]?token|token)["']?\s*[:=：]\s*["']?)([^\s"'\\,;}，。；、]+)/gi,
+    ];
+    const SECRET_WORD = /\b(?:Bearer\s+(?=[\w.~+/=-]*\d)[\w.~+/=-]{16,}|[spr]k[-_](?=[\w-]*\d)[\w-]{8,}|gh[po]_\w{16,}|github_pat_\w{16,}|xox[abpr]-[\w-]{8,}|AKIA[0-9A-Z]{12,}|AIza[\w-]{20,}|eyJ[\w-]{10,}\.[\w-]{4,}\.[\w-]*)/g;
+    const maskText = (s) => SECRET_TEXT.reduce((t, re) => t.replace(re, "$1[secret]"), s).replace(SECRET_WORD, "[secret]");
+    const scrub = (v, depth = 0) => (typeof v === "string" ? maskText(v)
+      : Array.isArray(v) ? v.map((x) => scrub(x, depth + 1))
+        : v && typeof v === "object" && depth < 12 ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, k === "headers" && depth === 0 ? x : scrub(x, depth + 1)])) : v);
+    // A door's body goes up to the run as JSON, so it is masked as JSON: its string values, never its shape.
+    const bodyScrubbed = (text) => { try { const b = JSON.parse(text); return b && typeof b === "object" ? JSON.stringify(scrub(b)) : maskText(text); } catch { return maskText(text); } };
+    const row = (value) => {
+      try {
+        const plain = value.hello || value.listen || value.routes;
+        const out = plain ? value : scrub(value);
+        if (!plain && typeof value.body === "string") out.body = bodyScrubbed(value.body);
+        fs.appendFileSync(FILE, JSON.stringify(out) + "\n", { mode: 0o600 });
+      } catch { /* the command is gone */ }
+    };
     const isBun = typeof Bun !== "undefined" && typeof Bun.serve === "function";
+    // Said once, so the command knows a message sent to this app can be seen arriving.
     row({ hello: isBun ? "bun" : "node", pid: process.pid });
     // Which port this process serves. Every process the start command spawns loads this file, turbo's
     // own launcher included, so "loaded" is not "watching your app": only a listener on the app's port is.
@@ -249,7 +270,10 @@ if (FILE) {
     // resends whole carries every earlier turn's tool answers too.
     const list = (v) => (Array.isArray(v) ? v.filter((x) => x && typeof x === "object") : []);
     const since = (items, isPerson) => { let at = -1; items.forEach((m, i) => { if (isPerson(m)) at = i; }); return items.slice(at + 1); };
-    const personSaid = (m) => m.role === "user" && !(Array.isArray(m.content) && m.content.some((c) => c && c.type === "tool_result"));
+    // A user message that only hands a tool's answer back ("Observation: ...", <tool_response>) is
+    // the agent loop talking, not the person.
+    const HANDED_BACK = /^\s*(?:Observation\s*:|<(?:tool_response|tool_result|function_results?)>)/i;
+    const personSaid = (m) => m.role === "user" && !(Array.isArray(m.content) && m.content.some((c) => c && c.type === "tool_result")) && !HANDED_BACK.test(textOf(m.content));
     const personAsked = (c) => c.role === "user" && !list(c.parts).some((p) => p.functionResponse);
     const toolsIn = (sent) => {
       let body; try { body = JSON.parse(sent); } catch { return undefined; }
@@ -269,6 +293,8 @@ if (FILE) {
       for (const it of items) if (it.type === "function_call" && it.call_id) names.set(it.call_id, it.name);
       for (const it of since(items, (x) => x.role === "user")) if (it.type === "function_call_output") add(names.get(it.call_id), it.output);
       for (const c of since(list(body.contents), personAsked)) for (const p of list(c.parts)) if (p.functionResponse) add(p.functionResponse.name, p.functionResponse.response);
+      const declared = declaredIn(body);
+      for (const o of observedIn(turnTexts(body, true, declared), declared)) add(o.name, o.text);
       return out.length ? out : undefined;
     };
 
@@ -279,13 +305,13 @@ if (FILE) {
     const CALLS_MAX = 12, VALUE_MAX = 200, ARGS_MAX = 1200;
     const SECRET_KEY = /(?:^|_)(?:pass(?:word|phrase)?|secret|token|api_?key|authorization|cookie|session(?:_id)?|credentials?|private_key)$/;
     const SECRET_VALUE = /^(?:Bearer\s|Basic\s|sk-|pk_|rk_|ghp_|gho_|github_pat_|xox[abpr]-|AKIA|AIza|eyJ[\w-]{10,}\.)/;
-    const clipped = (v, depth) => {
-      if (typeof v === "string") return SECRET_VALUE.test(v) ? "[secret]" : v.length > VALUE_MAX ? v.slice(0, VALUE_MAX) + "…" : v;
-      if (Array.isArray(v)) return depth > 4 ? [] : v.slice(0, 20).map((x) => clipped(x, depth + 1));
+    const clipped = (v, depth, max = VALUE_MAX) => {
+      if (typeof v === "string") return SECRET_VALUE.test(v) ? "[secret]" : v.length > max ? v.slice(0, max) + "…" : v;
+      if (Array.isArray(v)) return depth > 4 ? [] : v.slice(0, 20).map((x) => clipped(x, depth + 1, max));
       if (!v || typeof v !== "object") return v;
       const o = {};
       if (depth > 4) return o;
-      for (const [k, x] of Object.entries(v).slice(0, 40)) o[k] = SECRET_KEY.test(k.replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase()) ? "[secret]" : clipped(x, depth + 1);
+      for (const [k, x] of Object.entries(v).slice(0, 40)) o[k] = SECRET_KEY.test(k.replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase()) ? "[secret]" : clipped(x, depth + 1, max);
       return o;
     };
     const argsText = (raw) => {
@@ -342,10 +368,125 @@ if (FILE) {
       { output: since(list(body.input), (x) => x.role === "user") },
       ...since(list(body.contents), personAsked).filter((c) => c.role === "model").map((c) => ({ candidates: [{ content: c }] })),
     ]);
+    // Tool use a model writes in its words instead of as a structured call. A ReAct agent (CrewAI,
+    // LangChain) writes "Action: name" then "Action Input: {...}" and is handed "Observation: ..."
+    // back in its next prompt; others write <tool_call>{...}</tool_call>, <function=name>,
+    // <invoke name="..."> or a JSON object that names the tool. A name counts only when the request
+    // declares that tool, in its tools field or in the tool list its prompt carries, so a thought, a
+    // "Final Answer" or prose that says Action is never a call.
+    const FINAL = /^final[\s_-]*answer$/i;
+    const TOOL_NAME = /^[A-Za-z_][\w.-]{0,79}$/;
+    const promptText = (body) => [
+      ...list(body.messages).filter((m) => m.role !== "assistant").map((m) => textOf(m.content)),
+      textOf(body.system), textOf(body.instructions), typeof body.prompt === "string" ? body.prompt : "", typeof body.input === "string" ? body.input : "",
+      ...list(body.input).filter((x) => x.role && x.role !== "assistant").map((x) => textOf(x.content)),
+      textOf(body.systemInstruction && body.systemInstruction.parts), ...list(body.contents).filter((c) => c.role !== "model").map((c) => textOf(c.parts)),
+    ].join("\n");
+    const declaredIn = (body) => {
+      const out = new Set();
+      const add = (n) => { const s = typeof n === "string" ? n.trim().replace(/^["'`]+|["'`]+$/g, "") : ""; if (TOOL_NAME.test(s) && !FINAL.test(s)) out.add(s); };
+      for (const t of list(body.tools)) { add(t.name); add(t.function && t.function.name); add(t.toolSpec && t.toolSpec.name); for (const d of list(t.functionDeclarations || t.function_declarations)) add(d.name); }
+      for (const f of list(body.functions)) add(f.name);
+      for (const t of list(body.toolConfig && body.toolConfig.tools)) add(t.toolSpec && t.toolSpec.name);
+      const text = promptText(body).slice(0, 200000);
+      for (const m of text.matchAll(/^[ \t]*Tool Name:[ \t]*([^\n]+)/gim)) add(m[1]);
+      for (const m of text.matchAll(/\b(?:one of|name of|names? from)[ \t]*\[([^\]\n]{1,2000})\]/gi)) for (const x of m[1].split(",")) add(x);
+      for (const m of text.matchAll(/valid "?action"? values?:?[ \t]*([^\n]{1,2000})/gi)) for (const x of m[1].split(/,|\bor\b/)) add(x);
+      for (const m of text.matchAll(/<(tools|functions)>([\s\S]*?)<\/\1>/gi)) for (const k of m[2].matchAll(/"name"\s*:\s*"([^"]+)"/g)) add(k[1]);
+      for (const m of text.matchAll(/"name"\s*:\s*"([^"]+)"\s*,\s*"(?:description|parameters|input_schema)"/g)) add(m[1]);
+      for (const m of text.matchAll(/<(?:tool|function)\s+name\s*=\s*["']([^"']+)["']/gi)) add(m[1]);
+      return out;
+    };
+    const REACT = /(?:^|\n)[ \t>*_#]*Action[ \t]*\d*[ \t*_]*:[ \t*_`]*([^\n`*]*?)[ \t*_`]*\n+[ \t>*_#]*Action[ \t]*\d*[ \t_]*Input[ \t*_]*:[ \t*_]*([\s\S]*?)(?=\n[ \t>*_#]*(?:Observation|Thought|Final[ \t]*Answer|Action)\b|\u0000|$)/gi;
+    const TAG_CALL = /<(tool_call|function_call|tool_use)>([\s\S]*?)<\/\1>/gi;
+    const FN_TAG = /<function=([\w.-]+)>([\s\S]*?)<\/function>/gi;
+    const INVOKE = /<invoke\s+name\s*=\s*["']([^"']+)["'][^>]*>([\s\S]*?)<\/invoke>/gi;
+    const OBSERVED = /(?:^|\n)[ \t>*_]*Observation[ \t*_]*:[ \t]*([\s\S]*?)(?=\n[ \t>*_#]*(?:Thought|Action|Final[ \t]*Answer)\b|\u0000|$)|<(tool_response|tool_result|function_results?|observation)>([\s\S]*?)<\/\2>/gi;
+    const paramsOf = (s) => { const o = {}; for (const m of s.matchAll(/<parameter(?:=|\s+name\s*=\s*["'])([\w.-]+)["']?\s*>([\s\S]*?)<\/parameter>/gi)) o[m[1]] = m[2].trim(); return o; };
+    // Each balanced JSON object in the text, outermost first; its insides are not read again.
+    const objectsIn = (text) => {
+      const out = [];
+      for (let i = text.indexOf("{"); i !== -1 && out.length < 20; i = text.indexOf("{", i + 1)) {
+        let depth = 0, str = false, esc = false, end = -1;
+        for (let j = i; j < text.length && j < i + 20000; j++) {
+          const ch = text[j];
+          if (str) { if (esc) esc = false; else if (ch === "\\") esc = true; else if (ch === '"') str = false; continue; }
+          if (ch === '"') str = true; else if (ch === "{") depth++; else if (ch === "}" && --depth === 0) { end = j; break; }
+        }
+        const v = end === -1 ? null : parse(text.slice(i, end + 1));
+        if (v && typeof v === "object") { out.push({ at: i, v }); i = end; }
+      }
+      return out;
+    };
+    // An Action Input that opens with a JSON object is that object: a model that runs on past it
+    // ("Observ: ...") does not put its own invention into the arguments.
+    const leadingJson = (s) => { const t = s.trim(); const o = t[0] === "{" ? objectsIn(t)[0] : null; return o && o.at === 0 ? o.v : t; };
+    // A tool's schema carries a description; a call does not.
+    const jsonCall = (v) => {
+      const f = v.function && typeof v.function === "object" ? v.function : v;
+      if ("description" in f) return null;
+      const name = [f.name, v.tool, v.tool_name, v.action, typeof v.function === "string" ? v.function : undefined].find((x) => typeof x === "string");
+      return name ? { name, args: [f.arguments, f.args, f.parameters, v.action_input, v.tool_input, v.input, v.args, v.arguments, v.parameters].find((x) => x !== undefined) } : null;
+    };
+    const writtenCalls = (text, declared) => {
+      const found = [];
+      if (!text || !declared.size) return found;
+      const s = text.slice(0, 50000);
+      const take = (at, name, args) => { const n = String(name || "").trim(); if (declared.has(n)) found.push({ at, name: n, args: typeof args === "string" ? args.trim() : args }); };
+      for (const m of s.matchAll(REACT)) take(m.index, m[1], leadingJson(m[2]));
+      for (const m of s.matchAll(TAG_CALL)) { const v = parse(m[2].trim()); const c = v && typeof v === "object" && jsonCall(v); if (c) take(m.index, c.name, c.args); }
+      for (const m of s.matchAll(FN_TAG)) { const v = parse(m[2].trim()); take(m.index, m[1], v && typeof v === "object" ? v : paramsOf(m[2])); }
+      for (const m of s.matchAll(INVOKE)) take(m.index, m[1], paramsOf(m[2]));
+      for (const { at, v } of objectsIn(s)) { const c = jsonCall(v); if (c) take(at, c.name, c.args); }
+      return found.sort((a, b) => a.at - b.at);
+    };
+    // What a tool answered, named by the call written just before it.
+    const observedIn = (text, declared) => {
+      const s = String(text || "").slice(0, 50000);
+      const calls = writtenCalls(s, declared);
+      const out = [];
+      if (calls.length) for (const m of s.matchAll(OBSERVED)) { const call = calls.filter((c) => c.at < m.index).pop(); if (call) out.push({ name: call.name, text: (m[1] ?? m[3]).trim() }); }
+      return out;
+    };
+    // The model's own words: the text it answered, whole or streamed, in every provider's shape.
+    const replyText = (events) => {
+      let s = "";
+      for (const e of events) {
+        if (!e || typeof e !== "object") continue;
+        for (const c of list(e.choices)) { if (c.message) s += textOf(c.message.content); if (c.delta && typeof c.delta.content === "string") s += c.delta.content; if (typeof c.text === "string") s += c.text; }
+        if (e.type === "response.output_text.delta" && typeof e.delta === "string") s += e.delta;
+        for (const it of [...list(e.output), ...list(e.response && e.response.output)]) if (it.type === "message") for (const p of list(it.content)) if (typeof p.text === "string") s += p.text;
+        if (e.type === "content_block_delta" && e.delta && typeof e.delta.text === "string") s += e.delta.text;
+        for (const b of [...list(e.content), ...list(e.message && e.message.content), ...list(e.output && e.output.message && e.output.message.content)]) if (typeof b.text === "string") s += b.text;
+        if (e.message && typeof e.message.content === "string" && !e.choices) s += e.message.content;
+        for (const c of list(e.candidates)) for (const p of list(c.content && c.content.parts)) if (typeof p.text === "string" && !p.thought) s += p.text;
+        if (typeof e.response === "string") s += e.response;
+      }
+      return s;
+    };
+    // This turn as the prompt carries it: the model's earlier words after the person's latest
+    // message, and everything from that message on, where a single-prompt agent keeps its scratchpad.
+    // A user message right after the model wrote a tool call is the agent loop's nudge ("Analyze the
+    // tool result"), never the person: after a call, only the loop speaks.
+    const turnTexts = (body, withPerson, declared) => {
+      const from = (items, isPerson) => { const at = items.findLastIndex(isPerson); return items.slice(withPerson ? Math.max(0, at) : at + 1); };
+      const ours = (r) => withPerson ? r !== "system" && r !== "developer" : r === "assistant" || r === "model";
+      const ms = list(body.messages);
+      const person = (m) => personSaid(m) && !(ms[ms.indexOf(m) - 1] && ms[ms.indexOf(m) - 1].role === "assistant" && writtenCalls(textOf(ms[ms.indexOf(m) - 1].content), declared).length);
+      return [
+        ...from(ms, person).filter((m) => ours(m.role)).map((m) => textOf(m.content)),
+        ...from(list(body.input), (x) => x.role === "user").filter((x) => ours(x.role)).map((x) => textOf(x.content ?? x.output)),
+        ...from(list(body.contents), personAsked).filter((c) => ours(c.role)).map((c) => textOf(c.parts)),
+        ...(withPerson ? [typeof body.prompt === "string" ? body.prompt : "", typeof body.input === "string" ? body.input : ""] : []),
+      ].join("\u0000\n");
+    };
+
     const calledIn = (sent, events) => {
       const body = parse(sent);
       const out = [];
-      for (const c of [...(body && typeof body === "object" ? calledBefore(body) : []), ...callsOf(events)]) {
+      const declared = body && typeof body === "object" ? declaredIn(body) : new Set();
+      const written = [...writtenCalls(body && typeof body === "object" ? turnTexts(body, false, declared) : "", declared), ...writtenCalls(replyText(events), declared)].map((c) => ({ name: c.name, arguments: argsText(c.args) }));
+      for (const c of [...(body && typeof body === "object" ? calledBefore(body) : []), ...callsOf(events), ...written]) {
         if (out.length < CALLS_MAX && !out.some((o) => o.name === c.name && o.arguments === c.arguments)) out.push(c);
       }
       return out.length ? out : undefined;
@@ -422,33 +563,6 @@ if (FILE) {
       walk(parse(raw), "", 0);
       return out.length ? out : undefined;
     };
-    // A framework that folds its tools' answers into the next prompt as plain text (an agent loop
-    // writing "Observation: ..." into a user message) sends no tool message to find. Within one
-    // turn, what the second call's prompt carries that the first one did not, outside the model's
-    // own words, is that material. The first call of a turn holds the customer's message and the
-    // system prompt, and both are in every later call, so neither is ever counted.
-    const textsOf = (body) => {
-      const out = new Set();
-      const take = (v) => { const t = textOf(v).trim(); if (t) out.add(t.slice(0, TOOL_TEXT)); };
-      for (const m of Array.isArray(body.messages) ? body.messages : []) if (m && typeof m === "object" && m.role !== "assistant") take(m.content);
-      for (const it of Array.isArray(body.input) ? body.input : []) if (it && typeof it === "object" && it.role !== "assistant" && it.type !== "function_call") take(it.content ?? it.output);
-      if (typeof body.system === "string") take(body.system);
-      for (const c of Array.isArray(body.contents) ? body.contents : []) if (c && c.role !== "model") for (const p of Array.isArray(c.parts) ? c.parts : []) if (p && typeof p.text === "string") take(p.text);
-      return out;
-    };
-    const seenByTurn = new Map();
-    const materialIn = (sent, turn) => {
-      if (!turn) return undefined;
-      let body; try { body = JSON.parse(sent); } catch { return undefined; }
-      if (!body || typeof body !== "object") return undefined;
-      const now = textsOf(body);
-      const before = seenByTurn.get(turn);
-      seenByTurn.set(turn, now);
-      if (seenByTurn.size > 200) seenByTurn.delete(seenByTurn.keys().next().value);
-      if (!before) return undefined;
-      const fresh = [...now].filter((t) => !before.has(t)).slice(0, TOOLS_MAX).map((t) => ({ name: "", text: t }));
-      return fresh.length ? fresh : undefined;
-    };
     const rulesIn = (sent) => {
       const list = rulesNow();
       if (!list) return undefined;
@@ -482,7 +596,7 @@ if (FILE) {
     const emit = http.Server.prototype.emit;
     http.Server.prototype.emit = function (type, req, ...rest) {
       if (type === "listening") { try { listening(this.address()?.port); } catch { /* not a TCP server */ } }
-      if (type === "request") recheckRoutes();
+      if (type === "request") { recheckRoutes(); sweepTools(); }
       if (type !== "request" || !req || !req.method || /^(?:GET|HEAD|OPTIONS)$/.test(req.method)) return emit.call(this, type, req, ...rest);
       const { "x-cortad-turn": _turn, ...headers } = req.headers;
       const ctx = { method: req.method, path: req.url || "/", headers, chunks: [], size: 0, noted: false, turn: turnOf(req.headers) };
@@ -518,6 +632,7 @@ if (FILE) {
     if (isBun) {
       const inside = (handler, self) => function (req, server) {
         recheckRoutes();
+        sweepTools();
         if (!req || /^(?:GET|HEAD|OPTIONS)$/.test(req.method)) return handler.call(self, req, server);
         let path = "/";
         try { const u = new URL(req.url); path = u.pathname + u.search; } catch { /* keep "/" */ }
@@ -615,11 +730,10 @@ if (FILE) {
       try {
         const reply = readReply(type, String(body || "").slice(0, REPLY_MAX));
         const rules = rulesIn(sent);
-        const tools = toolsIn(sent) ?? materialIn(sent, turn);
+        const tools = toolsIn(sent);
         const called = calledIn(sent, reply.events);
         const passages = passagesIn(sent);
-        const row = { at: Date.now(), host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens), ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages ? { passages } : {}) };
-        fs.appendFileSync(FILE, JSON.stringify({ call: row }) + "\n", { mode: 0o600 });
+        row({ call: { at: Date.now(), host: String(host).replace(/:443$/, ""), model: String(askedFor(path, sent) || reply.model || "").slice(0, 160), status, ...(reply.tokens || { promptTokens: 0, cachedTokens: 0, completionTokens: 0 }), usage: Boolean(reply.tokens), ...(turn ? { turn } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages ? { passages } : {}) } });
       } catch { /* the command is gone */ }
     };
 
@@ -645,7 +759,161 @@ if (FILE) {
       } catch { return null; }
     };
     const depRow = (dep, status, code, passages, turn = (als.getStore() || {}).turn) => {
-      try { fs.appendFileSync(FILE, JSON.stringify({ dep: { at: Date.now(), ...(dep.env ? { env: dep.env } : {}), host: dep.host, status, code: code ? String(code).slice(0, 40) : undefined, ...(turn ? { turn } : {}), ...(passages ? { passages } : {}) } }) + "\n", { mode: 0o600 }); } catch { /* the command is gone */ }
+      row({ dep: { at: Date.now(), ...(dep.env ? { env: dep.env } : {}), host: dep.host, status, code: code ? String(code).slice(0, 40) : undefined, ...(turn ? { turn } : {}), ...(passages ? { passages } : {}) } });
+    };
+
+    // The app's own functions the read names as its tools. An app whose code picks the tool itself
+    // (a classifier answers {"intent":"order"} and the code calls queryOrder) never names one on the
+    // wire, so the function is wrapped where it is defined: each call writes its name, its clipped
+    // arguments and what it returned, on the turn it ran in. The run writes the list beside the rules
+    // and the command keeps the last one, so an app started again has it before its code loads.
+    const TOOLS_FILE = process.env.CORTAD_TOOLS_FILE;
+    const CODE_FILE = /\.(?:[cm]?[jt]sx?)$/;
+    const IDENT = /^[A-Za-z_$][\w$]*$/;
+    let targets = [], targetsAt = -1;
+    const targetsNow = () => {
+      if (!TOOLS_FILE) return targets;
+      try {
+        const at = fs.statSync(TOOLS_FILE).mtimeMs;
+        if (at !== targetsAt) {
+          targetsAt = at;
+          const raw = JSON.parse(fs.readFileSync(TOOLS_FILE, "utf8"));
+          targets = list(raw).filter((t) => typeof t.name === "string" && t.name && typeof t.file === "string" && CODE_FILE.test(t.file) && IDENT.test(String(t.function)) && !(t.function in globalThis))
+            .map((t) => ({ name: t.name.slice(0, 80), file: t.file.replace(/^\.?\/+/, ""), fn: t.function }))
+            // One function named twice (by its own name, and as "queryOrder (fallback)" where a
+            // router imported it) is recorded under its own name.
+            .sort((a, b) => (a.name !== a.fn) - (b.name !== b.fn));
+        }
+      } catch { /* not written yet */ }
+      return targets;
+    };
+    const targetsIn = (path) => targetsNow().filter((t) => path === t.file || path.endsWith("/" + t.file));
+    const unsaid = new Set();
+    const unwatched = (t, why) => {
+      const key = `${t.file}#${t.fn}`;
+      if (unsaid.has(key)) return;
+      unsaid.add(key);
+      try { process.stderr.write(`cortad: calls to ${t.name} in ${t.file} are not recorded: ${why}.\n`); } catch { /* no stderr */ }
+    };
+    // What a value is, as JSON can say it: a class instance (a client, a model) by its name only.
+    const plainOf = (v, depth = 0) => {
+      if (v === null || ["string", "number", "boolean"].includes(typeof v)) return v;
+      if (typeof v === "bigint") return String(v);
+      if (typeof v !== "object" || depth > 4) return v === undefined ? null : `[${typeof v}]`;
+      if (Array.isArray(v)) return v.slice(0, 20).map((x) => plainOf(x, depth + 1));
+      if (typeof v.toJSON === "function") { try { return plainOf(v.toJSON(), depth + 1); } catch { return "[object]"; } }
+      const proto = Object.getPrototypeOf(v);
+      if (proto !== Object.prototype && proto !== null) return `[${(v.constructor && v.constructor.name) || "object"}]`;
+      return Object.fromEntries(Object.entries(v).slice(0, 40).map(([k, x]) => [k, plainOf(x, depth + 1)]));
+    };
+    // The returned value's shape and a clipped text: a string as it is, anything else named by kind.
+    const returned = (v) => {
+      if (v === undefined || v === null) return "returned nothing";
+      if (typeof v === "string") return v;
+      const p = clipped(plainOf(v), 0, TOOL_TEXT);
+      const kind = Array.isArray(v) ? `list of ${v.length}` : typeof v === "object" ? "object" : "value";
+      return `${kind}: ${JSON.stringify(p)}`;
+    };
+    const paramNames = (fn) => {
+      let src = "";
+      try { src = Function.prototype.toString.call(fn); } catch { return []; }
+      const m = /^[^(=]*\(([^)]*)\)/.exec(src) || /^\s*(?:async\s+)?([A-Za-z_$][\w$]*)\s*=>/.exec(src);
+      if (!m || /[{[(]/.test(m[1])) return [];
+      return m[1].split(",").map((p) => p.replace(/=[\s\S]*$/, "").replace(/^\s*\.\.\./, "").replace(/:[\s\S]*$/, "").trim());
+    };
+    const toolRow = (name, names, args, value, error, turn) => {
+      const named = {};
+      args.forEach((a, i) => { named[IDENT.test(names[i] || "") ? names[i] : String(i)] = plainOf(a); });
+      let text;
+      try { text = error ? `raised ${(error && error.name) || "an error"}: ${(error && error.message) || String(error)}` : returned(value); } catch { text = "returned a value this hook cannot read"; }
+      row({ dep: { at: Date.now(), host: "in-app", status: 200, ...(turn ? { turn } : {}), called: [{ name, arguments: argsText(named) }], tools: [{ name, text: String(text).slice(0, TOOL_TEXT) }] } });
+    };
+    const TOOL = Symbol.for("cortad.wrapped");
+    const wrappers = new WeakMap();
+    const wrapTool = (fn, name) => {
+      if (typeof fn !== "function" || fn[TOOL]) return fn;
+      if (wrappers.has(fn)) return wrappers.get(fn);
+      try { if (/^class\b/.test(Function.prototype.toString.call(fn))) return fn; } catch { return fn; }
+      const names = paramNames(fn);
+      const w = function (...args) {
+        const turn = (als.getStore() || {}).turn;
+        let out;
+        try { out = new.target ? Reflect.construct(fn, args, new.target) : fn.apply(this, args); }
+        catch (e) { toolRow(name, names, args, undefined, e, turn); throw e; }
+        // A promise is followed, and the one handed back is a new one, so a rejection nobody
+        // handles still reaches the app as unhandled.
+        if (out instanceof Promise) return out.then((v) => { toolRow(name, names, args, v, undefined, turn); return v; }, (e) => { toolRow(name, names, args, undefined, e, turn); throw e; });
+        toolRow(name, names, args, out, undefined, turn);
+        return out;
+      };
+      try {
+        Object.defineProperty(w, "name", { value: fn.name });
+        Object.defineProperty(w, "length", { value: fn.length });
+        Object.setPrototypeOf(w, Object.getPrototypeOf(fn));
+        if (fn.prototype) w.prototype = fn.prototype;
+        for (const k of Object.keys(fn)) w[k] = fn[k];
+      } catch { /* the wrapper still calls through */ }
+      w[TOOL] = fn;
+      wrappers.set(fn, w);
+      return w;
+    };
+    // Called from the lines added to the end of a tool's file: the function by its own binding
+    // (an ES module's importers follow that binding) and by its CommonJS export.
+    const take = (fn, set, mod, t) => {
+      let done = false;
+      if (typeof fn === "function") { try { set(wrapTool(fn, t.name)); done = true; } catch { /* a binding that cannot change */ } }
+      const ex = mod && mod.exports;
+      if (ex && (typeof ex === "object" || typeof ex === "function") && typeof ex[t.fn] === "function") { try { ex[t.fn] = wrapTool(ex[t.fn], t.name); done = true; } catch { /* read-only */ } }
+      if (!done) unwatched(t, fn === undefined && !(ex && t.fn in ex) ? `the file has no function named ${t.fn}` : `${t.fn} there is not a function this hook can wrap`);
+    };
+    Object.defineProperty(globalThis, Symbol.for("cortad.tool"), { value: take, configurable: true });
+    const trailer = (text, mine) => {
+      let src = text;
+      // A const binding cannot be changed from inside the file; a let can, and nothing else differs.
+      for (const t of mine) src = src.replace(new RegExp(`(^|\\n)([ \\t]*(?:export[ \\t]+)?)const([ \\t]+${t.fn.replace(/\$/g, "\\$")}[ \\t]*[=:])`), "$1$2let$3");
+      const hook = 'globalThis[Symbol.for("cortad.tool")]';
+      return `${src}\n;${mine.map((t) => `try{${hook}(typeof ${t.fn}==="undefined"?undefined:${t.fn},(f)=>{${t.fn}=f},typeof module==="object"&&module?module:undefined,${JSON.stringify(t)})}catch{}`).join("")}\n`;
+    };
+    // Loading: a tool's file gets the lines above before it runs. Every other file is untouched.
+    const seenFiles = new Set();
+    const loadedFiles = new Set();
+    const { fileURLToPath } = require("node:url");
+    if (TOOLS_FILE && typeof Module.registerHooks === "function") {
+      try {
+        Module.registerHooks({
+          load(url, context, nextLoad) {
+            const out = nextLoad(url, context);
+            try {
+              if (typeof url !== "string" || !url.startsWith("file:") || url.includes("/node_modules/")) return out;
+              const path = fileURLToPath(url);
+              loadedFiles.add(path);
+              const mine = targetsIn(path);
+              if (!mine.length || out.source == null) return out;
+              seenFiles.add(path);
+              return { ...out, source: trailer(typeof out.source === "string" ? out.source : Buffer.from(out.source).toString("utf8"), mine) };
+            } catch { return out; }
+          },
+        });
+      } catch { /* this runtime has no module hooks */ }
+    }
+    // A list that arrives after the app loaded its tools: a CommonJS export is wrapped in place, which
+    // reaches code that calls it through the module, but a function already taken by name cannot be
+    // reached, so it is said once in the app's log. The command keeps the list for the next start.
+    let sweptAt = -1;
+    const sweepTools = () => {
+      if (!TOOLS_FILE) return;
+      const now = targetsNow();
+      if (targetsAt === sweptAt) return;
+      sweptAt = targetsAt;
+      const cache = require.cache || {};
+      for (const t of now) {
+        const mine = (k) => (k === t.file || k.endsWith("/" + t.file)) && !k.includes("/node_modules/");
+        const path = Object.keys(cache).find(mine) || [...loadedFiles].find(mine);
+        if (!path || seenFiles.has(path)) continue;
+        const ex = cache[path] && cache[path].exports;
+        if (ex && typeof ex[t.fn] === "function") { try { ex[t.fn] = wrapTool(ex[t.fn], t.name); } catch { /* read-only */ } }
+        unwatched(t, "the file was loaded before the list of tools arrived, and they are recorded from the app's next start");
+      }
     };
 
     // Outbound: the model call your app makes while it handles that request.

package/local.mjs

@@ -15,13 +15,13 @@
 import { holdsKeys, secretEnvValues } from "./lib/keys.mjs";
 import { spawn, execFile, execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, watch, writeFileSync } from "node:fs";
+import { appendFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, watch, writeFileSync } from "node:fs";
 import { homedir, hostname, tmpdir } from "node:os";
 import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import { createInterface } from "node:readline";
 import { promisify } from "node:util";
 import { COMMANDS, main as face } from "./lib/cli.mjs";
-import { clearRunner, projectOf, readToken, writeDigest, writeRunner, writeToken } from "./lib/home.mjs";
+import { clearRunner, homeOf, projectOf, readToken, writeDigest, writeRunner, writeToken } from "./lib/home.mjs";
 import { registerAll } from "./lib/register.mjs";
 import { finished } from "./lib/text.mjs";
 import { lockHolds, makeLock } from "./lib/lock.mjs";
@@ -420,7 +420,14 @@ async function verb(job) {
       // machine writes your files; the agent that edits them is your own.
       const mine = scratch(b.path);
       if (!mine) return { success: false, stderr: "this program does not write your files" };
-      try { mkdirSync(dirname(mine), { recursive: true }); if (b.append) appendFileSync(mine, bytes); else writeFileSync(mine, bytes); return { success: true, stderr: "" }; }
+      try {
+        mkdirSync(dirname(mine), { recursive: true });
+        if (b.append) appendFileSync(mine, bytes); else writeFileSync(mine, bytes);
+        // The read's tools, kept for this project too: the next start hands them to the hook before
+        // the app loads its code, which a Node app needs to have its tool functions wrapped.
+        if (mine === join(work, "tools.json")) { try { mkdirSync(homeOf(project), { recursive: true, mode: 0o700 }); writeFileSync(join(homeOf(project), "tools.json"), bytes, { mode: 0o600 }); } catch { /* kept for this session only */ } }
+        return { success: true, stderr: "" };
+      }
       catch (e) { return { success: false, stderr: String(e.message) }; }
     }
     case "mint": return identities ? JSON.parse(mask(JSON.stringify(await mintAcross({
@@ -910,6 +917,7 @@ if (!files.length) fail("no source files here to read.");
 // Which project this is, as a hash of where it lives: the same folder coming back resumes the same
 // connection, and the path itself never leaves this machine.
 const project = projectOf(root);
+try { copyFileSync(join(homeOf(project), "tools.json"), join(work, "tools.json")); } catch { /* no run has read this project's tools yet */ }
 // The key the last connect left for this project, if any: the token face signs in with it. A connect
 // from the screen always asks for a fresh one, since the code may belong to another account or site.
 const stored = readToken(project);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.1-rc.3",
+  "version": "0.3.1-rc.4",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"

package/skill/SKILL.md

@@ -56,6 +56,7 @@ Take the person through them worst first, each with its file and line. A long li
    - `no change` or `regressed`: the file goes back to how it was.
    - `inside the noise` or `unsettled`: the trials cannot tell yet, which is not a verdict against the change. `verify` again to add readings; a change the app's own tests support may stay, said as not yet verified by Cortad.
    - A verify that played no trial says so and compares nothing; the file stays as it was left.
+   - `Verification unavailable:` the two runs read the question on no common trial, and the line says why (the replay took another turn, the trial halted, the question's condition did not hold). It is not a verdict: the file stays, said as not yet verified by Cortad.
    - An `Overfit:` line: the visible trials moved and the held-out trials stayed where they were, so the change fits the trials it could see. The file goes back, and the next change aims at the behavior the question asks about.
    - `regressed`: the file goes back.
 4. Then the next finding.

package/skill/references/results.md

@@ -155,6 +155,17 @@ For the person: the report is at https://cortad.com/lab
 next: findings
 ```
 
+A verify with no pair: the two runs read the finding's question on no common trial, so there is no move to print, and the line says why. It is not a verdict on the change.
+
+```
+Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 1 of 1 trial played.
+Verify of finding:2 at backend/tools/refunds.py.
+Held-out trials: no pair for this question.
+Verification unavailable: the replay took a different turn at reply 2, and the question was not asked on the replay because its condition did not hold (the customer has reached this step of the journey: Agent states amount, asks consent). This is not a verdict on the change either way: the two runs share no reading of this question to compare.
+For the person: the report is at https://cortad.com/lab
+next: findings
+```
+
 ## findings
 
 Worst first, grouped by the line the rule lives at. A list longer than one page ends with `page 1 of 3, call findings with page 2`.