@senpi/runtime 3.0.187 → 3.0.190

package/dist/scanners/external/scaffold/preflight.py

@@ -31,14 +31,18 @@ Author stdout is captured as evidence and never parsed. Redirecting `sys.stdout`
 cover `print()` — `os.write(1, ...)`, a subprocess, or a C extension write to the same descriptor.
 """
 
+import ast
+import dis
 import io
+import itertools
 import json
 import math
 import os
 import sys
+import threading
 import time
 import traceback
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from . import http_post
 from .signal_envelope import candidate_to_signal
@@ -57,6 +61,390 @@ def _bounded(text: str, limit: int = _STREAM_LIMIT) -> str:
     return "{}\n… truncated {} characters …\n{}".format(text[:half], dropped, text[-half:])
 
 
+#: Longest key name echoed into a record. Key names come from the response.
+_KEY_LIMIT = 64
+
+#: Where an expression ends, (end_lineno, end_col_offset). The compiler can move a wrapped method
+#: call's start onto the `.get` line, but never its end, so the AST and `co_positions()` agree on it,
+#: and it is distinct for every call and subscript.
+Key = Tuple[int, int]
+
+_FILES: Dict[str, "_FileReads"] = {}
+_POSITIONS: Dict[Any, List[Any]] = {}
+_RESUMES: Dict[Any, int] = {}
+_SCOPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)
+_GENERATOR = 0x20 | 0x80 | 0x200  # CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR
+#: Literal defaults that stand in for a value rather than being one.
+_STAND_INS = (None, 0, 0.0, "", "0", "0.0")
+
+
+def _key(node: ast.AST) -> Key:
+    return (node.end_lineno, node.end_col_offset)  # type: ignore[attr-defined]
+
+
+def _keyed(node: ast.AST) -> bool:
+    return getattr(node, "end_col_offset", None) is not None
+
+
+def _is_get(node: ast.AST) -> bool:
+    return isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "get"
+
+
+def _is_or(node: ast.AST) -> bool:
+    return isinstance(node, ast.BoolOp) and isinstance(node.op, ast.Or)
+
+
+def _assigned(node: ast.AST) -> Optional[Tuple[str, ast.AST]]:
+    """(name, value) for `x = ...`, `x: T = ...` and `(x := ...)`."""
+    if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+        return node.targets[0].id, node.value
+    if isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and isinstance(node.target, ast.Name) and node.value:
+        return node.target.id, node.value
+    return None
+
+
+def _own_nodes(scope: ast.AST) -> Any:
+    """The nodes under `scope`, not those of the functions and classes inside it."""
+    stack = list(ast.iter_child_nodes(scope))
+    while stack:
+        node = stack.pop()
+        yield node
+        if not isinstance(node, _SCOPES):
+            stack.extend(ast.iter_child_nodes(node))
+
+
+def _within(stmts: List[ast.stmt]) -> Any:
+    for stmt in stmts:
+        yield stmt
+        yield from _own_nodes(stmt)
+
+
+def _reads_name(node: ast.AST, name: str) -> bool:
+    return any(isinstance(n, ast.Name) and n.id == name for n in ast.walk(node))
+
+
+def _truth_test(node: ast.AST, parents: Dict[Any, Any]) -> bool:
+    """Whether a read's result is only tested: `if row.get(k):`, `not row.get(k)`,
+    `row.get(k) is None`. Such a read checks an optional key; it does not take a value."""
+    parent = parents.get(node)
+    while _is_or(parent) or (isinstance(parent, ast.BoolOp) and isinstance(parent.op, ast.And)):
+        node, parent = parent, parents.get(parent)
+    if isinstance(parent, (ast.If, ast.While, ast.IfExp, ast.Assert)) and parent.test is node:
+        return True
+    if isinstance(parent, ast.UnaryOp) and isinstance(parent.op, ast.Not):
+        return True
+    return (isinstance(parent, ast.Compare) and len(parent.ops) == 1
+            and isinstance(parent.ops[0], (ast.Is, ast.IsNot))
+            and any(isinstance(c, ast.Constant) and c.value is None for c in [parent.left, *parent.comparators]))
+
+
+class _FileReads:
+    """How each `.get(...)` and subscript in one scanner file takes part in fallback chains.
+
+    `roots` maps each read to its chain. One chain: `a.get(k, b.get(k2, 0))`, `a.get(k) or b[k2]`,
+    `a.get(k) if ... else b.get(k2)`, helper calls as alternatives (`_f(a, k) or _f(b, k)`), and an
+    assignment with its fallback: `x = c.get(k)` then `if x is None: x = ...`, `x = x or ...`, the
+    same name in an `except`, or `x or ...` later. A miss in a chain is answered by a hit anywhere in it.
+
+    `receivers` are reads on the way to an alternative (`i["context"]` in `i["context"]["volume"]`,
+    a ternary's test). They join the chain; finding them does not answer it.
+
+    `solo` are `.get`s alone in their chain. Their caller's expression may be the chain.
+
+    `defaults` says, per `.get`, how a miss is judged from its default as written: "keep" (absent, or
+    a literal stand-in such as 0, None, "" or "0"), "drop" (a real value, a computed fallback such as
+    `a + b`, a bool, or a read only tested), or "value" (a name or another read: judged by its value).
+    """
+
+    def __init__(self, filename: str) -> None:
+        self.roots: Dict[Key, Any] = {}
+        self.receivers: set = set()
+        self.solo: set = set()
+        self.defaults: Dict[Key, str] = {}
+        self._labels = itertools.count()
+        try:
+            with open(filename, encoding="utf-8") as fh:
+                tree = ast.parse(fh.read(), filename)
+        except (OSError, SyntaxError, ValueError):
+            return
+        self._chains(tree)
+        for scope in [tree] + [n for n in ast.walk(tree) if isinstance(n, _SCOPES[:2])]:
+            self._join_assignments(scope)
+        counts: Dict[Any, int] = {}
+        for key, root in self.roots.items():
+            if key not in self.receivers:
+                counts[root] = counts.get(root, 0) + 1
+        self.solo = {k for k, r in self.roots.items() if k not in self.receivers and counts[r] == 1}
+        parents = {c: p for p in ast.walk(tree) for c in ast.iter_child_nodes(p)}
+        for node in ast.walk(tree):
+            if _is_get(node):
+                self.defaults[_key(node)] = "drop" if _truth_test(node, parents) else self._judge(node)
+
+    def _chains(self, tree: ast.AST) -> None:
+        claimed = set()
+
+        def receiver(node: ast.AST, root: Any) -> None:
+            claimed.add(id(node))
+            self.roots[_key(node)] = root
+            self.receivers.add(_key(node))
+
+        def through(node: ast.AST, root: Any) -> None:
+            while _is_get(node) or isinstance(node, ast.Subscript):
+                receiver(node, root)
+                node = node.func.value if _is_get(node) else node.value  # type: ignore[attr-defined]
+
+        def mark(node: ast.AST, root: Any) -> None:
+            claimed.add(id(node))
+            if _is_get(node):
+                self.roots[_key(node)] = root
+                through(node.func.value, root)  # type: ignore[attr-defined]
+                for alt in node.args[1:2]:  # type: ignore[attr-defined]
+                    mark(alt, root)
+            elif isinstance(node, ast.Subscript):
+                self.roots[_key(node)] = root
+                through(node.value, root)
+            elif _is_or(node):
+                for alt in node.values:  # type: ignore[attr-defined]
+                    mark(alt, root)
+            elif isinstance(node, ast.IfExp):
+                for read in ast.walk(node.test):
+                    if _is_get(read) or isinstance(read, ast.Subscript):
+                        receiver(read, root)
+                mark(node.body, root)
+                mark(node.orelse, root)
+            elif isinstance(node, ast.Call):  # `_f(a, k) or _f(b, k)`, `float(a.get(k)) or ...`
+                self.roots[_key(node)] = root
+                for arg in node.args:
+                    mark(arg, root)
+
+        # Breadth-first, so a chain is claimed from its outermost node before its inner ones.
+        for node in ast.walk(tree):
+            if id(node) not in claimed and (
+                _is_get(node) or _is_or(node) or isinstance(node, (ast.IfExp, ast.Subscript))
+            ):
+                mark(node, _key(node))
+
+    def _join(self, *groups: set) -> None:
+        members = set().union(*groups)
+        if len(members) > 1:
+            label = ("join", next(self._labels))
+            for key, root in list(self.roots.items()):
+                if root in members:
+                    self.roots[key] = label
+
+    def _chains_in(self, node: ast.AST) -> set:
+        return {self.roots[_key(n)] for n in ast.walk(node) if _keyed(n) and _key(n) in self.roots}
+
+    def _join_assignments(self, scope: ast.AST) -> None:
+        parents = {c: p for p in ast.walk(scope) for c in ast.iter_child_nodes(p)}
+        own = list(_own_nodes(scope))
+        assigns = sorted((n for n in own if _assigned(n) and self._chains_in(_assigned(n)[1])),
+                         key=lambda n: (n.lineno, n.col_offset))
+        latest: Dict[str, ast.AST] = {}
+        for node in assigns:
+            name, value = _assigned(node)  # type: ignore[misc]
+            if name in latest and self._falls_back(node, name, value, parents, scope):
+                self._join(self._chains_in(latest[name]), self._chains_in(value))
+            latest[name] = value
+        for node in own:
+            if isinstance(node, ast.Try):  # `try: x = a.get(k) ...` / `except ...: x = b[k]`
+                tried = {a[0]: a[1] for a in map(_assigned, _within(node.body)) if a}
+                for handler in node.handlers:
+                    for a in filter(None, map(_assigned, _within(handler.body))):
+                        if a[0] in tried:
+                            self._join(self._chains_in(tried[a[0]]), self._chains_in(a[1]))
+            elif _is_or(node):  # `legacy = c.get(k)` ... `legacy or c.get(k2)`
+                for operand in node.values:  # type: ignore[attr-defined]
+                    if isinstance(operand, ast.Name):
+                        before = [n for n in assigns if _assigned(n)[0] == operand.id  # type: ignore[index]
+                                  and (n.lineno, n.col_offset) < (node.lineno, node.col_offset)]
+                        if before:
+                            self._join(self._chains_in(_assigned(before[-1])[1]), self._chains_in(node))  # type: ignore[index]
+
+    @staticmethod
+    def _falls_back(assign: ast.AST, name: str, value: ast.AST, parents: Dict[Any, Any], scope: ast.AST) -> bool:
+        """Whether an assignment to `name` is the fallback of the one before it: `x = x or ...`, or it
+        sits in the body of an `if` that tests `x`. A walrus in that test is the first assignment.
+        `x = x + ...` accumulates; it is not a fallback."""
+        if _is_or(value) and isinstance(value.values[0], ast.Name) and value.values[0].id == name:  # type: ignore[attr-defined]
+            return True
+        child, node = assign, parents.get(assign)
+        while node is not None and node is not scope:
+            if isinstance(node, ast.If) and child is not node.test and _reads_name(node.test, name):
+                return True
+            child, node = node, parents.get(node)
+        return False
+
+    def _judge(self, call: ast.Call) -> str:
+        if len(call.args) < 2:
+            return "keep"
+        default = call.args[1]
+        if isinstance(default, ast.Constant):
+            value = default.value
+            return "keep" if not isinstance(value, bool) and value in _STAND_INS else "drop"
+        if isinstance(default, (ast.Dict, ast.List, ast.Tuple, ast.Set)):
+            return "drop" if (getattr(default, "keys", None) or getattr(default, "elts", None)) else "keep"
+        if _is_get(default) or _is_or(default) or isinstance(default, (ast.Name, ast.IfExp, ast.Subscript)):
+            return "value"
+        return "drop"
+
+
+def _file_reads(filename: str) -> _FileReads:
+    if filename not in _FILES:
+        _FILES[filename] = _FileReads(filename)
+    return _FILES[filename]
+
+
+def _position(frame: Any) -> Optional[Tuple[int, int, int, int]]:
+    """(lineno, end_lineno, col, end_col) of the expression a frame is running. 3.11+."""
+    code = frame.f_code
+    try:
+        positions = _POSITIONS.get(code)
+        if positions is None:
+            positions = _POSITIONS[code] = list(code.co_positions())
+        span = positions[frame.f_lasti // 2]
+    except (AttributeError, IndexError):
+        return None
+    return None if None in span else span
+
+
+def _first_resume(code: Any) -> int:
+    if code not in _RESUMES:
+        _RESUMES[code] = next((i.offset for i in dis.get_instructions(code) if i.opname == "RESUME"), -1)
+    return _RESUMES[code]
+
+
+def _placeholder(value: Any) -> bool:
+    """Whether a default's value stands in for one (0, None, "", {}, []). A bool is a flag's answer."""
+    if isinstance(value, bool):
+        return False
+    try:
+        return not value
+    except Exception:  # noqa: BLE001 — a value whose truth raises is not a placeholder
+        return False
+
+
+class _Reads:
+    """Row-key reads across every response of one tick: how often each (tool, path) was found, and
+    the misses that nothing else in their fallback chain answered.
+
+    Chains come from the source (`_FileReads`). A read alone in its chain in a helper is judged at
+    its caller's expression when that is a chain (`_f(a, k) or _f(b, k)`). A loop over spellings —
+    `for k in ("traderAddress", "address"): v = t.get(k)` — runs one `.get` once per spelling, so a
+    miss is also dropped when the same expression, in the same call, finds a key on the same row (or
+    a dict inside it), before or after it.
+
+    Calls are told apart by the number the preflight tracer gives each one as it starts, so no frame
+    is kept alive. Chains are per thread, so a scanner scoring rows on a pool does not interleave them.
+    """
+
+    def __init__(self, calls: Optional[Dict[int, int]] = None) -> None:
+        self.found: Dict[Tuple[str, str], int] = {}
+        self.missed: Dict[Tuple[str, str], Dict[str, Any]] = {}
+        self._calls = calls if calls is not None else {}
+        self._lock = threading.Lock()
+        self._chains: Dict[int, Dict[Any, Dict[str, Any]]] = {}  # per thread: open chains by (file, root)
+        # Rows are held for the tick, so a row's id is never reused. They are reachable from the
+        # response already, so this keeps nothing alive that would not be.
+        self._rows: Dict[int, Any] = {}
+        # Per row id: the (call, read) pairs that found a key there, and the misses each made.
+        self._row_hits: Dict[int, set] = {}
+        self._row_misses: Dict[int, Dict[Any, List[Tuple[str, str]]]] = {}
+
+    def read(self, tool: str, path: str, found: bool, default: Any, carried: Optional[List[str]], row: Any) -> None:
+        frame = sys._getframe(2)  # noqa: SLF001 — 0 is here, 1 the accessor, 2 the scanner
+        with self._lock:
+            if found:
+                self.found[(tool, path)] = self.found.get((tool, path), 0) + 1
+            span = _position(frame)
+            if span is None:
+                return  # nothing to tell a fallback from a guess by, so record no miss
+            file = frame.f_code.co_filename
+            reads = _file_reads(file)
+            key = (span[1], span[3])
+            rule = reads.defaults.get(key)
+            keep = not found and (rule == "keep" or (rule != "drop" and _placeholder(default)))
+            if key not in reads.roots or key in reads.solo:
+                outer = frame.f_back
+                while outer is not None and id(outer) in self._calls:
+                    outer_span = _position(outer)
+                    if outer_span is not None:
+                        outer_reads = _file_reads(outer.f_code.co_filename)
+                        outer_key = (outer_span[1], outer_span[3])
+                        if outer_key in outer_reads.roots and outer_key not in outer_reads.solo:
+                            frame, span, reads, key = outer, outer_span, outer_reads, outer_key
+                            file = outer.f_code.co_filename
+                            break
+                    outer = outer.f_back
+            owner = row._owner
+            call = (id(frame), self._calls.get(id(frame), 0))
+            here = (call, key)
+            self._rows.setdefault(id(owner), owner)
+            if found:
+                self._row_hits.setdefault(id(owner), set()).add(here)
+                for missed in self._row_misses.get(id(owner), {}).pop(here, []):
+                    self._forget(missed)
+            root = reads.roots.get(key, key)
+            # Every chain stays open until its own expression comes round again (the next row), its
+            # call changes, or the tick ends: an unrelated read between two halves of a fallback
+            # (`c = inst.get("context", {})`) does not end it.
+            open_chains = self._chains.setdefault(threading.get_ident(), {})
+            slot = (file, root)
+            chain = open_chains.get(slot)
+            if chain is not None and (chain["call"] != call or key in chain["keys"]):
+                self._close(open_chains.pop(slot))
+                chain = None
+            if chain is None:
+                chain = open_chains[slot] = {"call": call, "keys": set(), "found": False, "misses": []}
+            chain["keys"].add(key)
+            if found:
+                if key not in reads.receivers:
+                    chain["found"] = True
+            elif keep:
+                chain["misses"].append((tool, path, carried, (file, span[0]), id(owner), here, row._path))
+            if key == root:  # the outermost `.get` runs last, after every alternative
+                self._close(open_chains.pop(slot))
+
+    def _close(self, chain: Dict[str, Any]) -> None:
+        if chain["found"]:
+            return
+        for tool, path, carried, where, owner_id, here, row_path in chain["misses"]:
+            if here in self._row_hits.get(owner_id, ()):
+                continue  # this expression, in this call, found another spelling on this row
+            entry = self.missed.setdefault(
+                (tool, path), {"reads": 0, "carried": carried, "where": where, "row": row_path})
+            entry["reads"] += 1
+            self._row_misses.setdefault(owner_id, {}).setdefault(here, []).append((tool, path))
+
+    def _forget(self, missed: Tuple[str, str]) -> None:
+        entry = self.missed.get(missed)
+        if entry is not None:
+            entry["reads"] -= 1
+            if entry["reads"] <= 0:
+                del self.missed[missed]
+
+    def never_found(self, root: str) -> Dict[str, List[Dict[str, Any]]]:
+        """Per tool, keys read on rows of a list that no row of any of its responses carried.
+        Top-level keys are left out: an optional `error` or cursor is routinely absent."""
+        with self._lock:
+            for open_chains in self._chains.values():
+                for chain in open_chains.values():
+                    self._close(chain)
+            self._chains.clear()
+            prefix = os.path.abspath(root).rstrip(os.sep) + os.sep
+            out: Dict[str, List[Dict[str, Any]]] = {}
+            for (tool, path), entry in self.missed.items():
+                if "[]" not in path or self.found.get((tool, path)):
+                    continue
+                item: Dict[str, Any] = {
+                    "path": path, "row": entry["row"], "reads": entry["reads"], "carried": entry["carried"]}
+                file, line = entry["where"]
+                if os.path.abspath(file).startswith(prefix):
+                    item.update(file=os.path.relpath(file, root), line=line)
+                out.setdefault(tool, []).append(item)
+            return {tool: items[:10] for tool, items in out.items()}
+
+
 class _Watched(dict):
     """A response that remembers whether the scanner looked inside it.
 
@@ -64,22 +452,56 @@ class _Watched(dict):
     expects — a renamed key, a nesting level missed. That failure is silent: the read succeeded, the
     scanner carried on, and the score was computed from a default. Recording access is what turns it
     into something reportable.
+
+    The same failure one level down is a row key that does not exist: `resp["data"]` was read, so
+    the response counts as used, while every `row.get("volume", 0)` returned its default. Nested
+    dicts and lists are watched too, and every key read is reported to the tick's `_Reads`.
     """
 
-    def __init__(self, source: dict) -> None:
+    def __init__(self, source: Any = (), tool: str = "", path: str = "", reads: Optional[_Reads] = None,
+                 owner: Any = None) -> None:
         super().__init__(source)
         self.touched = False
+        self._tool = tool
+        self._path = path
+        self._reads = reads if reads is not None else _Reads()
+        # The list row this dict belongs to (itself for a row): `row["context"]` answers for `row`.
+        self._owner = owner if owner is not None else self
 
     def _touch(self) -> None:
         self.touched = True
 
+    def _at(self, key: Any) -> str:
+        key = str(key)[:_KEY_LIMIT]
+        return f"{self._path}.{key}" if self._path else key
+
+    def _child(self, key: Any, path: str, value: Any) -> Any:
+        # Wrapped once and stored back, so the scanner holds one object however often it looks.
+        if isinstance(value, dict) and not isinstance(value, _Watched):
+            value = _Watched(value, self._tool, path, self._reads, self._owner if "[]" in self._path else None)
+        elif isinstance(value, list) and not isinstance(value, _WatchedList):
+            value = _WatchedList(value, self._tool, path + "[]", self._reads)
+        else:
+            return value
+        dict.__setitem__(self, key, value)
+        return value
+
     def __getitem__(self, key: Any) -> Any:
         self._touch()
-        return super().__getitem__(key)
+        value = super().__getitem__(key)  # a missing key raises, which is already visible
+        path = self._at(key)
+        self._reads.read(self._tool, path, True, None, None, self)
+        return self._child(key, path, value)
 
-    def get(self, key: Any, default: Any = None) -> Any:
+    def get(self, key: Any, default: Any = None, /) -> Any:  # positional, as `dict.get` is
         self._touch()
-        return super().get(key, default)
+        path = self._at(key)
+        if dict.__contains__(self, key):
+            self._reads.read(self._tool, path, True, None, None, self)
+            return self._child(key, path, dict.__getitem__(self, key))
+        carried = sorted(str(k)[:_KEY_LIMIT] for k in dict.keys(self))[:20]
+        self._reads.read(self._tool, path, False, default, carried, self)
+        return default
 
     def __contains__(self, key: Any) -> bool:
         self._touch()
@@ -101,6 +523,47 @@ class _Watched(dict):
         self._touch()
         return super().items()
 
+    def __reduce_ex__(self, protocol: Any) -> Any:
+        # A copy or pickle is plain data, not the tick's bookkeeping.
+        return (dict, (dict(self),))
+
+
+class _WatchedList(list):
+    """A list inside a watched response. Its rows (dicts, and lists of rows) are watched as they are
+    handed out, and stored back, so a slice or a loop holds the same objects as the list."""
+
+    def __init__(self, source: Any = (), tool: str = "", path: str = "", reads: Optional[_Reads] = None) -> None:
+        super().__init__(source)
+        self._tool = tool
+        self._path = path
+        self._reads = reads if reads is not None else _Reads()
+
+    def _row(self, index: int, value: Any) -> Any:
+        if isinstance(value, dict) and not isinstance(value, _Watched):
+            value = _Watched(value, self._tool, self._path, self._reads)
+        elif isinstance(value, list) and not isinstance(value, _WatchedList):
+            value = _WatchedList(value, self._tool, self._path + "[]", self._reads)
+        else:
+            return value
+        list.__setitem__(self, index, value)
+        return value
+
+    def __getitem__(self, index: Any) -> Any:
+        if isinstance(index, slice):  # `rows[:20]` is how a universe gets capped
+            for i in range(*index.indices(len(self))):
+                self._row(i, list.__getitem__(self, i))
+            return _WatchedList(list.__getitem__(self, index), self._tool, self._path, self._reads)
+        return self._row(index, list.__getitem__(self, index))
+
+    def __iter__(self) -> Any:
+        index = 0
+        while index < len(self):  # a list iterator's own rule, so mid-loop removes and appends match
+            yield self._row(index, list.__getitem__(self, index))
+            index += 1
+
+    def __reduce_ex__(self, protocol: Any) -> Any:
+        return (list, (list(list.__iter__(self)),))
+
 
 class CountingMcp:
     """Records every call and forwards it unchanged.
@@ -114,10 +577,11 @@ class CountingMcp:
     place for it, since the same boundary protects production.
     """
 
-    def __init__(self, inner: Any) -> None:
+    def __init__(self, inner: Any, call_ids: Optional[Dict[int, int]] = None) -> None:
         self._inner = inner
         self.calls: List[Dict[str, Any]] = []
         self._watched: List[Dict[str, Any]] = []
+        self._reads = _Reads(call_ids)
 
     def call_tool(self, name: str, args: Optional[dict] = None) -> Any:
         started = time.monotonic()
@@ -163,16 +627,22 @@ class CountingMcp:
         self.calls.append(record)
 
         if isinstance(result, dict) and record["ok"]:
-            watched = _Watched(result)
+            watched = _Watched(result, name, "", self._reads)
             self._watched.append({"record": record, "response": watched})
             return watched
         return result
 
-    def settle_unread(self) -> None:
-        """Mark the responses nothing ever looked inside. Called once the tick is over."""
+    def settle(self, root: str) -> None:
+        """Mark the responses nothing looked inside, and the row keys looked for and never found.
+        Called once the tick is over. Missing keys are judged per tool, across all its responses,
+        and land on that tool's first record."""
+        missing = self._reads.never_found(root)
         for entry in self._watched:
+            record = entry["record"]
             if not entry["response"].touched:
-                entry["record"]["unread"] = True
+                record["unread"] = True
+            if record["tool"] in missing:
+                record["missing_keys"] = missing.pop(record["tool"])
 
     def __getattr__(self, name: str) -> Any:
         # Keeps the wrapper transparent for anything beyond the documented surface.
@@ -282,6 +752,10 @@ class _Tracer:
         # pin a chain of tracebacks.
         self._live_exc: Any = None
         self._group = 0
+        # The call each live scanner frame belongs to, numbered as calls start. A frame's id is
+        # reused once it returns; this tells two calls apart without keeping either frame alive.
+        self.call_ids: Dict[int, int] = {}
+        self._call_seq = itertools.count(1)
 
     def _own(self, frame: Any) -> bool:
         return os.path.abspath(frame.f_code.co_filename).startswith(self._prefix)
@@ -292,6 +766,11 @@ class _Tracer:
                 return None
             # Exceptions and returns still fire; per-line events do not.
             frame.f_trace_lines = False
+            # A generator fires 'call' on every resume; it keeps the number of its first entry.
+            code = frame.f_code
+            if not (code.co_flags & _GENERATOR and id(frame) in self.call_ids
+                    and frame.f_lasti != _first_resume(code)):
+                self.call_ids[id(frame)] = next(self._call_seq)
             return self
         if event == "exception":
             exc_type, exc_value, _tb = arg
@@ -378,6 +857,7 @@ def run_preflight(config: Dict[str, Any]) -> Dict[str, Any]:
     from . import scaffold as scaffold_mod
 
     root = config["path"]
+    _FILES.clear()  # the scanner's source is read fresh for each tick
     sink = _CollectingSink()
     tracer = _Tracer(root)
 
@@ -388,7 +868,7 @@ def run_preflight(config: Dict[str, Any]) -> Dict[str, Any]:
             "reason": "no_credentials",
             "detail": "no data-client credentials are configured in this environment",
         }
-    counter = CountingMcp(inner)
+    counter = CountingMcp(inner, tracer.call_ids)
 
     captured_out = io.StringIO()
     captured_err = io.StringIO()
@@ -430,7 +910,7 @@ def run_preflight(config: Dict[str, Any]) -> Dict[str, Any]:
         sys.settrace(None)
         sys.stdout, sys.stderr = real_stdout, real_stderr
         http_post.PREFLIGHT = False
-        counter.settle_unread()
+        counter.settle(root)
         tracer.release()
 
     # A tick that failed was reported through the sink rather than raised. The loop's error record

package/dist/senpi/__tests__/helpers/fake-paper-mcp.d.ts

@@ -0,0 +1,44 @@
+/**
+ * A fake Senpi MCP that holds a user's live strategies and simulations and answers `strategy_list`
+ * and `strategy_create_custom_strategy` the way the paper router does (SPEC §6.3, §6.5): one world
+ * per call, `paper_routing: true` at the head of every paper reply, `paper: true` on every paper
+ * row, and `PAPER_IDENTIFIER_MISMATCH` for a flag that disagrees with its filter.
+ *
+ * `router: false` plays an MCP without the router: it drops the unknown `paper` key and answers
+ * from the live world with no marker, and its strict UUID check refuses a `paper-` id.
+ *
+ * Every call is recorded so a test can assert what reached the MCP.
+ */
+import type { McpTransport } from "../../types.js";
+export interface FakeStrategyRow {
+    id: string;
+    strategyWalletAddress: string;
+    status: string;
+    strategyName: string;
+    skillName?: string;
+    totalFunded?: number;
+}
+export interface FakePaperMcpOptions {
+    router?: boolean;
+    live?: FakeStrategyRow[];
+    paper?: FakeStrategyRow[];
+    /** Statuses a new simulation reports on successive reads; the last one sticks. */
+    paperCreateStatuses?: string[];
+    /** The wallet a new simulation gets. Defaults to a hypaper-format address. */
+    paperCreateWallet?: string;
+    /** Rows the router forgot to flag `paper: true` (a broken router, for the row filter). */
+    unflaggedPaperRows?: boolean;
+}
+export interface FakeCall {
+    tool: string;
+    args: Record<string, unknown>;
+}
+export interface FakePaperMcp {
+    transport: McpTransport;
+    calls: FakeCall[];
+    callsTo(tool: string): Array<Record<string, unknown>>;
+    live: FakeStrategyRow[];
+    paper: FakeStrategyRow[];
+}
+export declare function createFakePaperMcp(options?: FakePaperMcpOptions): FakePaperMcp;
+//# sourceMappingURL=fake-paper-mcp.d.ts.map

package/dist/senpi/__tests__/helpers/fake-paper-mcp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fake-paper-mcp.d.ts","sourceRoot":"","sources":["../../../../src/senpi/__tests__/helpers/fake-paper-mcp.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAGnD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,EAAE,MAAM,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,kFAAkF;IAClF,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,8EAA8E;IAC9E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,0FAA0F;IAC1F,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,YAAY,CAAC;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,IAAI,EAAE,eAAe,EAAE,CAAC;IACxB,KAAK,EAAE,eAAe,EAAE,CAAC;CAC1B;AA2BD,wBAAgB,kBAAkB,CAAC,OAAO,GAAE,mBAAwB,GAAG,YAAY,CA0GlF"}