beatrina 0.8.6

package/engines/python/analyze.py

@@ -0,0 +1,814 @@
+"""analyze.py — the Python ANALYSIS child. It reads Python; it never runs Python.
+
+Behind Beatrix's analysis plane (host/planes/analyze.mjs), beside spike/analyze.R
+and for the same reason one language over: the evaluating session (the Python
+engine's ipykernel) answers nothing while a cell is inside a long computation,
+and that must not take syntax checking, navigation and rename down with it.
+
+    browser  ⇅  host/main.mjs  ⇅  ipykernel     — evaluates everything
+                               ⇅  analyze.py    — evaluates NOTHING
+
+Ported from carmapy's analyze.py (../carmar-python/py/carmapy/analyze.py), with
+one deliberate change recorded in docs/wp/wp8a-python-analyzer.md: the
+reference index is built from `ast`, not from `jedi`, and jedi is not imported.
+
+THE ONE RULE: nothing in this file may evaluate user source.
+
+  · `ast.parse()` builds a syntax tree and runs no line of it.
+  · `codeop.compile_command()` COMPILES (bytecode) and never executes; it is
+    here only to answer "would more input finish this?".
+  · There is no `eval`, no `exec`, no `compile(..., "exec")` handed to a
+    runner, no `import` of anything the user wrote, and none may be added.
+    test/python-analyzer.test.mjs greps this file for them.
+  · The process runs `python -I`: no PYTHONPATH, no user site-packages, no
+    script directory on sys.path. It imports the standard library and nothing
+    else, so there is no third-party module whose import could run code.
+
+── WHY AST AND NOT JEDI ─────────────────────────────────────────────────────
+The page's rename (lib/rename.js) rests on analyze.R's contract: two
+occurrences are the SAME symbol when name and `scope` match, where scope 0 is a
+document's top level and any other scope is the innermost FUNCTION THAT BINDS
+the name. carmapy took `scope` from jedi's `parent()`, which is the enclosing
+definition, not the binding — its own docstring says so and says the rule is
+therefore "not claimed for Python". Fed to lib/rename.js that is a silent
+partial rename: a read of a top-level name inside a function would carry the
+function's id and be skipped as "a local of the same name". A binder over the
+syntax tree answers the real question with Python's own rules (parameters,
+assignment, `global`, `nonlocal`, class bodies that do not enclose methods,
+comprehension scopes), costs nothing per name, and needs no third-party code.
+
+The ops this child answers are exactly the four the host forwards
+(`analyze`, `ping`, `workspace`, `workspace_references`) — the same as
+analyze.R. Hover, definition and rename are the PAGE's, computed from the
+reference index and the symbols, as they are for R.
+
+── POSITIONS ────────────────────────────────────────────────────────────────
+Every position emitted is 1-based line, 1-based CODE POINT column,
+`unit: "codepoint"` — analyze.R's contract, so lib/r-intelligence.js converts a
+Python reply with no change. `ast` reports columns as 0-based UTF-8 BYTE
+offsets, which are converted against the line's own bytes (`_cp`) and never
+emitted raw. `SyntaxError.offset` is already 1-based code points (measured in
+carmapy: `x = "👩" z` reports 9). Only "\n" starts a line, as in the page; the
+notebook normalises CRLF before a source is analysed.
+
+Python, like R, reports only the FIRST syntax error, so a reply carries at most
+one parse diagnostic.
+"""
+
+from __future__ import annotations
+
+import ast
+import codeop
+import json
+import os
+import re
+import sys
+import time
+import warnings
+
+
+def _budget() -> float:
+    try:
+        v = float(os.environ.get("CARMAR_ANALYZE_BUDGET", "3"))
+    except ValueError:
+        v = 3.0
+    if v != v or v in (float("inf"), float("-inf")):
+        v = 3.0
+    return max(0.0, min(v, 60.0))
+
+
+ANALYZE_BUDGET = _budget()
+
+MAX_FILES = 300
+MAX_DEPTH = 4
+MAX_BYTES = 512_000
+MAX_MATCHES = 50
+MAX_REPLY_BYTES = 4_194_304
+
+SKIP_DIRS = frozenset({
+    ".git", ".hg", ".svn", "node_modules", ".venv", "venv", "env",
+    "__pycache__", ".mypy_cache", ".pytest_cache", ".ruff_cache", ".tox",
+    ".ipynb_checkpoints", ".quarto", "_site", "dist", "build", ".eggs",
+    "site-packages", ".Rproj.user", "renv", "packrat",
+})
+
+
+# ── framing ────────────────────────────────────────────────────────────────
+
+class Emitter:
+    """One control frame: the session's random sentinel, then compact JSON."""
+
+    def __init__(self, sentinel: str, out=None):
+        self.sentinel = sentinel
+        self.out = out if out is not None else sys.stdout
+
+    def __call__(self, obj: dict) -> None:
+        self.out.write(self.sentinel + json.dumps(obj, separators=(",", ":"),
+                                                  ensure_ascii=False) + "\n")
+        self.out.flush()
+
+
+# ── positions ──────────────────────────────────────────────────────────────
+
+class Lines:
+    """A source's lines, with byte-offset -> code-point conversion per line."""
+
+    def __init__(self, source: str):
+        self.text = source.split("\n")
+        self._bytes: dict = {}
+
+    def line(self, lineno: int) -> str:
+        i = (lineno or 1) - 1
+        return self.text[i] if 0 <= i < len(self.text) else ""
+
+    def cp(self, lineno: int, byte_col: int) -> int:
+        """0-based UTF-8 byte offset on a line -> 0-based code-point offset."""
+        raw = self._bytes.get(lineno)
+        if raw is None:
+            raw = self.line(lineno).encode("utf-8")
+            self._bytes[lineno] = raw
+        return len(raw[:max(0, byte_col)].decode("utf-8", errors="ignore"))
+
+
+# ── diagnostics ────────────────────────────────────────────────────────────
+
+def _syntax_diagnostic(err: SyntaxError, incomplete: bool) -> dict:
+    line = err.lineno
+    col = err.offset                      # 1-based CODE POINTS
+    end_line = getattr(err, "end_lineno", None)
+    end_col = getattr(err, "end_offset", None)
+    # `end_offset` is -1 on an IndentationError and 0 on "'[' was never closed".
+    # Those are not positions: an end that does not lie strictly after the start
+    # is dropped, and the page widens the point to the token under it.
+    if col is None or end_col is None or end_line is None:
+        end_line, end_col = line, None
+    elif end_line < (line or 0) or (end_line == line and end_col <= col):
+        end_line, end_col = line, None
+    return {
+        "severity": "error", "rule": "parse",
+        "message": err.msg or "invalid syntax",
+        # A NUL byte is a SyntaxError with no position (3.12+); the raw text then
+        # says the message alone rather than "(None:None)".
+        "raw": f"{err.msg} ({err.lineno}:{err.offset})" if line is not None else str(err.msg),
+        "unit": "codepoint", "line": line, "col": col,
+        "endLine": end_line, "endCol": end_col, "incomplete": incomplete,
+    }
+
+
+def _is_incomplete(source: str) -> bool:
+    """Would more input make this parse? `codeop` compiles; it never executes."""
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            return codeop.compile_command(source, "<analyze>", "exec") is None
+    except (SyntaxError, ValueError, OverflowError, MemoryError, RecursionError):
+        return False
+
+
+def _parse(source: str):
+    # SyntaxWarnings (an invalid escape, `is` with a literal) are not errors and
+    # must not reach stderr as noise; they are silenced, never turned into
+    # diagnostics the page would underline as broken code.
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return ast.parse(source)
+
+
+# ── symbols (lines only, as analyze.R) ─────────────────────────────────────
+
+def _signature_of(node) -> tuple:
+    a = node.args
+    names: list = []
+    parts: list = []
+
+    def render(arg, default=None, prefix=""):
+        text = prefix + arg.arg
+        if arg.annotation is not None:
+            text += ": " + ast.unparse(arg.annotation)
+        if default is not None:
+            text += ("=" if arg.annotation is None else " = ") + ast.unparse(default)
+        names.append(arg.arg)
+        parts.append(text)
+
+    posonly = list(a.posonlyargs)
+    normal = list(a.args)
+    defaults = list(a.defaults)
+    positional = posonly + normal
+    pad = [None] * (len(positional) - len(defaults)) + defaults
+    for arg, dflt in zip(posonly, pad[:len(posonly)]):
+        render(arg, dflt)
+    if posonly:
+        parts.append("/")
+    for arg, dflt in zip(normal, pad[len(posonly):]):
+        render(arg, dflt)
+    if a.vararg is not None:
+        render(a.vararg, None, "*")
+    elif a.kwonlyargs:
+        parts.append("*")
+    for arg, dflt in zip(a.kwonlyargs, a.kw_defaults):
+        render(arg, dflt)
+    if a.kwarg is not None:
+        render(a.kwarg, None, "**")
+    return node.name + "(" + ", ".join(parts) + ")", names
+
+
+def top_symbols(tree) -> list:
+    """Top-level names a document defines, with the line each sits on."""
+    out = []
+    for node in getattr(tree, "body", []):
+        end = getattr(node, "end_lineno", node.lineno)
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            signature, params = _signature_of(node)
+            out.append({"name": node.name, "line": node.lineno, "endLine": end,
+                        "kind": "function", "params": params, "signature": signature})
+        elif isinstance(node, ast.ClassDef):
+            out.append({"name": node.name, "line": node.lineno, "endLine": end,
+                        "kind": "class", "params": [], "signature": node.name})
+        elif isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name):
+                    out.append({"name": target.id, "line": node.lineno, "endLine": end,
+                                "kind": "variable", "params": [], "signature": None})
+        elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
+            out.append({"name": node.target.id, "line": node.lineno, "endLine": end,
+                        "kind": "variable", "params": [], "signature": None})
+    return out
+
+
+# ── the reference index: a binder over the syntax tree ─────────────────────
+
+_FUNC_KINDS = ("function", "lambda", "comprehension")
+_COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)
+
+
+class Scope:
+    __slots__ = ("kind", "id", "parent", "bound", "globals", "nonlocals")
+
+    def __init__(self, kind, sid, parent):
+        self.kind = kind
+        self.id = sid
+        self.parent = parent
+        self.bound: set = set()
+        self.globals: set = set()
+        self.nonlocals: set = set()
+
+    def binder(self):
+        """Where a walrus binds: the nearest scope that is not a comprehension."""
+        s = self
+        while s is not None and s.kind == "comprehension":
+            s = s.parent
+        return s
+
+
+def _targets(node):
+    """Every plain name an assignment target binds (tuples, lists, stars)."""
+    if isinstance(node, ast.Name):
+        yield node.id
+    elif isinstance(node, (ast.Tuple, ast.List)):
+        for elt in node.elts:
+            yield from _targets(elt)
+    elif isinstance(node, ast.Starred):
+        yield from _targets(node.value)
+
+
+class References:
+    """Every parser-confirmed name occurrence, with its role and its binding scope.
+
+    Roles (analyze.R's list, analyze.R's reasons):
+      def    a binding — the thing to jump TO (assignment, def, class, import,
+             `except ... as`, a `for`/`with` target, a match capture).
+      formal a parameter: a binding, so it shadows an outer name.
+      field  `obj.attr` and `f(name=...)` — a name in ANOTHER namespace, kept in
+             the index so a rename can say what it declined to touch.
+      call   `f(...)` — a use, and a call.
+      use    any other mention, including `global x` / `nonlocal x`.
+
+    SCOPE is analyze.R's: 0 for a document's top level, otherwise the id of the
+    innermost function (or lambda, or comprehension) that BINDS the name. A
+    class body binds only for occurrences directly inside it — methods do not see
+    class names, which is Python's rule. Ids are `line * 10000 + column` of the
+    scope's node, stable within one parse, which is all analyze.R's are.
+    """
+
+    def __init__(self, source: str):
+        self.lines = Lines(source)
+        self.scopes: dict = {}
+        self.out: list = []
+
+    # -- pass 1: what each scope binds -------------------------------------
+
+    def _sid(self, node) -> int:
+        return (node.lineno or 0) * 10000 + self.lines.cp(node.lineno, node.col_offset)
+
+    def collect(self, tree) -> None:
+        module = Scope("module", 0, None)
+        self.scopes[id(tree)] = module
+        self._collect_body(tree.body, module)
+
+    def _collect_body(self, nodes, scope) -> None:
+        for node in nodes:
+            self._collect(node, scope)
+
+    def _collect_args(self, args, scope) -> None:
+        for a in (list(args.posonlyargs) + list(args.args) + list(args.kwonlyargs)
+                  + [x for x in (args.vararg, args.kwarg) if x is not None]):
+            scope.bound.add(a.arg)
+
+    def _collect(self, node, scope) -> None:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            scope.bound.add(node.name)
+            for d in node.decorator_list:
+                self._collect(d, scope)
+            for d in list(node.args.defaults) + [x for x in node.args.kw_defaults if x is not None]:
+                self._collect(d, scope)
+            inner = Scope("function", self._sid(node), scope)
+            self.scopes[id(node)] = inner
+            self._collect_args(node.args, inner)
+            self._collect_body(node.body, inner)
+            return
+        if isinstance(node, ast.ClassDef):
+            scope.bound.add(node.name)
+            for d in list(node.decorator_list) + list(node.bases) + [k.value for k in node.keywords]:
+                self._collect(d, scope)
+            inner = Scope("class", self._sid(node), scope)
+            self.scopes[id(node)] = inner
+            self._collect_body(node.body, inner)
+            return
+        if isinstance(node, ast.Lambda):
+            for d in list(node.args.defaults) + [x for x in node.args.kw_defaults if x is not None]:
+                self._collect(d, scope)
+            inner = Scope("lambda", self._sid(node), scope)
+            self.scopes[id(node)] = inner
+            self._collect_args(node.args, inner)
+            self._collect(node.body, inner)
+            return
+        if isinstance(node, _COMPREHENSIONS):
+            inner = Scope("comprehension", self._sid(node), scope)
+            self.scopes[id(node)] = inner
+            for gen in node.generators:
+                for name in _targets(gen.target):
+                    inner.bound.add(name)
+                self._collect(gen.iter, inner)
+                for cond in gen.ifs:
+                    self._collect(cond, inner)
+            for part in ("elt", "key", "value"):
+                if hasattr(node, part):
+                    self._collect(getattr(node, part), inner)
+            return
+        if isinstance(node, ast.Global):
+            scope.globals.update(node.names)
+            return
+        if isinstance(node, ast.Nonlocal):
+            scope.nonlocals.update(node.names)
+            return
+        if isinstance(node, ast.NamedExpr):
+            binder = scope.binder()
+            for name in _targets(node.target):
+                binder.bound.add(name)
+            self._collect(node.value, scope)
+            return
+        if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
+            scope.bound.add(node.id)
+            return
+        if isinstance(node, (ast.Import, ast.ImportFrom)):
+            for alias in node.names:
+                if alias.name == "*":
+                    continue
+                scope.bound.add(alias.asname or alias.name.split(".")[0])
+            return
+        if isinstance(node, ast.ExceptHandler) and node.name:
+            scope.bound.add(node.name)
+        if isinstance(node, ast.MatchAs) and node.name:
+            scope.bound.add(node.name)
+        if isinstance(node, ast.MatchStar) and node.name:
+            scope.bound.add(node.name)
+        if isinstance(node, ast.MatchMapping) and node.rest:
+            scope.bound.add(node.rest)
+        for child in ast.iter_child_nodes(node):
+            self._collect(child, scope)
+
+    # -- resolution ---------------------------------------------------------
+
+    def resolve(self, name: str, scope):
+        """The id and kind of the scope that binds `name` as seen from `scope`."""
+        # Python's rule, in the order Python applies it. `global x` sends x to the
+        # module; `nonlocal x` says "not bound HERE", so the walk moves outward;
+        # a class body binds only what is written directly in it (a method does
+        # not see its class's names).
+        s = scope
+        while s is not None:
+            if s.kind == "module":
+                return 0, "module"
+            if s.kind == "class":
+                if s is scope and name in s.bound:
+                    return s.id, "class"
+            else:
+                if name in s.globals:
+                    return 0, "module"
+                if name not in s.nonlocals and name in s.bound:
+                    return s.id, s.kind
+            s = s.parent
+        return 0, "module"
+
+    # -- pass 2: occurrences ------------------------------------------------
+
+    def emit(self, name, lineno, cp_col, role, scope) -> None:
+        if lineno is None or cp_col is None:
+            return
+        sid, kind = self.resolve(name, scope)
+        self.out.append({
+            "name": name, "line": lineno, "col": cp_col + 1,
+            "endCol": cp_col + 1 + len(name), "role": role, "unit": "codepoint",
+            "scope": sid, "scopeKind": kind,
+        })
+
+    def emit_field(self, name, lineno, cp_col, scope) -> None:
+        if lineno is None or cp_col is None:
+            return
+        self.out.append({
+            "name": name, "line": lineno, "col": cp_col + 1,
+            "endCol": cp_col + 1 + len(name), "role": "field", "unit": "codepoint",
+            "scope": scope.id if scope.kind != "module" else 0, "scopeKind": scope.kind,
+        })
+
+    def _find(self, lineno, byte_start, pattern, name):
+        """The code-point column of `name` matched by `pattern` on a line."""
+        line = self.lines.line(lineno)
+        start = self.lines.cp(lineno, byte_start)
+        m = re.compile(pattern.format(re.escape(name))).search(line, start)
+        return m.start(1) if m else None
+
+    def walk(self, tree) -> list:
+        self._walk_body(tree.body, self.scopes[id(tree)])
+        self.out.sort(key=lambda r: (r["line"], r["col"]))
+        return self.out
+
+    def _walk_body(self, nodes, scope) -> None:
+        for node in nodes:
+            self._walk(node, scope, None)
+
+    def _walk_args(self, args, scope) -> None:
+        for a in (list(args.posonlyargs) + list(args.args) + list(args.kwonlyargs)
+                  + [x for x in (args.vararg, args.kwarg) if x is not None]):
+            if a.annotation is not None:
+                self._walk(a.annotation, scope.parent, None)
+            self.emit(a.arg, a.lineno, self.lines.cp(a.lineno, a.col_offset), "formal", scope)
+
+    def _walk(self, node, scope, parent) -> None:
+        if node is None:
+            return
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            for d in node.decorator_list:
+                self._walk(d, scope, node)
+            col = self._find(node.lineno, node.col_offset, r"(?:async\s+)?def\s+({})\b", node.name)
+            self.emit(node.name, node.lineno, col, "def", scope)
+            inner = self.scopes[id(node)]
+            for d in list(node.args.defaults) + [x for x in node.args.kw_defaults if x is not None]:
+                self._walk(d, scope, node)
+            self._walk_args(node.args, inner)
+            if node.returns is not None:
+                self._walk(node.returns, scope, node)
+            self._walk_body(node.body, inner)
+            return
+        if isinstance(node, ast.ClassDef):
+            for d in node.decorator_list:
+                self._walk(d, scope, node)
+            col = self._find(node.lineno, node.col_offset, r"class\s+({})\b", node.name)
+            self.emit(node.name, node.lineno, col, "def", scope)
+            for b in node.bases:
+                self._walk(b, scope, node)
+            for k in node.keywords:
+                self._walk(k, scope, node)
+            self._walk_body(node.body, self.scopes[id(node)])
+            return
+        if isinstance(node, ast.Lambda):
+            inner = self.scopes[id(node)]
+            for d in list(node.args.defaults) + [x for x in node.args.kw_defaults if x is not None]:
+                self._walk(d, scope, node)
+            self._walk_args(node.args, inner)
+            self._walk(node.body, inner, node)
+            return
+        if isinstance(node, _COMPREHENSIONS):
+            inner = self.scopes[id(node)]
+            for gen in node.generators:
+                self._walk(gen.target, inner, gen)
+                self._walk(gen.iter, inner, gen)
+                for cond in gen.ifs:
+                    self._walk(cond, inner, gen)
+            for part in ("elt", "key", "value"):
+                if hasattr(node, part):
+                    self._walk(getattr(node, part), inner, node)
+            return
+        if isinstance(node, ast.NamedExpr):
+            binder = scope.binder()
+            if isinstance(node.target, ast.Name):
+                t = node.target
+                self.emit(t.id, t.lineno, self.lines.cp(t.lineno, t.col_offset), "def", binder)
+            self._walk(node.value, scope, node)
+            return
+        if isinstance(node, ast.Name):
+            col = self.lines.cp(node.lineno, node.col_offset)
+            if isinstance(node.ctx, (ast.Store, ast.Del)):
+                role = "def"
+            elif isinstance(parent, ast.Call) and parent.func is node:
+                role = "call"
+            else:
+                role = "use"
+            self.emit(node.id, node.lineno, col, role, scope)
+            return
+        if isinstance(node, ast.Attribute):
+            self._walk(node.value, scope, node)
+            end_line = getattr(node, "end_lineno", None)
+            end_col = getattr(node, "end_col_offset", None)
+            if end_line is not None and end_col is not None:
+                cp_end = self.lines.cp(end_line, end_col)
+                start = cp_end - len(node.attr)
+                if self.lines.line(end_line)[start:cp_end] == node.attr:
+                    self.emit_field(node.attr, end_line, start, scope)
+            return
+        if isinstance(node, ast.keyword):
+            if node.arg is not None and getattr(node, "lineno", None) is not None:
+                col = self.lines.cp(node.lineno, node.col_offset)
+                if self.lines.line(node.lineno)[col:col + len(node.arg)] == node.arg:
+                    self.emit_field(node.arg, node.lineno, col, scope)
+            self._walk(node.value, scope, node)
+            return
+        if isinstance(node, (ast.Import, ast.ImportFrom)):
+            for alias in node.names:
+                if alias.name == "*":
+                    continue
+                bound = alias.asname or alias.name.split(".")[0]
+                lineno = getattr(alias, "lineno", node.lineno)
+                byte = getattr(alias, "col_offset", node.col_offset)
+                if alias.asname:
+                    col = self._find(lineno, byte, r"\bas\s+({})\b", bound)
+                else:
+                    col = self.lines.cp(lineno, byte)
+                    if self.lines.line(lineno)[col:col + len(bound)] != bound:
+                        col = self._find(lineno, byte, r"\b({})\b", bound)
+                self.emit(bound, lineno, col, "def", scope)
+            return
+        if isinstance(node, (ast.Global, ast.Nonlocal)):
+            keyword = "global" if isinstance(node, ast.Global) else "nonlocal"
+            line = self.lines.line(node.lineno)
+            pos = self.lines.cp(node.lineno, node.col_offset) + len(keyword)
+            for name in node.names:
+                m = re.compile(r"\b({})\b".format(re.escape(name))).search(line, pos)
+                if m:
+                    self.emit(name, node.lineno, m.start(1), "use", scope)
+                    pos = m.end(1)
+            return
+        if isinstance(node, ast.ExceptHandler):
+            if node.type is not None:
+                self._walk(node.type, scope, node)
+            if node.name:
+                col = self._find(node.lineno, node.col_offset, r"\bas\s+({})\b", node.name)
+                self.emit(node.name, node.lineno, col, "def", scope)
+            self._walk_body(node.body, scope)
+            return
+        if isinstance(node, ast.MatchAs):
+            if node.pattern is not None:
+                self._walk(node.pattern, scope, node)
+            if node.name:
+                if node.pattern is None:
+                    col = self.lines.cp(node.lineno, node.col_offset)
+                else:
+                    col = self._find(node.lineno, node.col_offset, r"\bas\s+({})\b", node.name)
+                self.emit(node.name, node.lineno, col, "def", scope)
+            return
+        if isinstance(node, ast.MatchStar):
+            if node.name:
+                col = self._find(node.lineno, node.col_offset, r"\*\s*({})\b", node.name)
+                self.emit(node.name, node.lineno, col, "def", scope)
+            return
+        if isinstance(node, ast.MatchMapping):
+            for key in node.keys:
+                self._walk(key, scope, node)
+            for pat in node.patterns:
+                self._walk(pat, scope, node)
+            if node.rest:
+                end_line = getattr(node, "end_lineno", node.lineno)
+                col = self._find(node.lineno, node.col_offset, r"\*\*\s*({})\b", node.rest)
+                self.emit(node.rest, node.lineno if col is not None else end_line, col, "def", scope)
+            return
+        for child in ast.iter_child_nodes(node):
+            self._walk(child, scope, node)
+
+
+def document_references(tree, source: str) -> list:
+    refs = References(source)
+    refs.collect(tree)
+    return refs.walk(tree)
+
+
+# ── the ops ────────────────────────────────────────────────────────────────
+
+def analyze_source(source) -> dict:
+    """Diagnostics always; symbols and the reference index when it parses."""
+    src = source if isinstance(source, str) else ""
+    diagnostics: list = []
+    symbols: list = []
+    references: list = []
+    complete = True
+    failed = None
+    try:
+        tree = _parse(src)
+    except SyntaxError as err:
+        complete = not _is_incomplete(src)
+        diagnostics.append(_syntax_diagnostic(err, incomplete=not complete))
+        tree = None
+    except (ValueError, OverflowError, MemoryError, RecursionError) as err:
+        diagnostics.append({
+            "severity": "error", "rule": "parse", "message": str(err),
+            "raw": f"{type(err).__name__}: {err}", "unit": "codepoint",
+            "line": None, "col": None, "endLine": None, "endCol": None,
+            "incomplete": False,
+        })
+        tree = None
+    if tree is not None:
+        symbols = top_symbols(tree)
+        try:
+            references = document_references(tree, src)
+        except RecursionError:
+            failed = "references unavailable: the document nests too deeply to index"
+    return {"diagnostics": diagnostics, "symbols": symbols,
+            "references": references, "complete": complete, "failed": failed}
+
+
+def _walk_files(root: str, deadline: float, max_files: int, max_depth: int):
+    found: list = []
+    truncated = False
+    stack = [(root, 1)]
+    while stack:
+        directory, depth = stack.pop()
+        if depth > max_depth:
+            continue
+        if len(found) >= max_files or time.monotonic() >= deadline:
+            truncated = True
+            break
+        try:
+            entries = sorted(os.scandir(directory), key=lambda e: e.name)
+        except OSError:
+            continue
+        for entry in entries:
+            if len(found) >= max_files or time.monotonic() >= deadline:
+                truncated = True
+                break
+            try:
+                if entry.is_dir(follow_symlinks=False):
+                    if entry.name in SKIP_DIRS or entry.name.startswith("."):
+                        continue
+                    stack.append((entry.path, depth + 1))
+                elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".py"):
+                    found.append(entry.path)
+            except OSError:
+                continue
+    return found, truncated
+
+
+def _read_text(path: str, max_bytes: int):
+    try:
+        size = os.path.getsize(path)
+    except OSError:
+        return None
+    if size > max_bytes:
+        return None
+    try:
+        with open(path, "rb") as fh:
+            raw = fh.read(size)
+    except OSError:
+        return None
+    if b"\0" in raw:
+        return None
+    try:
+        return raw.decode("utf-8").replace("\r\n", "\n")
+    except UnicodeDecodeError:
+        return None
+
+
+def workspace_symbols(root, max_files=MAX_FILES, max_depth=MAX_DEPTH,
+                      max_bytes=MAX_BYTES, max_seconds=None) -> dict:
+    """Every top-level definition in the `.py` files below a root. Read, never run."""
+    budget = ANALYZE_BUDGET if max_seconds is None else max_seconds
+    if not root or not os.path.isdir(root):
+        return {"files": 0, "symbols": [], "truncated": False, "error": "no such directory"}
+    deadline = time.monotonic() + budget
+    base = os.path.realpath(root)
+    files, truncated = _walk_files(base, deadline, max_files, max_depth)
+    out = []
+    for path in files:
+        if time.monotonic() >= deadline:
+            truncated = True
+            break
+        text = _read_text(path, max_bytes)
+        if text is None:
+            continue
+        try:
+            tree = _parse(text)
+        except (SyntaxError, ValueError, OverflowError, MemoryError, RecursionError):
+            continue
+        rel = os.path.relpath(path, base)
+        for sym in top_symbols(tree):
+            out.append({**sym, "path": path, "file": rel})
+    return {"files": len(files), "symbols": out, "truncated": truncated, "error": None}
+
+
+def workspace_references(root, name, max_files=MAX_FILES, max_depth=MAX_DEPTH,
+                         max_bytes=MAX_BYTES, max_matches=MAX_MATCHES,
+                         max_reply_bytes=MAX_REPLY_BYTES, max_seconds=None) -> dict:
+    """Every parser-confirmed occurrence of one name in bounded `.py` files."""
+    budget = ANALYZE_BUDGET if max_seconds is None else max_seconds
+    if not root or not os.path.isdir(root):
+        return {"files": 0, "documents": [], "truncated": False, "error": "no such directory"}
+    want = str(name or "")
+    if not want:
+        return {"files": 0, "documents": [], "truncated": False, "error": "name is required"}
+    deadline = time.monotonic() + budget
+    base = os.path.realpath(root)
+    files, truncated = _walk_files(base, deadline, max_files, max_depth)
+    documents = []
+    total = 0
+    for path in files:
+        if time.monotonic() >= deadline:
+            truncated = True
+            break
+        text = _read_text(path, max_bytes)
+        if text is None or want not in text:
+            continue
+        try:
+            tree = _parse(text)
+            refs = document_references(tree, text)
+        except (SyntaxError, ValueError, OverflowError, MemoryError, RecursionError):
+            continue
+        hits = [r for r in refs if r["name"] == want]
+        if not hits:
+            continue
+        size = len(text.encode("utf-8"))
+        if len(documents) >= max_matches or total + size > max_reply_bytes:
+            truncated = True
+            break
+        documents.append({"path": path, "file": os.path.relpath(path, base),
+                          "source": text, "references": hits, "symbols": top_symbols(tree)})
+        total += size
+    return {"files": len(files), "documents": documents, "truncated": truncated, "error": None}
+
+
+# ── the command loop ───────────────────────────────────────────────────────
+# NDJSON in, sentinel-framed JSON out, exactly as analyze.R: `analyze` echoes
+# `uri` and `version` so the page can discard a stale answer by strict equality.
+
+def handle(cmd: dict) -> dict:
+    op = cmd.get("type")
+    rid = cmd.get("id")
+    if op == "ping":
+        return {"type": "pong", "id": rid, "python": sys.version.split()[0]}
+    if op == "analyze":
+        return {"type": "analyze", "id": rid, "uri": cmd.get("uri"),
+                "version": cmd.get("version"), **analyze_source(cmd.get("source"))}
+    if op == "workspace":
+        root = cmd.get("root")
+        res = workspace_symbols(str(root) if root else "")
+        return {"type": "workspace", "id": rid, "root": root, **res}
+    if op == "workspace_references":
+        root = cmd.get("root")
+        res = workspace_references(str(root) if root else "", cmd.get("name"))
+        return {"type": "workspace_references", "id": rid, "root": root,
+                "name": cmd.get("name"), **res}
+    return {"type": "analyze-error", "id": rid, "message": f"unknown command: {op}"}
+
+
+def main(argv=None, stdin=None, stdout=None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    if not argv or not argv[0]:
+        print("analyze.py needs a frame sentinel as its first argument", file=sys.stderr)
+        return 2
+    emit = Emitter(argv[0], stdout)
+    stdin = stdin if stdin is not None else sys.stdin
+    emit({"type": "analyze-ready", "pid": os.getpid(),
+          "python": sys.version.split()[0], "unit": "codepoint", "budget": ANALYZE_BUDGET})
+    for line in stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            cmd = json.loads(line)
+        except ValueError:
+            continue
+        if not isinstance(cmd, dict) or not isinstance(cmd.get("type"), str):
+            continue
+        try:
+            emit(handle(cmd))
+        except RecursionError:
+            emit({"type": cmd["type"], "id": cmd.get("id"), "uri": cmd.get("uri"),
+                  "version": cmd.get("version"),
+                  "failed": "the analyzer ran out of stack on this document"})
+        except Exception as err:                      # noqa: BLE001
+            # Reported, never swallowed: `failed` is how the editor says what it
+            # could not do, and the process lives on for the next request.
+            emit({"type": cmd["type"], "id": cmd.get("id"), "uri": cmd.get("uri"),
+                  "version": cmd.get("version"), "failed": f"{type(err).__name__}: {err}"})
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())