docbridge 0.11.0 → 0.12.0

package/dist/workers/python/docbridge_python_scanner/__init__.py

@@ -0,0 +1,6 @@
+"""DocBridge Python scanner worker.
+
+Implements schemaVersion 1 of the DocBridge scanner worker protocol on top of
+the standard library's ``ast`` and ``tokenize`` modules. The package has no
+third-party dependency and runs on CPython 3.10 or newer.
+"""

package/dist/workers/python/docbridge_python_scanner/comments.py

@@ -0,0 +1,177 @@
+"""Annotation sources: leading ``#`` comment blocks and docstrings.
+
+Both sources are read from the original source text through ``tokenize``
+positions and ``ast`` node offsets, never from ``ast.get_docstring()``, so
+every ``@doc`` target keeps the position it has in the file.
+"""
+
+from __future__ import annotations
+
+import ast
+import bisect
+import io
+import re
+import tokenize
+from dataclasses import dataclass
+
+from .links import DocTarget, find_doc_matches
+from .positions import LineTable
+
+STRING_PREFIX_PATTERN = re.compile(r"[a-zA-Z]*")
+QUOTES = ('"""', "'''", '"', "'")
+
+
+@dataclass(frozen=True)
+class CommentLine:
+    """A comment that is the only content of its line."""
+
+    row: int
+    col: int
+    text: str
+
+
+class TokenIndex:
+    """Tokens of one successfully parsed file, addressable by position."""
+
+    def __init__(self, tokens: list[tokenize.TokenInfo]) -> None:
+        self.tokens = tokens
+        self._starts = [token.start for token in tokens]
+        self.own_line_comments: dict[int, CommentLine] = {}
+        for index, token in enumerate(tokens):
+            if token.type != tokenize.COMMENT:
+                continue
+            follower = tokens[index + 1] if index + 1 < len(tokens) else None
+            if follower is not None and follower.type == tokenize.NL:
+                row, col = token.start
+                self.own_line_comments[row] = CommentLine(row, col, token.string[1:])
+
+    @classmethod
+    def from_text(cls, parse_text: str) -> TokenIndex:
+        return cls(list(tokenize.generate_tokens(io.StringIO(parse_text).readline)))
+
+    def index_at(self, row: int, col: int) -> int:
+        """Return the index of the first token starting at or after ``(row, col)``."""
+        return bisect.bisect_left(self._starts, (row, col))
+
+    def token_at(self, row: int, col: int) -> tokenize.TokenInfo | None:
+        index = self.index_at(row, col)
+        if index < len(self.tokens) and self.tokens[index].start == (row, col):
+            return self.tokens[index]
+        return None
+
+    def leading_comment_block(self, row_above: int, col: int) -> list[CommentLine]:
+        """Return the contiguous comment-only lines ending at ``row_above``.
+
+        Every line of the block starts its comment at ``col``; a blank line, a
+        code line, or a comment at another column ends the block.
+        """
+        block: list[CommentLine] = []
+        row = row_above
+        while row >= 1:
+            comment = self.own_line_comments.get(row)
+            if comment is None or comment.col != col:
+                break
+            block.append(comment)
+            row -= 1
+        block.reverse()
+        return block
+
+
+def comment_block_targets(block: list[CommentLine], table: LineTable) -> list[DocTarget]:
+    """Find ``@doc`` targets in each comment line's text after ``#``."""
+    targets: list[DocTarget] = []
+    for comment in block:
+        body_col = comment.col + 1
+        for match in find_doc_matches(comment.text):
+            targets.append(
+                DocTarget(
+                    target=match.target,
+                    start=table.position_from_chars(comment.row, body_col + match.start),
+                    end=table.position_from_chars(comment.row, body_col + match.end),
+                )
+            )
+    return targets
+
+
+def docstring_node(body: list[ast.stmt]) -> ast.Expr | None:
+    """Return the docstring statement of a body, when its first statement is one."""
+    if not body:
+        return None
+    first = body[0]
+    if (
+        isinstance(first, ast.Expr)
+        and isinstance(first.value, ast.Constant)
+        and isinstance(first.value.value, str)
+    ):
+        return first
+    return None
+
+
+def _source_text(table: LineTable, start: tuple[int, int], end: tuple[int, int]) -> str:
+    """Return the parse text between two (row, code point col) positions."""
+    start_row, start_col = start
+    end_row, end_col = end
+    if start_row == end_row:
+        return table.line_text(start_row)[start_col:end_col]
+    pieces = [table.line_text(start_row)[start_col:]]
+    pieces.extend(table.line_text(row) for row in range(start_row + 1, end_row))
+    pieces.append(table.line_text(end_row)[:end_col])
+    return "\n".join(pieces)
+
+
+def _string_body_span(text: str) -> tuple[int, int]:
+    """Return the offsets of a string literal's content, excluding its delimiters."""
+    prefix_match = STRING_PREFIX_PATTERN.match(text)
+    prefix_length = prefix_match.end() if prefix_match else 0
+    rest = text[prefix_length:]
+    for quote in QUOTES:
+        if rest.startswith(quote) and rest.endswith(quote) and len(rest) >= 2 * len(quote):
+            return prefix_length + len(quote), len(text) - len(quote)
+    return prefix_length, len(text)
+
+
+def docstring_targets(node: ast.Expr, table: LineTable, tokens: TokenIndex) -> list[DocTarget]:
+    """Find ``@doc`` targets in each literal of a docstring, delimiters excluded.
+
+    A docstring may be parenthesized or implicitly concatenated, so its span
+    holds several ``STRING`` tokens between brackets and comments. Each
+    literal's body, without its prefix and quotes, is searched on its own, so
+    a target never absorbs a closing quote or bracket or continues into the
+    next literal. A ``str`` constant never contains an f-string, so every
+    literal is a plain ``STRING`` token.
+    """
+    start_row = node.lineno
+    start_col = table.chars_from_bytes(start_row, node.col_offset)
+    end_row = node.end_lineno if node.end_lineno is not None else start_row
+    end_col = table.chars_from_bytes(
+        end_row, node.end_col_offset if node.end_col_offset is not None else 0
+    )
+    targets: list[DocTarget] = []
+    for token in tokens.tokens[tokens.index_at(start_row, start_col) :]:
+        if token.start >= (end_row, end_col):
+            break
+        if token.type == tokenize.STRING:
+            targets.extend(_literal_targets(token, table))
+    return targets
+
+
+def _literal_targets(token: tokenize.TokenInfo, table: LineTable) -> list[DocTarget]:
+    """Find ``@doc`` targets in one string literal's body, at their source positions."""
+    start_row, start_col = token.start
+    text = _source_text(table, token.start, token.end)
+    body_start, body_end = _string_body_span(text)
+
+    targets: list[DocTarget] = []
+    for match in find_doc_matches(text[body_start:body_end]):
+        offset = body_start + match.start
+        line_break = text.rfind("\n", 0, offset)
+        row = start_row + text.count("\n", 0, offset)
+        col = offset - (line_break + 1) if line_break >= 0 else start_col + offset
+        targets.append(
+            DocTarget(
+                target=match.target,
+                start=table.position_from_chars(row, col),
+                end=table.position_from_chars(row, col + len(match.target)),
+            )
+        )
+    return targets

package/dist/workers/python/docbridge_python_scanner/declarations.py

@@ -0,0 +1,318 @@
+"""Declaration walk: supported definitions, grouping, visibility, and ranges.
+
+The walk visits ``def``, ``async def``, and ``class`` at module level and in
+class bodies, recursively for classes, descending into ``if``, ``try``,
+``with``, ``for``, ``while``, and ``match`` blocks at those levels and never
+into function bodies.
+
+Each endpoint collects its declarations in source order. A declaration is a
+single definition or a group, and a group is either a property chain (a
+function decorated ``property``, ``cached_property``, or ``@<name>.getter``,
+``.setter``, or ``.deleter`` with its own name, followed by functions of that
+name decorated with such an accessor) or a run of consecutive ``overload``
+stubs plus at most one implementation, the first following function of that
+name not decorated ``overload``. A same-name definition that a group does not
+accept closes it and starts the next declaration of the endpoint, which
+repeats it.
+"""
+
+from __future__ import annotations
+
+import ast
+import enum
+import tokenize
+from dataclasses import dataclass
+
+from .comments import (
+    TokenIndex,
+    comment_block_targets,
+    docstring_node,
+    docstring_targets,
+)
+from .links import DocTarget
+from .positions import LineTable
+
+PROPERTY_DECORATORS = frozenset({"property", "cached_property"})
+ACCESSOR_ATTRIBUTES = frozenset({"getter", "setter", "deleter"})
+OVERLOAD_DECORATORS = frozenset({"overload"})
+OPENING_BRACKETS = frozenset({"(", "[", "{"})
+CLOSING_BRACKETS = frozenset({")", "]", "}"})
+
+DefinitionNode = ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef
+
+
+@dataclass
+class Member:
+    """One ``def`` / ``async def`` / ``class`` node contributing to an endpoint."""
+
+    name_range: dict[str, dict[str, int]]
+    declaration_range: dict[str, dict[str, int]]
+    signature_range: dict[str, dict[str, int]]
+    targets: list[DocTarget]
+
+
+class GroupState(enum.Enum):
+    """Whether a declaration still accepts later same-name definitions."""
+
+    CLOSED = enum.auto()
+    PROPERTY_CHAIN = enum.auto()
+    OVERLOAD_STUBS = enum.auto()
+
+
+@dataclass
+class Declaration:
+    """One declaration of an endpoint: a single definition or a group."""
+
+    members: list[Member]
+    state: GroupState
+
+    @classmethod
+    def start(cls, member: Member, node: DefinitionNode) -> Declaration:
+        """Start a declaration; a property function or overload stub opens a group."""
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            return cls([member], GroupState.CLOSED)
+        if has_named_decorator(node, OVERLOAD_DECORATORS):
+            return cls([member], GroupState.OVERLOAD_STUBS)
+        if has_named_decorator(node, PROPERTY_DECORATORS) or is_property_accessor(node):
+            return cls([member], GroupState.PROPERTY_CHAIN)
+        return cls([member], GroupState.CLOSED)
+
+    @property
+    def targets(self) -> list[DocTarget]:
+        return [target for member in self.members for target in member.targets]
+
+    @property
+    def first_annotated_member(self) -> Member:
+        return next(member for member in self.members if member.targets)
+
+    def accepts(self, node: DefinitionNode) -> bool:
+        """Whether ``node``, a later definition of the same name, joins this group."""
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            return False
+        if self.state is GroupState.PROPERTY_CHAIN:
+            return is_property_accessor(node)
+        return self.state is GroupState.OVERLOAD_STUBS
+
+    def join(self, member: Member, node: DefinitionNode) -> None:
+        """Add an accepted definition; an implementation closes an overload group."""
+        self.members.append(member)
+        if self.state is GroupState.OVERLOAD_STUBS and not has_named_decorator(
+            node, OVERLOAD_DECORATORS
+        ):
+            self.state = GroupState.CLOSED
+
+
+@dataclass
+class DeclarationEntry:
+    """A supported endpoint: its first member owns the ranges."""
+
+    symbol_name: str
+    canonical_id: str
+    private: bool
+    declarations: list[Declaration]
+
+    @property
+    def members(self) -> list[Member]:
+        return [member for declaration in self.declarations for member in declaration.members]
+
+    def add(self, member: Member, node: DefinitionNode) -> None:
+        """Join the open group that accepts ``node``, else start a new declaration."""
+        last = self.declarations[-1] if self.declarations else None
+        if last is not None and last.accepts(node):
+            last.join(member, node)
+        else:
+            self.declarations.append(Declaration.start(member, node))
+
+
+@dataclass
+class UnsupportedEntry:
+    """A statement that is not a supported declaration but carries ``@doc``."""
+
+    range: dict[str, dict[str, int]]
+    targets: list[DocTarget]
+
+
+Entry = DeclarationEntry | UnsupportedEntry
+
+
+def is_dunder(name: str) -> bool:
+    return len(name) > 4 and name.startswith("__") and name.endswith("__")
+
+
+def is_private_name(name: str) -> bool:
+    return name.startswith("_") and not is_dunder(name)
+
+
+def decorator_name(decorator: ast.expr) -> str | None:
+    """The last segment of a ``Name`` or ``Attribute`` decorator; ``Call`` has none."""
+    if isinstance(decorator, ast.Name):
+        return decorator.id
+    if isinstance(decorator, ast.Attribute):
+        return decorator.attr
+    return None
+
+
+def has_named_decorator(node: DefinitionNode, names: frozenset[str]) -> bool:
+    return any(decorator_name(decorator) in names for decorator in node.decorator_list)
+
+
+def is_property_accessor(node: DefinitionNode) -> bool:
+    """Decorated ``@<name>.getter``, ``.setter``, or ``.deleter`` with its own name."""
+    return any(
+        isinstance(decorator, ast.Attribute)
+        and decorator.attr in ACCESSOR_ATTRIBUTES
+        and isinstance(decorator.value, ast.Name)
+        and decorator.value.id == node.name
+        for decorator in node.decorator_list
+    )
+
+
+class DeclarationCollector:
+    """Collects entries in source order for one parsed file.
+
+    Endpoints are kept per file, keyed by canonical ID, so a class declared
+    twice (an ``if``/``else`` pair, say) contributes the members of both
+    bodies to the same member endpoints.
+    """
+
+    def __init__(self, table: LineTable, tokens: TokenIndex) -> None:
+        self.table = table
+        self.tokens = tokens
+        self.entries: list[Entry] = []
+        self.endpoints: dict[str, DeclarationEntry] = {}
+
+    def collect(self, module: ast.Module) -> list[Entry]:
+        self._module_docstring(module)
+        self._walk(module.body, prefix="", private=False)
+        return self.entries
+
+    def _module_docstring(self, module: ast.Module) -> None:
+        docstring = docstring_node(module.body)
+        if docstring is None:
+            return
+        targets = docstring_targets(docstring, self.table, self.tokens)
+        if targets:
+            self.entries.append(UnsupportedEntry(self._node_range(docstring), targets))
+
+    def _walk(self, body: list[ast.stmt], prefix: str, private: bool) -> None:
+        for statement in body:
+            if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                self._definition(statement, prefix, private)
+                continue
+            self._unsupported_statement(statement)
+            for block in self._nested_blocks(statement):
+                self._walk(block, prefix, private)
+
+    @staticmethod
+    def _nested_blocks(statement: ast.stmt) -> list[list[ast.stmt]]:
+        if isinstance(statement, (ast.If, ast.For, ast.AsyncFor, ast.While)):
+            return [statement.body, statement.orelse]
+        if isinstance(statement, (ast.With, ast.AsyncWith)):
+            return [statement.body]
+        if isinstance(statement, ast.Try) or (
+            hasattr(ast, "TryStar") and isinstance(statement, ast.TryStar)
+        ):
+            handlers = [handler.body for handler in statement.handlers]
+            return [statement.body, *handlers, statement.orelse, statement.finalbody]
+        if isinstance(statement, ast.Match):
+            return [case.body for case in statement.cases]
+        return []
+
+    def _unsupported_statement(self, statement: ast.stmt) -> None:
+        """Report a leading ``@doc`` comment on a statement that is no declaration."""
+        column = self.table.chars_from_bytes(statement.lineno, statement.col_offset)
+        block = self.tokens.leading_comment_block(statement.lineno - 1, column)
+        targets = comment_block_targets(block, self.table)
+        if not targets:
+            return
+        token = self.tokens.token_at(statement.lineno, column)
+        end = token.end if token is not None else (statement.lineno, column)
+        self.entries.append(
+            UnsupportedEntry(
+                {
+                    "start": self.table.position_from_chars(statement.lineno, column),
+                    "end": self.table.position_from_chars(*end),
+                },
+                targets,
+            )
+        )
+
+    def _definition(self, node: DefinitionNode, prefix: str, private: bool) -> None:
+        canonical_id = prefix + node.name
+        entry = self.endpoints.get(canonical_id)
+        if entry is None:
+            entry = DeclarationEntry(
+                symbol_name=node.name,
+                canonical_id=canonical_id,
+                private=private or is_private_name(node.name),
+                declarations=[],
+            )
+            self.endpoints[canonical_id] = entry
+            self.entries.append(entry)
+        entry.add(self._member(node), node)
+
+        if isinstance(node, ast.ClassDef):
+            self._walk(
+                node.body,
+                prefix=canonical_id + ".",
+                private=private or is_private_name(node.name),
+            )
+
+    def _member(self, node: DefinitionNode) -> Member:
+        keyword_col = self.table.chars_from_bytes(node.lineno, node.col_offset)
+        first_line = node.decorator_list[0].lineno if node.decorator_list else node.lineno
+        block = self.tokens.leading_comment_block(first_line - 1, keyword_col)
+        targets = comment_block_targets(block, self.table)
+        docstring = docstring_node(node.body)
+        if docstring is not None:
+            targets.extend(docstring_targets(docstring, self.table, self.tokens))
+
+        start_row = block[0].row if block else first_line
+        start = self.table.position_from_chars(start_row, keyword_col)
+        name_token, signature_end = self._header_tokens(node, keyword_col)
+        end_row = node.end_lineno if node.end_lineno is not None else node.lineno
+        end_col = node.end_col_offset if node.end_col_offset is not None else 0
+
+        return Member(
+            name_range={
+                "start": self.table.position_from_chars(*name_token.start),
+                "end": self.table.position_from_chars(*name_token.end),
+            },
+            declaration_range={
+                "start": start,
+                "end": self.table.position_from_bytes(end_row, end_col),
+            },
+            signature_range={"start": start, "end": self.table.position_from_chars(*signature_end)},
+            targets=targets,
+        )
+
+    def _header_tokens(
+        self, node: DefinitionNode, keyword_col: int
+    ) -> tuple[tokenize.TokenInfo, tuple[int, int]]:
+        """Return the name token and the end of the ``:`` that closes the header."""
+        keyword = "class" if isinstance(node, ast.ClassDef) else "def"
+        tokens = self.tokens.tokens
+        index = self.tokens.index_at(node.lineno, keyword_col)
+        while tokens[index].type != tokenize.NAME or tokens[index].string != keyword:
+            index += 1
+        name_token = tokens[index + 1]
+
+        depth = 0
+        for token in tokens[index + 2 :]:
+            if token.type != tokenize.OP:
+                continue
+            if token.string in OPENING_BRACKETS:
+                depth += 1
+            elif token.string in CLOSING_BRACKETS:
+                depth -= 1
+            elif token.string == ":" and depth == 0:
+                return name_token, token.end
+        return name_token, name_token.end
+
+    def _node_range(self, node: ast.stmt) -> dict[str, dict[str, int]]:
+        end_row = node.end_lineno if node.end_lineno is not None else node.lineno
+        end_col = node.end_col_offset if node.end_col_offset is not None else 0
+        return {
+            "start": self.table.position_from_bytes(node.lineno, node.col_offset),
+            "end": self.table.position_from_bytes(end_row, end_col),
+        }

package/dist/workers/python/docbridge_python_scanner/links.py

@@ -0,0 +1,62 @@
+"""``@doc`` target extraction and link-target validation."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# Whitespace means the ASCII set: space, tab, LF, CR, FF, and VT. A Unicode
+# space such as U+00A0 neither separates a target nor ends one.
+DOC_TARGET_PATTERN = re.compile(r"@doc\s+(\S+)", re.ASCII)
+WHITESPACE_PATTERN = re.compile(r"\s", re.ASCII)
+
+INVALID_LINK_TARGET_MESSAGE = (
+    "Link target must be a project-root-relative file path and fragment in file#fragment form."
+)
+
+
+@dataclass(frozen=True)
+class DocTarget:
+    """One ``@doc`` target with the 1-based UTF-16 range of its text."""
+
+    target: str
+    start: dict[str, int]
+    end: dict[str, int]
+
+
+@dataclass(frozen=True)
+class DocMatch:
+    """A target and its code point offsets, relative to the searched text."""
+
+    target: str
+    start: int
+    end: int
+
+
+def find_doc_matches(text: str) -> list[DocMatch]:
+    """Match ``@doc\\s+(\\S+)`` over ``text`` and return each target's offsets."""
+    return [
+        DocMatch(target=match.group(1), start=match.start(1), end=match.end(1))
+        for match in DOC_TARGET_PATTERN.finditer(text)
+    ]
+
+
+def _has_whitespace(value: str) -> bool:
+    return WHITESPACE_PATTERN.search(value) is not None
+
+
+def is_valid_link_target(target: str, source_file_path: str) -> bool:
+    """Mirror the core's ``parseLinkTarget`` rules for a ``file#fragment`` target."""
+    parts = target.split("#")
+    if len(parts) != 2:
+        return False
+    file_path, fragment = parts
+    if not file_path or not fragment:
+        return False
+    if file_path.startswith(("/", "./", "../")):
+        return False
+    if "\\" in file_path or _has_whitespace(file_path) or _has_whitespace(fragment):
+        return False
+    if ".." in file_path.split("/"):
+        return False
+    return file_path != source_file_path

package/dist/workers/python/docbridge_python_scanner/positions.py

@@ -0,0 +1,75 @@
+"""Position conversion: three column systems into 1-based UTF-16 columns.
+
+CPython reports columns in three ways that must be kept apart:
+
+- ``ast`` ``col_offset`` / ``end_col_offset`` are 0-based UTF-8 byte offsets
+  into the line;
+- ``tokenize`` columns are 0-based Unicode code point indexes;
+- ``SyntaxError.offset`` is 1-based in code points.
+
+DocBridge positions are 1-based lines and 1-based UTF-16 code unit columns,
+with end-exclusive ranges. Lines split on ``\\n`` only, so a CRLF file keeps
+its ``\\r`` inside the line, after every token. A UTF-8 BOM is removed before
+parsing, because CPython rejects it inside a ``str``, and is counted back in as
+the first character of line 1.
+"""
+
+from __future__ import annotations
+
+BOM = ""
+
+
+def utf16_length(text: str) -> int:
+    """Return the number of UTF-16 code units needed to encode ``text``."""
+    return len(text) + sum(1 for character in text if ord(character) >= 0x10000)
+
+
+class LineTable:
+    """Line-based column conversion for one file's content."""
+
+    def __init__(self, content: str) -> None:
+        self.has_bom = content.startswith(BOM)
+        self.parse_text = content[len(BOM) :] if self.has_bom else content
+        self.lines = self.parse_text.split("\n")
+
+    def line_text(self, line: int) -> str:
+        """Return the parse text of a 1-based line, or ``""`` outside the file."""
+        if 1 <= line <= len(self.lines):
+            return self.lines[line - 1]
+        return ""
+
+    def column_from_chars(self, line: int, chars: int) -> int:
+        """Convert a 0-based code point index (``tokenize``) into a column."""
+        text = self.line_text(line)
+        chars = max(0, min(chars, len(text)))
+        bom_units = 1 if self.has_bom and line == 1 else 0
+        return utf16_length(text[:chars]) + 1 + bom_units
+
+    def column_from_bytes(self, line: int, byte_offset: int) -> int:
+        """Convert a 0-based UTF-8 byte offset (``ast``) into a column."""
+        encoded = self.line_text(line).encode("utf-8")
+        prefix = encoded[: max(0, byte_offset)].decode("utf-8", errors="ignore")
+        return self.column_from_chars(line, len(prefix))
+
+    def column_from_offset(self, line: int, offset: int) -> int:
+        """Convert a 1-based code point offset (``SyntaxError``) into a column."""
+        return self.column_from_chars(line, offset - 1)
+
+    def chars_from_bytes(self, line: int, byte_offset: int) -> int:
+        """Convert a 0-based UTF-8 byte offset into a 0-based code point index."""
+        encoded = self.line_text(line).encode("utf-8")
+        return len(encoded[: max(0, byte_offset)].decode("utf-8", errors="ignore"))
+
+    def position_from_chars(self, line: int, chars: int) -> dict[str, int]:
+        return {"line": line, "column": self.column_from_chars(line, chars)}
+
+    def position_from_bytes(self, line: int, byte_offset: int) -> dict[str, int]:
+        return {"line": line, "column": self.column_from_bytes(line, byte_offset)}
+
+
+def make_range(start: dict[str, int], end: dict[str, int]) -> dict[str, dict[str, int]]:
+    return {"start": start, "end": end}
+
+
+def make_location(file_path: str, position: dict[str, int]) -> dict[str, object]:
+    return {"filePath": file_path, "line": position["line"], "column": position["column"]}