docbridge 0.11.0 → 0.12.0

package/dist/workers/python/docbridge_python_scanner/protocol.py

@@ -0,0 +1,105 @@
+"""Worker protocol: one JSON request on stdin, one JSON response on stdout.
+
+``--probe`` prints one JSON line describing the runtime and exits 0 whether or
+not the runtime is usable, so this module stays importable on older Pythons:
+it uses no 3.10-only syntax and imports the scanner only for a scan.
+"""
+
+from __future__ import annotations
+
+import json
+import platform
+import sys
+
+SCHEMA_VERSION = 1
+LANGUAGE = "python"
+RUNTIME_FLOOR = (3, 10)
+
+
+class ProtocolError(Exception):
+    """The request cannot be scanned; the worker exits non-zero."""
+
+
+def probe() -> dict[str, object]:
+    """Describe the interpreter: CPython identity and the 3.10 floor."""
+    implementation = platform.python_implementation()
+    version = platform.python_version()
+    if implementation != "CPython":
+        return {"ok": False, "reason": f"expected CPython, found {implementation} {version}"}
+    if sys.version_info < RUNTIME_FLOOR:
+        floor = ".".join(str(part) for part in RUNTIME_FLOOR)
+        return {"ok": False, "reason": f"CPython {version} is below the {floor} floor"}
+    return {"ok": True, "runtime": "cpython", "version": version}
+
+
+def parse_request(raw: bytes) -> dict[str, object]:
+    """Decode and validate the request shape the scanner relies on."""
+    try:
+        request = json.loads(raw.decode("utf-8"))
+    except (UnicodeDecodeError, ValueError) as error:
+        raise ProtocolError(f"request is not valid JSON: {error}") from error
+    if not isinstance(request, dict):
+        raise ProtocolError("request must be a JSON object")
+    if request.get("schemaVersion") != SCHEMA_VERSION:
+        raise ProtocolError(
+            f"unsupported schemaVersion {request.get('schemaVersion')!r}; expected {SCHEMA_VERSION}"
+        )
+    files = request.get("files")
+    if not isinstance(files, list):
+        raise ProtocolError("request.files must be an array")
+    for index, file in enumerate(files):
+        if (
+            not isinstance(file, dict)
+            or not isinstance(file.get("filePath"), str)
+            or not isinstance(file.get("content"), str)
+        ):
+            raise ProtocolError(f"request.files[{index}] must hold filePath and content strings")
+    options = request.get("options")
+    if options is not None and not isinstance(options, dict):
+        raise ProtocolError("request.options must be an object")
+    visibility = (options or {}).get("visibility")
+    if visibility is not None and (
+        not isinstance(visibility, list) or not all(isinstance(v, str) for v in visibility)
+    ):
+        raise ProtocolError("request.options.visibility must be an array of strings")
+    return request
+
+
+def scan_request(request: dict[str, object]) -> dict[str, object]:
+    """Scan every requested file in request order."""
+    from .scanner import scan_file
+
+    options = request.get("options") or {}
+    visibility = options.get("visibility") if isinstance(options, dict) else None
+    files = request.get("files")
+    scanned = [
+        scan_file(file["filePath"], file["content"], visibility)
+        for file in (files if isinstance(files, list) else [])
+    ]
+    return {
+        "schemaVersion": SCHEMA_VERSION,
+        "requestId": request.get("requestId"),
+        "language": LANGUAGE,
+        "files": scanned,
+    }
+
+
+def _write_json_line(payload: dict[str, object]) -> None:
+    sys.stdout.buffer.write(json.dumps(payload, ensure_ascii=False).encode("utf-8") + b"\n")
+    sys.stdout.buffer.flush()
+
+
+def main(argv: list[str]) -> int:
+    if argv == ["--probe"]:
+        _write_json_line(probe())
+        return 0
+    if argv:
+        sys.stderr.write("docbridge-python-scanner: usage: [--probe] < request.json\n")
+        return 2
+    try:
+        request = parse_request(sys.stdin.buffer.read())
+    except ProtocolError as error:
+        sys.stderr.write(f"docbridge-python-scanner: {error}\n")
+        return 1
+    _write_json_line(scan_request(request))
+    return 0

package/dist/workers/python/docbridge_python_scanner/scanner.py

@@ -0,0 +1,214 @@
+"""Per-file scanning: parse, collect declarations, and build the response file."""
+
+from __future__ import annotations
+
+import ast
+import tokenize
+
+from .comments import TokenIndex
+from .declarations import DeclarationCollector, DeclarationEntry, Entry, Member, UnsupportedEntry
+from .links import INVALID_LINK_TARGET_MESSAGE, DocTarget, is_valid_link_target
+from .positions import LineTable, make_location
+
+LANGUAGE = "python"
+DEFAULT_VISIBILITY = ("public",)
+
+ResponseFile = dict[str, object]
+
+
+def _empty_response(file_path: str) -> ResponseFile:
+    return {
+        "filePath": file_path,
+        "symbols": [],
+        "undocumentedSymbols": [],
+        "links": [],
+        "diagnostics": [],
+    }
+
+
+def _sentence(message: str) -> str:
+    return message if message.endswith(".") else message + "."
+
+
+def _parse_error(file_path: str, message: str, position: dict[str, int]) -> ResponseFile:
+    response = _empty_response(file_path)
+    response["diagnostics"] = [
+        {
+            "severity": "error",
+            "code": "code_parse_error",
+            "target": file_path,
+            "language": LANGUAGE,
+            "message": "Python parse error: " + _sentence(message),
+            "location": make_location(file_path, position),
+        }
+    ]
+    return response
+
+
+def _syntax_error_position(error: SyntaxError, table: LineTable) -> dict[str, int]:
+    if error.lineno is None or error.offset is None:
+        return {"line": 1, "column": 1}
+    return {"line": error.lineno, "column": table.column_from_offset(error.lineno, error.offset)}
+
+
+def _tokenize_error_position(error: tokenize.TokenError, table: LineTable) -> dict[str, int]:
+    if len(error.args) >= 2 and isinstance(error.args[1], tuple) and len(error.args[1]) == 2:
+        row, col = error.args[1]
+        return table.position_from_chars(row, col)
+    return {"line": 1, "column": 1}
+
+
+def scan_file(file_path: str, content: str, visibility: list[str] | None) -> ResponseFile:
+    """Scan one file. ``None`` visibility means the default ``["public"]``."""
+    table = LineTable(content)
+    try:
+        module = ast.parse(table.parse_text, file_path, type_comments=False)
+    except SyntaxError as error:
+        return _parse_error(
+            file_path, error.msg or str(error), _syntax_error_position(error, table)
+        )
+    except (ValueError, RecursionError) as error:
+        return _parse_error(file_path, str(error), {"line": 1, "column": 1})
+    try:
+        tokens = TokenIndex.from_text(table.parse_text)
+    except tokenize.TokenError as error:
+        message = error.args[0] if error.args else str(error)
+        return _parse_error(file_path, str(message), _tokenize_error_position(error, table))
+    except SyntaxError as error:
+        return _parse_error(
+            file_path, error.msg or str(error), _syntax_error_position(error, table)
+        )
+
+    entries = DeclarationCollector(table, tokens).collect(module)
+    visible = set(DEFAULT_VISIBILITY if visibility is None else visibility)
+    return ResponseBuilder(file_path, visible).build(entries)
+
+
+class ResponseBuilder:
+    """Turns collected entries into the worker's per-file response."""
+
+    def __init__(self, file_path: str, visible: set[str]) -> None:
+        self.file_path = file_path
+        self.visible = visible
+        self.response = _empty_response(file_path)
+        self.symbols: list[dict[str, object]] = []
+        self.undocumented: list[dict[str, object]] = []
+        self.links: list[dict[str, object]] = []
+        self.diagnostics: list[dict[str, object]] = []
+
+    def build(self, entries: list[Entry]) -> ResponseFile:
+        for entry in entries:
+            if isinstance(entry, UnsupportedEntry):
+                self._unsupported(entry.range)
+            else:
+                self._declaration(entry)
+        self.response["symbols"] = self.symbols
+        self.response["undocumentedSymbols"] = self.undocumented
+        self.response["links"] = self.links
+        self.response["diagnostics"] = self.diagnostics
+        return self.response
+
+    def _unsupported(self, name_range: dict[str, dict[str, int]]) -> None:
+        self.diagnostics.append(
+            {
+                "severity": "warning",
+                "code": "unsupported_declaration",
+                "target": self.file_path,
+                "language": LANGUAGE,
+                "message": "Python declaration annotated with @doc is not supported.",
+                "location": make_location(self.file_path, name_range["start"]),
+                "range": name_range,
+            }
+        )
+
+    def _declaration(self, entry: DeclarationEntry) -> None:
+        visibility_class = "private" if entry.private else "public"
+        if visibility_class not in self.visible:
+            for member in entry.members:
+                if member.targets:
+                    self._unsupported(member.name_range)
+            return
+
+        endpoint = f"{self.file_path}#{entry.canonical_id}"
+        symbol = self._symbol(entry, entry.members[0], endpoint)
+        # A group's members document the endpoint together. The first annotated
+        # declaration owns the links; the endpoint is reported once, at the
+        # first annotated repeat, and later repeats are dropped silently.
+        annotated = [declaration for declaration in entry.declarations if declaration.targets]
+        if not annotated:
+            self.undocumented.append(symbol)
+            return
+        self.symbols.append(symbol)
+        self._links(endpoint, annotated[0].targets)
+        if len(annotated) > 1:
+            self._duplicate_code_symbol(endpoint, annotated[1].first_annotated_member)
+
+    def _symbol(self, entry: DeclarationEntry, member: Member, endpoint: str) -> dict[str, object]:
+        return {
+            "kind": "code",
+            "language": LANGUAGE,
+            "filePath": self.file_path,
+            "symbolName": entry.symbol_name,
+            "canonicalId": entry.canonical_id,
+            "endpoint": endpoint,
+            "location": make_location(self.file_path, member.name_range["start"]),
+            "nameRange": member.name_range,
+            "declarationRange": member.declaration_range,
+            "signatureRange": member.signature_range,
+        }
+
+    def _duplicate_code_symbol(self, endpoint: str, member: Member) -> None:
+        self.diagnostics.append(
+            {
+                "severity": "error",
+                "code": "duplicate_code_symbol",
+                "target": endpoint,
+                "language": LANGUAGE,
+                "message": "Duplicate Python code symbol endpoint: " + endpoint,
+                "location": make_location(self.file_path, member.name_range["start"]),
+                "range": member.name_range,
+            }
+        )
+
+    def _links(self, endpoint: str, targets: list[DocTarget]) -> None:
+        seen: set[str] = set()
+        for target in targets:
+            location = make_location(self.file_path, target.start)
+            target_range = {"start": target.start, "end": target.end}
+            if not is_valid_link_target(target.target, self.file_path):
+                self.diagnostics.append(
+                    {
+                        "severity": "error",
+                        "code": "invalid_link_target",
+                        "target": target.target,
+                        "language": LANGUAGE,
+                        "source": endpoint,
+                        "message": INVALID_LINK_TARGET_MESSAGE,
+                        "location": location,
+                        "range": target_range,
+                    }
+                )
+                continue
+            if target.target in seen:
+                self.diagnostics.append(
+                    {
+                        "severity": "warning",
+                        "code": "duplicate_link",
+                        "target": target.target,
+                        "language": LANGUAGE,
+                        "source": endpoint,
+                        "message": f"Duplicate @doc link from {endpoint} to {target.target}.",
+                        "location": location,
+                        "range": target_range,
+                    }
+                )
+                continue
+            seen.add(target.target)
+            self.links.append(
+                {
+                    "source": endpoint,
+                    "target": target.target,
+                    "location": location,
+                    "targetRange": target_range,
+                }
+            )

package/dist/workers/python/docbridge_python_scanner.py

@@ -0,0 +1,16 @@
+"""DocBridge Python scanner worker entrypoint.
+
+The core runs this file as ``python3 -I -S <absolute path>``. Isolated mode
+drops the script directory from ``sys.path``, so the directory is inserted
+explicitly before the package import; nothing else is ever added.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from docbridge_python_scanner.protocol import main
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/dist/workers/ruby/bin/docbridge-ruby-scanner

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# DocBridge Ruby scanner worker entrypoint. The core always starts it as
+# `ruby --disable=gems,did_you_mean,error_highlight -W0 <this file>`, so it
+# adds its own lib/ to the load path instead of relying on RubyGems.
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "docbridge_ruby_scanner/cli"
+
+exit(DocbridgeRubyScanner::CLI.run(ARGV, $stdin, $stdout, $stderr))

package/dist/workers/ruby/lib/docbridge_ruby_scanner/cli.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "json"
+require "docbridge_ruby_scanner/probe"
+
+module DocbridgeRubyScanner
+  # The executable's behaviour: `--probe` prints one JSON line and exits 0;
+  # otherwise one request is read from stdin and one response is written to
+  # stdout. Stderr carries only error text.
+  module CLI
+    module_function
+
+    def run(argv, stdin, stdout, stderr)
+      stdout.binmode
+      if argv.include?("--probe")
+        stdout.write("#{JSON.generate(Probe.result)}\n")
+        return 0
+      end
+
+      require "docbridge_ruby_scanner/protocol"
+      stdin.binmode
+      request = JSON.parse(stdin.read.force_encoding(Encoding::UTF_8))
+      stdout.write("#{JSON.generate(Protocol.scan_request(request))}\n")
+      0
+    rescue JSON::ParserError, KeyError, TypeError, NoMethodError => error
+      stderr.write("docbridge-ruby-scanner: invalid request: #{error.message}\n")
+      1
+    end
+  end
+end

package/dist/workers/ruby/lib/docbridge_ruby_scanner/comments.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "docbridge_ruby_scanner/compat"
+require "docbridge_ruby_scanner/links"
+
+module DocbridgeRubyScanner
+  # The contiguous run of full-line `#` comments that ends on the line directly
+  # above a declaration. Trailing comments (code before the `#` on the same
+  # line), `=begin`/`=end` blocks, and blank lines break the run.
+  CommentBlock = Struct.new(:start_offset, :targets)
+
+  class CommentIndex
+    def initialize(parse_result, converter)
+      @converter = converter
+      @by_line = {}
+      content = converter.content
+      parse_result.comments.each do |comment|
+        next unless Compat.inline_comment?(comment, content)
+
+        location = comment.location
+        next unless full_line?(location.start_offset)
+
+        @by_line[converter.position(location.start_offset)[:line]] = location
+      end
+    end
+
+    # The comment block attached to a declaration that starts at
+    # `start_offset`, or nil when the line above it is not a comment.
+    def block_above(start_offset)
+      line = @converter.position(start_offset)[:line] - 1
+      return nil unless @by_line.key?(line)
+
+      line -= 1 while @by_line.key?(line - 1)
+      lines = []
+      while (location = @by_line[line])
+        lines << location
+        line += 1
+      end
+      targets = lines.flat_map do |location|
+        body_start = location.start_offset + 1
+        Links.doc_targets(@converter.slice(body_start, location.end_offset), body_start)
+      end
+      CommentBlock.new(lines.first.start_offset, targets)
+    end
+
+    private
+
+    def full_line?(offset)
+      content = @converter.content
+      line_start = content.byterindex("\n", offset)
+      line_start = line_start.nil? ? 0 : line_start + 1
+      @converter.slice(line_start, offset).match?(/\A[ \t]*\z/)
+    end
+  end
+end

package/dist/workers/ruby/lib/docbridge_ruby_scanner/compat.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "prism"
+
+module DocbridgeRubyScanner
+  # Every Prism accessor whose shape differs between the Prism bundled with
+  # CRuby 3.3 (0.19) and the 1.x series bundled with 3.4 and later lives here,
+  # so the rest of the worker reads one stable API.
+  #
+  # Prism 0.19 exposes the last segment of a constant path as
+  # `ConstantPathNode#child` (a ConstantReadNode); 0.28 replaced it with
+  # `#name` and `#name_loc` and 1.x deprecates `#child`.
+  module Compat
+    module_function
+
+    # The name of the last segment of a constant path, or nil when the parser
+    # recovered from a missing segment.
+    def constant_path_name(node)
+      if node.respond_to?(:name_loc)
+        node.name
+      else
+        child = node.child
+        child.respond_to?(:name) ? child.name : nil
+      end
+    end
+
+    # The location of the last segment of a constant path.
+    def constant_path_name_loc(node)
+      node.respond_to?(:name_loc) ? node.name_loc : node.child.location
+    end
+
+    # True for a `#` comment; `=begin`/`=end` blocks and `__END__` data are
+    # not. The class exists in both versions, but checking the source byte is
+    # independent of the comment class hierarchy.
+    def inline_comment?(comment, content)
+      content.byteslice(comment.location.start_offset, 1) == "#"
+    end
+  end
+end