@guanlin4924/devflow-pi 0.1.76 → 0.1.78

package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/java_ast.py

@@ -0,0 +1,37 @@
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import List, Tuple
+
+from adapters.native_process import HELPER_DIR, NativeProcessAdapter
+
+
+class JavaAstAdapter(NativeProcessAdapter):
+    name = "javac-tree-ast"
+    languages = ("java",)
+    native_description = "JDK javac com.sun.source Tree API; declarations, imports, and calls"
+    transport = "base64-tsv"
+
+    def __init__(self, fallback):
+        super().__init__(fallback)
+        self.classes = Path(tempfile.gettempdir()) / "devflow-java-ast-helper-v1"
+
+    def version(self) -> str:
+        return "1"
+
+    def check_available(self, codebase=None) -> Tuple[bool, str]:
+        javac, java = shutil.which("javac"), shutil.which("java")
+        if not javac or not java:
+            return False, "JDK javac/java executables not found"
+        source = HELPER_DIR / "DevflowJavaAstHelper.java"
+        target = self.classes / "DevflowJavaAstHelper.class"
+        if not target.exists() or target.stat().st_mtime < source.stat().st_mtime:
+            self.classes.mkdir(parents=True, exist_ok=True)
+            result = subprocess.run([javac, "-d", str(self.classes), str(source)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
+            if result.returncode != 0:
+                return False, f"javac Tree API helper compilation failed: {result.stderr.strip()[:500]}"
+        return True, "JDK javac Tree API available"
+
+    def command(self, codebase=None) -> List[str]:
+        return [shutil.which("java") or "java", "-cp", str(self.classes), "DevflowJavaAstHelper"]

package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/native_process.py

@@ -0,0 +1,328 @@
+import base64
+import json
+import os
+import select
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from adapters.base import Adapter, ParseResult
+from adapters.generic_text import _relation, _symbol
+
+
+HELPER_DIR = Path(__file__).resolve().parent / "helpers"
+
+
+class NativeProcessAdapter(Adapter):
+    """Base class for language-native parsers exposed through JSONL helpers.
+
+    Helpers stay alive for the lifetime of one index scan, which avoids the
+    process-startup cost for every source file.  They return syntax facts only;
+    this class converts those facts into the stable shared index schema.
+    """
+
+    languages: Tuple[str, ...] = ()
+    native_description = "language-native syntax parser"
+    transport = "json"
+
+    def __init__(self, fallback: Adapter):
+        self.fallback = fallback
+        self._processes: Dict[str, subprocess.Popen] = {}
+        self._availability: Dict[str, Tuple[bool, str]] = {}
+
+    def detect(self, codebase, files, manifests) -> int:
+        return 120 if any(self._language_for_path(path) in self.languages for path in files) else 0
+
+    def supports(self, language: str) -> bool:
+        return language in self.languages
+
+    def check_available(self, codebase=None) -> Tuple[bool, str]:
+        raise NotImplementedError
+
+    def command(self, codebase=None) -> List[str]:
+        raise NotImplementedError
+
+    def _availability_for(self, codebase=None) -> Tuple[bool, str]:
+        key = self._availability_key(codebase)
+        if key not in self._availability:
+            self._availability[key] = self.check_available(codebase)
+        return self._availability[key]
+
+    @staticmethod
+    def _availability_key(codebase=None) -> str:
+        if codebase is None:
+            return "<global>"
+        root = getattr(codebase, "root", None)
+        if root is not None:
+            try:
+                return str(Path(root).resolve())
+            except OSError:
+                return str(root)
+        return f"<codebase:{getattr(codebase, 'name', id(codebase))}>"
+
+    def parser_for_file(self, codebase, file_record) -> str:
+        available, _ = self._availability_for(codebase)
+        return self.parser_id if available else self.fallback.parser_for_file(codebase, file_record)
+
+    def capability(self) -> Dict:
+        codebase_results = [value for key, value in self._availability.items() if key != "<global>"]
+        if codebase_results:
+            available = any(value[0] for value in codebase_results)
+            reasons = sorted({value[1] for value in codebase_results})
+            reason = "; ".join(reasons)
+        else:
+            available, reason = self._availability_for(None)
+        return {
+            "parser": self.parser_id,
+            "languages": list(self.languages),
+            "native": True,
+            "available": available,
+            "reason": reason,
+            "fallback": self.fallback.parser_id,
+            "scope": self.native_description,
+        }
+
+    def parse_file(self, codebase, file_record, text) -> ParseResult:
+        if not self.supports(file_record["language"]):
+            return self.fallback.parse_file(codebase, file_record, text)
+        available, reason = self._availability_for(codebase)
+        if not available:
+            return self._fallback_result(codebase, file_record, text, reason)
+        try:
+            payload = self._request(codebase, file_record, text)
+        except Exception as error:
+            self._stop_process(codebase)
+            return self._fallback_result(codebase, file_record, text, f"native parser failed: {error}", kind="parse_failed")
+        if not payload.get("ok", False):
+            message = payload.get("error") or "native parser returned no result"
+            return self._fallback_result(codebase, file_record, text, message, kind="parse_failed")
+        return self._convert(codebase, file_record, text, payload)
+
+    def _fallback_result(self, codebase, file_record, text, reason: str, kind: str = "adapter_missing") -> ParseResult:
+        result = self.fallback.parse_file(codebase, file_record, text)
+        result.actual_parser = result.actual_parser or self.fallback.parser_for_file(codebase, file_record)
+        result.diagnostics.append({
+            "codebase": codebase.name,
+            "path": file_record["path"],
+            "line": 1,
+            "severity": "warning",
+            "kind": kind,
+            "message": f"native AST unavailable for {file_record['language']}; {self.fallback.parser_id} fallback used: {reason}",
+            "parser": self.parser_id,
+            "metadata": {
+                "capability": "native_unavailable",
+                "native_parser": self.parser_id,
+                "fallback_parser": self.fallback.parser_id,
+            },
+        })
+        return result
+
+    def _start_process(self, codebase=None) -> subprocess.Popen:
+        command = self.command(codebase)
+        key = self._process_key(codebase, command)
+        current = self._processes.get(key)
+        if current is not None and current.poll() is None:
+            return current
+        process = subprocess.Popen(
+            command,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            text=True,
+            encoding="utf-8",
+            bufsize=1,
+            env={**os.environ, "NO_COLOR": "1"},
+        )
+        self._processes[key] = process
+        return process
+
+    def _process_key(self, codebase=None, command: Optional[List[str]] = None) -> str:
+        actual_command = command if command is not None else self.command(codebase)
+        return f"{self._availability_key(codebase)}\0{json.dumps(actual_command, ensure_ascii=False)}"
+
+    def _request(self, codebase, file_record, text) -> Dict:
+        process = self._start_process(codebase)
+        if self.transport == "base64-tsv":
+            request = "\t".join([
+                base64.b64encode(file_record["path"].encode("utf-8")).decode("ascii"),
+                base64.b64encode(text.encode("utf-8")).decode("ascii"),
+            ])
+        else:
+            request = json.dumps({"path": file_record["path"], "source": text}, ensure_ascii=False)
+        assert process.stdin is not None and process.stdout is not None
+        process.stdin.write(request + "\n")
+        process.stdin.flush()
+        readable, _, _ = select.select([process.stdout], [], [], 30.0)
+        if not readable:
+            raise TimeoutError("native parser timed out after 30 seconds")
+        response = process.stdout.readline()
+        if not response:
+            raise RuntimeError(f"native parser exited with status {process.poll()}")
+        return json.loads(response)
+
+    def _convert(self, codebase, record, text: str, payload: Dict) -> ParseResult:
+        path = record["path"]
+        language = record["language"]
+        content_hash = record["content_hash"]
+        file_symbol = _symbol(
+            codebase.name, language, "file", Path(path).name, path, 1,
+            max(1, len(text.splitlines())), content_hash,
+            qualified_name=path, parser=self.parser_id,
+        )
+        symbols = [file_symbol]
+        by_key: Dict[str, Dict] = {"": file_symbol}
+        pending = list(payload.get("declarations") or [])
+        # Helpers emit parents before children, but tolerate out-of-order data.
+        while pending:
+            remaining = []
+            progressed = False
+            for declaration in pending:
+                parent_key = str(declaration.get("parent") or "")
+                parent = by_key.get(parent_key)
+                if parent is None:
+                    remaining.append(declaration)
+                    continue
+                kind = declaration.get("kind") or "function"
+                name = declaration.get("name") or "anonymous"
+                signature = declaration.get("signature") or ""
+                qualified = declaration.get("qualified_name") or f"{parent['qualified_name']}::{name}"
+                symbol = _symbol(
+                    codebase.name, language, kind, name, path,
+                    max(1, int(declaration.get("line_start") or 1)),
+                    max(1, int(declaration.get("line_end") or declaration.get("line_start") or 1)),
+                    content_hash, qualified_name=qualified, signature=signature[:500],
+                    parser=self.parser_id, visibility=declaration.get("visibility") or "",
+                )
+                symbol["container_id"] = parent["id"]
+                symbol["metadata"].update({"scope_resolution": "native_ast", "native_kind": declaration.get("native_kind", "")})
+                symbols.append(symbol)
+                by_key[str(declaration.get("key"))] = symbol
+                progressed = True
+            if not progressed:
+                # Malformed helper output must not create dangling containers.
+                for declaration in remaining:
+                    declaration["parent"] = ""
+                pending = remaining
+                if pending and all(str(item.get("parent") or "") == "" for item in pending):
+                    continue
+                break
+            pending = remaining
+
+        relations = []
+        for key, symbol in by_key.items():
+            if not key:
+                continue
+            relations.append(_relation(
+                symbol["container_id"] or file_symbol["id"], "contains", path,
+                symbol["line_start"], self.parser_id, "exact", to_id=symbol["id"],
+                metadata={"extraction": "native_ast"},
+            ))
+        for item in payload.get("relations") or []:
+            owner = by_key.get(str(item.get("owner") or ""), file_symbol)
+            target_key = str(item.get("target_key") or "")
+            target = by_key.get(target_key) if target_key else None
+            relations.append(_relation(
+                owner["id"], item.get("type") or "calls", path,
+                max(1, int(item.get("line") or 1)), self.parser_id,
+                item.get("confidence") or "heuristic",
+                to_id=target["id"] if target else "",
+                to_external="" if target else str(item.get("target") or ""),
+                metadata={"extraction": "native_ast", **(item.get("metadata") or {})},
+            ))
+        diagnostics = []
+        for item in payload.get("diagnostics") or []:
+            diagnostics.append({
+                "codebase": codebase.name,
+                "path": path,
+                "line": max(1, int(item.get("line") or 1)),
+                "severity": item.get("severity") or "warning",
+                "kind": item.get("kind") or "parse_failed",
+                "message": item.get("message") or "native parser diagnostic",
+                "parser": self.parser_id,
+            })
+        return ParseResult(symbols=symbols, relations=relations, diagnostics=diagnostics, actual_parser=self.parser_id)
+
+    def _stop_process(self, codebase=None) -> None:
+        if codebase is None:
+            processes = list(self._processes.values())
+            self._processes.clear()
+        else:
+            prefix = f"{self._availability_key(codebase)}\0"
+            keys = [key for key in self._processes if key.startswith(prefix)]
+            processes = [self._processes.pop(key) for key in keys]
+        for process in processes:
+            self._close_process(process)
+
+    @staticmethod
+    def _close_process(process: subprocess.Popen) -> None:
+        try:
+            if process.stdin:
+                process.stdin.close()
+            process.terminate()
+            process.wait(timeout=1)
+        except Exception:
+            process.kill()
+        finally:
+            if process.stdout:
+                process.stdout.close()
+
+    def __del__(self):
+        self._stop_process()
+
+    @staticmethod
+    def _language_for_path(path: Path) -> str:
+        from adapters.generic_text import language_for_path
+        return language_for_path(path)
+
+
+def executable(name: str) -> Tuple[bool, str]:
+    path = shutil.which(name)
+    return (bool(path), path or f"{name} executable not found")
+
+
+class UnavailableNativeAdapter(Adapter):
+    """Explicit native capability gap with a provenance-safe fallback."""
+
+    name = "native-unavailable"
+
+    def __init__(self, language: str, expected_parser: str, fallback: Adapter, reason: str):
+        self.language = language
+        self.expected_parser = expected_parser
+        self.fallback = fallback
+        self.reason = reason
+
+    def version(self) -> str:
+        return "1"
+
+    def parser_for_file(self, codebase, file_record) -> str:
+        return self.fallback.parser_for_file(codebase, file_record)
+
+    def parse_file(self, codebase, file_record, text) -> ParseResult:
+        result = self.fallback.parse_file(codebase, file_record, text)
+        result.actual_parser = result.actual_parser or self.fallback.parser_for_file(codebase, file_record)
+        result.diagnostics.append({
+            "codebase": codebase.name,
+            "path": file_record["path"],
+            "line": 1,
+            "severity": "warning",
+            "kind": "adapter_missing",
+            "message": f"native AST unavailable for {self.language}; {self.fallback.parser_id} fallback used: {self.reason}",
+            "parser": self.parser_id,
+            "metadata": {
+                "capability": "native_unavailable",
+                "expected_parser": self.expected_parser,
+                "fallback_parser": self.fallback.parser_id,
+            },
+        })
+        return result
+
+    def capability(self) -> Dict:
+        return {
+            "parser": self.expected_parser,
+            "languages": [self.language],
+            "native": True,
+            "available": False,
+            "reason": self.reason,
+            "fallback": self.fallback.parser_id,
+        }