@guanlin4924/devflow-pi 0.1.76 → 0.1.78

package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/registry.py

@@ -1,6 +1,16 @@
+from .capabilities import load_capability_manifest
 from .generic_text import GenericTextAdapter
+from .go_ast import GoAstAdapter
+from .java_ast import JavaAstAdapter
+from .native_process import UnavailableNativeAdapter
+from .php_ast import PhpAstAdapter
 from .python_ast import PythonAstAdapter
 from .structured_scope import StructuredScopeAdapter
+from .tree_sitter_wasm import TreeSitterWasmAdapter
+from .typescript_ast import TypeScriptAstAdapter
+
+
+CAPABILITY_MANIFEST = load_capability_manifest()
 
 
 class AdapterRegistry:
@@ -8,61 +18,56 @@ class AdapterRegistry:
         self.python = PythonAstAdapter()
         self.structured = StructuredScopeAdapter()
         self.generic = GenericTextAdapter()
-        self.adapters = [self.python, self.structured, self.generic]
-
-        # Keep language selection explicit.  The parser implementation is
-        # conservative and shared for now, but every supported language has a
-        # deliberate registration instead of silently falling through to text
-        # regexes.  This is the extension point for native AST adapters.
+        self.typescript = TypeScriptAstAdapter(self.structured)
+        self.go = GoAstAdapter(self.structured)
+        self.php = PhpAstAdapter(self.structured)
+        self.java = JavaAstAdapter(self.structured)
+        self.native_adapters = [self.python, self.typescript, self.go, self.php, self.java]
+        self.adapters = [*self.native_adapters, self.structured, self.generic]
         self.language_adapters = {
             "python": self.python,
-            "typescript": self.structured,
-            "javascript": self.structured,
-            "ruby": self.structured,
-            "go": self.structured,
-            "rust": self.structured,
-            "php": self.structured,
-            "java": self.structured,
-            "kotlin": self.structured,
-            "c": self.structured,
-            "cpp": self.structured,
-            "csharp": self.structured,
-            "swift": self.structured,
-            "dart": self.structured,
-            "scala": self.structured,
-            "lua": self.structured,
-            "shell": self.structured,
-            "vue": self.structured,
-            "svelte": self.structured,
-            # Data, configuration, and template types are explicitly registered too. They
-            # currently use the safe text adapter until a native parser is
-            # available; this keeps their parser provenance explicit and the
-            # unified output contract unchanged.
-            "sql": self.generic,
-            "html": self.generic,
-            "css": self.generic,
-            "json": self.generic,
-            "yaml": self.generic,
-            "toml": self.generic,
-            "xml": self.generic,
-            "graphql": self.generic,
-            "protobuf": self.generic,
-            "hcl": self.generic,
-            "terraform": self.generic,
-            "dockerfile": self.generic,
-            "markdown": self.generic,
+            "typescript": self.typescript,
+            "javascript": self.typescript,
+            "go": self.go,
+            "php": self.php,
+            "java": self.java,
         }
+        structured_languages = {
+            "ruby", "rust", "kotlin", "c", "cpp", "csharp", "swift",
+            "dart", "scala", "lua", "shell", "vue", "svelte",
+        }
+        for language, declaration in CAPABILITY_MANIFEST["languages"].items():
+            if language in self.language_adapters:
+                continue
+            expected = declaration["parser"]
+            reason = declaration.get("todo") or f"{expected} integration is not installed"
+            fallback = self.structured if language in structured_languages else self.generic
+            if declaration.get("strategy") == "tree-sitter-wasm":
+                self.language_adapters[language] = TreeSitterWasmAdapter(language, fallback)
+            else:
+                self.language_adapters[language] = UnavailableNativeAdapter(language, expected, fallback, reason)
 
     def adapter_for_file(self, codebase, file_record):
         language = file_record.get("language")
-        return self.language_adapters.get(language, self.generic)
+        return self.language_adapters.get(
+            language,
+            UnavailableNativeAdapter(language or "unknown", "unregistered-native-ast", self.generic, "no native adapter is registered"),
+        )
 
     def adapters_for(self, codebase, files, manifests):
-        scored = []
-        for adapter in self.adapters:
-            score = adapter.detect(codebase, files, manifests)
-            if score > 0:
-                scored.append((score, adapter))
-        if not scored:
-            return [GenericTextAdapter()]
-        return [adapter for _, adapter in sorted(scored, key=lambda item: item[0], reverse=True)]
+        return [self.adapter_for_file(codebase, {"language": self._language(path)}) for path in files]
+
+    def capabilities(self):
+        capabilities = {}
+        for language, adapter in sorted(self.language_adapters.items()):
+            declaration = dict(CAPABILITY_MANIFEST["languages"].get(language, {}))
+            runtime = adapter.capability()
+            declaration["declaredStatus"] = declaration.pop("status", "unknown")
+            declaration.update(runtime)
+            capabilities[language] = declaration
+        return capabilities
+
+    @staticmethod
+    def _language(path):
+        from .generic_text import language_for_path
+        return language_for_path(path)

package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/structured_scope.py

@@ -127,7 +127,7 @@ class StructuredScopeAdapter(Adapter):
             for target in _extract_calls(line, record["language"]):
                 receiver = bool(re.search(rf"\.\s*{re.escape(target)}\s*\(", line))
                 relations.append(_relation(owner, "calls", record["path"], number, self.parser_id, "heuristic", to_external=f"call:{target}", metadata={"resolution": "receiver_unresolved" if receiver else "name_only", "allow_unique_name_resolution": not receiver, "dynamic_limitations": True}))
-        return ParseResult(symbols=[file_symbol, *symbols], relations=relations, diagnostics=[])
+        return ParseResult(symbols=[file_symbol, *symbols], relations=relations, diagnostics=[], actual_parser=self.parser_id)
 
 
 def _without_strings_and_comments(line: str) -> str:

package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/tree_sitter_wasm.py

@@ -0,0 +1,253 @@
+"""Optional Tree-sitter WASM adapter.
+
+The indexer remains usable without a grammar bundle.  In that case this
+adapter deliberately returns the configured fallback and a diagnostic; it
+never labels lexical facts as Tree-sitter facts.  A grammar is supplied by the
+runtime (or DEVFLOW_TREE_SITTER_WASM_GRAMMAR_<LANG>) so the npm package does
+not silently depend on a machine-specific native installation.
+"""
+
+import json
+import hashlib
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Dict, Tuple
+
+from adapters.base import Adapter, ParseResult
+from adapters.capabilities import load_capability_manifest
+from adapters.generic_text import _relation, _symbol
+
+
+HELPER = Path(__file__).resolve().parent / "helpers" / "tree_sitter_wasm_helper.mjs"
+
+
+class TreeSitterWasmAdapter(Adapter):
+    name = "tree-sitter-wasm"
+
+    def __init__(self, language: str, fallback: Adapter):
+        self.language = language
+        self.fallback = fallback
+        self.declaration = load_capability_manifest()["languages"].get(language, {})
+        self._availability: Tuple[bool, str] | None = None
+
+    def version(self) -> str:
+        return str(self.declaration.get("adapter_version", 1))
+
+    @property
+    def parser_id(self) -> str:
+        grammar = self.declaration.get("grammar", {}).get("id", self.language)
+        return f"tree-sitter-wasm:{grammar}@{self.version()}"
+
+    def parser_for_file(self, codebase, file_record) -> str:
+        available, _ = self._availability_for(codebase)
+        return self.parser_id if available else self.fallback.parser_for_file(codebase, file_record)
+
+    def check_available(self, codebase=None) -> Tuple[bool, str]:
+        grammar = self._grammar_path(codebase)
+        if not grammar:
+            return False, "Tree-sitter WASM grammar is not configured"
+        pin = self._grammar_pin()
+        if not pin["version"] or not pin["sha256"]:
+            return False, "Tree-sitter WASM grammar is unpinned; configure version and sha256"
+        actual_hash = self._sha256(grammar)
+        expected_hash = pin["sha256"].removeprefix("sha256:").lower()
+        if actual_hash != expected_hash:
+            return False, f"Tree-sitter WASM grammar hash mismatch: expected {expected_hash}, got {actual_hash}"
+        runtime = self._runtime()
+        if runtime is None:
+            return False, "Node/Bun runtime is not available"
+        if not self._module_available(runtime):
+            return False, "web-tree-sitter module is not importable"
+        if not HELPER.is_file():
+            return False, "Tree-sitter WASM helper is missing"
+        return True, f"Tree-sitter WASM grammar available: {grammar} ({pin['version']}, sha256:{actual_hash})"
+
+    def capability(self) -> Dict:
+        available, reason = self._availability_for(None)
+        grammar = dict(self.declaration.get("grammar", {}))
+        return {
+            "parser": self.parser_id,
+            "strategy": "tree-sitter-wasm",
+            "native": False,
+            "available": available,
+            "status": "implemented" if available else "parser_unavailable",
+            "support_level": "supported" if available else ("unpinned" if self._grammar_path(None) else "parser_unavailable"),
+            "pin": self._grammar_pin(),
+            "reason": reason,
+            "grammar": grammar,
+            "fallback": self.fallback.parser_id,
+            "scope": "Tree-sitter WASM syntax tree; framework semantics remain a separate linker stage",
+        }
+
+    def parse_file(self, codebase, file_record, text) -> ParseResult:
+        if file_record.get("language") != self.language:
+            return self.fallback.parse_file(codebase, file_record, text)
+        available, reason = self._availability_for(codebase)
+        if not available:
+            result = self.fallback.parse_file(codebase, file_record, text)
+            result.diagnostics.append({
+                "codebase": codebase.name,
+                "path": file_record["path"],
+                "line": 1,
+                "severity": "warning",
+                "kind": "parser_unavailable",
+                "message": f"Tree-sitter WASM unavailable for {self.language}; fallback used: {reason}",
+                "parser": self.parser_id,
+                "metadata": {"support_level": "parser_unavailable", "fallback_parser": self.fallback.parser_id},
+            })
+            return result
+        try:
+            payload = self._parse(codebase, file_record, text)
+        except Exception as error:
+            result = self.fallback.parse_file(codebase, file_record, text)
+            result.diagnostics.append({
+                "codebase": codebase.name, "path": file_record["path"], "line": 1,
+                "severity": "error", "kind": "parse_failed",
+                "message": f"Tree-sitter WASM parse failed; fallback used: {error}",
+                "parser": self.parser_id,
+            })
+            return result
+        return self._convert(codebase, file_record, text, payload)
+
+    def _availability_for(self, codebase=None):
+        # A grammar/runtime is global for a scan.  Codebase-specific grammar
+        # overrides are intentionally checked only when parsing that codebase.
+        if codebase is not None:
+            return self.check_available(codebase)
+        if self._availability is None:
+            self._availability = self.check_available(None)
+        return self._availability
+
+    def _grammar_path(self, codebase=None):
+        key = f"DEVFLOW_TREE_SITTER_WASM_GRAMMAR_{self.language.upper().replace('-', '_')}"
+        configured = os.environ.get(key) or self.declaration.get("grammar", {}).get("path")
+        if configured:
+            path = Path(configured)
+            if not path.is_absolute() and codebase is not None:
+                path = Path(codebase.root) / path
+            return str(path.resolve()) if path.is_file() else None
+        package = self.declaration.get("grammar", {}).get("package")
+        package_path = self._resolve_package_grammar(package) if package else None
+        return package_path
+
+    @staticmethod
+    def _resolve_package_grammar(package):
+        """Resolve a declared npm grammar without downloading at runtime.
+
+        The package is an installation-time dependency.  Searching the
+        workspace and adapter ancestors keeps this working from both a
+        checkout and an installed plugin while ensuring no network fallback
+        can occur during indexing.
+        """
+        if not package:
+            return None
+        roots = [Path.cwd(), Path(__file__).resolve()]
+        seen = set()
+        for root in roots:
+            for candidate in [root, *root.parents]:
+                package_root = candidate / "node_modules" / package
+                candidates = [package_root / "grammar.wasm"]
+                if package_root.is_dir():
+                    candidates.extend(sorted(package_root.glob("*.wasm")))
+                for node_modules in candidates:
+                    key = str(node_modules)
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    if node_modules.is_file():
+                        return str(node_modules.resolve())
+        return None
+
+    def _grammar_pin(self):
+        grammar = self.declaration.get("grammar", {})
+        prefix = f"DEVFLOW_TREE_SITTER_GRAMMAR_{self.language.upper().replace('-', '_')}"
+        return {
+            "version": os.environ.get(f"{prefix}_VERSION") or grammar.get("version"),
+            "sha256": os.environ.get(f"{prefix}_SHA256") or grammar.get("sha256"),
+        }
+
+    @staticmethod
+    def _sha256(path):
+        digest = hashlib.sha256()
+        with Path(path).open("rb") as handle:
+            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                digest.update(chunk)
+        return digest.hexdigest()
+
+    @staticmethod
+    def _module_available(runtime):
+        # Probe the module before claiming availability.  The helper import is
+        # deliberately not the first place this is discovered: otherwise the
+        # snapshot would advertise a parser which only fails per file.
+        try:
+            probe = subprocess.run(
+                [runtime, "--input-type=module", "-e",
+                 "import('web-tree-sitter').then(()=>process.exit(0)).catch(()=>process.exit(1))"],
+                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                timeout=5, check=False,
+            )
+        except (OSError, subprocess.TimeoutExpired):
+            return False
+        return probe.returncode == 0
+
+    @staticmethod
+    def _runtime():
+        configured = os.environ.get("DEVFLOW_TREE_SITTER_WASM_RUNTIME")
+        if configured:
+            return configured if shutil.which(configured) else None
+        return shutil.which("bun") or shutil.which("node")
+
+    def _parse(self, codebase, record, text):
+        runtime = self._runtime()
+        grammar = self._grammar_path(codebase)
+        max_bytes = int(os.environ.get("DEVFLOW_TREE_SITTER_WASM_MAX_BYTES", str(2 * 1024 * 1024)))
+        source_bytes = len(text.encode("utf-8"))
+        if source_bytes > max_bytes:
+            raise RuntimeError(f"source exceeds Tree-sitter WASM limit ({source_bytes} > {max_bytes} bytes)")
+        timeout = float(os.environ.get("DEVFLOW_TREE_SITTER_WASM_TIMEOUT", "10"))
+        proc = subprocess.run(
+            [runtime, str(HELPER), grammar],
+            input=json.dumps({"path": record["path"], "source": text}, ensure_ascii=False) + "\n",
+            text=True, encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            timeout=timeout, check=False,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(proc.stderr.strip()[:500] or f"helper exited {proc.returncode}")
+        payload = json.loads(proc.stdout.splitlines()[-1])
+        if not payload.get("ok"):
+            raise RuntimeError(payload.get("error") or "Tree-sitter helper returned no result")
+        return payload
+
+    def _convert(self, codebase, record, text, payload):
+        path, content_hash = record["path"], record["content_hash"]
+        file_symbol = _symbol(codebase.name, record["language"], "file", Path(path).name, path, 1,
+                              max(1, len(text.splitlines())), content_hash, qualified_name=path,
+                              parser=self.parser_id)
+        symbols = [file_symbol]
+        by_key = {"": file_symbol}
+        for item in payload.get("declarations", []):
+            parent = by_key.get(str(item.get("parent") or ""), file_symbol)
+            symbol = _symbol(codebase.name, record["language"], item.get("kind", "symbol"),
+                             item.get("name", "anonymous"), path, max(1, int(item.get("line_start", 1))),
+                             max(1, int(item.get("line_end", item.get("line_start", 1)))), content_hash,
+                             qualified_name=item.get("qualified_name") or f"{parent['qualified_name']}::{item.get('name', 'anonymous')}",
+                             parser=self.parser_id)
+            symbol["container_id"] = parent["id"]
+            symbol["metadata"].update({"parse_status": payload.get("parse_status", "complete"), "grammar": self.declaration.get("grammar", {})})
+            symbols.append(symbol)
+            by_key[str(item.get("key"))] = symbol
+        relations = []
+        for symbol in symbols[1:]:
+            relations.append(_relation(symbol["container_id"], "contains", path, symbol["line_start"], self.parser_id, "exact", to_id=symbol["id"]))
+        for item in payload.get("calls", []):
+            owner = by_key.get(str(item.get("owner") or ""), file_symbol)
+            relations.append(_relation(owner["id"], "calls", path, max(1, int(item.get("line", 1))), self.parser_id,
+                                        "heuristic", to_external=f"call:{item.get('target', 'unknown')}"))
+        diagnostics = list(payload.get("diagnostics", []))
+        if payload.get("parse_status") != "complete":
+            diagnostics.append({"codebase": codebase.name, "path": path, "line": 1, "severity": "warning",
+                                "kind": "parse_partial", "message": "Tree-sitter grammar produced a partial tree",
+                                "parser": self.parser_id, "metadata": payload.get("metadata", {})})
+        return ParseResult(symbols=symbols, relations=relations, diagnostics=diagnostics, actual_parser=self.parser_id)

package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/typescript_ast.py

@@ -0,0 +1,31 @@
+import shutil
+import subprocess
+from pathlib import Path
+from typing import List, Tuple
+
+from adapters.native_process import HELPER_DIR, NativeProcessAdapter
+
+
+class TypeScriptAstAdapter(NativeProcessAdapter):
+    name = "typescript-compiler-ast"
+    languages = ("typescript", "javascript")
+    native_description = "TypeScript Compiler API syntax tree; declarations, imports, and calls"
+
+    def version(self) -> str:
+        return "1"
+
+    def check_available(self, codebase=None) -> Tuple[bool, str]:
+        node = shutil.which("node")
+        if not node:
+            return False, "node executable not found"
+        helper = HELPER_DIR / "typescript_ast_helper.cjs"
+        probe = subprocess.run(
+            [node, "-e", "require.resolve('typescript')"], cwd=str(helper.parent),
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
+        )
+        if probe.returncode != 0:
+            return False, "typescript compiler API module not found"
+        return True, "typescript compiler API available"
+
+    def command(self, codebase=None) -> List[str]:
+        return [shutil.which("node") or "node", str(HELPER_DIR / "typescript_ast_helper.cjs")]

package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py

@@ -22,6 +22,7 @@ if str(SCRIPT_DIR) not in sys.path:
     sys.path.insert(0, str(SCRIPT_DIR))
 
 from adapters.generic_text import language_for_path
+from adapters.capabilities import confidence_for_parser
 from adapters.registry import AdapterRegistry
 from semantic import SEMANTIC_EXTRACTOR_VERSION, build_semantic_facts
 from core.discovery import discover_codebases
@@ -99,7 +100,7 @@ def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int], works
         os.close(directory_fd)
 
 
-def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovered_paths: Iterable[str] | None = None) -> Dict[str, object]:
+def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovered_paths: Iterable[str] | None = None, capabilities: Dict[str, object] | None = None) -> Dict[str, object]:
     indexed_paths = sorted(record["path"] for record in files)
     discovered = sorted(set(discovered_paths or indexed_paths))
     source_manifest_hash = _hash_lines(f"{record['path']}\0{record['content_hash']}" for record in files)
@@ -112,11 +113,7 @@ def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovere
         "dirtyPathsAtBuild": sorted(changed),
         "extractors": extractors,
         "extractorSetHash": _hash_lines(extractors),
-        "capabilities": {
-            "python-ast@1": {"scope": "exact syntax tree", "targets": "lexical names; dynamic dispatch unresolved"},
-            "structured-scope@1": {"scope": "inferred lexical braces/end blocks", "targets": "name-only heuristic"},
-            "generic-text@2": {"scope": "file-level", "targets": "heuristic only"},
-        },
+        "capabilities": capabilities or {},
     }
 
 
@@ -290,14 +287,27 @@ def command_scan(args) -> int:
                         "parser": "pending",
                     }
                     adapter = registry.adapter_for_file(codebase, file_record)
-                    file_record["parser"] = adapter.parser_id
+                    parser_for_file = getattr(adapter, "parser_for_file", None)
+                    file_record["parser"] = parser_for_file(codebase, file_record) if parser_for_file else adapter.parser_id
                     result = adapter.parse_file(codebase, file_record, text)
+                    file_record["parser"] = result.actual_parser or file_record["parser"]
                     files.append(file_record)
                     symbols.extend(result.symbols)
                     relations.extend(result.relations)
                     diagnostics.extend(result.diagnostics)
-                    adapter_entities.extend(getattr(result, "entities", []) or [])
-                    adapter_flows.extend(getattr(result, "flows", []) or [])
+                    for entity in getattr(result, "entities", []) or []:
+                        entity = dict(entity)
+                        entity.setdefault("extractor", file_record["parser"])
+                        entity.setdefault("confidence", confidence_for_parser(file_record["parser"], "symbol"))
+                        entity.setdefault("metadata", {})
+                        adapter_entities.append(entity)
+                    for flow in getattr(result, "flows", []) or []:
+                        flow = dict(flow)
+                        flow.setdefault("extractor", file_record["parser"])
+                        flow.setdefault("confidence", "heuristic")
+                        flow.setdefault("evidence_level", "syntax")
+                        flow.setdefault("metadata", {})
+                        adapter_flows.append(flow)
                     parsed += 1
                     if parsed == 1 or parsed % 250 == 0 or parsed == len(all_files):
                         progress.emit("parse", f"parsed {parsed}/{len(all_files)} files in {codebase.name}", codebase=codebase.name, parsed=parsed, total=len(all_files))
@@ -319,12 +329,19 @@ def command_scan(args) -> int:
         staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.build-", dir=str(output.parent)))
         try:
             generation = str(uuid.uuid4())
-            workspace_state = _workspace_state(root, files, changed, discovered_paths)
+            registry_capabilities = getattr(registry, "capabilities", None)
+            workspace_state = _workspace_state(
+                root, files, changed, discovered_paths,
+                registry_capabilities() if registry_capabilities else {},
+            )
             workspace_state["generatedAt"] = datetime.now(timezone.utc).isoformat()
             workspace_state["semanticExtractor"] = SEMANTIC_EXTRACTOR_VERSION
             workspace_state["extractorSetHash"] = _hash_lines([*workspace_state.get("extractors", []), SEMANTIC_EXTRACTOR_VERSION])
             progress.emit("semantic", "building semantic facts")
-            semantic_entities, semantic_flows, feature_bindings = build_semantic_facts(root, files, symbols, relations)
+            semantic_stats = {}
+            semantic_entities, semantic_flows, feature_bindings = build_semantic_facts(root, files, symbols, relations, stats=semantic_stats)
+            workspace_state["semanticScan"] = semantic_stats
+            progress.emit("semantic", "semantic facts completed", **semantic_stats)
             semantic_entities.extend(adapter_entities)
             semantic_flows.extend(adapter_flows)
             write_jsonl(staging / INDEX_FILES["files"], files)
@@ -484,7 +501,7 @@ def command_relations(args) -> int:
             truncated = len(rows) > limit
             for row in rows[:limit]:
                 item = dict(row)
-                if args.change_type:
+                if getattr(args, "change_type", None):
                     item["impact_kind"] = args.change_type
                     item["severity"] = _impact_severity(args.change_type, item["flow_type"])
                 print(json.dumps(item, ensure_ascii=False, sort_keys=True))