@guanlin4924/devflow-pi 0.1.77 → 0.1.79

package/plugins/devflow-intel/skills/workflow-code-index/scripts/semantic.py

@@ -5,31 +5,58 @@ does not replace framework parsers; it gives the relational index a stable
 schema so API, database, UI and feature facts can be linked incrementally.
 """
 import re
+import time
 from pathlib import Path
 from typing import Dict, Iterable, List, Tuple
 
 from core.schema import short_hash, stable_key
 from core.hashing import file_sha256
 
-SEMANTIC_EXTRACTOR_VERSION = "semantic-linker@1"
+SEMANTIC_EXTRACTOR_VERSION = "semantic-linker@2"
+SEMANTIC_MAX_BYTES = 512_000
+SEMANTIC_FILE_BUDGET_SECONDS = 2.0
+SEMANTIC_MAX_LINE_CHARS = 100_000
+SEMANTIC_LANGUAGES = {
+    "python", "javascript", "typescript", "vue", "svelte", "go", "rust",
+    "php", "java", "kotlin", "dart", "c", "cpp", "csharp", "ruby",
+    "swift", "scala", "lua", "shell", "sql", "html", "css",
+}
+SEMANTIC_LOCK_FILES = {
+    "package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "yarn.lock",
+    "composer.lock", "cargo.lock", "poetry.lock", "pdm.lock", "go.sum",
+}
+SEMANTIC_GENERATED_DIRS = {
+    "node_modules", "vendor", "dist", "build", "coverage", "target", ".next",
+    ".nuxt", "_nuxt", ".cache", "generated", ".generated", "__generated__", "snapshots",
+}
 
 
 def build_semantic_facts(root: Path, files: Iterable[dict], symbols: Iterable[dict],
-                         relations: Iterable[dict]) -> Tuple[List[dict], List[dict], List[dict]]:
+                         relations: Iterable[dict], stats: Dict | None = None) -> Tuple[List[dict], List[dict], List[dict]]:
     entities: List[dict] = []
     flows: List[dict] = []
     bindings: List[dict] = []
     records = list(files)
+    summary = {"processed": 0, "timedOut": 0, "skipped": {}}
     symbol_by_path = {}
     for symbol in symbols:
         symbol_by_path.setdefault(symbol.get("path"), []).append(symbol)
 
     for file_number, file_record in enumerate(records, 1):
         if file_number == 1 or file_number % 250 == 0 or file_number == len(records):
-            print(f"[code-index][semantic] processed {file_number}/{len(records)} files", flush=True)
+            skipped_total = sum(summary["skipped"].values())
+            print(f"[code-index][semantic] visited {file_number}/{len(records)} files; processed={summary['processed']} skipped={skipped_total} timed_out={summary['timedOut']}", flush=True)
         path = file_record["path"]
+        skip_reason = semantic_skip_reason(file_record)
+        if skip_reason:
+            summary["skipped"][skip_reason] = summary["skipped"].get(skip_reason, 0) + 1
+            continue
         source = _read(root, path)
         if source is None:
+            summary["skipped"]["unreadable"] = summary["skipped"].get("unreadable", 0) + 1
+            continue
+        if _looks_minified_or_compressed(path, source):
+            summary["skipped"]["minified_or_compressed"] = summary["skipped"].get("minified_or_compressed", 0) + 1
             continue
         try:
             if file_sha256(root / path) != file_record.get("content_hash"):
@@ -38,21 +65,36 @@ def build_semantic_facts(root: Path, files: Iterable[dict], symbols: Iterable[di
                 continue
         except OSError:
             continue
+        deadline = time.monotonic() + SEMANTIC_FILE_BUDGET_SECONDS
+        file_entities: List[dict] = []
+        file_flows: List[dict] = []
+        file_bindings: List[dict] = []
         language = file_record.get("language", "")
         feature = _feature_from_path(path)
         if feature:
             feature_id = f"feature:{stable_key(feature)}"
-            entities.append(_entity(feature_id, "feature", feature, file_record, 1, 1,
-                                    "heuristic", "path_feature_inference"))
+            file_entities.append(_entity(feature_id, "feature", feature, file_record, 1, 1,
+                                         "heuristic", "path_feature_inference"))
             for symbol in symbol_by_path.get(path, []):
                 if symbol.get("kind") in {"route", "component", "class", "function", "method"}:
-                    bindings.append(_binding(feature_id, symbol["id"], "static_inferred", "heuristic", path, symbol.get("line_start", 1)))
+                    file_bindings.append(_binding(feature_id, symbol["id"], "static_inferred", "heuristic", path, symbol.get("line_start", 1)))
 
         state_ids = set()
+        timed_out = False
         for line_no, line in enumerate(source.splitlines(), 1):
-            _extract_api(line, path, file_record, line_no, entities, flows)
-            _extract_db(line, path, file_record, line_no, entities, flows)
-            _extract_ui(line, path, file_record, line_no, entities, flows, state_ids)
+            if time.monotonic() > deadline:
+                timed_out = True
+                break
+            _extract_api(line, path, file_record, line_no, file_entities, file_flows)
+            _extract_db(line, path, file_record, line_no, file_entities, file_flows)
+            _extract_ui(line, path, file_record, line_no, file_entities, file_flows, state_ids)
+        if timed_out:
+            summary["timedOut"] += 1
+            continue
+        entities.extend(file_entities)
+        flows.extend(file_flows)
+        bindings.extend(file_bindings)
+        summary["processed"] += 1
 
     # Existing structural relations remain the source of method call edges;
     # expose them in the semantic graph without changing their meaning.
@@ -67,11 +109,48 @@ def build_semantic_facts(root: Path, files: Iterable[dict], symbols: Iterable[di
                 "line_start": relation.get("evidence_line"),
                 "confidence": relation.get("confidence", "heuristic"),
                 "evidence_level": "syntax",
+                "extractor": relation.get("parser", "unknown"),
                 "metadata": relation.get("metadata", {}),
             })
+    if stats is not None:
+        stats.clear()
+        stats.update(summary)
     return _dedupe_entities(entities), _dedupe_flows(flows), _dedupe_bindings(bindings)
 
 
+def semantic_skip_reason(record: dict):
+    path = Path(record.get("path", ""))
+    parts = tuple(part.lower() for part in path.parts)
+    name = path.name.lower()
+    if name in SEMANTIC_LOCK_FILES or name.endswith((".lock", ".lockb")):
+        return "lock_file"
+    if any(part in SEMANTIC_GENERATED_DIRS for part in parts):
+        return "generated_path"
+    path_text = "/".join(parts)
+    if any(marker in path_text for marker in ("public/assets/", "public/build/", "public/_nuxt/")):
+        return "generated_path"
+    if "public" in parts:
+        public_index = parts.index("public")
+        if any(part in {"assets", "build", "dist", "static", "_nuxt"} for part in parts[public_index + 1:-1]):
+            return "generated_path"
+    if name.endswith((".min.js", ".min.css", ".bundle.js", ".chunk.js", ".map", ".g.dart", ".freezed.dart", ".pb.go", ".pb.cc", "_pb2.py", ".designer.cs")) or ".generated." in name:
+        return "generated_asset"
+    if int(record.get("size_bytes") or 0) > SEMANTIC_MAX_BYTES:
+        return "too_large"
+    if record.get("language", "") not in SEMANTIC_LANGUAGES:
+        return "unsupported_language"
+    return None
+
+
+def _looks_minified_or_compressed(path: str, source: str) -> bool:
+    if not source:
+        return False
+    lines = source.splitlines()
+    if any(len(line) > SEMANTIC_MAX_LINE_CHARS for line in lines):
+        return True
+    return len(source) > 100_000 and len(lines) <= 5
+
+
 def _extract_api(line, path, record, line_no, entities, flows):
     # Framework-neutral route declarations and client URLs.
     match = re.search(r"(?:@(?:app|router|api)\.(get|post|put|patch|delete)\s*\(|(?:GET|POST|PUT|PATCH|DELETE)\s+)(['\"])(/[^'\"]+)\2", line, re.I)
@@ -144,15 +223,17 @@ def _extract_ui(line, path, record, line_no, entities, flows, state_ids=None):
 
 
 def _entity(entity_id, entity_type, name, record, start, end, confidence, extractor, metadata=None):
-    return {"id": entity_id, "entity_type": entity_type, "name": name, "codebase": record["codebase"], "path": record["path"], "line_start": start, "line_end": end, "content_hash": record["content_hash"], "confidence": confidence, "extractor": extractor, "metadata": metadata or {}}
+    extractor_id = extractor if "@" in extractor else f"{SEMANTIC_EXTRACTOR_VERSION}:{extractor}"
+    return {"id": entity_id, "entity_type": entity_type, "name": name, "codebase": record["codebase"], "path": record["path"], "line_start": start, "line_end": end, "content_hash": record["content_hash"], "confidence": confidence, "extractor": extractor_id, "metadata": metadata or {}}
 
 
 def _flow(source_id, target_id, flow_type, path, line, confidence, extractor, metadata=None):
-    return {"source_id": source_id, "target_id": target_id, "flow_type": flow_type, "source_path": path, "line_start": line, "confidence": confidence, "evidence_level": "static_source", "extractor": extractor, "metadata": metadata or {}}
+    extractor_id = extractor if "@" in extractor else f"{SEMANTIC_EXTRACTOR_VERSION}:{extractor}"
+    return {"source_id": source_id, "target_id": target_id, "flow_type": flow_type, "source_path": path, "line_start": line, "confidence": confidence, "evidence_level": "static_source", "extractor": extractor_id, "metadata": metadata or {}}
 
 
 def _binding(feature_id, entity_id, source, confidence, path, line):
-    return {"feature_id": feature_id, "entity_id": entity_id, "binding_type": source, "confidence": confidence, "source_path": path, "line_start": line}
+    return {"feature_id": feature_id, "entity_id": entity_id, "binding_type": source, "confidence": confidence, "source_path": path, "line_start": line, "extractor": f"{SEMANTIC_EXTRACTOR_VERSION}:path_feature_binding"}
 
 
 def _read(root, path):

package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/sqlite_writer.py

@@ -2,6 +2,7 @@ import sqlite3
 import os
 from pathlib import Path
 
+from adapters.capabilities import confidence_for_parser
 from core.schema import json_dumps
 
 
@@ -31,6 +32,7 @@ CREATE TABLE IF NOT EXISTS symbols (
   language TEXT NOT NULL,
   content_hash TEXT NOT NULL,
   visibility TEXT,
+  parser TEXT NOT NULL,
   metadata_json TEXT
 );
 
@@ -113,6 +115,7 @@ CREATE TABLE IF NOT EXISTS feature_bindings (
   confidence TEXT NOT NULL,
   source_path TEXT,
   line_start INTEGER,
+  extractor TEXT NOT NULL,
   PRIMARY KEY(snapshot_id, feature_id, entity_id)
 );
 
@@ -207,19 +210,27 @@ def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id
         parsed_count = sum(1 for item in files if item.get("path") not in parse_failed_paths)
         conn.execute(
             "INSERT INTO analysis_snapshots(snapshot_id, schema_version, source_revision, content_manifest_hash, extractor_set_hash, created_at, status) VALUES(?,?,?,?,?,?,?)",
-            (snapshot_id, 2, workspace_state.get("repositoryRevision", ""), workspace_state.get("contentManifestHash", ""),
+            (snapshot_id, 3, workspace_state.get("repositoryRevision", ""), workspace_state.get("contentManifestHash", ""),
              workspace_state.get("extractorSetHash", ""), workspace_state.get("generatedAt", ""), "current"),
         )
         # Structural symbols and files are first-class graph nodes as well.
         # This prevents semantic flows and feature bindings from dangling when
         # a framework linker points at an existing code symbol.
-        structural_entities = [
-            {"id": item["id"], "entity_type": item.get("kind", "symbol"), "name": item.get("name", item["id"]),
-             "codebase": item.get("codebase"), "path": item.get("path"), "line_start": item.get("line_start"),
-             "line_end": item.get("line_end"), "content_hash": item.get("content_hash"), "confidence": "exact",
-             "extractor": item.get("parser", "index"), "metadata": item.get("metadata", {})}
-            for item in [*files, *symbols]
-        ]
+        parser_by_path = {item.get("path"): item.get("parser", "unknown") for item in files}
+        structural_entities = []
+        for item in [*files, *symbols]:
+            is_file = "size_bytes" in item
+            metadata = dict(item.get("metadata", {}))
+            parser = item.get("parser") or metadata.get("parser") or parser_by_path.get(item.get("path"), "unknown")
+            metadata.setdefault("parser", parser)
+            structural_entities.append({
+                "id": item["id"], "entity_type": item.get("kind", "file" if is_file else "symbol"),
+                "name": item.get("name", item.get("path", item["id"])), "codebase": item.get("codebase"),
+                "path": item.get("path"), "line_start": item.get("line_start", 1 if is_file else None),
+                "line_end": item.get("line_end", 1 if is_file else None), "content_hash": item.get("content_hash"),
+                "confidence": confidence_for_parser(parser, "file" if is_file else "symbol"),
+                "extractor": parser, "metadata": metadata,
+            })
         by_id = {item["id"]: item for item in semantic_entities}
         for item in structural_entities:
             by_id.setdefault(item["id"], item)
@@ -247,8 +258,8 @@ def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id
             [(snapshot_id, item["source_id"], item["target_id"], item["flow_type"], item.get("source_path"), item.get("line_start"), item["confidence"], item.get("evidence_level", "static_source"), item.get("extractor", "unknown"), json_dumps(item.get("metadata", {}))) for item in semantic_flows],
         )
         conn.executemany(
-            "INSERT INTO feature_bindings(snapshot_id, feature_id, entity_id, binding_type, confidence, source_path, line_start) VALUES(?,?,?,?,?,?,?)",
-            [(snapshot_id, item["feature_id"], item["entity_id"], item["binding_type"], item["confidence"], item.get("source_path"), item.get("line_start")) for item in feature_bindings],
+            "INSERT INTO feature_bindings(snapshot_id, feature_id, entity_id, binding_type, confidence, source_path, line_start, extractor) VALUES(?,?,?,?,?,?,?,?)",
+            [(snapshot_id, item["feature_id"], item["entity_id"], item["binding_type"], item["confidence"], item.get("source_path"), item.get("line_start"), item.get("extractor", "unknown")) for item in feature_bindings],
         )
         conn.execute(
             "INSERT INTO coverage_metrics(snapshot_id, files_scanned, files_parsed, diagnostics, semantic_entities, semantic_flows, dynamic_limitations, metadata_json) VALUES(?,?,?,?,?,?,?,?)",
@@ -264,9 +275,9 @@ def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id
         conn.executemany(
             """
             INSERT INTO symbols(id, codebase, kind, name, qualified_name, container_id, path, line_start, line_end,
-              signature, language, content_hash, visibility, metadata_json)
+              signature, language, content_hash, visibility, parser, metadata_json)
             VALUES(:id, :codebase, :kind, :name, :qualified_name, :container_id, :path, :line_start, :line_end,
-              :signature, :language, :content_hash, :visibility, :metadata_json)
+              :signature, :language, :content_hash, :visibility, :parser, :metadata_json)
             """,
             [_symbol_row(record) for record in symbols],
         )
@@ -302,7 +313,9 @@ def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id
 
 def _symbol_row(record):
     row = dict(record)
-    row["metadata_json"] = json_dumps(row.pop("metadata", {}))
+    metadata = row.pop("metadata", {})
+    row["parser"] = row.get("parser") or metadata.get("parser") or "unknown"
+    row["metadata_json"] = json_dumps(metadata)
     return row
 
 

package/src/constants.ts

@@ -1,5 +1,5 @@
 export const PRODUCT_NAME = "Devflow Pi";
-export const PRODUCT_VERSION = "0.1.77";
+export const PRODUCT_VERSION = "0.1.79";
 export const PROJECT_SCHEMA_VERSION = 1;
 export const OMP_RUNTIME_VERSION = "17.2.7";