@comfanion/usethis_search 3.0.0-dev.16 → 3.0.0-dev.18

package/vectorizer/index.ts

@@ -16,7 +16,7 @@ import { mergeResults, DEFAULT_HYBRID_CONFIG } from "./hybrid-search.ts";
 import { QueryCache, DEFAULT_CACHE_CONFIG } from "./query-cache.ts";
 import { SearchMetrics } from "./search-metrics.ts";
 import { GraphDB } from "./graph-db.ts";
-import { GraphBuilder } from "./graph-builder.ts";
+import { GraphBuilder, isStructuralPredicate } from "./graph-builder.ts";
 import { UsageTracker } from "./usage-tracker.ts";
 
 // Suppress transformers.js logs unless DEBUG is set
@@ -85,6 +85,19 @@ let HYBRID_CONFIG = { ...DEFAULT_HYBRID_CONFIG };
 let METRICS_ENABLED = false;
 let CACHE_ENABLED = true;
 
+// ── Graph config (v3) ───────────────────────────────────────────────────────
+const DEFAULT_GRAPH_CONFIG = {
+  enabled: true,
+  max_related: 4,
+  min_relevance: 0.5,
+  lsp: {
+    enabled: true,
+    timeout_ms: 5000,
+  },
+  read_intercept: true,
+};
+let GRAPH_CONFIG = { ...DEFAULT_GRAPH_CONFIG, lsp: { ...DEFAULT_GRAPH_CONFIG.lsp } };
+
 function defaultVectorizerYaml() {
   return (
     `vectorizer:\n` +
@@ -121,6 +134,16 @@ function defaultVectorizerYaml() {
     `    hybrid: true\n` +
     `    bm25_weight: 0.3\n` +
     `\n` +
+    `  # Graph-based context (v3)\n` +
+    `  graph:\n` +
+    `    enabled: true\n` +
+    `    max_related: 4\n` +
+    `    min_relevance: 0.5\n` +
+    `    lsp:\n` +
+    `      enabled: true\n` +
+    `      timeout_ms: 5000\n` +
+    `    read_intercept: true\n` +
+    `\n` +
     `  # Quality monitoring\n` +
     `  quality:\n` +
     `    enable_metrics: false\n` +
@@ -282,6 +305,26 @@ async function loadConfig(projectRoot) {
       CACHE_ENABLED = parseBool(qs, "enable_cache", true);
     }
 
+    // ── Parse graph config (v3) ──────────────────────────────────────────────
+    const graphMatch = section.match(/^\s{2}graph:\s*\n([\s\S]*?)(?=^\s{2}[a-zA-Z_\-]+:|\Z)/m);
+    if (graphMatch) {
+      const gs = graphMatch[1];
+      GRAPH_CONFIG.enabled = parseBool(gs, "enabled", DEFAULT_GRAPH_CONFIG.enabled);
+      GRAPH_CONFIG.max_related = parseNumber(gs, "max_related", DEFAULT_GRAPH_CONFIG.max_related);
+      GRAPH_CONFIG.min_relevance = parseNumber(gs, "min_relevance", DEFAULT_GRAPH_CONFIG.min_relevance);
+      GRAPH_CONFIG.read_intercept = parseBool(gs, "read_intercept", DEFAULT_GRAPH_CONFIG.read_intercept);
+
+      // Nested lsp: section
+      const lspMatch = gs.match(/^\s+lsp:\s*\n([\s\S]*?)(?=^\s{4}[a-zA-Z_\-]+:|\Z)/m);
+      if (lspMatch) {
+        const ls = lspMatch[1];
+        GRAPH_CONFIG.lsp.enabled = parseBool(ls, "enabled", DEFAULT_GRAPH_CONFIG.lsp.enabled);
+        GRAPH_CONFIG.lsp.timeout_ms = parseNumber(ls, "timeout_ms", DEFAULT_GRAPH_CONFIG.lsp.timeout_ms);
+      }
+
+      if (DEBUG) console.log("[vectorizer] Graph config:", GRAPH_CONFIG);
+    }
+
     // Parse global exclude
     const excludeMatch = section.match(/^\s{2}exclude:\s*\n((?:\s{4}-\s+.+\n?)*)/m);
     if (excludeMatch) {
@@ -392,11 +435,19 @@ class CodebaseIndexer {
     this.db = await lancedb.connect(path.join(this.cacheDir, "lancedb"));
     await this.loadHashes();
 
-    const graphType = this.indexName === "docs" ? "doc_graph" : "code_graph";
-    const graphPath = path.join(this.root, ".opencode", "graph", graphType);
-    await fs.mkdir(path.dirname(graphPath), { recursive: true });
-    this.graphDB = await new GraphDB(graphPath).init();
-    this.graphBuilder = new GraphBuilder(this.graphDB, this.root);
+    // Graph DB — only if graph is enabled in config
+    if (GRAPH_CONFIG.enabled) {
+      const graphType = this.indexName === "docs" ? "doc_graph" : "code_graph";
+      const graphPath = path.join(this.root, ".opencode", "graph", graphType);
+      await fs.mkdir(path.dirname(graphPath), { recursive: true });
+      this.graphDB = await new GraphDB(graphPath).init();
+      this.graphBuilder = new GraphBuilder(
+        this.graphDB,
+        this.root,
+        GRAPH_CONFIG.lsp.enabled,
+        GRAPH_CONFIG.lsp.timeout_ms,
+      );
+    }
 
     // Usage tracker — provenance & usage stats
     this.usageTracker = new UsageTracker(this.cacheDir);
@@ -557,18 +608,39 @@ class CodebaseIndexer {
     // Semantic chunking
     const chunks = chunkContent(cleaned, fileMeta.file_type, fileMeta.language, CHUNKING_CONFIG);
 
-    // v3: Assign chunk IDs for graph tracking
-    const chunksWithIds = this.graphBuilder.assignChunkIds(relPath, chunks);
+    // v3: Assign chunk IDs for graph tracking (works without graph — just adds IDs)
+    const chunksWithIds = this.graphBuilder
+      ? this.graphBuilder.assignChunkIds(relPath, chunks)
+      : chunks.map((c, i) => ({ ...c, chunk_id: `chunk:${relPath}::_chunk_${i}` }));
 
     // v3: Delete old edges for this file and build new ones
-    await this.graphDB.deleteByFile(relPath);
-    await this.graphBuilder.buildEdges(relPath, content, chunksWithIds, fileMeta.file_type);
+    let graphEdgesBuilt = 0;
+    if (this.graphBuilder && this.graphDB) {
+      await this.graphDB.deleteByFile(relPath);
+      graphEdgesBuilt = await this.graphBuilder.buildEdges(relPath, content, chunksWithIds, fileMeta.file_type);
+
+      // Log graph creation to indexer.log
+      if (graphEdgesBuilt > 0 || DEBUG) {
+        const timestamp = new Date().toISOString().slice(11, 19);
+        const logMsg = `${timestamp} Graph built: ${relPath} (${chunksWithIds.length} chunks)`;
+        if (DEBUG) console.log(`[vectorizer] ${logMsg}`);
+        
+        // Write to indexer.log in .opencode directory
+        try {
+          const logPath = path.join(this.root, ".opencode", "indexer.log");
+          const fsSync = await import("fs");
+          fsSync.appendFileSync(logPath, `${logMsg}\n`);
+        } catch {
+          // non-fatal — logging is advisory
+        }
+      }
 
-    // FR-054: Store graph build timestamp + file hash as metadata triple
-    try {
-      await this.graphDB.setFileMeta(relPath, hash, Date.now());
-    } catch {
-      // non-fatal — metadata is advisory
+      // FR-054: Store graph build timestamp + file hash as metadata triple
+      try {
+        await this.graphDB.setFileMeta(relPath, hash, Date.now());
+      } catch {
+        // non-fatal — metadata is advisory
+      }
     }
 
     const data = [];
@@ -590,6 +662,9 @@ class CodebaseIndexer {
         function_name: chunksWithIds[i].function_name || "",
         class_name: chunksWithIds[i].class_name || "",
         tags: (fileMeta.tags || []).join(","),
+        // Line numbers for "from-to" extraction (default to -1 when unknown)
+        start_line: chunksWithIds[i].start_line ?? -1,
+        end_line: chunksWithIds[i].end_line ?? -1,
       });
     }
 
@@ -629,7 +704,7 @@ class CodebaseIndexer {
     const table = await this.db.openTable(tableName);
     let allRows;
     try {
-      allRows = await table.filter("").limit(100000).execute();
+      allRows = await table.filter("true").limit(100000).execute();
     } catch (e) {
       if (DEBUG) console.log("[vectorizer] BM25 index build failed (corrupted table?):", e.message);
       return null;
@@ -693,10 +768,15 @@ class CodebaseIndexer {
           const bm25Results = bm25.search(query, fetchLimit);
 
           // Build score maps
+          // LanceDB _distance is L2 (euclidean). For normalized vectors,
+          // L2 ∈ [0, 2]. Convert to similarity ∈ [0, 1]:
+          //   similarity = 1 - (distance / 2)
+          const distanceToScore = (d: number | null | undefined) =>
+            d != null ? Math.max(0, 1 - d / 2) : 0.5;
+
           const vectorScores = new Map();
           for (let i = 0; i < results.length; i++) {
-            const score = results[i]._distance != null ? 1 - results[i]._distance : 0.5;
-            vectorScores.set(i, score);
+            vectorScores.set(i, distanceToScore(results[i]._distance));
           }
 
           const bm25Scores = new Map();
@@ -711,7 +791,7 @@ class CodebaseIndexer {
 
           for (let i = 0; i < results.length; i++) {
             const key = `${results[i].file}:${results[i].chunk_index}`;
-            const vs = results[i]._distance != null ? 1 - results[i]._distance : 0.5;
+            const vs = distanceToScore(results[i]._distance);
             resultMap.set(key, { row: results[i], vectorScore: vs, bm25Score: 0 });
           }
 
@@ -812,7 +892,10 @@ class CodebaseIndexer {
 
         const outgoing = await this.graphDB.getOutgoing(result.chunk_id);
         const incoming = await this.graphDB.getIncoming(result.chunk_id);
-        const allEdges = [...outgoing, ...incoming];
+        // Filter out structural and meta edges — only relation edges are useful for context
+        const allEdges = [...outgoing, ...incoming].filter(
+          e => e.predicate !== "belongs_to" && e.predicate !== "graph_built" && !isStructuralPredicate(e.predicate)
+        );
 
         const neighbors = [];
         for (const edge of allEdges) {
@@ -833,8 +916,13 @@ class CodebaseIndexer {
           });
         }
 
+        // Apply min_relevance filter, then cap at max_related
         neighbors.sort((a, b) => b.score - a.score);
-        result.relatedContext = neighbors.slice(0, 3);
+        const minRelevance = GRAPH_CONFIG.min_relevance ?? 0.5;
+        const maxRelated = GRAPH_CONFIG.max_related ?? 4;
+        result.relatedContext = neighbors
+          .filter(n => n.score >= minRelevance)
+          .slice(0, maxRelated);
 
         // FR-060: Record provenance for each attached chunk
         if (this.usageTracker) {
@@ -875,7 +963,7 @@ class CodebaseIndexer {
       const table = await this.db.openTable(tableName);
       let rows;
       try {
-        rows = await table.filter("").limit(100000).execute();
+        rows = await table.filter("true").limit(100000).execute();
       } catch (e) {
         if (DEBUG) console.log("[vectorizer] Chunk cache build failed (corrupted table?):", e.message);
         return null;
@@ -1013,7 +1101,7 @@ class CodebaseIndexer {
         const tables = await this.db.tableNames();
         if (tables.includes(tableName)) {
           const table = await this.db.openTable(tableName);
-          const allRows = await table.filter("").limit(100000).execute();
+          const allRows = await table.filter("true").limit(100000).execute();
           const chunkData = allRows
             .filter(r => r.chunk_id && r.vector)
             .map(r => ({ chunk_id: r.chunk_id, vector: Array.from(r.vector), file: r.file }));

package/vectorizer.yaml

@@ -61,6 +61,22 @@ vectorizer:
   # Indexes to maintain - each has pattern (what to include) and ignore (what to skip)
   indexes:
 
+    # Source code index - all common programming languages
+    code:
+      enabled: true
+      pattern: "**/*.{js,ts,jsx,tsx,mjs,cjs,py,go,rs,java,kt,swift,c,cpp,h,hpp,cs,rb,php,scala,clj}"
+      ignore:
+        - "**/node_modules/**"
+        - "**/.git/**"
+        - "**/dist/**"
+        - "**/build/**"
+        - "**/.opencode/**"
+        - "**/docs/**"
+        - "**/vendor/**"
+        - "**/__pycache__/**"
+      hybrid: true
+      bm25_weight: 0.3
+
     # Documentation index - markdown, text files
     docs:
       enabled: true