@comfanion/usethis_search 0.2.0-dev.0 → 3.0.0-dev.1

package/vectorizer/graph-db.ts

@@ -0,0 +1,289 @@
+import levelgraph from "levelgraph"
+import { Level } from "level"
+
+export interface Triple {
+  subject: string
+  predicate: string
+  object: string
+  weight: number
+  source: string
+  file: string
+  line?: number
+}
+
+export class GraphDB {
+  private db: any
+  private initialized: boolean = false
+
+  constructor(private dbPath: string) {}
+
+  async init(): Promise<this> {
+    const levelDb = new Level(this.dbPath)
+    this.db = levelgraph(levelDb)
+    this.initialized = true
+    return this
+  }
+
+  async putEdges(triples: Triple[]): Promise<void> {
+    if (!this.initialized) {
+      throw new Error("GraphDB not initialized. Call init() first.")
+    }
+    await new Promise<void>((resolve, reject) => {
+      this.db.put(triples, (err: Error | undefined) => {
+        if (err) reject(err)
+        else resolve()
+      })
+    })
+  }
+
+  async getOutgoing(chunkId: string): Promise<Triple[]> {
+    if (!this.initialized) {
+      throw new Error("GraphDB not initialized. Call init() first.")
+    }
+    return new Promise<Triple[]>((resolve, reject) => {
+      this.db.get({ subject: chunkId }, (err: Error | undefined, triples: Triple[]) => {
+        if (err) reject(err)
+        else resolve(triples || [])
+      })
+    })
+  }
+
+  async getIncoming(chunkId: string): Promise<Triple[]> {
+    if (!this.initialized) {
+      throw new Error("GraphDB not initialized. Call init() first.")
+    }
+    return new Promise<Triple[]>((resolve, reject) => {
+      this.db.get({ object: chunkId }, (err: Error | undefined, triples: Triple[]) => {
+        if (err) reject(err)
+        else resolve(triples || [])
+      })
+    })
+  }
+
+  async deleteByFile(filePath: string): Promise<void> {
+    if (!this.initialized) {
+      throw new Error("GraphDB not initialized. Call init() first.")
+    }
+    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
+        if (err) reject(err)
+        else resolve(triples || [])
+      })
+    })
+
+    const toDelete = allTriples.filter(t => t.file === filePath)
+
+    for (const t of toDelete) {
+      await new Promise<void>((resolve, reject) => {
+        this.db.del(t, (err: Error | undefined) => {
+          if (err) reject(err)
+          else resolve()
+        })
+      })
+    }
+  }
+
+  async close(): Promise<void> {
+    if (this.initialized && this.db) {
+      await new Promise<void>((resolve, reject) => {
+        this.db.close((err: Error | undefined) => {
+          if (err) reject(err)
+          else resolve()
+        })
+      })
+      this.initialized = false
+    }
+  }
+
+  // ---- FR-054: File metadata triples for incremental updates -----------------
+
+  /**
+   * Store graph build metadata for a file as a special triple.
+   * Subject: `meta:<filePath>`, Predicate: `graph_built`, Object: `<hash>`.
+   * Weight encodes the timestamp (seconds since epoch).
+   */
+  async setFileMeta(filePath: string, hash: string, timestamp: number): Promise<void> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    // Remove old meta triple for this file first
+    await this.deleteFileMeta(filePath)
+
+    const triple: Triple = {
+      subject: `meta:${filePath}`,
+      predicate: "graph_built",
+      object: hash,
+      weight: Math.floor(timestamp / 1000), // seconds since epoch fits in weight
+      source: "meta",
+      file: filePath,
+    }
+    await this.putEdges([triple])
+  }
+
+  /**
+   * Get the stored graph build metadata for a file.
+   * Returns { hash, timestamp } or null if not found.
+   */
+  async getFileMeta(filePath: string): Promise<{ hash: string; timestamp: number } | null> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const triples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get(
+        { subject: `meta:${filePath}`, predicate: "graph_built" },
+        (err: Error | undefined, result: Triple[]) => {
+          if (err) reject(err)
+          else resolve(result || [])
+        },
+      )
+    })
+
+    if (triples.length === 0) return null
+    return {
+      hash: triples[0].object,
+      timestamp: triples[0].weight * 1000, // back to ms
+    }
+  }
+
+  /**
+   * Delete file meta triple.
+   */
+  async deleteFileMeta(filePath: string): Promise<void> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const triples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get(
+        { subject: `meta:${filePath}`, predicate: "graph_built" },
+        (err: Error | undefined, result: Triple[]) => {
+          if (err) reject(err)
+          else resolve(result || [])
+        },
+      )
+    })
+
+    for (const t of triples) {
+      await new Promise<void>((resolve, reject) => {
+        this.db.del(t, (err: Error | undefined) => {
+          if (err) reject(err)
+          else resolve()
+        })
+      })
+    }
+  }
+
+  /**
+   * Get all file metadata triples (for validation / stats).
+   */
+  async getAllFileMeta(): Promise<Array<{ filePath: string; hash: string; timestamp: number }>> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const triples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get({ predicate: "graph_built" }, (err: Error | undefined, result: Triple[]) => {
+        if (err) reject(err)
+        else resolve(result || [])
+      })
+    })
+
+    return triples.map((t) => ({
+      filePath: t.subject.replace(/^meta:/, ""),
+      hash: t.object,
+      timestamp: t.weight * 1000,
+    }))
+  }
+
+  /**
+   * Get all triples in the graph (for validation/stats).
+   * Excludes meta triples (predicate === "graph_built").
+   */
+  async getAllTriples(): Promise<Triple[]> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
+        if (err) reject(err)
+        else resolve(triples || [])
+      })
+    })
+
+    return allTriples.filter(t => t.predicate !== "graph_built")
+  }
+
+  async getRelatedFiles(chunkId: string, maxDepth: number = 1): Promise<{path: string, relation: string, weight: number}[]> {
+    if (!this.initialized) {
+      throw new Error("GraphDB not initialized. Call init() first.")
+    }
+
+    const relatedFiles: Map<string, {relation: string, weight: number}> = new Map()
+    const visited = new Set<string>()
+    
+    async function traverse(currentId: string, currentDepth: number, currentRelation: string) {
+      if (currentDepth > maxDepth || visited.has(currentId)) {
+        return
+      }
+      
+      visited.add(currentId)
+      
+      try {
+        const outgoing = await new Promise<Triple[]>((resolve, reject) => {
+          this.db.get({ subject: currentId }, (err: Error | undefined, triples: Triple[]) => {
+            if (err) reject(err)
+            else resolve(triples || [])
+          })
+        })
+        
+        for (const triple of outgoing) {
+          const fileId = triple.object
+          
+          // Aggregate relations and weights
+          const existing = relatedFiles.get(fileId)
+          if (existing) {
+            existing.weight = Math.max(existing.weight, triple.weight)
+          } else {
+            relatedFiles.set(fileId, {
+              relation: currentRelation || triple.predicate,
+              weight: triple.weight
+            })
+          }
+          
+          // Recurse for imports/extends relations
+          if (triple.predicate === "imports" || triple.predicate === "extends") {
+            await traverse(fileId, currentDepth + 1, triple.predicate)
+          }
+        }
+        
+        const incoming = await new Promise<Triple[]>((resolve, reject) => {
+          this.db.get({ object: currentId }, (err: Error | undefined, triples: Triple[]) => {
+            if (err) reject(err)
+            else resolve(triples || [])
+          })
+        })
+        
+        for (const triple of incoming) {
+          const fileId = triple.subject
+          
+          const existing = relatedFiles.get(fileId)
+          if (existing) {
+            existing.weight = Math.max(existing.weight, triple.weight)
+          } else {
+            relatedFiles.set(fileId, {
+              relation: `used_by`,
+              weight: triple.weight
+            })
+          }
+        }
+      } catch (error) {
+        console.error(`Error traversing graph for ${currentId}:`, error)
+      }
+    }
+    
+    await traverse(chunkId, 0, "")
+    
+    const result = Array.from(relatedFiles.entries())
+      .map(([path, data]) => ({
+        path,
+        relation: data.relation,
+        weight: data.weight
+      }))
+      .sort((a, b) => b.weight - a.weight)
+    
+    return result
+  }
+}

package/vectorizer/index.js

@@ -15,6 +15,9 @@ import { BM25Index } from "./bm25-index.ts";
 import { mergeResults, DEFAULT_HYBRID_CONFIG } from "./hybrid-search.ts";
 import { QueryCache, DEFAULT_CACHE_CONFIG } from "./query-cache.ts";
 import { SearchMetrics } from "./search-metrics.ts";
+import { GraphDB } from "./graph-db.ts";
+import { GraphBuilder } from "./graph-builder.ts";
+import { UsageTracker } from "./usage-tracker.ts";
 
 // Suppress transformers.js logs unless DEBUG is set
 const DEBUG = process.env.DEBUG?.includes("vectorizer") || process.env.DEBUG === "*";
@@ -350,6 +353,10 @@ class CodebaseIndexer {
     this.configLoaded = false;
     this.bm25 = null;          // lazy-built BM25 index
     this.metrics = null;       // lazy-loaded SearchMetrics
+    this.graphDB = null;       // Graph DB for relationships
+    this.graphBuilder = null;   // Graph builder orchestrator
+    this._chunkCache = null;   // Lazy Map<chunk_id, row> for findChunkById
+    this.usageTracker = null;  // Usage tracking & provenance (v3)
   }
 
   async init() {
@@ -360,6 +367,17 @@ class CodebaseIndexer {
     await fs.mkdir(this.cacheDir, { recursive: true });
     this.db = await lancedb.connect(path.join(this.cacheDir, "lancedb"));
     await this.loadHashes();
+
+    const graphType = this.indexName === "docs" ? "doc_graph" : "code_graph";
+    const graphPath = path.join(this.root, ".opencode", "graph", graphType);
+    await fs.mkdir(path.dirname(graphPath), { recursive: true });
+    this.graphDB = await new GraphDB(graphPath).init();
+    this.graphBuilder = new GraphBuilder(this.graphDB, this.root);
+
+    // Usage tracker — provenance & usage stats
+    this.usageTracker = new UsageTracker(this.cacheDir);
+    await this.usageTracker.load();
+
     return this;
   }
 
@@ -388,6 +406,18 @@ class CodebaseIndexer {
     }
     this._bm25Rows = null;
     this.metrics = null;
+    // Close graph DB to release LevelDB lock
+    if (this.graphDB) {
+      try { await this.graphDB.close(); } catch { /* best effort */ }
+      this.graphDB = null;
+      this.graphBuilder = null;
+    }
+    // Save & release usage tracker
+    if (this.usageTracker) {
+      try { await this.usageTracker.save(); } catch { /* best effort */ }
+      this.usageTracker = null;
+    }
+    this._chunkCache = null;
     clearQueryCache();
     if (global.gc) global.gc();
   }
@@ -503,13 +533,28 @@ class CodebaseIndexer {
     // Semantic chunking
     const chunks = chunkContent(cleaned, fileMeta.file_type, fileMeta.language, CHUNKING_CONFIG);
 
+    // v3: Assign chunk IDs for graph tracking
+    const chunksWithIds = this.graphBuilder.assignChunkIds(relPath, chunks);
+
+    // v3: Delete old edges for this file and build new ones
+    await this.graphDB.deleteByFile(relPath);
+    await this.graphBuilder.buildEdges(relPath, content, chunksWithIds, fileMeta.file_type);
+
+    // FR-054: Store graph build timestamp + file hash as metadata triple
+    try {
+      await this.graphDB.setFileMeta(relPath, hash, Date.now());
+    } catch {
+      // non-fatal — metadata is advisory
+    }
+
     const data = [];
-    for (let i = 0; i < chunks.length; i++) {
-      const embedding = await this.embed(chunks[i].content);
+    for (let i = 0; i < chunksWithIds.length; i++) {
+      const embedding = await this.embed(chunksWithIds[i].content);
       data.push({
+        chunk_id: chunksWithIds[i].chunk_id,
         file: relPath,
         chunk_index: i,
-        content: chunks[i].content,
+        content: chunksWithIds[i].content,
         vector: embedding,
         archived: archived,
         // v2 metadata
@@ -517,9 +562,9 @@ class CodebaseIndexer {
         language: fileMeta.language,
         last_modified: fileMeta.last_modified,
         file_size: fileMeta.file_size,
-        heading_context: chunks[i].heading_context || "",
-        function_name: chunks[i].function_name || "",
-        class_name: chunks[i].class_name || "",
+        heading_context: chunksWithIds[i].heading_context || "",
+        function_name: chunksWithIds[i].function_name || "",
+        class_name: chunksWithIds[i].class_name || "",
         tags: (fileMeta.tags || []).join(","),
       });
     }
@@ -720,9 +765,97 @@ class CodebaseIndexer {
       }
     }
 
+    // ── Graph context expansion (v3) ───────────────────────────────────────
+    if (this.graphDB) {
+      for (const result of finalResults) {
+        if (!result.chunk_id) continue;
+
+        const outgoing = await this.graphDB.getOutgoing(result.chunk_id);
+        const incoming = await this.graphDB.getIncoming(result.chunk_id);
+        const allEdges = [...outgoing, ...incoming];
+
+        const neighbors = [];
+        for (const edge of allEdges) {
+          const neighborId = edge.subject === result.chunk_id ? edge.object : edge.subject;
+          const neighborChunk = await this.findChunkById(neighborId);
+          if (!neighborChunk) continue;
+
+          const similarity = this.cosineSimilarity(neighborChunk.vector, queryEmbedding);
+          const score = edge.weight * similarity;
+
+          neighbors.push({
+            chunk_id: neighborId,
+            file: neighborChunk.file,
+            content: neighborChunk.content,
+            relation: edge.predicate,
+            score,
+            via: edge.source
+          });
+        }
+
+        neighbors.sort((a, b) => b.score - a.score);
+        result.relatedContext = neighbors.slice(0, 3);
+
+        // FR-060: Record provenance for each attached chunk
+        if (this.usageTracker) {
+          for (const n of result.relatedContext) {
+            this.usageTracker.recordProvenance(query, result.chunk_id, n.chunk_id, n.relation);
+          }
+        }
+      }
+    }
+
+    // FR-061: Record usage counts for all returned chunks (main + attached)
+    if (this.usageTracker) {
+      const allChunkIds = [];
+      for (const r of finalResults) {
+        if (r.chunk_id) allChunkIds.push(r.chunk_id);
+        if (r.relatedContext) {
+          for (const rc of r.relatedContext) {
+            if (rc.chunk_id) allChunkIds.push(rc.chunk_id);
+          }
+        }
+      }
+      this.usageTracker.recordSearchResults(allChunkIds);
+      // Save asynchronously (non-blocking)
+      this.usageTracker.save().catch(() => {});
+    }
+
     return finalResults;
   }
 
+  async findChunkById(chunkId) {
+    // Lazy-build an in-memory Map keyed by chunk_id on first call.
+    // The cache lives until unloadModel() clears it.
+    if (!this._chunkCache) {
+      const tableName = "chunks";
+      const tables = await this.db.tableNames();
+      if (!tables.includes(tableName)) return null;
+
+      const table = await this.db.openTable(tableName);
+      const rows = await table.search([0]).limit(100000).execute();
+      this._chunkCache = new Map();
+      for (const row of rows) {
+        if (row.chunk_id) {
+          this._chunkCache.set(row.chunk_id, row);
+        }
+      }
+    }
+    return this._chunkCache.get(chunkId) || null;
+  }
+
+  cosineSimilarity(vecA, vecB) {
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < vecA.length; i++) {
+      dotProduct += vecA[i] * vecB[i];
+      normA += vecA[i] * vecA[i];
+      normB += vecB[i] * vecB[i];
+    }
+    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+  }
+
   async checkHealth(extraIgnore = []) {
     const { glob } = await import("glob");
     const preset = INDEX_PRESETS[this.indexName] || DEFAULT_PRESETS.code;
@@ -806,14 +939,17 @@ class CodebaseIndexer {
 
     let indexed = 0;
     let skipped = 0;
+    const total = files.length;
 
-    for (const relPath of files) {
+    for (let i = 0; i < files.length; i++) {
+      const relPath = files[i];
       const filePath = path.join(this.root, relPath);
       try {
         const wasIndexed = await this.indexFile(filePath);
         if (wasIndexed) {
           indexed++;
-          if (onProgress) onProgress(indexed, files.length, relPath);
+          // FR-053: progress indicator includes graph building phase
+          if (onProgress) onProgress(indexed, total, relPath, i + 1);
         } else {
           skipped++;
         }
@@ -822,7 +958,29 @@ class CodebaseIndexer {
       }
     }
 
-    return { indexed, skipped, total: files.length };
+    // FR-005: Build semantic similarity edges as post-pass
+    // Only if we actually indexed new files and have a graph builder
+    let semanticEdges = 0;
+    if (indexed > 0 && this.graphBuilder && this.graphDB) {
+      try {
+        const tableName = "chunks";
+        const tables = await this.db.tableNames();
+        if (tables.includes(tableName)) {
+          const table = await this.db.openTable(tableName);
+          const allRows = await table.search([0]).limit(100000).execute();
+          const chunkData = allRows
+            .filter(r => r.chunk_id && r.vector)
+            .map(r => ({ chunk_id: r.chunk_id, vector: Array.from(r.vector), file: r.file }));
+          semanticEdges = await this.graphBuilder.buildSemanticEdges(chunkData, 0.8, 3);
+          if (DEBUG) console.log(`[vectorizer] Built ${semanticEdges} semantic similarity edges`);
+        }
+      } catch (e) {
+        if (DEBUG) console.log(`[vectorizer] Semantic edge building failed:`, e.message);
+        // non-fatal — explicit edges still work
+      }
+    }
+
+    return { indexed, skipped, total, semanticEdges };
   }
 
   async indexSingleFile(filePath) {