@comfanion/usethis_search 3.0.0-dev.25 → 3.0.0-dev.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comfanion/usethis_search",
-  "version": "3.0.0-dev.25",
+  "version": "3.0.0-dev.27",
   "description": "OpenCode plugin: semantic search with graph-based context (v3: graph relations, 1-hop context, LSP + regex analyzers)",
   "type": "module",
   "main": "./index.ts",
@@ -48,8 +48,6 @@
     "@opencode-ai/plugin": ">=1.1.0",
     "@xenova/transformers": "^2.17.0",
     "glob": "^10.3.10",
-    "level": "^8.0.1",
-    "levelgraph": "^4.0.0",
     "vectordb": "^0.4.0"
   },
   "peerDependencies": {

package/vectorizer/graph-db.ts

@@ -1,5 +1,4 @@
-import levelgraph from "levelgraph"
-import { Level } from "level"
+import { Database } from "bun:sqlite"
 import { filePathFromNodeId, isStructuralPredicate } from "./graph-builder"
 
 export interface Triple {
@@ -13,98 +12,137 @@ export interface Triple {
 }
 
 export class GraphDB {
-  private db: any
+  private db: Database | null = null
   private initialized: boolean = false
 
+  // Prepared statements (cached for performance)
+  private _stmtInsert: any = null
+  private _stmtBySubject: any = null
+  private _stmtByObject: any = null
+  private _stmtByFile: any = null
+  private _stmtDeleteByFile: any = null
+  private _stmtBySubjectPredicate: any = null
+  private _stmtByPredicate: any = null
+  private _stmtAll: any = null
+
   constructor(private dbPath: string) {}
 
   async init(): Promise<this> {
-    const levelDb = new Level(this.dbPath)
-    this.db = levelgraph(levelDb)
+    // bun:sqlite uses a file path; append .db if not already
+    const fullPath = this.dbPath.endsWith(".db") ? this.dbPath : this.dbPath + ".db"
+    this.db = new Database(fullPath)
+
+    // WAL mode for concurrent readers
+    this.db.exec("PRAGMA journal_mode = WAL")
+    this.db.exec("PRAGMA synchronous = NORMAL")  // faster writes, safe with WAL
+    this.db.exec("PRAGMA cache_size = -2000")     // 2MB cache
+
+    // Create triples table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS triples (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        subject TEXT NOT NULL,
+        predicate TEXT NOT NULL,
+        object TEXT NOT NULL,
+        weight REAL NOT NULL DEFAULT 0,
+        source TEXT NOT NULL DEFAULT '',
+        file TEXT NOT NULL DEFAULT '',
+        line INTEGER
+      )
+    `)
+
+    // Indexes for fast lookups
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_subject ON triples(subject)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_object ON triples(object)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_file ON triples(file)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_predicate ON triples(predicate)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_subject_predicate ON triples(subject, predicate)")
+
+    // Prepare statements
+    this._stmtInsert = this.db.prepare(
+      "INSERT INTO triples (subject, predicate, object, weight, source, file, line) VALUES (?, ?, ?, ?, ?, ?, ?)"
+    )
+    this._stmtBySubject = this.db.prepare("SELECT * FROM triples WHERE subject = ?")
+    this._stmtByObject = this.db.prepare("SELECT * FROM triples WHERE object = ?")
+    this._stmtByFile = this.db.prepare("SELECT * FROM triples WHERE file = ?")
+    this._stmtDeleteByFile = this.db.prepare("DELETE FROM triples WHERE file = ?")
+    this._stmtBySubjectPredicate = this.db.prepare("SELECT * FROM triples WHERE subject = ? AND predicate = ?")
+    this._stmtByPredicate = this.db.prepare("SELECT * FROM triples WHERE predicate = ?")
+    this._stmtAll = this.db.prepare("SELECT * FROM triples")
+
     this.initialized = true
     return this
   }
 
+  private toTriple(row: any): Triple {
+    return {
+      subject: row.subject,
+      predicate: row.predicate,
+      object: row.object,
+      weight: row.weight,
+      source: row.source,
+      file: row.file,
+      line: row.line ?? undefined,
+    }
+  }
+
   async putEdges(triples: Triple[]): Promise<void> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    await new Promise<void>((resolve, reject) => {
-      this.db.put(triples, (err: Error | undefined) => {
-        if (err) reject(err)
-        else resolve()
-      })
+
+    // Batch insert in a single transaction — much faster than individual inserts
+    const insertMany = this.db.transaction((items: Triple[]) => {
+      for (const t of items) {
+        this._stmtInsert.run(t.subject, t.predicate, t.object, t.weight, t.source, t.file, t.line ?? null)
+      }
     })
+    insertMany(triples)
   }
 
   async getOutgoing(chunkId: string): Promise<Triple[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    return new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ subject: chunkId }, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
+    const rows = this._stmtBySubject.all(chunkId)
+    return rows.map((r: any) => this.toTriple(r))
   }
 
   async getIncoming(chunkId: string): Promise<Triple[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    return new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ object: chunkId }, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
+    const rows = this._stmtByObject.all(chunkId)
+    return rows.map((r: any) => this.toTriple(r))
   }
 
   async deleteByFile(filePath: string): Promise<void> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
-
-    const toDelete = allTriples.filter(t => t.file === filePath)
-
-    for (const t of toDelete) {
-      await new Promise<void>((resolve, reject) => {
-        this.db.del(t, (err: Error | undefined) => {
-          if (err) reject(err)
-          else resolve()
-        })
-      })
-    }
+    this._stmtDeleteByFile.run(filePath)
   }
 
   async close(): Promise<void> {
     if (this.initialized && this.db) {
-      await new Promise<void>((resolve, reject) => {
-        this.db.close((err: Error | undefined) => {
-          if (err) reject(err)
-          else resolve()
-        })
-      })
+      this.db.close()
+      this.db = null
+      this._stmtInsert = null
+      this._stmtBySubject = null
+      this._stmtByObject = null
+      this._stmtByFile = null
+      this._stmtDeleteByFile = null
+      this._stmtBySubjectPredicate = null
+      this._stmtByPredicate = null
+      this._stmtAll = null
       this.initialized = false
     }
   }
 
   // ---- FR-054: File metadata triples for incremental updates -----------------
 
-  /**
-   * Store graph build metadata for a file as a special triple.
-   * Subject: `meta:<filePath>`, Predicate: `graph_built`, Object: `<hash>`.
-   * Weight encodes the timestamp (seconds since epoch).
-   */
   async setFileMeta(filePath: string, hash: string, timestamp: number): Promise<void> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
     // Remove old meta triple for this file first
     await this.deleteFileMeta(filePath)
@@ -113,111 +151,61 @@ export class GraphDB {
       subject: `meta:${filePath}`,
       predicate: "graph_built",
       object: hash,
-      weight: Math.floor(timestamp / 1000), // seconds since epoch fits in weight
+      weight: Math.floor(timestamp / 1000),
       source: "meta",
       file: filePath,
     }
     await this.putEdges([triple])
   }
 
-  /**
-   * Get the stored graph build metadata for a file.
-   * Returns { hash, timestamp } or null if not found.
-   */
   async getFileMeta(filePath: string): Promise<{ hash: string; timestamp: number } | null> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const triples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get(
-        { subject: `meta:${filePath}`, predicate: "graph_built" },
-        (err: Error | undefined, result: Triple[]) => {
-          if (err) reject(err)
-          else resolve(result || [])
-        },
-      )
-    })
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    if (triples.length === 0) return null
+    const rows = this._stmtBySubjectPredicate.all(`meta:${filePath}`, "graph_built")
+    if (rows.length === 0) return null
     return {
-      hash: triples[0].object,
-      timestamp: triples[0].weight * 1000, // back to ms
+      hash: rows[0].object,
+      timestamp: rows[0].weight * 1000,
     }
   }
 
-  /**
-   * Delete file meta triple.
-   */
   async deleteFileMeta(filePath: string): Promise<void> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
     try {
-      const triples = await new Promise<Triple[]>((resolve, reject) => {
-        this.db.get(
-          { subject: `meta:${filePath}`, predicate: "graph_built" },
-          (err: Error | undefined, result: Triple[]) => {
-            if (err) reject(err)
-            else resolve(result || [])
-          },
-        )
-      })
-
-      for (const t of triples) {
-        await new Promise<void>((resolve, reject) => {
-          this.db.del(t, (err: Error | undefined) => {
-            if (err) reject(err)
-            else resolve()
-          })
-        })
-      }
-    } catch (err) {
-      // Silently ignore errors (e.g., no meta triple exists)
+      this.db!.prepare("DELETE FROM triples WHERE subject = ? AND predicate = ?")
+        .run(`meta:${filePath}`, "graph_built")
+    } catch {
+      // Silently ignore errors
     }
   }
 
-  /**
-   * Get all file metadata triples (for validation / stats).
-   */
   async getAllFileMeta(): Promise<Array<{ filePath: string; hash: string; timestamp: number }>> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const triples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ predicate: "graph_built" }, (err: Error | undefined, result: Triple[]) => {
-        if (err) reject(err)
-        else resolve(result || [])
-      })
-    })
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    return triples.map((t) => ({
-      filePath: t.subject.replace(/^meta:/, ""),
-      hash: t.object,
-      timestamp: t.weight * 1000,
+    const rows = this._stmtByPredicate.all("graph_built")
+    return rows.map((r: any) => ({
+      filePath: r.subject.replace(/^meta:/, ""),
+      hash: r.object,
+      timestamp: r.weight * 1000,
     }))
   }
 
-  /**
-   * Get all triples in the graph (for validation/stats).
-   * Excludes meta, anchor, and structural triples by default.
-   * Pass includeStructural=true to also get structural edges.
-   */
   async getAllTriples(includeStructural: boolean = false): Promise<Triple[]> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const allRows = this._stmtAll.all()
+    return allRows
+      .map((r: any) => this.toTriple(r))
+      .filter((t: Triple) => {
+        if (t.predicate === "graph_built" || t.predicate === "belongs_to") return false
+        if (!includeStructural && isStructuralPredicate(t.predicate)) return false
+        return true
       })
-    })
-
-    return allTriples.filter(t => {
-      if (t.predicate === "graph_built" || t.predicate === "belongs_to") return false
-      if (!includeStructural && isStructuralPredicate(t.predicate)) return false
-      return true
-    })
   }
 
   async getRelatedFiles(chunkId: string, maxDepth: number = 1): Promise<{path: string, relation: string, weight: number}[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
 
@@ -225,7 +213,6 @@ export class GraphDB {
     const visited = new Set<string>()
     const self = this
 
-    // Resolve the caller's file directly from the node ID
     const callerFile = filePathFromNodeId(chunkId)
 
     async function traverse(currentId: string, currentDepth: number, currentRelation: string) {
@@ -236,19 +223,12 @@ export class GraphDB {
       visited.add(currentId)
 
       try {
-        const outgoing = await new Promise<Triple[]>((resolve, reject) => {
-          self.db.get({ subject: currentId }, (err: Error | undefined, triples: Triple[]) => {
-            if (err) reject(err)
-            else resolve(triples || [])
-          })
-        })
+        const outgoing = self._stmtBySubject.all(currentId).map((r: any) => self.toTriple(r))
 
         for (const triple of outgoing) {
-          // Skip meta, anchor, and structural-only edges
           if (triple.predicate === "graph_built" || triple.predicate === "belongs_to") continue
           if (isStructuralPredicate(triple.predicate)) continue
 
-          // Resolve file for the target node directly from its ID
           const targetFile = filePathFromNodeId(triple.object)
           if (!targetFile) continue
 
@@ -267,12 +247,7 @@ export class GraphDB {
           }
         }
 
-        const incoming = await new Promise<Triple[]>((resolve, reject) => {
-          self.db.get({ object: currentId }, (err: Error | undefined, triples: Triple[]) => {
-            if (err) reject(err)
-            else resolve(triples || [])
-          })
-        })
+        const incoming = self._stmtByObject.all(currentId).map((r: any) => self.toTriple(r))
 
         for (const triple of incoming) {
           if (triple.predicate === "graph_built" || triple.predicate === "belongs_to") continue
@@ -298,7 +273,6 @@ export class GraphDB {
 
     await traverse(chunkId, 0, "")
 
-    // Remove the caller's own file from results
     if (callerFile) relatedFiles.delete(callerFile)
 
     return Array.from(relatedFiles.entries())

package/vectorizer/index.ts

@@ -612,7 +612,148 @@ class CodebaseIndexer {
     return this.hashes[relPath] !== currentHash;
   }
 
-  // ── Index a single file (v2: cleaning + semantic chunking + metadata) ─────
+  // ── Phase 1: Prepare file (chunk + graph, NO embedding) ─────────────────
+  // Returns prepared chunk data ready for embedding, or null if skipped.
+
+  async prepareFile(filePath) {
+    const relPath = path.relative(this.root, filePath);
+
+    let content;
+    try {
+      content = await fs.readFile(filePath, "utf8");
+    } catch {
+      return null;
+    }
+
+    const hash = this.fileHash(content);
+    if (this.hashes[relPath] === hash) {
+      return null; // unchanged
+    }
+
+    // Extract metadata
+    const fileMeta = await extractFileMetadata(filePath, content);
+    const archived = this.isArchived(relPath, content);
+
+    // Clean content before chunking
+    const cleaned = cleanContent(content, fileMeta.file_type, CLEANING_CONFIG);
+
+    // Semantic chunking
+    const chunks = chunkContent(cleaned, fileMeta.file_type, fileMeta.language, CHUNKING_CONFIG);
+
+    // Assign chunk IDs
+    const chunksWithIds = this.graphBuilder
+      ? this.graphBuilder.assignChunkIds(relPath, chunks)
+      : chunks.map((c, i) => ({ ...c, chunk_id: `chunk:${relPath}::_chunk_${i}` }));
+
+    // Build graph edges (Phase 1 — no embedding needed)
+    if (this.graphBuilder && this.graphDB) {
+      await this.graphDB.deleteByFile(relPath);
+      const edgesBuilt = await this.graphBuilder.buildEdges(relPath, content, chunksWithIds, fileMeta.file_type);
+
+      if (edgesBuilt > 0 || DEBUG) {
+        const timestamp = new Date().toISOString().slice(11, 19);
+        const logMsg = `${timestamp} Graph built: ${relPath} (${chunksWithIds.length} chunks)`;
+        if (DEBUG) console.log(`[vectorizer] ${logMsg}`);
+        try {
+          const logPath = path.join(this.root, ".opencode", "indexer.log");
+          const fsSync = await import("fs");
+          fsSync.appendFileSync(logPath, `${logMsg}\n`);
+        } catch { /* non-fatal */ }
+      }
+
+      try {
+        await this.graphDB.setFileMeta(relPath, hash, Date.now());
+      } catch { /* non-fatal */ }
+    }
+
+    // Return prepared rows (without vector — Phase 2 fills it)
+    const rows = chunksWithIds.map((chunk, i) => ({
+      chunk_id: chunk.chunk_id,
+      file: relPath,
+      chunk_index: i,
+      content: chunk.content,
+      archived,
+      file_type: fileMeta.file_type,
+      language: fileMeta.language,
+      last_modified: fileMeta.last_modified,
+      file_size: fileMeta.file_size,
+      heading_context: chunk.heading_context || "",
+      function_name: chunk.function_name || "",
+      class_name: chunk.class_name || "",
+      tags: (fileMeta.tags || []).join(","),
+      start_line: chunk.start_line ?? -1,
+      end_line: chunk.end_line ?? -1,
+    }));
+
+    return { relPath, hash, rows };
+  }
+
+  // ── Phase 2: Batch embed + store ──────────────────────────────────────────
+  // Takes prepared rows from prepareFile(), embeds in batches, stores in LanceDB.
+
+  async embedAndStore(preparedFiles, batchSize = 32, onProgress = null) {
+    if (preparedFiles.length === 0) return 0;
+
+    // Collect all rows with their content for batch embedding
+    const allRows = [];
+    for (const pf of preparedFiles) {
+      for (const row of pf.rows) {
+        allRows.push(row);
+      }
+    }
+
+    if (allRows.length === 0) return 0;
+
+    // Load model once
+    const model = await this.loadModel();
+
+    // Batch embed
+    const allData = [];
+    for (let i = 0; i < allRows.length; i += batchSize) {
+      const batch = allRows.slice(i, i + batchSize);
+      const texts = batch.map(r => r.content);
+
+      // Embed batch — @xenova/transformers processes array inputs efficiently
+      const embeddings = [];
+      for (const text of texts) {
+        const result = await model(text, { pooling: "mean", normalize: true });
+        embeddings.push(Array.from(result.data));
+      }
+
+      for (let j = 0; j < batch.length; j++) {
+        allData.push({ ...batch[j], vector: embeddings[j] });
+      }
+
+      if (onProgress) {
+        onProgress(Math.min(i + batchSize, allRows.length), allRows.length, "embedding");
+      }
+    }
+
+    // Bulk store in LanceDB
+    const tableName = "chunks";
+    const tables = await this.db.tableNames();
+    if (tables.includes(tableName)) {
+      const table = await this.db.openTable(tableName);
+      await table.add(allData);
+    } else {
+      await this.db.createTable(tableName, allData);
+    }
+
+    // Update hashes for all prepared files
+    for (const pf of preparedFiles) {
+      this.hashes[pf.relPath] = pf.hash;
+    }
+    await this.saveHashes();
+
+    // Invalidate caches
+    if (this.bm25) { this.bm25.clear(); this.bm25 = null; }
+    this._bm25Rows = null;
+    this._chunkCache = null;
+
+    return allData.length;
+  }
+
+  // ── Index a single file (legacy — used by freshen/on-change) ───────────
 
   async indexFile(filePath) {
     const relPath = path.relative(this.root, filePath);
@@ -1170,31 +1311,64 @@ class CodebaseIndexer {
       }
     }
 
-    let indexed = 0;
-    let skipped = 0;
     const total = files.length;
+    const CONCURRENCY = 5;
 
-    for (let i = 0; i < files.length; i++) {
-      const relPath = files[i];
-      const filePath = path.join(this.root, relPath);
-      try {
-        const wasIndexed = await this.indexFile(filePath);
-        if (wasIndexed) {
-          indexed++;
-          // FR-053: progress indicator includes graph building phase
-          if (onProgress) onProgress(indexed, total, relPath, i + 1);
+    // ══════════════════════════════════════════════════════════════════════════
+    // Phase 1: Prepare files in parallel (chunk + graph, no embedding)
+    // ══════════════════════════════════════════════════════════════════════════
+    const preparedFiles = [];
+    let prepared = 0;
+    let skipped = 0;
+
+    // Process in batches of CONCURRENCY
+    for (let i = 0; i < files.length; i += CONCURRENCY) {
+      const batch = files.slice(i, i + CONCURRENCY);
+      const promises = batch.map(async (relPath) => {
+        const filePath = path.join(this.root, relPath);
+        try {
+          const result = await this.prepareFile(filePath);
+          return result;
+        } catch {
+          return null;
+        }
+      });
+
+      const results = await Promise.all(promises);
+      for (let j = 0; j < results.length; j++) {
+        if (results[j]) {
+          preparedFiles.push(results[j]);
+          prepared++;
+          if (onProgress) onProgress(prepared, total, results[j].relPath, i + j + 1, "prepare");
         } else {
           skipped++;
         }
-      } catch {
-        skipped++;
       }
     }
 
+    if (DEBUG) console.log(`[vectorizer] Phase 1 done: ${prepared} files prepared, ${skipped} skipped`);
+
+    // ══════════════════════════════════════════════════════════════════════════
+    // Phase 2: Batch embed + store (sequential, batch forward pass)
+    // ══════════════════════════════════════════════════════════════════════════
+    let chunksEmbedded = 0;
+    if (preparedFiles.length > 0) {
+      const totalChunks = preparedFiles.reduce((sum, pf) => sum + pf.rows.length, 0);
+      if (DEBUG) console.log(`[vectorizer] Phase 2: embedding ${totalChunks} chunks from ${preparedFiles.length} files`);
+
+      chunksEmbedded = await this.embedAndStore(preparedFiles, 32, (done, embedTotal, phase) => {
+        if (onProgress) onProgress(done, embedTotal, `embedding`, done, "embed");
+      });
+
+      if (DEBUG) console.log(`[vectorizer] Phase 2 done: ${chunksEmbedded} chunks embedded and stored`);
+    }
+
+    const indexed = prepared; // file count for backward compat
+
     // FR-005: Build semantic similarity edges as post-pass
     // Disabled by default (O(n²) — slow on large repos). Enable via graph.semantic_edges: true
     let semanticEdges = 0;
-    if (indexed > 0 && this.graphBuilder && this.graphDB && GRAPH_CONFIG.semantic_edges) {
+    if (chunksEmbedded > 0 && this.graphBuilder && this.graphDB && GRAPH_CONFIG.semantic_edges) {
       try {
         const tableName = "chunks";
         const tables = await this.db.tableNames();