@comfanion/usethis_search 3.0.0-dev.24 → 3.0.0-dev.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comfanion/usethis_search",
-  "version": "3.0.0-dev.24",
+  "version": "3.0.0-dev.26",
   "description": "OpenCode plugin: semantic search with graph-based context (v3: graph relations, 1-hop context, LSP + regex analyzers)",
   "type": "module",
   "main": "./index.ts",
@@ -48,8 +48,6 @@
     "@opencode-ai/plugin": ">=1.1.0",
     "@xenova/transformers": "^2.17.0",
     "glob": "^10.3.10",
-    "level": "^8.0.1",
-    "levelgraph": "^4.0.0",
     "vectordb": "^0.4.0"
   },
   "peerDependencies": {

package/tools/search.ts

@@ -10,7 +10,7 @@ import { tool } from "@opencode-ai/plugin"
 import path from "path"
 import fs from "fs/promises"
 
-import { CodebaseIndexer, getSearchConfig } from "../vectorizer/index.ts"
+import { CodebaseIndexer, getSearchConfig, getIndexer, releaseIndexer } from "../vectorizer/index.ts"
 
 // ── Extension → language mapping (for filter parsing) ─────────────────────
 const EXT_TO_LANG: Record<string, string> = {
@@ -133,9 +133,9 @@ Examples:
       // Freshen from config (default: false — auto_index handles it)
       if (cfg.freshen) {
         try {
-          const tempIndexer = await new CodebaseIndexer(projectRoot, indexName).init()
-          await tempIndexer.freshen()
-          await tempIndexer.unloadModel()
+          const indexer = await getIndexer(projectRoot, indexName)
+          await indexer.freshen()
+          releaseIndexer(projectRoot, indexName)
         } catch {
           // non-fatal — search can proceed without freshen
         }
@@ -144,19 +144,22 @@ Examples:
       let allResults: any[] = []
 
       if (args.searchAll) {
-        const tempIndexer = await new CodebaseIndexer(projectRoot, "code").init()
+        const tempIndexer = await getIndexer(projectRoot, "code")
         const indexes = await tempIndexer.listIndexes()
-        await tempIndexer.unloadModel()
+        releaseIndexer(projectRoot, "code")
 
         if (indexes.length === 0) {
           return `No indexes found. The codebase needs to be indexed first.\n\nRun the CLI: bunx usethis_search reindex`
         }
 
         for (const idx of indexes) {
-          const indexer = await new CodebaseIndexer(projectRoot, idx).init()
-          const results = await indexer.search(args.query, limit, includeArchived, searchOptions)
-          allResults.push(...results.map((r: any) => ({ ...r, _index: idx })))
-          await indexer.unloadModel()
+          const indexer = await getIndexer(projectRoot, idx)
+          try {
+            const results = await indexer.search(args.query, limit, includeArchived, searchOptions)
+            allResults.push(...results.map((r: any) => ({ ...r, _index: idx })))
+          } finally {
+            releaseIndexer(projectRoot, idx)
+          }
         }
 
         allResults.sort((a, b) => {
@@ -171,9 +174,9 @@ Examples:
           await fs.access(hashesFile)
         } catch {
           // Index doesn't exist — check what indexes ARE available
-          const tempIndexer = await new CodebaseIndexer(projectRoot, "code").init()
+          const tempIndexer = await getIndexer(projectRoot, "code")
           const available = await tempIndexer.listIndexes()
-          await tempIndexer.unloadModel()
+          releaseIndexer(projectRoot, "code")
 
           if (available.length > 0) {
             const list = available.map(i => `"${i}"`).join(", ")
@@ -182,10 +185,13 @@ Examples:
           return `No indexes found. The codebase needs to be indexed first.\n\nRun the CLI: bunx usethis_search reindex`
         }
 
-        const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
-        const results = await indexer.search(args.query, limit, includeArchived, searchOptions)
-        allResults = results.map((r: any) => ({ ...r, _index: indexName }))
-        await indexer.unloadModel()
+        const indexer = await getIndexer(projectRoot, indexName)
+        try {
+          const results = await indexer.search(args.query, limit, includeArchived, searchOptions)
+          allResults = results.map((r: any) => ({ ...r, _index: indexName }))
+        } finally {
+          releaseIndexer(projectRoot, indexName)
+        }
       }
 
       // ── Score cutoff — drop low-relevance results ──────────────────────────

package/vectorizer/graph-db.ts

@@ -1,5 +1,4 @@
-import levelgraph from "levelgraph"
-import { Level } from "level"
+import { Database } from "bun:sqlite"
 import { filePathFromNodeId, isStructuralPredicate } from "./graph-builder"
 
 export interface Triple {
@@ -13,98 +12,137 @@ export interface Triple {
 }
 
 export class GraphDB {
-  private db: any
+  private db: Database | null = null
   private initialized: boolean = false
 
+  // Prepared statements (cached for performance)
+  private _stmtInsert: any = null
+  private _stmtBySubject: any = null
+  private _stmtByObject: any = null
+  private _stmtByFile: any = null
+  private _stmtDeleteByFile: any = null
+  private _stmtBySubjectPredicate: any = null
+  private _stmtByPredicate: any = null
+  private _stmtAll: any = null
+
   constructor(private dbPath: string) {}
 
   async init(): Promise<this> {
-    const levelDb = new Level(this.dbPath)
-    this.db = levelgraph(levelDb)
+    // bun:sqlite uses a file path; append .db if not already
+    const fullPath = this.dbPath.endsWith(".db") ? this.dbPath : this.dbPath + ".db"
+    this.db = new Database(fullPath)
+
+    // WAL mode for concurrent readers
+    this.db.exec("PRAGMA journal_mode = WAL")
+    this.db.exec("PRAGMA synchronous = NORMAL")  // faster writes, safe with WAL
+    this.db.exec("PRAGMA cache_size = -2000")     // 2MB cache
+
+    // Create triples table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS triples (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        subject TEXT NOT NULL,
+        predicate TEXT NOT NULL,
+        object TEXT NOT NULL,
+        weight REAL NOT NULL DEFAULT 0,
+        source TEXT NOT NULL DEFAULT '',
+        file TEXT NOT NULL DEFAULT '',
+        line INTEGER
+      )
+    `)
+
+    // Indexes for fast lookups
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_subject ON triples(subject)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_object ON triples(object)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_file ON triples(file)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_predicate ON triples(predicate)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_subject_predicate ON triples(subject, predicate)")
+
+    // Prepare statements
+    this._stmtInsert = this.db.prepare(
+      "INSERT INTO triples (subject, predicate, object, weight, source, file, line) VALUES (?, ?, ?, ?, ?, ?, ?)"
+    )
+    this._stmtBySubject = this.db.prepare("SELECT * FROM triples WHERE subject = ?")
+    this._stmtByObject = this.db.prepare("SELECT * FROM triples WHERE object = ?")
+    this._stmtByFile = this.db.prepare("SELECT * FROM triples WHERE file = ?")
+    this._stmtDeleteByFile = this.db.prepare("DELETE FROM triples WHERE file = ?")
+    this._stmtBySubjectPredicate = this.db.prepare("SELECT * FROM triples WHERE subject = ? AND predicate = ?")
+    this._stmtByPredicate = this.db.prepare("SELECT * FROM triples WHERE predicate = ?")
+    this._stmtAll = this.db.prepare("SELECT * FROM triples")
+
     this.initialized = true
     return this
   }
 
+  private toTriple(row: any): Triple {
+    return {
+      subject: row.subject,
+      predicate: row.predicate,
+      object: row.object,
+      weight: row.weight,
+      source: row.source,
+      file: row.file,
+      line: row.line ?? undefined,
+    }
+  }
+
   async putEdges(triples: Triple[]): Promise<void> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    await new Promise<void>((resolve, reject) => {
-      this.db.put(triples, (err: Error | undefined) => {
-        if (err) reject(err)
-        else resolve()
-      })
+
+    // Batch insert in a single transaction — much faster than individual inserts
+    const insertMany = this.db.transaction((items: Triple[]) => {
+      for (const t of items) {
+        this._stmtInsert.run(t.subject, t.predicate, t.object, t.weight, t.source, t.file, t.line ?? null)
+      }
     })
+    insertMany(triples)
   }
 
   async getOutgoing(chunkId: string): Promise<Triple[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    return new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ subject: chunkId }, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
+    const rows = this._stmtBySubject.all(chunkId)
+    return rows.map((r: any) => this.toTriple(r))
   }
 
   async getIncoming(chunkId: string): Promise<Triple[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    return new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ object: chunkId }, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
+    const rows = this._stmtByObject.all(chunkId)
+    return rows.map((r: any) => this.toTriple(r))
   }
 
   async deleteByFile(filePath: string): Promise<void> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
-
-    const toDelete = allTriples.filter(t => t.file === filePath)
-
-    for (const t of toDelete) {
-      await new Promise<void>((resolve, reject) => {
-        this.db.del(t, (err: Error | undefined) => {
-          if (err) reject(err)
-          else resolve()
-        })
-      })
-    }
+    this._stmtDeleteByFile.run(filePath)
   }
 
   async close(): Promise<void> {
     if (this.initialized && this.db) {
-      await new Promise<void>((resolve, reject) => {
-        this.db.close((err: Error | undefined) => {
-          if (err) reject(err)
-          else resolve()
-        })
-      })
+      this.db.close()
+      this.db = null
+      this._stmtInsert = null
+      this._stmtBySubject = null
+      this._stmtByObject = null
+      this._stmtByFile = null
+      this._stmtDeleteByFile = null
+      this._stmtBySubjectPredicate = null
+      this._stmtByPredicate = null
+      this._stmtAll = null
       this.initialized = false
     }
   }
 
   // ---- FR-054: File metadata triples for incremental updates -----------------
 
-  /**
-   * Store graph build metadata for a file as a special triple.
-   * Subject: `meta:<filePath>`, Predicate: `graph_built`, Object: `<hash>`.
-   * Weight encodes the timestamp (seconds since epoch).
-   */
   async setFileMeta(filePath: string, hash: string, timestamp: number): Promise<void> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
     // Remove old meta triple for this file first
     await this.deleteFileMeta(filePath)
@@ -113,111 +151,61 @@ export class GraphDB {
       subject: `meta:${filePath}`,
       predicate: "graph_built",
       object: hash,
-      weight: Math.floor(timestamp / 1000), // seconds since epoch fits in weight
+      weight: Math.floor(timestamp / 1000),
       source: "meta",
       file: filePath,
     }
     await this.putEdges([triple])
   }
 
-  /**
-   * Get the stored graph build metadata for a file.
-   * Returns { hash, timestamp } or null if not found.
-   */
   async getFileMeta(filePath: string): Promise<{ hash: string; timestamp: number } | null> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const triples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get(
-        { subject: `meta:${filePath}`, predicate: "graph_built" },
-        (err: Error | undefined, result: Triple[]) => {
-          if (err) reject(err)
-          else resolve(result || [])
-        },
-      )
-    })
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    if (triples.length === 0) return null
+    const rows = this._stmtBySubjectPredicate.all(`meta:${filePath}`, "graph_built")
+    if (rows.length === 0) return null
     return {
-      hash: triples[0].object,
-      timestamp: triples[0].weight * 1000, // back to ms
+      hash: rows[0].object,
+      timestamp: rows[0].weight * 1000,
     }
   }
 
-  /**
-   * Delete file meta triple.
-   */
   async deleteFileMeta(filePath: string): Promise<void> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
     try {
-      const triples = await new Promise<Triple[]>((resolve, reject) => {
-        this.db.get(
-          { subject: `meta:${filePath}`, predicate: "graph_built" },
-          (err: Error | undefined, result: Triple[]) => {
-            if (err) reject(err)
-            else resolve(result || [])
-          },
-        )
-      })
-
-      for (const t of triples) {
-        await new Promise<void>((resolve, reject) => {
-          this.db.del(t, (err: Error | undefined) => {
-            if (err) reject(err)
-            else resolve()
-          })
-        })
-      }
-    } catch (err) {
-      // Silently ignore errors (e.g., no meta triple exists)
+      this.db!.prepare("DELETE FROM triples WHERE subject = ? AND predicate = ?")
+        .run(`meta:${filePath}`, "graph_built")
+    } catch {
+      // Silently ignore errors
     }
   }
 
-  /**
-   * Get all file metadata triples (for validation / stats).
-   */
   async getAllFileMeta(): Promise<Array<{ filePath: string; hash: string; timestamp: number }>> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const triples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ predicate: "graph_built" }, (err: Error | undefined, result: Triple[]) => {
-        if (err) reject(err)
-        else resolve(result || [])
-      })
-    })
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    return triples.map((t) => ({
-      filePath: t.subject.replace(/^meta:/, ""),
-      hash: t.object,
-      timestamp: t.weight * 1000,
+    const rows = this._stmtByPredicate.all("graph_built")
+    return rows.map((r: any) => ({
+      filePath: r.subject.replace(/^meta:/, ""),
+      hash: r.object,
+      timestamp: r.weight * 1000,
     }))
   }
 
-  /**
-   * Get all triples in the graph (for validation/stats).
-   * Excludes meta, anchor, and structural triples by default.
-   * Pass includeStructural=true to also get structural edges.
-   */
   async getAllTriples(includeStructural: boolean = false): Promise<Triple[]> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const allRows = this._stmtAll.all()
+    return allRows
+      .map((r: any) => this.toTriple(r))
+      .filter((t: Triple) => {
+        if (t.predicate === "graph_built" || t.predicate === "belongs_to") return false
+        if (!includeStructural && isStructuralPredicate(t.predicate)) return false
+        return true
       })
-    })
-
-    return allTriples.filter(t => {
-      if (t.predicate === "graph_built" || t.predicate === "belongs_to") return false
-      if (!includeStructural && isStructuralPredicate(t.predicate)) return false
-      return true
-    })
   }
 
   async getRelatedFiles(chunkId: string, maxDepth: number = 1): Promise<{path: string, relation: string, weight: number}[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
 
@@ -225,7 +213,6 @@ export class GraphDB {
     const visited = new Set<string>()
     const self = this
 
-    // Resolve the caller's file directly from the node ID
     const callerFile = filePathFromNodeId(chunkId)
 
     async function traverse(currentId: string, currentDepth: number, currentRelation: string) {
@@ -236,19 +223,12 @@ export class GraphDB {
       visited.add(currentId)
 
       try {
-        const outgoing = await new Promise<Triple[]>((resolve, reject) => {
-          self.db.get({ subject: currentId }, (err: Error | undefined, triples: Triple[]) => {
-            if (err) reject(err)
-            else resolve(triples || [])
-          })
-        })
+        const outgoing = self._stmtBySubject.all(currentId).map((r: any) => self.toTriple(r))
 
         for (const triple of outgoing) {
-          // Skip meta, anchor, and structural-only edges
           if (triple.predicate === "graph_built" || triple.predicate === "belongs_to") continue
           if (isStructuralPredicate(triple.predicate)) continue
 
-          // Resolve file for the target node directly from its ID
           const targetFile = filePathFromNodeId(triple.object)
           if (!targetFile) continue
 
@@ -267,12 +247,7 @@ export class GraphDB {
           }
         }
 
-        const incoming = await new Promise<Triple[]>((resolve, reject) => {
-          self.db.get({ object: currentId }, (err: Error | undefined, triples: Triple[]) => {
-            if (err) reject(err)
-            else resolve(triples || [])
-          })
-        })
+        const incoming = self._stmtByObject.all(currentId).map((r: any) => self.toTriple(r))
 
         for (const triple of incoming) {
           if (triple.predicate === "graph_built" || triple.predicate === "belongs_to") continue
@@ -298,7 +273,6 @@ export class GraphDB {
 
     await traverse(chunkId, 0, "")
 
-    // Remove the caller's own file from results
     if (callerFile) relatedFiles.delete(callerFile)
 
     return Array.from(relatedFiles.entries())

package/vectorizer/index.ts

@@ -460,17 +460,24 @@ class CodebaseIndexer {
     await this.loadHashes();
 
     // Graph DB — only if graph is enabled in config
+    // Non-fatal: if LevelDB lock fails (parallel access), search works without graph
     if (GRAPH_CONFIG.enabled) {
-      const graphType = this.indexName === "docs" ? "doc_graph" : "code_graph";
-      const graphPath = path.join(this.root, ".opencode", "graph", graphType);
-      await fs.mkdir(path.dirname(graphPath), { recursive: true });
-      this.graphDB = await new GraphDB(graphPath).init();
-      this.graphBuilder = new GraphBuilder(
-        this.graphDB,
-        this.root,
-        GRAPH_CONFIG.lsp.enabled,
-        GRAPH_CONFIG.lsp.timeout_ms,
-      );
+      try {
+        const graphType = this.indexName === "docs" ? "doc_graph" : "code_graph";
+        const graphPath = path.join(this.root, ".opencode", "graph", graphType);
+        await fs.mkdir(path.dirname(graphPath), { recursive: true });
+        this.graphDB = await new GraphDB(graphPath).init();
+        this.graphBuilder = new GraphBuilder(
+          this.graphDB,
+          this.root,
+          GRAPH_CONFIG.lsp.enabled,
+          GRAPH_CONFIG.lsp.timeout_ms,
+        );
+      } catch (e) {
+        if (DEBUG) console.log(`[vectorizer] GraphDB init failed (lock?): ${e.message || e}`);
+        this.graphDB = null;
+        this.graphBuilder = null;
+      }
     }
 
     // Usage tracker — provenance & usage stats
@@ -1327,4 +1334,65 @@ function getSearchConfig() {
   return SEARCH_CONFIG;
 }
 
-export { CodebaseIndexer, INDEX_PRESETS, getEmbeddingModel, getSearchConfig };
+// ── Singleton indexer pool ──────────────────────────────────────────────────
+// Prevents LevelDB lock conflicts when parallel searches hit the same index.
+// Each unique (projectRoot, indexName) gets one shared CodebaseIndexer.
+const _indexerPool = new Map<string, { indexer: CodebaseIndexer; refCount: number; initPromise: Promise<CodebaseIndexer> }>();
+
+/**
+ * Get or create a shared CodebaseIndexer for the given project + index.
+ * Multiple callers get the same instance — no LevelDB lock conflicts.
+ *
+ * Usage:
+ *   const indexer = await getIndexer(projectRoot, "code");
+ *   try {
+ *     const results = await indexer.search(...);
+ *   } finally {
+ *     releaseIndexer(projectRoot, "code");
+ *   }
+ */
+async function getIndexer(projectRoot: string, indexName: string = "code"): Promise<CodebaseIndexer> {
+  const key = `${projectRoot}::${indexName}`;
+  const existing = _indexerPool.get(key);
+  if (existing) {
+    existing.refCount++;
+    return existing.initPromise;
+  }
+
+  const indexer = new CodebaseIndexer(projectRoot, indexName);
+  const initPromise = indexer.init();
+  _indexerPool.set(key, { indexer, refCount: 1, initPromise });
+  return initPromise;
+}
+
+/**
+ * Release a reference to a shared indexer. When refCount reaches 0,
+ * the indexer is kept alive (for future reuse) but model memory is freed.
+ * Call destroyIndexer() to fully close and remove from pool.
+ */
+function releaseIndexer(projectRoot: string, indexName: string = "code") {
+  const key = `${projectRoot}::${indexName}`;
+  const entry = _indexerPool.get(key);
+  if (!entry) return;
+  entry.refCount = Math.max(0, entry.refCount - 1);
+  // Keep in pool — don't unload. Next search reuses the same instance.
+}
+
+/**
+ * Fully close and remove an indexer from the pool.
+ * Used by CLI clear/reindex operations that need a fresh state.
+ */
+async function destroyIndexer(projectRoot: string, indexName: string = "code") {
+  const key = `${projectRoot}::${indexName}`;
+  const entry = _indexerPool.get(key);
+  if (!entry) return;
+  _indexerPool.delete(key);
+  try {
+    const indexer = await entry.initPromise;
+    await indexer.unloadModel();
+  } catch {
+    // best effort
+  }
+}
+
+export { CodebaseIndexer, INDEX_PRESETS, getEmbeddingModel, getSearchConfig, getIndexer, releaseIndexer, destroyIndexer };