@comfanion/usethis_search 3.0.0-dev.8 → 3.0.0

package/vectorizer/graph-db.ts

@@ -1,5 +1,5 @@
-import levelgraph from "levelgraph"
-import { Level } from "level"
+import { Database } from "bun:sqlite"
+import { filePathFromNodeId, isStructuralPredicate } from "./graph-builder"
 
 export interface Triple {
   subject: string
@@ -12,98 +12,137 @@ export interface Triple {
 }
 
 export class GraphDB {
-  private db: any
+  private db: Database | null = null
   private initialized: boolean = false
 
+  // Prepared statements (cached for performance)
+  private _stmtInsert: any = null
+  private _stmtBySubject: any = null
+  private _stmtByObject: any = null
+  private _stmtByFile: any = null
+  private _stmtDeleteByFile: any = null
+  private _stmtBySubjectPredicate: any = null
+  private _stmtByPredicate: any = null
+  private _stmtAll: any = null
+
   constructor(private dbPath: string) {}
 
   async init(): Promise<this> {
-    const levelDb = new Level(this.dbPath)
-    this.db = levelgraph(levelDb)
+    // bun:sqlite uses a file path; append .db if not already
+    const fullPath = this.dbPath.endsWith(".db") ? this.dbPath : this.dbPath + ".db"
+    this.db = new Database(fullPath)
+
+    // WAL mode for concurrent readers
+    this.db.exec("PRAGMA journal_mode = WAL")
+    this.db.exec("PRAGMA synchronous = NORMAL")  // faster writes, safe with WAL
+    this.db.exec("PRAGMA cache_size = -2000")     // 2MB cache
+
+    // Create triples table
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS triples (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        subject TEXT NOT NULL,
+        predicate TEXT NOT NULL,
+        object TEXT NOT NULL,
+        weight REAL NOT NULL DEFAULT 0,
+        source TEXT NOT NULL DEFAULT '',
+        file TEXT NOT NULL DEFAULT '',
+        line INTEGER
+      )
+    `)
+
+    // Indexes for fast lookups
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_subject ON triples(subject)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_object ON triples(object)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_file ON triples(file)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_predicate ON triples(predicate)")
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_subject_predicate ON triples(subject, predicate)")
+
+    // Prepare statements
+    this._stmtInsert = this.db.prepare(
+      "INSERT INTO triples (subject, predicate, object, weight, source, file, line) VALUES (?, ?, ?, ?, ?, ?, ?)"
+    )
+    this._stmtBySubject = this.db.prepare("SELECT * FROM triples WHERE subject = ?")
+    this._stmtByObject = this.db.prepare("SELECT * FROM triples WHERE object = ?")
+    this._stmtByFile = this.db.prepare("SELECT * FROM triples WHERE file = ?")
+    this._stmtDeleteByFile = this.db.prepare("DELETE FROM triples WHERE file = ?")
+    this._stmtBySubjectPredicate = this.db.prepare("SELECT * FROM triples WHERE subject = ? AND predicate = ?")
+    this._stmtByPredicate = this.db.prepare("SELECT * FROM triples WHERE predicate = ?")
+    this._stmtAll = this.db.prepare("SELECT * FROM triples")
+
     this.initialized = true
     return this
   }
 
+  private toTriple(row: any): Triple {
+    return {
+      subject: row.subject,
+      predicate: row.predicate,
+      object: row.object,
+      weight: row.weight,
+      source: row.source,
+      file: row.file,
+      line: row.line ?? undefined,
+    }
+  }
+
   async putEdges(triples: Triple[]): Promise<void> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    await new Promise<void>((resolve, reject) => {
-      this.db.put(triples, (err: Error | undefined) => {
-        if (err) reject(err)
-        else resolve()
-      })
+
+    // Batch insert in a single transaction — much faster than individual inserts
+    const insertMany = this.db.transaction((items: Triple[]) => {
+      for (const t of items) {
+        this._stmtInsert.run(t.subject, t.predicate, t.object, t.weight, t.source, t.file, t.line ?? null)
+      }
     })
+    insertMany(triples)
   }
 
   async getOutgoing(chunkId: string): Promise<Triple[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    return new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ subject: chunkId }, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
+    const rows = this._stmtBySubject.all(chunkId)
+    return rows.map((r: any) => this.toTriple(r))
   }
 
   async getIncoming(chunkId: string): Promise<Triple[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    return new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ object: chunkId }, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
+    const rows = this._stmtByObject.all(chunkId)
+    return rows.map((r: any) => this.toTriple(r))
   }
 
   async deleteByFile(filePath: string): Promise<void> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
-    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
-
-    const toDelete = allTriples.filter(t => t.file === filePath)
-
-    for (const t of toDelete) {
-      await new Promise<void>((resolve, reject) => {
-        this.db.del(t, (err: Error | undefined) => {
-          if (err) reject(err)
-          else resolve()
-        })
-      })
-    }
+    this._stmtDeleteByFile.run(filePath)
   }
 
   async close(): Promise<void> {
     if (this.initialized && this.db) {
-      await new Promise<void>((resolve, reject) => {
-        this.db.close((err: Error | undefined) => {
-          if (err) reject(err)
-          else resolve()
-        })
-      })
+      this.db.close()
+      this.db = null
+      this._stmtInsert = null
+      this._stmtBySubject = null
+      this._stmtByObject = null
+      this._stmtByFile = null
+      this._stmtDeleteByFile = null
+      this._stmtBySubjectPredicate = null
+      this._stmtByPredicate = null
+      this._stmtAll = null
       this.initialized = false
     }
   }
 
   // ---- FR-054: File metadata triples for incremental updates -----------------
 
-  /**
-   * Store graph build metadata for a file as a special triple.
-   * Subject: `meta:<filePath>`, Predicate: `graph_built`, Object: `<hash>`.
-   * Weight encodes the timestamp (seconds since epoch).
-   */
   async setFileMeta(filePath: string, hash: string, timestamp: number): Promise<void> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
     // Remove old meta triple for this file first
     await this.deleteFileMeta(filePath)
@@ -112,159 +151,117 @@ export class GraphDB {
       subject: `meta:${filePath}`,
       predicate: "graph_built",
       object: hash,
-      weight: Math.floor(timestamp / 1000), // seconds since epoch fits in weight
+      weight: Math.floor(timestamp / 1000),
       source: "meta",
       file: filePath,
     }
     await this.putEdges([triple])
   }
 
-  /**
-   * Get the stored graph build metadata for a file.
-   * Returns { hash, timestamp } or null if not found.
-   */
   async getFileMeta(filePath: string): Promise<{ hash: string; timestamp: number } | null> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const triples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get(
-        { subject: `meta:${filePath}`, predicate: "graph_built" },
-        (err: Error | undefined, result: Triple[]) => {
-          if (err) reject(err)
-          else resolve(result || [])
-        },
-      )
-    })
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    if (triples.length === 0) return null
+    const rows = this._stmtBySubjectPredicate.all(`meta:${filePath}`, "graph_built")
+    if (rows.length === 0) return null
     return {
-      hash: triples[0].object,
-      timestamp: triples[0].weight * 1000, // back to ms
+      hash: rows[0].object,
+      timestamp: rows[0].weight * 1000,
     }
   }
 
-  /**
-   * Delete file meta triple.
-   */
   async deleteFileMeta(filePath: string): Promise<void> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const triples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get(
-        { subject: `meta:${filePath}`, predicate: "graph_built" },
-        (err: Error | undefined, result: Triple[]) => {
-          if (err) reject(err)
-          else resolve(result || [])
-        },
-      )
-    })
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    for (const t of triples) {
-      await new Promise<void>((resolve, reject) => {
-        this.db.del(t, (err: Error | undefined) => {
-          if (err) reject(err)
-          else resolve()
-        })
-      })
+    try {
+      this.db!.prepare("DELETE FROM triples WHERE subject = ? AND predicate = ?")
+        .run(`meta:${filePath}`, "graph_built")
+    } catch {
+      // Silently ignore errors
     }
   }
 
-  /**
-   * Get all file metadata triples (for validation / stats).
-   */
   async getAllFileMeta(): Promise<Array<{ filePath: string; hash: string; timestamp: number }>> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const triples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({ predicate: "graph_built" }, (err: Error | undefined, result: Triple[]) => {
-        if (err) reject(err)
-        else resolve(result || [])
-      })
-    })
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    return triples.map((t) => ({
-      filePath: t.subject.replace(/^meta:/, ""),
-      hash: t.object,
-      timestamp: t.weight * 1000,
+    const rows = this._stmtByPredicate.all("graph_built")
+    return rows.map((r: any) => ({
+      filePath: r.subject.replace(/^meta:/, ""),
+      hash: r.object,
+      timestamp: r.weight * 1000,
     }))
   }
 
-  /**
-   * Get all triples in the graph (for validation/stats).
-   * Excludes meta triples (predicate === "graph_built").
-   */
-  async getAllTriples(): Promise<Triple[]> {
-    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
-
-    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
-      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
-        if (err) reject(err)
-        else resolve(triples || [])
-      })
-    })
+  async getAllTriples(includeStructural: boolean = false): Promise<Triple[]> {
+    if (!this.initialized || !this.db) throw new Error("GraphDB not initialized. Call init() first.")
 
-    return allTriples.filter(t => t.predicate !== "graph_built")
+    const allRows = this._stmtAll.all()
+    return allRows
+      .map((r: any) => this.toTriple(r))
+      .filter((t: Triple) => {
+        if (t.predicate === "graph_built" || t.predicate === "belongs_to") return false
+        if (!includeStructural && isStructuralPredicate(t.predicate)) return false
+        return true
+      })
   }
 
   async getRelatedFiles(chunkId: string, maxDepth: number = 1): Promise<{path: string, relation: string, weight: number}[]> {
-    if (!this.initialized) {
+    if (!this.initialized || !this.db) {
       throw new Error("GraphDB not initialized. Call init() first.")
     }
 
     const relatedFiles: Map<string, {relation: string, weight: number}> = new Map()
     const visited = new Set<string>()
-    
+    const self = this
+
+    const callerFile = filePathFromNodeId(chunkId)
+
     async function traverse(currentId: string, currentDepth: number, currentRelation: string) {
-      if (currentDepth > maxDepth || visited.has(currentId)) {
+      if (currentDepth >= maxDepth || visited.has(currentId)) {
         return
       }
-      
+
       visited.add(currentId)
-      
+
       try {
-        const outgoing = await new Promise<Triple[]>((resolve, reject) => {
-          this.db.get({ subject: currentId }, (err: Error | undefined, triples: Triple[]) => {
-            if (err) reject(err)
-            else resolve(triples || [])
-          })
-        })
-        
+        const outgoing = self._stmtBySubject.all(currentId).map((r: any) => self.toTriple(r))
+
         for (const triple of outgoing) {
-          const fileId = triple.object
-          
-          // Aggregate relations and weights
-          const existing = relatedFiles.get(fileId)
+          if (triple.predicate === "graph_built" || triple.predicate === "belongs_to") continue
+          if (isStructuralPredicate(triple.predicate)) continue
+
+          const targetFile = filePathFromNodeId(triple.object)
+          if (!targetFile) continue
+
+          const existing = relatedFiles.get(targetFile)
           if (existing) {
             existing.weight = Math.max(existing.weight, triple.weight)
           } else {
-            relatedFiles.set(fileId, {
+            relatedFiles.set(targetFile, {
               relation: currentRelation || triple.predicate,
               weight: triple.weight
             })
           }
-          
-          // Recurse for imports/extends relations
+
           if (triple.predicate === "imports" || triple.predicate === "extends") {
-            await traverse(fileId, currentDepth + 1, triple.predicate)
+            await traverse(triple.object, currentDepth + 1, triple.predicate)
           }
         }
-        
-        const incoming = await new Promise<Triple[]>((resolve, reject) => {
-          this.db.get({ object: currentId }, (err: Error | undefined, triples: Triple[]) => {
-            if (err) reject(err)
-            else resolve(triples || [])
-          })
-        })
-        
+
+        const incoming = self._stmtByObject.all(currentId).map((r: any) => self.toTriple(r))
+
         for (const triple of incoming) {
-          const fileId = triple.subject
-          
-          const existing = relatedFiles.get(fileId)
+          if (triple.predicate === "graph_built" || triple.predicate === "belongs_to") continue
+          if (isStructuralPredicate(triple.predicate)) continue
+
+          const sourceFile = filePathFromNodeId(triple.subject)
+          if (!sourceFile) continue
+
+          const existing = relatedFiles.get(sourceFile)
           if (existing) {
             existing.weight = Math.max(existing.weight, triple.weight)
           } else {
-            relatedFiles.set(fileId, {
-              relation: `used_by`,
+            relatedFiles.set(sourceFile, {
+              relation: "used_by",
               weight: triple.weight
             })
           }
@@ -273,17 +270,17 @@ export class GraphDB {
         console.error(`Error traversing graph for ${currentId}:`, error)
       }
     }
-    
+
     await traverse(chunkId, 0, "")
-    
-    const result = Array.from(relatedFiles.entries())
-      .map(([path, data]) => ({
-        path,
+
+    if (callerFile) relatedFiles.delete(callerFile)
+
+    return Array.from(relatedFiles.entries())
+      .map(([filePath, data]) => ({
+        path: filePath,
         relation: data.relation,
         weight: data.weight
       }))
       .sort((a, b) => b.weight - a.weight)
-    
-    return result
   }
 }

package/vectorizer/hybrid-search.ts

@@ -13,7 +13,7 @@ export interface HybridSearchConfig {
 }
 
 export const DEFAULT_HYBRID_CONFIG: HybridSearchConfig = {
-  enabled: false,
+  enabled: true,
   bm25_weight: 0.3,
 }