@comfanion/usethis_search 4.3.1 → 4.5.0

package/api.ts

@@ -7,16 +7,38 @@
 
 import { GraphDB } from "./vectorizer/graph-db"
 
-// Global GraphDB instance (shared across plugins)
+// Global GraphDB instance (shared across plugins).
+// NOTE: This is a weak reference — the GraphDB may be closed if the indexer
+// is evicted from the pool. All API methods guard against stale references.
 let graphDBInstance: GraphDB | null = null
 
 /**
- * Initialize API with GraphDB instance
+ * Initialize API with GraphDB instance.
+ * Called by file-indexer on startup. The instance may become stale
+ * if the indexer pool evicts the owning indexer — API methods handle this.
  */
 export function initGraphAPI(db: GraphDB): void {
   graphDBInstance = db
 }
 
+/**
+ * Clear the API reference (called when the owning GraphDB is closed).
+ */
+export function clearGraphAPI(): void {
+  graphDBInstance = null
+}
+
+/** Check if the stored GraphDB reference is still usable. */
+function isGraphAlive(): boolean {
+  if (!graphDBInstance) return false
+  // GraphDB sets db=null in close() — use that as liveness check
+  try {
+    return (graphDBInstance as any).initialized === true
+  } catch {
+    return false
+  }
+}
+
 /**
  * Get related files for a given file path
  * 
@@ -39,21 +61,18 @@ export async function getRelatedFiles(
   filePath: string,
   maxDepth: number = 1
 ): Promise<{path: string, relation: string, weight: number}[]> {
-  if (!graphDBInstance) {
-    console.warn("[usethis_search API] GraphDB not initialized. Returning empty array.")
-    return []
-  }
+  if (!isGraphAlive()) return []
 
   try {
     const chunkId = `file:${filePath}`
-    const related = await graphDBInstance.getRelatedFiles(chunkId, maxDepth)
+    const related = await graphDBInstance!.getRelatedFiles(chunkId, maxDepth)
     
     // Filter out the input file itself (it might appear in the graph)
     const filtered = related.filter(r => r.path !== filePath)
     
     return filtered
-  } catch (error) {
-    console.error(`[usethis_search API] Error getting related files for ${filePath}:`, error)
+  } catch {
+    // GraphDB may have been closed between the alive check and usage — non-fatal
     return []
   }
 }
@@ -62,22 +81,20 @@ export async function getRelatedFiles(
  * Check if graph API is available
  */
 export function isGraphAPIAvailable(): boolean {
-  return graphDBInstance !== null
+  return isGraphAlive()
 }
 
 /**
  * Get all graph entries for a file (both incoming and outgoing)
  */
 export async function getGraphEntries(filePath: string) {
-  if (!graphDBInstance) {
-    return null
-  }
+  if (!isGraphAlive()) return null
 
   try {
     const chunkId = `file:${filePath}`
     const [outgoing, incoming] = await Promise.all([
-      graphDBInstance.getOutgoing(chunkId),
-      graphDBInstance.getIncoming(chunkId),
+      graphDBInstance!.getOutgoing(chunkId),
+      graphDBInstance!.getIncoming(chunkId),
     ])
 
     return {
@@ -85,8 +102,8 @@ export async function getGraphEntries(filePath: string) {
       extends: outgoing.filter(t => t.predicate === "extends"),
       used_by: incoming,
     }
-  } catch (error) {
-    console.error(`[usethis_search API] Error getting graph entries for ${filePath}:`, error)
+  } catch {
+    // GraphDB may have been closed — non-fatal
     return null
   }
 }

package/cache/manager.ts

@@ -224,7 +224,12 @@ class WorkspaceCache {
     if (this._shutdownRegistered) return
     this._shutdownRegistered = true
 
+     let flushed = false
      const flush = () => {
+       // Guard against double flush (SIGINT/SIGTERM → exit → flush again)
+       if (flushed) return
+       flushed = true
+
        // Synchronous-ish flush — best effort
        // Node process is exiting, so we can't await.
        // Use writeFileSync as last resort.
@@ -555,7 +560,7 @@ class WorkspaceCache {
      let removed = 0
 
      for (const [chunkId, entry] of this.entries) {
-       if (entry.path === filePath) {
+       if (entry.path === filePath || entry.path.startsWith(filePath)) {
          this.entries.delete(chunkId)
          this._totalTokens -= entry.tokens
          removed++
@@ -570,11 +575,11 @@ class WorkspaceCache {
     * Get all chunks sorted by: search-main first (by score desc), then search-graph, then manual.
     */
    getAll(): WorkspaceEntry[] {
-     return Array.from(this.entries.values()).sort((a, b) => {
-       // Main chunks first
-       const roleOrder = { "search-main": 0, "search-graph": 1, manual: 2 }
-       const ra = roleOrder[a.role] ?? 2
-       const rb = roleOrder[b.role] ?? 2
+      return Array.from(this.entries.values()).sort((a, b) => {
+        // Main chunks first, then context, then graph, then manual
+        const roleOrder: Record<string, number> = { "search-main": 0, "search-context": 1, "search-graph": 2, manual: 3 }
+        const ra = roleOrder[a.role] ?? 3
+        const rb = roleOrder[b.role] ?? 3
        if (ra !== rb) return ra - rb
 
        // Within same role: higher score first
@@ -704,8 +709,8 @@ class WorkspaceCache {
     * This method only detects changes and removes orphaned chunks.
     * For full content refresh, chunks must be re-attached from search.
     *
-    * Called by message.before hook — ensures AI sees fresh file status.
-    * Cost: ~2ms for 30 files (stat only, no readFile).
+     * Called by message.before hook — ensures AI sees fresh file status.
+     * Cost: ~5-50ms for 30 files (reads file content to check chunk presence).
     *
     * Returns { updated, removed } counts.
     */
@@ -729,16 +734,22 @@ class WorkspaceCache {
        try {
          const fullPath = path.join(this.projectRoot, filePath)
          const content = await fs.readFile(fullPath, "utf-8")
-         const newHash = this.hashContent(content)
 
-         // Check if file changed
-         const fileChanged = chunks.some(c => c.contentHash !== newHash)
+          // Check if any chunk's content is no longer present in the file.
+          // contentHash is per-chunk, not per-file, so direct hash comparison
+          // doesn't work. Instead we check if each chunk's text is still in the file.
+          let fileChanged = false
+         for (const chunk of chunks) {
+           if (!content.includes(chunk.content.trim())) {
+             fileChanged = true
+             break
+           }
+         }
+
          if (fileChanged) {
            // Mark all chunks from this file as stale
            // (they need re-attachment from search to get fresh content)
            stale += chunks.length
-           // TODO(chunk-7): Implement stale flag on WorkspaceEntry
-           // For now, chunks remain in cache but are marked as needing refresh
          }
        } catch {
          // File no longer exists — remove all chunks from this file
@@ -776,12 +787,12 @@ class WorkspaceCache {
        return
      }
 
-     // Build eviction priority list
-     const candidates = Array.from(this.entries.values()).sort((a, b) => {
-       // Evict graph chunks before main chunks before manual chunks
-       const roleOrder = { "search-graph": 0, "search-main": 1, manual: 2 }
-       const ra = roleOrder[a.role] ?? 2
-       const rb = roleOrder[b.role] ?? 2
+      // Build eviction priority list
+      const candidates = Array.from(this.entries.values()).sort((a, b) => {
+        // Evict graph first, then context, then main, then manual (manual evicted last)
+        const roleOrder: Record<string, number> = { "search-graph": 0, "search-context": 1, "search-main": 2, manual: 3 }
+        const ra = roleOrder[a.role] ?? 3
+        const rb = roleOrder[b.role] ?? 3
        if (ra !== rb) return ra - rb
 
        // Within same role: lowest score first

package/cli.ts

@@ -22,7 +22,7 @@
 
 import path from "path"
 import fs from "fs/promises"
-import { CodebaseIndexer } from "./vectorizer/index.ts"
+import { CodebaseIndexer, disposeSharedModel } from "./vectorizer/index.ts"
 
 const args = process.argv.slice(2)
 const action = args[0]
@@ -257,7 +257,10 @@ async function main() {
   }
 }
 
-main().catch((e) => {
-  console.error(`\n  Fatal error: ${e.message}\n`)
-  process.exit(1)
-})
+main()
+  .then(() => disposeSharedModel())
+  .catch(async (e) => {
+    await disposeSharedModel()
+    console.error(`\n  Fatal error: ${e.message}\n`)
+    process.exit(1)
+  })

package/file-indexer.ts

@@ -3,7 +3,7 @@ import path from "path"
 import fs from "fs/promises"
 import fsSync from "fs"
 
-import { CodebaseIndexer } from "./vectorizer/index.ts"
+import { CodebaseIndexer, getIndexer, releaseIndexer } from "./vectorizer/index.ts"
 import { initGraphAPI } from "./api"
 
 /**
@@ -161,6 +161,7 @@ async function loadConfig(projectRoot: string): Promise<VectorizerConfig> {
       const tempIndexer = new CodebaseIndexer(projectRoot, "code")
       // @ts-ignore - accessing internal method for config creation
       await tempIndexer.init()
+      await tempIndexer.unloadModel()
       try {
         content = await fs.readFile(configPath, "utf8")
       } catch {
@@ -252,7 +253,7 @@ async function ensureIndexOnSessionStart(
 
   for (const [indexName, indexConfig] of Object.entries(config.indexes)) {
     if (!indexConfig.enabled) continue
-    const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+    const indexer = await getIndexer(projectRoot, indexName)
     
     try {
       // Initialize graph API for Mind plugin integration
@@ -274,7 +275,7 @@ async function ensureIndexOnSessionStart(
         needsWork = true
       }
     } finally {
-      await indexer.unloadModel()
+      releaseIndexer(projectRoot, indexName)
     }
   }
 
@@ -289,13 +290,13 @@ async function ensureIndexOnSessionStart(
     const indexExists = await hasIndex(projectRoot, indexName)
     const startTime = Date.now()
 
-    const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+    const indexer = await getIndexer(projectRoot, indexName)
     try {
       if (!indexExists) {
         log(`Creating "${indexName}" index...`)
-        const stats = await indexer.indexAll((indexed: number, total: number, file: string) => {
+        const stats = await indexer.indexAll((indexed: number, total: number, file: string, _fileNum?: number, phase?: string) => {
           if (indexed % 10 === 0 || indexed === total) {
-            logFile(`"${indexName}": ${indexed}/${total} - ${file}`)
+            logFile(`"${indexName}" [${phase || "?"}]: ${indexed}/${total} - ${file}`)
           }
         }, config.exclude)
         const elapsed = ((Date.now() - startTime) / 1000).toFixed(1)
@@ -307,9 +308,9 @@ async function ensureIndexOnSessionStart(
 
         if (health.needsReindex) {
           log(`Rebuilding "${indexName}" (${health.reason}: ${health.currentCount} vs ${health.expectedCount} files)...`)
-          const stats = await indexer.indexAll((indexed: number, total: number, file: string) => {
+          const stats = await indexer.indexAll((indexed: number, total: number, file: string, _fileNum?: number, phase?: string) => {
             if (indexed % 10 === 0 || indexed === total) {
-              logFile(`"${indexName}": ${indexed}/${total} - ${file}`)
+              logFile(`"${indexName}" [${phase || "?"}]: ${indexed}/${total} - ${file}`)
             }
           }, config.exclude)
           const elapsed = ((Date.now() - startTime) / 1000).toFixed(1)
@@ -329,8 +330,13 @@ async function ensureIndexOnSessionStart(
           }
         }
       }
+    } catch (e) {
+      const elapsed = ((Date.now() - startTime) / 1000).toFixed(1)
+      log(`ERROR indexing "${indexName}" after ${elapsed}s: ${(e as Error).message}`)
+      logFile(`ERROR "${indexName}" stack: ${(e as Error).stack || "no stack"}`)
+      // Continue with other indexes — don't let one failure stop everything
     } finally {
-      await indexer.unloadModel()
+      releaseIndexer(projectRoot, indexName)
     }
   }
 
@@ -374,7 +380,7 @@ async function processPendingFiles(projectRoot: string, config: VectorizerConfig
   debug(`Processing ${filesToProcess.size} index(es)...`)
 
   for (const [indexName, files] of filesToProcess.entries()) {
-    const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+    const indexer = await getIndexer(projectRoot, indexName)
     try {
       for (const filePath of files) {
         try {
@@ -389,7 +395,7 @@ async function processPendingFiles(projectRoot: string, config: VectorizerConfig
         }
       }
     } finally {
-      await indexer.unloadModel()
+      releaseIndexer(projectRoot, indexName)
     }
   }
 }
@@ -420,6 +426,15 @@ export const FileIndexerPlugin: Plugin = async ({ directory, client }) => {
 
   log(`Plugin ACTIVE`)
 
+  // Catch unhandled errors that would silently kill indexing
+  process.on("uncaughtException", (err) => {
+    logFile(`FATAL uncaughtException: ${err.message}\n${err.stack || "no stack"}`)
+  })
+  process.on("unhandledRejection", (reason) => {
+    const msg = reason instanceof Error ? `${reason.message}\n${reason.stack}` : String(reason)
+    logFile(`FATAL unhandledRejection: ${msg}`)
+  })
+
   const lang = await getLanguage(directory)
   const messages = FUN_MESSAGES[lang]
 
@@ -444,6 +459,8 @@ export const FileIndexerPlugin: Plugin = async ({ directory, client }) => {
           toast(messages.done(result.totalFiles, duration), "success")
         }
       } catch (e: any) {
+        log(`ERROR ensureIndexOnSessionStart: ${e.message}`)
+        logFile(`ERROR stack: ${e.stack || "no stack"}`)
         toast(messages.error(e.message), "error")
       }
     }, 1000)

package/hooks/message-before.ts

@@ -51,10 +51,10 @@ interface Message {
 /** Tools that return full workspace state in their output. */
 const WORKSPACE_TOOLS = new Set([
   "search",
-  "workspace_list",
-  "workspace_forget",
-  "workspace_clear",
-  "workspace_restore",
+  "list",
+  "forget",
+  "clear",
+  "explore",
 ])
 
 /** Minimum output length to consider pruning. Short outputs are kept as-is. */
@@ -80,7 +80,7 @@ export function createWorkspaceInjectionHandler(state: SessionState) {
 /**
  * Replace old workspace tool outputs with compact summaries.
  *
- * Workspace tools (search, workspace_list, etc.) return full workspace
+ * Workspace tools (search, list, forget, etc.) return full workspace
  * state in their output. Only the LAST such output is kept — all previous
  * ones are replaced with a 1-line summary.
  *

package/hooks/tool-substitution.ts

@@ -1,6 +1,10 @@
 /**
  * Tool Output Substitution Hook
  *
+ * STATUS: DISABLED — not registered in plugin index.ts.
+ * The workspace-injection approach (v6) replaced tool substitution.
+ * Kept for potential future re-enablement.
+ *
  * Intercepts read() tool outputs and replaces them with compact messages
  * when the file is in the workspace cache.
  *
@@ -278,69 +282,6 @@ function substituteReadOutput(output: { title: string; output: string; metadata:
   }
 }
 
-/**
- * Substitute grep() output if ALL matched files are in workspace.
- *
- * Input: { pattern: "auth", include?: "*.ts" }
- * Output: "src/auth.ts:10:export function login(...)\nsrc/types.ts:5:interface User {...}"
- *
- * Parse output to extract file paths, check if ALL are in workspace.
- * If yes: Replace with "[Pattern "auth" matched N files, all in workspace context:\n- file1\n- file2\n...]"
- * If partial: Keep original
- */
-function substituteGrepOutput(output: { title: string; output: string; metadata: any }, cache: WorkspaceCache): void {
-  try {
-    const pattern = output.metadata?.pattern || extractPatternFromTitle(output.title)
-    if (!pattern) return
-
-    // Parse grep output to extract file paths
-    // Format: "path:line:content" or just "path"
-    const filePaths = parseGrepOutput(output.output)
-    if (filePaths.length === 0) return
-
-    // Check if ALL files are in workspace
-    const allInWorkspace = filePaths.every(fp => cache.has(fp))
-    if (!allInWorkspace) return
-
-    // Replace with compact message
-    const fileList = filePaths.map(fp => `- ${fp}`).join("\n")
-    output.output = `[Pattern "${pattern}" matched ${filePaths.length} files, all in workspace context:\n${fileList}]`
-  } catch {
-    // Silently fail
-  }
-}
-
-/**
- * Substitute glob() output if ALL matched files are in workspace.
- *
- * Input: { pattern: "src/[glob-pattern].ts" }
- * Output: "src/auth.ts\nsrc/types.ts\nsrc/utils.ts"
- *
- * Parse output (newline-separated paths), check if ALL are in workspace.
- * If yes: Replace with "[Pattern matched N files, all in workspace context:\n- file1\n- file2\n...]"
- * If partial: Keep original
- */
-function substituteGlobOutput(output: { title: string; output: string; metadata: any }, cache: WorkspaceCache): void {
-  try {
-    const pattern = output.metadata?.pattern || extractPatternFromTitle(output.title)
-    if (!pattern) return
-
-    // Parse glob output (newline-separated file paths)
-    const filePaths = parseGlobOutput(output.output)
-    if (filePaths.length === 0) return
-
-    // Check if ALL files are in workspace
-    const allInWorkspace = filePaths.every(fp => cache.has(fp))
-    if (!allInWorkspace) return
-
-    // Replace with compact message
-    const fileList = filePaths.map(fp => `- ${fp}`).join("\n")
-    output.output = `[Pattern "${pattern}" matched ${filePaths.length} files, all in workspace context:\n${fileList}]`
-  } catch {
-    // Silently fail
-  }
-}
-
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
 /**
@@ -365,61 +306,4 @@ function extractFilePathFromTitle(title: string): string | null {
   return null
 }
 
-/**
- * Extract pattern from grep() or glob() title.
- * Title format: "Search for: auth" or "Find files: src/[pattern].ts" or similar
- */
-function extractPatternFromTitle(title: string): string | null {
-  // Try common patterns
-  const patterns = [
-    /(?:search|find|pattern|glob).*?:\s*(.+?)(?:\s*\(|$)/i,
-    /(?:search|find|pattern|glob)\s+(.+?)(?:\s*\(|$)/i,
-  ]
-
-  for (const pattern of patterns) {
-    const match = title.match(pattern)
-    if (match) {
-      return match[1].trim()
-    }
-  }
 
-  return null
-}
-
-/**
- * Parse grep output to extract file paths.
- *
- * Format variations:
- *   - "path:line:content" (standard grep)
- *   - "path:line" (grep -n without content)
- *   - "path" (grep -l, list files only)
- *
- * Returns unique file paths.
- */
-function parseGrepOutput(output: string): string[] {
-  const lines = output.split("\n").filter(l => l.trim())
-  const paths = new Set<string>()
-
-  for (const line of lines) {
-    // Extract path (everything before first colon, or entire line if no colon)
-    const colonIndex = line.indexOf(":")
-    const path = colonIndex >= 0 ? line.substring(0, colonIndex) : line
-
-    if (path.trim()) {
-      paths.add(path.trim())
-    }
-  }
-
-  return Array.from(paths)
-}
-
-/**
- * Parse glob output to extract file paths.
- *
- * Format: newline-separated file paths
- * Returns unique file paths.
- */
-function parseGlobOutput(output: string): string[] {
-  const lines = output.split("\n").filter(l => l.trim())
-  return Array.from(new Set(lines.map(l => l.trim())))
-}

package/index.ts

@@ -1,7 +1,10 @@
 import type { Plugin } from "@opencode-ai/plugin"
+import path from "path"
+import fsSync from "fs"
 
 import search from "./tools/search"
-import { workspace_list, workspace_forget, workspace_clear, workspace_restore } from "./tools/workspace"
+import { list, forget, clear } from "./tools/workspace"
+import { explore } from "./tools/graph"
 import FileIndexerPlugin from "./file-indexer"
 import { workspaceCache } from "./cache/manager"
 import { createWorkspaceInjectionHandler } from "./hooks/message-before"
@@ -29,18 +32,26 @@ const UsethisSearchPlugin: Plugin = async ({ directory, client }) => {
   try {
     const hooks = await FileIndexerPlugin({ directory, client } as any)
     fileIndexerEvent = hooks?.event || null
-  } catch {
+  } catch (e) {
     // file indexer init failed — tools still work, just no auto-indexing
+    // Log to file so the error is not lost silently
+    try {
+      const logPath = path.join(directory, ".opencode", "indexer.log")
+      const msg = `${new Date().toISOString()} ERROR FileIndexerPlugin init: ${(e as Error).message}\n${(e as Error).stack || ""}\n`
+      fsSync.appendFileSync(logPath, msg)
+    } catch {
+      // can't even write log — truly silent
+    }
   }
 
   return {
     // ── Tools ───────────────────────────────────────────────────────────
     tool: {
       search,
-      workspace_list,
-      workspace_forget,
-      workspace_clear,
-      workspace_restore,
+      list,
+      forget,
+      clear,
+      explore,
     },
 
     // ── Hooks ───────────────────────────────────────────────────────────

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@comfanion/usethis_search",
-  "version": "4.3.1",
-  "description": "OpenCode plugin: semantic search with context-efficient workspace state (v4.3.1: improved tool description - chunks contain direct file content, no verification needed)",
+  "version": "4.5.0",
+  "description": "OpenCode plugin: semantic search with query decomposition, RRF merge, and context-efficient workspace (v4.5.0)",
   "type": "module",
   "main": "./index.ts",
   "exports": {
@@ -27,6 +27,7 @@
     "tools/workspace.ts",
     "tools/workspace-state.ts",
     "tools/read-interceptor.ts",
+    "tools/graph.ts",
     "cache/manager.ts",
     "hooks/message-before.ts",
     "hooks/tool-substitution.ts",
@@ -41,6 +42,7 @@
     "vectorizer/graph-db.ts",
     "vectorizer/chunk-store.ts",
     "vectorizer/usage-tracker.ts",
+    "vectorizer/query-decomposer.ts",
     "vectorizer/graph-builder.ts",
     "vectorizer/analyzers/regex-analyzer.ts",
     "vectorizer/analyzers/lsp-analyzer.ts",