@comfanion/usethis_search 3.0.0-dev.9 → 3.0.0

package/cli.ts

@@ -0,0 +1,263 @@
+#!/usr/bin/env bun
+/**
+ * CLI for usethis_search index management.
+ *
+ * Usage:
+ *   bun run cli.ts <action> [options]
+ *
+ * Actions:
+ *   reindex [--index code|docs|config]  Re-index files
+ *   status  [--index code|docs|config]  Show index stats
+ *   list                                List all indexes
+ *   clear   [--index code|docs|config]  Clear index and rebuild
+ *   validate-graph [--index code]       Validate graph consistency
+ *
+ * Examples:
+ *   bun run cli.ts reindex
+ *   bun run cli.ts reindex --index docs
+ *   bun run cli.ts status --index code
+ *   bun run cli.ts list
+ *   bun run cli.ts clear --index code
+ */
+
+import path from "path"
+import fs from "fs/promises"
+import { CodebaseIndexer } from "./vectorizer/index.ts"
+
+const args = process.argv.slice(2)
+const action = args[0]
+const indexFlag = args.indexOf("--index")
+const indexName = indexFlag !== -1 && args[indexFlag + 1] ? args[indexFlag + 1] : "code"
+const projectRoot = process.cwd()
+
+function formatDuration(ms: number): string {
+  const seconds = ms / 1000
+  if (seconds < 60) return `${seconds.toFixed(1)}s`
+  const mins = Math.floor(seconds / 60)
+  const secs = Math.round(seconds % 60)
+  return secs > 0 ? `${mins}m ${secs}s` : `${mins}m`
+}
+
+async function reindex() {
+  console.log(`\n  Reindexing "${indexName}"...\n`)
+  const start = Date.now()
+
+  const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+  const stats = await indexer.indexAll(
+    (indexed: number, total: number, file: string, fileNum: number) => {
+      const pct = Math.round((fileNum / total) * 100)
+      process.stdout.write(`\r  [${pct}%] ${fileNum}/${total} — ${file}`)
+    },
+  )
+  await indexer.unloadModel()
+
+  const elapsed = formatDuration(Date.now() - start)
+  console.log(`\n\n  Done in ${elapsed}`)
+  console.log(`  Files indexed: ${stats.indexed}`)
+  console.log(`  Files unchanged: ${stats.skipped}`)
+  console.log(`  Total files: ${stats.total}`)
+  if (stats.semanticEdges > 0) {
+    console.log(`  Semantic edges: ${stats.semanticEdges}`)
+  }
+  console.log()
+}
+
+async function status() {
+  try {
+    const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+    const stats = await indexer.getStats()
+    await indexer.unloadModel()
+
+    console.log(`\n  Index: "${indexName}"`)
+    console.log(`  Description: ${stats.description}`)
+    console.log(`  Model: ${stats.model}`)
+    console.log(`  Files: ${stats.fileCount}`)
+    console.log(`  Chunks: ${stats.chunkCount}`)
+    if (stats.features) {
+      console.log(`  Chunking: ${stats.features.chunking}`)
+      console.log(`  Hybrid: ${stats.features.hybrid ? "on" : "off"}`)
+      console.log(`  BM25 weight: ${stats.features.bm25_weight}`)
+      console.log(`  Metrics: ${stats.features.metrics ? "on" : "off"}`)
+      console.log(`  Cache: ${stats.features.cache ? "on" : "off"}`)
+    }
+    console.log()
+  } catch {
+    console.log(`\n  Index "${indexName}" not found. Run: bun run cli.ts reindex --index ${indexName}\n`)
+  }
+}
+
+async function list() {
+  const vectorsDir = path.join(projectRoot, ".opencode", "vectors")
+  let indexes: string[] = []
+
+  try {
+    const entries = await fs.readdir(vectorsDir, { withFileTypes: true })
+    indexes = entries.filter(e => e.isDirectory()).map(e => e.name)
+  } catch {
+    console.log(`\n  No indexes found. Run: bun run cli.ts reindex\n`)
+    return
+  }
+
+  if (indexes.length === 0) {
+    console.log(`\n  No indexes found. Run: bun run cli.ts reindex\n`)
+    return
+  }
+
+  console.log(`\n  Indexes:\n`)
+  for (const idx of indexes) {
+    try {
+      const indexer = await new CodebaseIndexer(projectRoot, idx).init()
+      const stats = await indexer.getStats()
+      await indexer.unloadModel()
+      const hybrid = stats.features?.hybrid ? " [hybrid]" : ""
+      console.log(`  ${idx.padEnd(10)} ${stats.fileCount} files, ${stats.chunkCount} chunks${hybrid}`)
+    } catch {
+      console.log(`  ${idx.padEnd(10)} (error reading)`)
+    }
+  }
+  console.log()
+}
+
+async function clear() {
+  console.log(`\n  Clearing "${indexName}" index...`)
+  const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+  await indexer.clear()
+  await indexer.unloadModel()
+  console.log(`  Cleared. Run: bun run cli.ts reindex --index ${indexName}\n`)
+}
+
+async function validateGraph() {
+  console.log(`\n  Validating graph for "${indexName}"...\n`)
+
+  const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+  const graphDB = (indexer as any).graphDB
+  const db = (indexer as any).db
+
+  if (!graphDB) {
+    await indexer.unloadModel()
+    console.log(`  No graph database found. Run reindex first.\n`)
+    return
+  }
+
+  // Get all triples
+  let allTriples: any[] = []
+  try {
+    allTriples = await graphDB.getAllTriples()
+  } catch (e: any) {
+    await indexer.unloadModel()
+    console.log(`  Error reading graph: ${e.message}\n`)
+    return
+  }
+
+  // Get all chunk IDs from vector DB
+  const knownChunkIds = new Set<string>()
+  const tables = await db.tableNames()
+  if (tables.includes("chunks")) {
+    const table = await db.openTable("chunks")
+    try {
+      const rows = await table.filter("true").limit(100000).execute()
+      for (const row of rows) {
+        if (row.chunk_id) knownChunkIds.add(row.chunk_id)
+      }
+    } catch (e: any) {
+      await indexer.unloadModel()
+      console.log(`  Error reading vector DB: ${e.message}\n`)
+      return
+    }
+  }
+
+  // Find orphaned triples
+  const predicateCounts: Record<string, number> = {}
+  const sourceCounts: Record<string, number> = {}
+  let orphaned = 0
+
+  for (const t of allTriples) {
+    predicateCounts[t.predicate] = (predicateCounts[t.predicate] || 0) + 1
+    sourceCounts[t.source] = (sourceCounts[t.source] || 0) + 1
+
+    if (t.subject.startsWith("chunk:") && !knownChunkIds.has(t.subject)) orphaned++
+    if (t.object.startsWith("chunk:") && !knownChunkIds.has(t.object)) orphaned++
+  }
+
+  await indexer.unloadModel()
+
+  console.log(`  Total triples: ${allTriples.length}`)
+  console.log(`  Known chunks:  ${knownChunkIds.size}`)
+  console.log(`  Orphaned refs: ${orphaned}`)
+  console.log()
+
+  console.log(`  Edge types:`)
+  for (const [pred, count] of Object.entries(predicateCounts).sort((a, b) => b[1] - a[1])) {
+    console.log(`    ${pred.padEnd(22)} ${count}`)
+  }
+  console.log()
+
+  console.log(`  Edge sources:`)
+  for (const [source, count] of Object.entries(sourceCounts).sort((a, b) => b[1] - a[1])) {
+    console.log(`    ${source.padEnd(22)} ${count}`)
+  }
+  console.log()
+
+  if (orphaned > 0) {
+    console.log(`  ⚠ Found ${orphaned} orphaned references. Run: bun run cli.ts reindex --index ${indexName}`)
+  } else {
+    console.log(`  ✓ Graph is healthy`)
+  }
+  console.log()
+}
+
+// ── Main ────────────────────────────────────────────────────────────────────
+
+async function main() {
+  if (!action || action === "help" || action === "--help" || action === "-h") {
+    console.log(`
+  usethis_search CLI — index management
+
+  Usage: bun run cli.ts <action> [--index <name>]
+
+  Actions:
+    reindex          Re-index files (default: code)
+    status           Show index statistics
+    list             List all indexes
+    clear            Clear index data
+    validate-graph   Check graph consistency
+
+  Options:
+    --index <name>   Index name: code, docs, config (default: code)
+
+  Examples:
+    bun run cli.ts reindex
+    bun run cli.ts reindex --index docs
+    bun run cli.ts status
+    bun run cli.ts list
+    bun run cli.ts clear --index code
+    bun run cli.ts validate-graph
+`)
+    return
+  }
+
+  switch (action) {
+    case "reindex":
+      await reindex()
+      break
+    case "status":
+      await status()
+      break
+    case "list":
+      await list()
+      break
+    case "clear":
+      await clear()
+      break
+    case "validate-graph":
+      await validateGraph()
+      break
+    default:
+      console.log(`\n  Unknown action: "${action}". Run: bun run cli.ts help\n`)
+  }
+}
+
+main().catch((e) => {
+  console.error(`\n  Fatal error: ${e.message}\n`)
+  process.exit(1)
+})

package/file-indexer.ts

@@ -3,7 +3,7 @@ import path from "path"
 import fs from "fs/promises"
 import fsSync from "fs"
 
-import { CodebaseIndexer } from "./vectorizer/index.js"
+import { CodebaseIndexer } from "./vectorizer/index.ts"
 import { initGraphAPI } from "./api"
 
 /**

package/index.ts

@@ -1,7 +1,6 @@
 import type { Plugin } from "@opencode-ai/plugin"
 
 import search from "./tools/search"
-import codeindex from "./tools/codeindex"
 import FileIndexerPlugin from "./file-indexer"
 
 const UsethisSearchPlugin: Plugin = async ({ directory, client }) => {
@@ -17,7 +16,6 @@ const UsethisSearchPlugin: Plugin = async ({ directory, client }) => {
   return {
     tool: {
       search,
-      codeindex,
     },
 
     event: async (args: any) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comfanion/usethis_search",
-  "version": "3.0.0-dev.9",
+  "version": "3.0.0",
   "description": "OpenCode plugin: semantic search with graph-based context (v3: graph relations, 1-hop context, LSP + regex analyzers)",
   "type": "module",
   "main": "./index.ts",
@@ -8,15 +8,23 @@
     ".": "./index.ts"
   },
   "scripts": {
-    "test": "bun test"
+    "test": "bun test",
+    "reindex": "bun run cli.ts reindex",
+    "status": "bun run cli.ts status",
+    "index:list": "bun run cli.ts list",
+    "index:clear": "bun run cli.ts clear"
+  },
+  "bin": {
+    "usethis-search": "./cli.ts"
   },
   "files": [
     "index.ts",
+    "cli.ts",
     "api.ts",
     "file-indexer.ts",
     "tools/search.ts",
     "tools/codeindex.ts",
-    "vectorizer/index.js",
+    "vectorizer/index.ts",
     "vectorizer/content-cleaner.ts",
     "vectorizer/metadata-extractor.ts",
     "vectorizer/bm25-index.ts",
@@ -24,6 +32,7 @@
     "vectorizer/query-cache.ts",
     "vectorizer/search-metrics.ts",
     "vectorizer/graph-db.ts",
+    "vectorizer/chunk-store.ts",
     "vectorizer/usage-tracker.ts",
     "vectorizer/graph-builder.ts",
     "vectorizer/analyzers/regex-analyzer.ts",
@@ -40,8 +49,6 @@
     "@opencode-ai/plugin": ">=1.1.0",
     "@xenova/transformers": "^2.17.0",
     "glob": "^10.3.10",
-    "level": "^8.0.1",
-    "levelgraph": "^4.0.0",
     "vectordb": "^0.4.0"
   },
   "peerDependencies": {

package/tools/codeindex.ts

@@ -9,7 +9,7 @@ import { tool } from "@opencode-ai/plugin"
 import path from "path"
 import fs from "fs/promises"
 
-import { CodebaseIndexer } from "../vectorizer/index.js"
+import { CodebaseIndexer } from "../vectorizer/index.ts"
 
 const INDEX_EXTENSIONS: Record<string, string[]> = {
   code: [".js", ".ts", ".jsx", ".tsx", ".go", ".py", ".rs", ".java", ".kt", ".swift", ".c", ".cpp", ".h", ".cs", ".rb", ".php"],
@@ -327,7 +327,7 @@ Available indexes:
         if (tables.includes("chunks")) {
           const table = await db.openTable("chunks")
           try {
-            const rows = await table.search([0]).limit(100000).execute()
+            const rows = await table.filter("").limit(100000).execute()
             for (const row of rows) {
               if (row.chunk_id) knownChunkIds.add(row.chunk_id)
             }