@comfanion/usethis_search 3.0.0-dev.0 → 3.0.0-dev.10

package/vectorizer/analyzers/lsp-client.ts

@@ -0,0 +1,369 @@
+/**
+ * Lightweight LSP client over JSON-RPC / stdio.
+ *
+ * Spawns a language server as a child process, speaks the Language Server
+ * Protocol, and exposes high-level helpers used by LSPAnalyzer.
+ *
+ * Zero external dependencies — implements JSON-RPC framing inline.
+ */
+
+import { spawn, ChildProcess } from "child_process"
+import path from "path"
+import fs from "fs"
+
+// ---------------------------------------------------------------------------
+// LSP types (minimal subset we need)
+// ---------------------------------------------------------------------------
+
+export interface LSPPosition {
+  line: number
+  character: number
+}
+
+export interface LSPRange {
+  start: LSPPosition
+  end: LSPPosition
+}
+
+export interface LSPLocation {
+  uri: string
+  range: LSPRange
+}
+
+export interface LSPSymbolInformation {
+  name: string
+  kind: number
+  range: LSPRange
+  selectionRange: LSPRange
+  children?: LSPSymbolInformation[]
+}
+
+// SymbolKind constants we care about
+export const SymbolKind = {
+  File: 1,
+  Module: 2,
+  Namespace: 3,
+  Package: 4,
+  Class: 5,
+  Method: 6,
+  Property: 7,
+  Field: 8,
+  Constructor: 9,
+  Enum: 10,
+  Interface: 11,
+  Function: 12,
+  Variable: 13,
+  Constant: 14,
+  TypeParameter: 26,
+} as const
+
+// ---------------------------------------------------------------------------
+// Server binary resolution
+// ---------------------------------------------------------------------------
+
+interface ServerConfig {
+  command: string
+  args: string[]
+  languages: string[]
+}
+
+const SERVER_CONFIGS: Record<string, ServerConfig> = {
+  typescript: {
+    command: "typescript-language-server",
+    args: ["--stdio"],
+    languages: ["typescript", "javascript", "typescriptreact", "javascriptreact"],
+  },
+  python: {
+    command: "pylsp",
+    args: [],
+    languages: ["python"],
+  },
+  go: {
+    command: "gopls",
+    args: ["serve"],
+    languages: ["go"],
+  },
+  rust: {
+    command: "rust-analyzer",
+    args: [],
+    languages: ["rust"],
+  },
+}
+
+function languageToServerId(language: string): string | null {
+  for (const [id, cfg] of Object.entries(SERVER_CONFIGS)) {
+    if (cfg.languages.includes(language)) return id
+  }
+  return null
+}
+
+function findBinary(name: string): string | null {
+  // 1. Check PATH via which-style lookup
+  const pathEnv = process.env.PATH || ""
+  const dirs = pathEnv.split(path.delimiter)
+  for (const dir of dirs) {
+    const full = path.join(dir, name)
+    try {
+      fs.accessSync(full, fs.constants.X_OK)
+      return full
+    } catch { /* not found */ }
+  }
+  return null
+}
+
+// ---------------------------------------------------------------------------
+// JSON-RPC framing
+// ---------------------------------------------------------------------------
+
+function encodeMessage(body: object): Buffer {
+  const json = JSON.stringify(body)
+  const content = Buffer.from(json, "utf-8")
+  const header = `Content-Length: ${content.byteLength}\r\n\r\n`
+  return Buffer.concat([Buffer.from(header, "ascii"), content])
+}
+
+// ---------------------------------------------------------------------------
+// LSPClient
+// ---------------------------------------------------------------------------
+
+export class LSPClient {
+  private proc: ChildProcess | null = null
+  private requestId = 0
+  private pending = new Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }>()
+  private buffer = Buffer.alloc(0)
+  private initialized = false
+  private serverConfig: ServerConfig | null = null
+  private rootUri: string
+
+  constructor(
+    private projectRoot: string,
+    private timeoutMs: number = 10_000,
+  ) {
+    this.rootUri = `file://${this.projectRoot}`
+  }
+
+  // ---- lifecycle ----------------------------------------------------------
+
+  /** Check if a server binary exists for the given language. */
+  static isAvailable(language: string): boolean {
+    const serverId = languageToServerId(language)
+    if (!serverId) return false
+    const cfg = SERVER_CONFIGS[serverId]
+    return findBinary(cfg.command) !== null
+  }
+
+  /** Start the language server for `language`. Rejects if unavailable. */
+  async start(language: string): Promise<void> {
+    if (this.initialized) return
+
+    const serverId = languageToServerId(language)
+    if (!serverId) throw new Error(`No LSP server config for language: ${language}`)
+
+    const cfg = SERVER_CONFIGS[serverId]
+    const bin = findBinary(cfg.command)
+    if (!bin) throw new Error(`LSP server binary not found: ${cfg.command}`)
+
+    this.serverConfig = cfg
+
+    this.proc = spawn(bin, cfg.args, {
+      cwd: this.projectRoot,
+      stdio: ["pipe", "pipe", "pipe"],
+      env: { ...process.env },
+    })
+
+    this.proc.stdout!.on("data", (chunk: Buffer) => this.onData(chunk))
+    this.proc.stderr!.on("data", (chunk: Buffer) => {
+      // Silently consume stderr — language servers are chatty
+    })
+    this.proc.on("error", (err) => {
+      // Reject all pending
+      for (const p of this.pending.values()) p.reject(err)
+      this.pending.clear()
+    })
+    this.proc.on("exit", () => {
+      for (const p of this.pending.values()) p.reject(new Error("LSP server exited"))
+      this.pending.clear()
+      this.initialized = false
+    })
+
+    // LSP initialize handshake
+    const initResult = await this.sendRequest("initialize", {
+      processId: process.pid,
+      rootUri: this.rootUri,
+      capabilities: {
+        textDocument: {
+          documentSymbol: { hierarchicalDocumentSymbolSupport: true },
+          definition: { linkSupport: false },
+          references: {},
+          implementation: {},
+        },
+      },
+    })
+
+    // Send initialized notification (no response expected)
+    this.sendNotification("initialized", {})
+    this.initialized = true
+  }
+
+  /** Shut down gracefully. */
+  async stop(): Promise<void> {
+    if (!this.proc || !this.initialized) return
+    try {
+      await this.sendRequest("shutdown", null)
+      this.sendNotification("exit", null)
+    } catch { /* best effort */ }
+    this.proc.kill()
+    this.proc = null
+    this.initialized = false
+  }
+
+  // ---- LSP helpers --------------------------------------------------------
+
+  async openDocument(filePath: string, content: string, languageId: string): Promise<void> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    this.sendNotification("textDocument/didOpen", {
+      textDocument: {
+        uri,
+        languageId,
+        version: 1,
+        text: content,
+      },
+    })
+    // Brief pause to let the notification reach the server.
+    // Callers that need full type info should wait separately.
+    await new Promise(r => setTimeout(r, 200))
+  }
+
+  async closeDocument(filePath: string): Promise<void> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    this.sendNotification("textDocument/didClose", {
+      textDocument: { uri },
+    })
+  }
+
+  async documentSymbol(filePath: string): Promise<LSPSymbolInformation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/documentSymbol", {
+      textDocument: { uri },
+    })
+    return result || []
+  }
+
+  async definition(filePath: string, line: number, character: number): Promise<LSPLocation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/definition", {
+      textDocument: { uri },
+      position: { line, character },
+    })
+    return this.normalizeLocations(result)
+  }
+
+  async references(filePath: string, line: number, character: number): Promise<LSPLocation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/references", {
+      textDocument: { uri },
+      position: { line, character },
+      context: { includeDeclaration: false },
+    })
+    return this.normalizeLocations(result)
+  }
+
+  async implementation(filePath: string, line: number, character: number): Promise<LSPLocation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/implementation", {
+      textDocument: { uri },
+      position: { line, character },
+    })
+    return this.normalizeLocations(result)
+  }
+
+  // ---- transport ----------------------------------------------------------
+
+  private sendRequest(method: string, params: any): Promise<any> {
+    return new Promise((resolve, reject) => {
+      if (!this.proc?.stdin?.writable) {
+        return reject(new Error("LSP server not running"))
+      }
+      const id = ++this.requestId
+      const msg = { jsonrpc: "2.0", id, method, params }
+
+      const timer = setTimeout(() => {
+        this.pending.delete(id)
+        reject(new Error(`LSP request timed out: ${method} (${this.timeoutMs}ms)`))
+      }, this.timeoutMs)
+
+      this.pending.set(id, {
+        resolve: (v: any) => { clearTimeout(timer); resolve(v) },
+        reject: (e: Error) => { clearTimeout(timer); reject(e) },
+      })
+
+      this.proc.stdin.write(encodeMessage(msg))
+    })
+  }
+
+  private sendNotification(method: string, params: any): void {
+    if (!this.proc?.stdin?.writable) return
+    const msg = { jsonrpc: "2.0", method, params }
+    this.proc.stdin.write(encodeMessage(msg))
+  }
+
+  private onData(chunk: Buffer): void {
+    this.buffer = Buffer.concat([this.buffer, chunk])
+    this.processBuffer()
+  }
+
+  private processBuffer(): void {
+    while (true) {
+      // Look for Content-Length header
+      const headerEnd = this.buffer.indexOf("\r\n\r\n")
+      if (headerEnd === -1) break
+
+      const headerStr = this.buffer.subarray(0, headerEnd).toString("ascii")
+      const match = headerStr.match(/Content-Length:\s*(\d+)/i)
+      if (!match) {
+        // Corrupt header — skip past it
+        this.buffer = this.buffer.subarray(headerEnd + 4)
+        continue
+      }
+
+      const contentLength = parseInt(match[1], 10)
+      const bodyStart = headerEnd + 4
+      const bodyEnd = bodyStart + contentLength
+
+      if (this.buffer.length < bodyEnd) break // not enough data yet
+
+      const bodyStr = this.buffer.subarray(bodyStart, bodyEnd).toString("utf-8")
+      this.buffer = this.buffer.subarray(bodyEnd)
+
+      try {
+        const msg = JSON.parse(bodyStr)
+        this.handleMessage(msg)
+      } catch {
+        // malformed JSON — skip
+      }
+    }
+  }
+
+  private handleMessage(msg: any): void {
+    // Response to a request we sent
+    if (msg.id != null && this.pending.has(msg.id)) {
+      const p = this.pending.get(msg.id)!
+      this.pending.delete(msg.id)
+      if (msg.error) {
+        p.reject(new Error(`LSP error ${msg.error.code}: ${msg.error.message}`))
+      } else {
+        p.resolve(msg.result)
+      }
+      return
+    }
+    // Server notifications / requests — ignore for indexing purposes
+  }
+
+  private normalizeLocations(result: any): LSPLocation[] {
+    if (!result) return []
+    if (Array.isArray(result)) return result
+    // Single Location
+    if (result.uri) return [result]
+    return []
+  }
+}

package/vectorizer/graph-builder.ts

@@ -52,10 +52,13 @@ export class GraphBuilder {
       if (lspAvailable) {
         try {
           relations = await this.lspAnalyzer.analyzeFile(filePath, chunks)
-        } catch (error) {
-          relations = this.regexAnalyzer.analyzeCode(filePath, content, chunks)
+        } catch {
+          // LSP threw — fall through to regex
         }
-      } else {
+      }
+
+      // Fallback: if LSP unavailable, threw, or returned nothing → use regex
+      if (relations.length === 0) {
         relations = this.regexAnalyzer.analyzeCode(filePath, content, chunks)
       }
     }
@@ -92,4 +95,104 @@ export class GraphBuilder {
 
     return result
   }
+
+  // ---- FR-005: Semantic similarity edges ------------------------------------
+
+  /**
+   * Build "similar_to" edges between chunks whose cosine similarity > threshold.
+   * Only creates edges where no explicit link already exists.
+   *
+   * @param chunks — array of { chunk_id, vector } (all indexed chunks)
+   * @param threshold — minimum cosine similarity (default 0.8)
+   * @param maxEdgesPerChunk — limit outgoing similarity edges per chunk (default 3)
+   * @returns number of similarity edges created
+   */
+  async buildSemanticEdges(
+    chunks: Array<{ chunk_id: string; vector: number[]; file: string }>,
+    threshold: number = 0.8,
+    maxEdgesPerChunk: number = 3,
+  ): Promise<number> {
+    if (chunks.length < 2) return 0
+
+    // Pre-collect all existing edges so we can skip pairs that already have links
+    const existingPairs = new Set<string>()
+    for (const chunk of chunks) {
+      try {
+        const outgoing = await this.graphDB.getOutgoing(chunk.chunk_id)
+        for (const t of outgoing) {
+          existingPairs.add(`${t.subject}|${t.object}`)
+          existingPairs.add(`${t.object}|${t.subject}`) // bidirectional check
+        }
+      } catch {
+        // skip — chunk may not have edges yet
+      }
+    }
+
+    const newTriples: Triple[] = []
+
+    // For each chunk, find top-N most similar chunks above threshold
+    for (let i = 0; i < chunks.length; i++) {
+      const a = chunks[i]
+      if (!a.vector || a.vector.length === 0) continue
+
+      const candidates: Array<{ idx: number; similarity: number }> = []
+
+      for (let j = i + 1; j < chunks.length; j++) {
+        const b = chunks[j]
+        if (!b.vector || b.vector.length === 0) continue
+
+        // Skip same-file chunks (intra-file similarity is less useful)
+        if (a.file === b.file) continue
+
+        // Skip if explicit edge already exists
+        const pairKey = `${a.chunk_id}|${b.chunk_id}`
+        if (existingPairs.has(pairKey)) continue
+
+        const similarity = this.cosineSimilarity(a.vector, b.vector)
+        if (similarity > threshold) {
+          candidates.push({ idx: j, similarity })
+        }
+      }
+
+      // Sort by similarity descending, take top N
+      candidates.sort((x, y) => y.similarity - x.similarity)
+      const top = candidates.slice(0, maxEdgesPerChunk)
+
+      for (const c of top) {
+        const b = chunks[c.idx]
+        newTriples.push({
+          subject: a.chunk_id,
+          predicate: "similar_to",
+          object: b.chunk_id,
+          weight: parseFloat(c.similarity.toFixed(4)),
+          source: "semantic",
+          file: a.file,
+        })
+        // Mark as existing so reverse pair isn't duplicated
+        existingPairs.add(`${a.chunk_id}|${b.chunk_id}`)
+        existingPairs.add(`${b.chunk_id}|${a.chunk_id}`)
+      }
+    }
+
+    if (newTriples.length > 0) {
+      // Batch insert in groups of 100 to avoid overwhelming LevelDB
+      for (let i = 0; i < newTriples.length; i += 100) {
+        const batch = newTriples.slice(i, i + 100)
+        await this.graphDB.putEdges(batch)
+      }
+    }
+
+    return newTriples.length
+  }
+
+  private cosineSimilarity(vecA: number[], vecB: number[]): number {
+    let dot = 0, normA = 0, normB = 0
+    for (let i = 0; i < vecA.length; i++) {
+      dot += vecA[i] * vecB[i]
+      normA += vecA[i] * vecA[i]
+      normB += vecB[i] * vecB[i]
+    }
+    const denom = Math.sqrt(normA) * Math.sqrt(normB)
+    return denom === 0 ? 0 : dot / denom
+  }
 }

package/vectorizer/graph-db.ts

@@ -94,4 +94,196 @@ export class GraphDB {
       this.initialized = false
     }
   }
+
+  // ---- FR-054: File metadata triples for incremental updates -----------------
+
+  /**
+   * Store graph build metadata for a file as a special triple.
+   * Subject: `meta:<filePath>`, Predicate: `graph_built`, Object: `<hash>`.
+   * Weight encodes the timestamp (seconds since epoch).
+   */
+  async setFileMeta(filePath: string, hash: string, timestamp: number): Promise<void> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    // Remove old meta triple for this file first
+    await this.deleteFileMeta(filePath)
+
+    const triple: Triple = {
+      subject: `meta:${filePath}`,
+      predicate: "graph_built",
+      object: hash,
+      weight: Math.floor(timestamp / 1000), // seconds since epoch fits in weight
+      source: "meta",
+      file: filePath,
+    }
+    await this.putEdges([triple])
+  }
+
+  /**
+   * Get the stored graph build metadata for a file.
+   * Returns { hash, timestamp } or null if not found.
+   */
+  async getFileMeta(filePath: string): Promise<{ hash: string; timestamp: number } | null> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const triples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get(
+        { subject: `meta:${filePath}`, predicate: "graph_built" },
+        (err: Error | undefined, result: Triple[]) => {
+          if (err) reject(err)
+          else resolve(result || [])
+        },
+      )
+    })
+
+    if (triples.length === 0) return null
+    return {
+      hash: triples[0].object,
+      timestamp: triples[0].weight * 1000, // back to ms
+    }
+  }
+
+  /**
+   * Delete file meta triple.
+   */
+  async deleteFileMeta(filePath: string): Promise<void> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const triples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get(
+        { subject: `meta:${filePath}`, predicate: "graph_built" },
+        (err: Error | undefined, result: Triple[]) => {
+          if (err) reject(err)
+          else resolve(result || [])
+        },
+      )
+    })
+
+    for (const t of triples) {
+      await new Promise<void>((resolve, reject) => {
+        this.db.del(t, (err: Error | undefined) => {
+          if (err) reject(err)
+          else resolve()
+        })
+      })
+    }
+  }
+
+  /**
+   * Get all file metadata triples (for validation / stats).
+   */
+  async getAllFileMeta(): Promise<Array<{ filePath: string; hash: string; timestamp: number }>> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const triples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get({ predicate: "graph_built" }, (err: Error | undefined, result: Triple[]) => {
+        if (err) reject(err)
+        else resolve(result || [])
+      })
+    })
+
+    return triples.map((t) => ({
+      filePath: t.subject.replace(/^meta:/, ""),
+      hash: t.object,
+      timestamp: t.weight * 1000,
+    }))
+  }
+
+  /**
+   * Get all triples in the graph (for validation/stats).
+   * Excludes meta triples (predicate === "graph_built").
+   */
+  async getAllTriples(): Promise<Triple[]> {
+    if (!this.initialized) throw new Error("GraphDB not initialized. Call init() first.")
+
+    const allTriples = await new Promise<Triple[]>((resolve, reject) => {
+      this.db.get({}, (err: Error | undefined, triples: Triple[]) => {
+        if (err) reject(err)
+        else resolve(triples || [])
+      })
+    })
+
+    return allTriples.filter(t => t.predicate !== "graph_built")
+  }
+
+  async getRelatedFiles(chunkId: string, maxDepth: number = 1): Promise<{path: string, relation: string, weight: number}[]> {
+    if (!this.initialized) {
+      throw new Error("GraphDB not initialized. Call init() first.")
+    }
+
+    const relatedFiles: Map<string, {relation: string, weight: number}> = new Map()
+    const visited = new Set<string>()
+    
+    async function traverse(currentId: string, currentDepth: number, currentRelation: string) {
+      if (currentDepth > maxDepth || visited.has(currentId)) {
+        return
+      }
+      
+      visited.add(currentId)
+      
+      try {
+        const outgoing = await new Promise<Triple[]>((resolve, reject) => {
+          this.db.get({ subject: currentId }, (err: Error | undefined, triples: Triple[]) => {
+            if (err) reject(err)
+            else resolve(triples || [])
+          })
+        })
+        
+        for (const triple of outgoing) {
+          const fileId = triple.object
+          
+          // Aggregate relations and weights
+          const existing = relatedFiles.get(fileId)
+          if (existing) {
+            existing.weight = Math.max(existing.weight, triple.weight)
+          } else {
+            relatedFiles.set(fileId, {
+              relation: currentRelation || triple.predicate,
+              weight: triple.weight
+            })
+          }
+          
+          // Recurse for imports/extends relations
+          if (triple.predicate === "imports" || triple.predicate === "extends") {
+            await traverse(fileId, currentDepth + 1, triple.predicate)
+          }
+        }
+        
+        const incoming = await new Promise<Triple[]>((resolve, reject) => {
+          this.db.get({ object: currentId }, (err: Error | undefined, triples: Triple[]) => {
+            if (err) reject(err)
+            else resolve(triples || [])
+          })
+        })
+        
+        for (const triple of incoming) {
+          const fileId = triple.subject
+          
+          const existing = relatedFiles.get(fileId)
+          if (existing) {
+            existing.weight = Math.max(existing.weight, triple.weight)
+          } else {
+            relatedFiles.set(fileId, {
+              relation: `used_by`,
+              weight: triple.weight
+            })
+          }
+        }
+      } catch (error) {
+        console.error(`Error traversing graph for ${currentId}:`, error)
+      }
+    }
+    
+    await traverse(chunkId, 0, "")
+    
+    const result = Array.from(relatedFiles.entries())
+      .map(([path, data]) => ({
+        path,
+        relation: data.relation,
+        weight: data.weight
+      }))
+      .sort((a, b) => b.weight - a.weight)
+    
+    return result
+  }
 }