@comfanion/usethis_search 0.2.0-dev.0 → 3.0.0-dev.1

package/vectorizer/analyzers/lsp-analyzer.ts

@@ -0,0 +1,293 @@
+/**
+ * LSPAnalyzer — extracts code relations via Language Server Protocol.
+ *
+ * Uses LSPClient (lsp-client.ts) to communicate with real language servers
+ * (typescript-language-server, pylsp, gopls, etc.).
+ *
+ * Strategy: LSP-first with graceful degradation.
+ * - If LSP server binary not found → isAvailable() returns false → GraphBuilder uses regex fallback
+ * - If LSP server crashes or times out during analysis → analyzeFile() returns [] → GraphBuilder uses regex
+ * - Timeout: 5 seconds per file (configurable)
+ */
+
+import path from "path"
+import fs from "fs/promises"
+import { ChunkWithId } from "../graph-builder"
+import { LSPClient, LSPSymbolInformation, SymbolKind } from "./lsp-client"
+
+export interface Relation {
+  from: string
+  to: string
+  predicate: string
+  weight: number
+  source: "lsp"
+  line?: number
+}
+
+/** Language ID mapping from file extension. */
+const EXT_TO_LANGUAGE: Record<string, string> = {
+  ts: "typescript",
+  js: "javascript",
+  tsx: "typescriptreact",
+  jsx: "javascriptreact",
+  py: "python",
+  go: "go",
+  rs: "rust",
+  java: "java",
+  cpp: "cpp",
+  c: "c",
+  cs: "csharp",
+}
+
+/** Symbol kinds that represent types (classes, interfaces) — check implementations. */
+const TYPE_SYMBOL_KINDS = new Set([
+  SymbolKind.Class,
+  SymbolKind.Interface,
+])
+
+/** Symbol kinds worth analyzing for references/definitions. */
+const ANALYZABLE_SYMBOL_KINDS = new Set([
+  SymbolKind.Class,
+  SymbolKind.Interface,
+  SymbolKind.Function,
+  SymbolKind.Method,
+  SymbolKind.Variable,
+  SymbolKind.Constant,
+  SymbolKind.Enum,
+])
+
+export class LSPAnalyzer {
+  private client: LSPClient | null = null
+  private currentLanguage: string | null = null
+  private filesOpened = 0
+  private readonly timeoutMs: number
+
+  constructor(
+    private projectRoot?: string,
+    timeoutMs?: number,
+  ) {
+    this.timeoutMs = timeoutMs ?? 5000
+  }
+
+  // ---- public API ---------------------------------------------------------
+
+  /** Check if an LSP server is available for this file's language. */
+  async isAvailable(filePath: string): Promise<boolean> {
+    try {
+      const language = this.getLanguage(filePath)
+      if (!language) return false
+      return LSPClient.isAvailable(language)
+    } catch {
+      return false
+    }
+  }
+
+  /**
+   * Analyze a file using LSP and return relations.
+   *
+   * Flow:
+   * 1. Start/reuse server for the file's language
+   * 2. Open document
+   * 3. Get document symbols (functions, classes, interfaces)
+   * 4. For each symbol: get definitions, references, implementations
+   * 5. Map LSP locations → chunk IDs → Relation objects
+   * 6. Close document
+   */
+  async analyzeFile(filePath: string, chunks: ChunkWithId[]): Promise<Relation[]> {
+    const language = this.getLanguage(filePath)
+    if (!language) return []
+
+    const root = this.projectRoot || process.cwd()
+    const absPath = path.isAbsolute(filePath) ? filePath : path.resolve(root, filePath)
+
+    let content: string
+    try {
+      content = await fs.readFile(absPath, "utf-8")
+    } catch {
+      return []
+    }
+
+    // Ensure client is started for this language
+    try {
+      await this.ensureClient(language, root)
+    } catch {
+      return []
+    }
+
+    const client = this.client!
+    const relations: Relation[] = []
+
+    try {
+      // Open the document
+      await client.openDocument(filePath, content, language)
+      this.filesOpened++
+
+      // First file needs extra warmup for project loading / type graph build
+      if (this.filesOpened === 1) {
+        await new Promise(r => setTimeout(r, 2000))
+      } else {
+        await new Promise(r => setTimeout(r, 500))
+      }
+
+      // Get all symbols in the file
+      const rawSymbols = await client.documentSymbol(filePath)
+      const symbols = this.flattenSymbols(rawSymbols)
+
+      // Analyze each symbol
+      for (const sym of symbols) {
+        if (!ANALYZABLE_SYMBOL_KINDS.has(sym.kind)) continue
+
+        const fromChunkId = this.findChunkForPosition(chunks, sym.selectionRange.start.line)
+        if (!fromChunkId) continue
+
+        const selLine = sym.selectionRange.start.line
+        const selChar = sym.selectionRange.start.character
+
+        // --- implementations (for classes/interfaces) ---
+        if (TYPE_SYMBOL_KINDS.has(sym.kind)) {
+          try {
+            const impls = await client.implementation(filePath, selLine, selChar)
+            for (const impl of impls) {
+              const toChunkId = this.locationToChunkId(filePath, impl.uri, impl.range.start.line, root)
+              if (toChunkId && toChunkId !== fromChunkId) {
+                relations.push({
+                  from: fromChunkId,
+                  to: toChunkId,
+                  predicate: "implements",
+                  weight: 1.0,
+                  source: "lsp",
+                  line: selLine,
+                })
+              }
+            }
+          } catch { /* server may not support implementation */ }
+        }
+
+        // --- references ---
+        try {
+          const refs = await client.references(filePath, selLine, selChar)
+          for (const ref of refs) {
+            const toChunkId = this.locationToChunkId(filePath, ref.uri, ref.range.start.line, root)
+            if (toChunkId && toChunkId !== fromChunkId) {
+              relations.push({
+                from: toChunkId,
+                to: fromChunkId,
+                predicate: "used_by",
+                weight: 1.0,
+                source: "lsp",
+                line: ref.range.start.line,
+              })
+            }
+          }
+        } catch { /* server may not support references */ }
+
+        // --- definitions ---
+        try {
+          const defs = await client.definition(filePath, selLine, selChar)
+          for (const def of defs) {
+            const toChunkId = this.locationToChunkId(filePath, def.uri, def.range.start.line, root)
+            if (toChunkId && toChunkId !== fromChunkId) {
+              relations.push({
+                from: fromChunkId,
+                to: toChunkId,
+                predicate: "references",
+                weight: 1.0,
+                source: "lsp",
+                line: selLine,
+              })
+            }
+          }
+        } catch { /* server may not support definition */ }
+      }
+
+      await client.closeDocument(filePath)
+    } catch {
+      // Any error — return whatever we collected so far (or empty)
+    }
+
+    return this.deduplicateRelations(relations)
+  }
+
+  /** Stop the LSP server. Call when done indexing. */
+  async shutdown(): Promise<void> {
+    if (this.client) {
+      try { await this.client.stop() } catch { /* best effort */ }
+      this.client = null
+      this.currentLanguage = null
+      this.filesOpened = 0
+    }
+  }
+
+  // ---- internals ----------------------------------------------------------
+
+  private getLanguage(filePath: string): string | null {
+    const ext = filePath.split(".").pop()
+    if (!ext) return null
+    return EXT_TO_LANGUAGE[ext] || null
+  }
+
+  /** Start or reuse a client for the given language. */
+  private async ensureClient(language: string, root: string): Promise<void> {
+    // If we already have a client for a different language, shut it down
+    if (this.client && this.currentLanguage !== language) {
+      await this.shutdown()
+    }
+    if (!this.client) {
+      this.client = new LSPClient(root, this.timeoutMs)
+      await this.client.start(language)
+      this.currentLanguage = language
+    }
+  }
+
+  /** Flatten hierarchical DocumentSymbol tree into flat array. */
+  private flattenSymbols(symbols: LSPSymbolInformation[]): LSPSymbolInformation[] {
+    const result: LSPSymbolInformation[] = []
+    const walk = (list: LSPSymbolInformation[]) => {
+      for (const sym of list) {
+        result.push(sym)
+        if (sym.children?.length) walk(sym.children)
+      }
+    }
+    walk(symbols)
+    return result
+  }
+
+  /** Convert LSP location URI + line → chunk_id. */
+  private locationToChunkId(currentFile: string, uri: string, line: number, root: string): string | null {
+    // uri = file:///absolute/path/to/file.ts
+    const filePath = uri.startsWith("file://") ? uri.slice(7) : uri
+    const relPath = path.relative(root, filePath)
+
+    // Skip external files (node_modules, etc.)
+    if (relPath.startsWith("..") || relPath.includes("node_modules")) return null
+
+    const withoutExt = relPath.replace(/\.[^/.]+$/, "")
+    const normalized = withoutExt.replace(/[^a-zA-Z0-9]/g, "_")
+    // For cross-file references, point to chunk 0 (first chunk of target file)
+    // For same-file, we could be more precise but chunk 0 is sufficient for graph
+    return `chunk_${normalized}_0`
+  }
+
+  private findChunkForPosition(chunks: ChunkWithId[], line: number): string | null {
+    for (const chunk of chunks) {
+      if (chunk.start_line !== undefined && chunk.end_line !== undefined) {
+        if (line >= chunk.start_line && line <= chunk.end_line) {
+          return chunk.chunk_id
+        }
+      }
+    }
+    // Fallback: if no chunk matches by line, return first chunk
+    return chunks.length > 0 ? chunks[0].chunk_id : null
+  }
+
+  /** Remove duplicate relations (same from+to+predicate). */
+  private deduplicateRelations(relations: Relation[]): Relation[] {
+    const seen = new Set<string>()
+    return relations.filter(r => {
+      const key = `${r.from}|${r.to}|${r.predicate}`
+      if (seen.has(key)) return false
+      seen.add(key)
+      return true
+    })
+  }
+}

package/vectorizer/analyzers/lsp-client.ts

@@ -0,0 +1,369 @@
+/**
+ * Lightweight LSP client over JSON-RPC / stdio.
+ *
+ * Spawns a language server as a child process, speaks the Language Server
+ * Protocol, and exposes high-level helpers used by LSPAnalyzer.
+ *
+ * Zero external dependencies — implements JSON-RPC framing inline.
+ */
+
+import { spawn, ChildProcess } from "child_process"
+import path from "path"
+import fs from "fs"
+
+// ---------------------------------------------------------------------------
+// LSP types (minimal subset we need)
+// ---------------------------------------------------------------------------
+
+export interface LSPPosition {
+  line: number
+  character: number
+}
+
+export interface LSPRange {
+  start: LSPPosition
+  end: LSPPosition
+}
+
+export interface LSPLocation {
+  uri: string
+  range: LSPRange
+}
+
+export interface LSPSymbolInformation {
+  name: string
+  kind: number
+  range: LSPRange
+  selectionRange: LSPRange
+  children?: LSPSymbolInformation[]
+}
+
+// SymbolKind constants we care about
+export const SymbolKind = {
+  File: 1,
+  Module: 2,
+  Namespace: 3,
+  Package: 4,
+  Class: 5,
+  Method: 6,
+  Property: 7,
+  Field: 8,
+  Constructor: 9,
+  Enum: 10,
+  Interface: 11,
+  Function: 12,
+  Variable: 13,
+  Constant: 14,
+  TypeParameter: 26,
+} as const
+
+// ---------------------------------------------------------------------------
+// Server binary resolution
+// ---------------------------------------------------------------------------
+
+interface ServerConfig {
+  command: string
+  args: string[]
+  languages: string[]
+}
+
+const SERVER_CONFIGS: Record<string, ServerConfig> = {
+  typescript: {
+    command: "typescript-language-server",
+    args: ["--stdio"],
+    languages: ["typescript", "javascript", "typescriptreact", "javascriptreact"],
+  },
+  python: {
+    command: "pylsp",
+    args: [],
+    languages: ["python"],
+  },
+  go: {
+    command: "gopls",
+    args: ["serve"],
+    languages: ["go"],
+  },
+  rust: {
+    command: "rust-analyzer",
+    args: [],
+    languages: ["rust"],
+  },
+}
+
+function languageToServerId(language: string): string | null {
+  for (const [id, cfg] of Object.entries(SERVER_CONFIGS)) {
+    if (cfg.languages.includes(language)) return id
+  }
+  return null
+}
+
+function findBinary(name: string): string | null {
+  // 1. Check PATH via which-style lookup
+  const pathEnv = process.env.PATH || ""
+  const dirs = pathEnv.split(path.delimiter)
+  for (const dir of dirs) {
+    const full = path.join(dir, name)
+    try {
+      fs.accessSync(full, fs.constants.X_OK)
+      return full
+    } catch { /* not found */ }
+  }
+  return null
+}
+
+// ---------------------------------------------------------------------------
+// JSON-RPC framing
+// ---------------------------------------------------------------------------
+
+function encodeMessage(body: object): Buffer {
+  const json = JSON.stringify(body)
+  const content = Buffer.from(json, "utf-8")
+  const header = `Content-Length: ${content.byteLength}\r\n\r\n`
+  return Buffer.concat([Buffer.from(header, "ascii"), content])
+}
+
+// ---------------------------------------------------------------------------
+// LSPClient
+// ---------------------------------------------------------------------------
+
+export class LSPClient {
+  private proc: ChildProcess | null = null
+  private requestId = 0
+  private pending = new Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }>()
+  private buffer = Buffer.alloc(0)
+  private initialized = false
+  private serverConfig: ServerConfig | null = null
+  private rootUri: string
+
+  constructor(
+    private projectRoot: string,
+    private timeoutMs: number = 10_000,
+  ) {
+    this.rootUri = `file://${this.projectRoot}`
+  }
+
+  // ---- lifecycle ----------------------------------------------------------
+
+  /** Check if a server binary exists for the given language. */
+  static isAvailable(language: string): boolean {
+    const serverId = languageToServerId(language)
+    if (!serverId) return false
+    const cfg = SERVER_CONFIGS[serverId]
+    return findBinary(cfg.command) !== null
+  }
+
+  /** Start the language server for `language`. Rejects if unavailable. */
+  async start(language: string): Promise<void> {
+    if (this.initialized) return
+
+    const serverId = languageToServerId(language)
+    if (!serverId) throw new Error(`No LSP server config for language: ${language}`)
+
+    const cfg = SERVER_CONFIGS[serverId]
+    const bin = findBinary(cfg.command)
+    if (!bin) throw new Error(`LSP server binary not found: ${cfg.command}`)
+
+    this.serverConfig = cfg
+
+    this.proc = spawn(bin, cfg.args, {
+      cwd: this.projectRoot,
+      stdio: ["pipe", "pipe", "pipe"],
+      env: { ...process.env },
+    })
+
+    this.proc.stdout!.on("data", (chunk: Buffer) => this.onData(chunk))
+    this.proc.stderr!.on("data", (chunk: Buffer) => {
+      // Silently consume stderr — language servers are chatty
+    })
+    this.proc.on("error", (err) => {
+      // Reject all pending
+      for (const p of this.pending.values()) p.reject(err)
+      this.pending.clear()
+    })
+    this.proc.on("exit", () => {
+      for (const p of this.pending.values()) p.reject(new Error("LSP server exited"))
+      this.pending.clear()
+      this.initialized = false
+    })
+
+    // LSP initialize handshake
+    const initResult = await this.sendRequest("initialize", {
+      processId: process.pid,
+      rootUri: this.rootUri,
+      capabilities: {
+        textDocument: {
+          documentSymbol: { hierarchicalDocumentSymbolSupport: true },
+          definition: { linkSupport: false },
+          references: {},
+          implementation: {},
+        },
+      },
+    })
+
+    // Send initialized notification (no response expected)
+    this.sendNotification("initialized", {})
+    this.initialized = true
+  }
+
+  /** Shut down gracefully. */
+  async stop(): Promise<void> {
+    if (!this.proc || !this.initialized) return
+    try {
+      await this.sendRequest("shutdown", null)
+      this.sendNotification("exit", null)
+    } catch { /* best effort */ }
+    this.proc.kill()
+    this.proc = null
+    this.initialized = false
+  }
+
+  // ---- LSP helpers --------------------------------------------------------
+
+  async openDocument(filePath: string, content: string, languageId: string): Promise<void> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    this.sendNotification("textDocument/didOpen", {
+      textDocument: {
+        uri,
+        languageId,
+        version: 1,
+        text: content,
+      },
+    })
+    // Brief pause to let the notification reach the server.
+    // Callers that need full type info should wait separately.
+    await new Promise(r => setTimeout(r, 200))
+  }
+
+  async closeDocument(filePath: string): Promise<void> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    this.sendNotification("textDocument/didClose", {
+      textDocument: { uri },
+    })
+  }
+
+  async documentSymbol(filePath: string): Promise<LSPSymbolInformation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/documentSymbol", {
+      textDocument: { uri },
+    })
+    return result || []
+  }
+
+  async definition(filePath: string, line: number, character: number): Promise<LSPLocation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/definition", {
+      textDocument: { uri },
+      position: { line, character },
+    })
+    return this.normalizeLocations(result)
+  }
+
+  async references(filePath: string, line: number, character: number): Promise<LSPLocation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/references", {
+      textDocument: { uri },
+      position: { line, character },
+      context: { includeDeclaration: false },
+    })
+    return this.normalizeLocations(result)
+  }
+
+  async implementation(filePath: string, line: number, character: number): Promise<LSPLocation[]> {
+    const uri = `file://${path.resolve(this.projectRoot, filePath)}`
+    const result = await this.sendRequest("textDocument/implementation", {
+      textDocument: { uri },
+      position: { line, character },
+    })
+    return this.normalizeLocations(result)
+  }
+
+  // ---- transport ----------------------------------------------------------
+
+  private sendRequest(method: string, params: any): Promise<any> {
+    return new Promise((resolve, reject) => {
+      if (!this.proc?.stdin?.writable) {
+        return reject(new Error("LSP server not running"))
+      }
+      const id = ++this.requestId
+      const msg = { jsonrpc: "2.0", id, method, params }
+
+      const timer = setTimeout(() => {
+        this.pending.delete(id)
+        reject(new Error(`LSP request timed out: ${method} (${this.timeoutMs}ms)`))
+      }, this.timeoutMs)
+
+      this.pending.set(id, {
+        resolve: (v: any) => { clearTimeout(timer); resolve(v) },
+        reject: (e: Error) => { clearTimeout(timer); reject(e) },
+      })
+
+      this.proc.stdin.write(encodeMessage(msg))
+    })
+  }
+
+  private sendNotification(method: string, params: any): void {
+    if (!this.proc?.stdin?.writable) return
+    const msg = { jsonrpc: "2.0", method, params }
+    this.proc.stdin.write(encodeMessage(msg))
+  }
+
+  private onData(chunk: Buffer): void {
+    this.buffer = Buffer.concat([this.buffer, chunk])
+    this.processBuffer()
+  }
+
+  private processBuffer(): void {
+    while (true) {
+      // Look for Content-Length header
+      const headerEnd = this.buffer.indexOf("\r\n\r\n")
+      if (headerEnd === -1) break
+
+      const headerStr = this.buffer.subarray(0, headerEnd).toString("ascii")
+      const match = headerStr.match(/Content-Length:\s*(\d+)/i)
+      if (!match) {
+        // Corrupt header — skip past it
+        this.buffer = this.buffer.subarray(headerEnd + 4)
+        continue
+      }
+
+      const contentLength = parseInt(match[1], 10)
+      const bodyStart = headerEnd + 4
+      const bodyEnd = bodyStart + contentLength
+
+      if (this.buffer.length < bodyEnd) break // not enough data yet
+
+      const bodyStr = this.buffer.subarray(bodyStart, bodyEnd).toString("utf-8")
+      this.buffer = this.buffer.subarray(bodyEnd)
+
+      try {
+        const msg = JSON.parse(bodyStr)
+        this.handleMessage(msg)
+      } catch {
+        // malformed JSON — skip
+      }
+    }
+  }
+
+  private handleMessage(msg: any): void {
+    // Response to a request we sent
+    if (msg.id != null && this.pending.has(msg.id)) {
+      const p = this.pending.get(msg.id)!
+      this.pending.delete(msg.id)
+      if (msg.error) {
+        p.reject(new Error(`LSP error ${msg.error.code}: ${msg.error.message}`))
+      } else {
+        p.resolve(msg.result)
+      }
+      return
+    }
+    // Server notifications / requests — ignore for indexing purposes
+  }
+
+  private normalizeLocations(result: any): LSPLocation[] {
+    if (!result) return []
+    if (Array.isArray(result)) return result
+    // Single Location
+    if (result.uri) return [result]
+    return []
+  }
+}