edhindex 1.0.0

package/src/retrieval/reranker.ts

@@ -0,0 +1,64 @@
+import { TransformersReranker } from '../embeddings/transformers.js';
+import { MetadataStore, ChunkMetadata } from '../storage/metadata.js';
+import { logger } from '../logging.js';
+
+export interface RerankedResult {
+  id: string;
+  score: number;
+  metadata: ChunkMetadata | null;
+}
+
+export async function rerankResults(
+  query: string,
+  candidates: Array<{ id: string }>,
+  metadataStore: MetadataStore,
+  reranker: TransformersReranker | null,
+  topK: number,
+  useRerank: boolean,
+): Promise<RerankedResult[]> {
+  if (candidates.length === 0) return [];
+
+  if (!useRerank || !reranker) {
+    return candidates.slice(0, topK).map(c => ({
+      id: c.id,
+      score: 1,
+      metadata: metadataStore.getChunk(c.id) || null,
+    }));
+  }
+
+  const texts: string[] = [];
+  const validCandidates: Array<{ id: string }> = [];
+
+  for (const c of candidates) {
+    const meta = metadataStore.getChunk(c.id);
+    if (meta) {
+      texts.push(meta.imports + ' ' + meta.exports + ' ' + (meta.symbol || ''));
+      validCandidates.push(c);
+    }
+  }
+
+  if (texts.length === 0) {
+    return candidates.slice(0, topK).map(c => ({
+      id: c.id,
+      score: 1,
+      metadata: metadataStore.getChunk(c.id) || null,
+    }));
+  }
+
+  try {
+    const scores = await reranker.rerank(query, texts);
+    const results: RerankedResult[] = scores.slice(0, topK).map(s => ({
+      id: validCandidates[s.index].id,
+      score: s.score,
+      metadata: metadataStore.getChunk(validCandidates[s.index].id) || null,
+    }));
+    return results;
+  } catch (e) {
+    logger.debug('Reranking failed, falling back to unranked:', e);
+    return validCandidates.slice(0, topK).map(c => ({
+      id: c.id,
+      score: 1,
+      metadata: metadataStore.getChunk(c.id) || null,
+    }));
+  }
+}

package/src/retrieval/search.ts

@@ -0,0 +1,144 @@
+import { FTSStore } from '../storage/fts.js';
+import { MetadataStore, ChunkMetadata } from '../storage/metadata.js';
+import { VectorStore } from '../vector/store.js';
+import { EmbeddingProvider } from '../embeddings/provider.js';
+import { TransformersReranker } from '../embeddings/transformers.js';
+import { searchBM25 } from './bm25.js';
+import { rerankResults, RerankedResult } from './reranker.js';
+import { logger } from '../logging.js';
+
+export interface SearchOptions {
+  query: string;
+  maxResults?: number;
+  rerank?: boolean;
+  fileFilter?: string;
+  languageFilter?: string;
+}
+
+export interface SearchResult {
+  id: string;
+  file: string;
+  language: string;
+  symbol: string;
+  kind: string | null;
+  startLine: number;
+  endLine: number;
+  content: string;
+  score: number;
+  matchType: 'keyword' | 'vector' | 'hybrid';
+}
+
+const BM25_TOP_K = 30;
+const VECTOR_TOP_K = 30;
+
+export class SearchEngine {
+  private fts: FTSStore;
+  private metadata: MetadataStore;
+  private vectors: VectorStore;
+  private embedder: EmbeddingProvider;
+  private reranker: TransformersReranker | null;
+  private useRerank: boolean;
+
+  constructor(
+    fts: FTSStore,
+    metadata: MetadataStore,
+    vectors: VectorStore,
+    embedder: EmbeddingProvider,
+    reranker: TransformersReranker | null,
+    useRerank: boolean,
+  ) {
+    this.fts = fts;
+    this.metadata = metadata;
+    this.vectors = vectors;
+    this.embedder = embedder;
+    this.reranker = reranker;
+    this.useRerank = useRerank;
+  }
+
+  async search(opts: SearchOptions): Promise<SearchResult[]> {
+    const maxResults = opts.maxResults || 10;
+    const query = opts.query.trim();
+    if (!query) return [];
+
+    // Normalize query
+    const normalized = query.replace(/\s+/g, ' ').trim();
+
+    // BM25
+    const bm25Results = searchBM25(this.fts, normalized, BM25_TOP_K);
+    logger.debug(`BM25 returned ${bm25Results.length} results`);
+
+    // Vector search
+    let vectorResults: Array<{ id: string; score: number }> = [];
+    try {
+      const embedding = await this.embedder.embed([normalized]);
+      if (embedding.length > 0 && embedding[0].length > 0) {
+        vectorResults = await this.vectors.search(embedding[0], VECTOR_TOP_K);
+      }
+    } catch (e) {
+      logger.debug('Vector search failed:', e);
+    }
+    logger.debug(`Vector search returned ${vectorResults.length} results`);
+
+    // Merge and deduplicate
+    const seen = new Set<string>();
+    const merged: Array<{ id: string; score: number }> = [];
+
+    for (const r of bm25Results) {
+      if (!seen.has(r.id)) {
+        seen.add(r.id);
+        merged.push(r);
+      }
+    }
+
+    for (const r of vectorResults) {
+      if (!seen.has(r.id)) {
+        seen.add(r.id);
+        merged.push({ id: r.id, score: r.score * 0.8 });
+      } else {
+        const existing = merged.find(m => m.id === r.id);
+        if (existing) {
+          existing.score = Math.max(existing.score, r.score);
+        }
+      }
+    }
+
+    logger.debug(`Merged ${merged.length} unique results`);
+
+    // Rerank
+    const reranked = await rerankResults(
+      normalized,
+      merged,
+      this.metadata,
+      this.reranker,
+      maxResults,
+      this.useRerank,
+    );
+
+    // Build final results
+    const results: SearchResult[] = [];
+    for (const r of reranked) {
+      if (!r.metadata) continue;
+
+      let matchType: SearchResult['matchType'] = 'hybrid';
+      const inBM25 = bm25Results.some(b => b.id === r.id);
+      const inVector = vectorResults.some(v => v.id === r.id);
+      if (inBM25 && !inVector) matchType = 'keyword';
+      else if (!inBM25 && inVector) matchType = 'vector';
+
+      results.push({
+        id: r.id,
+        file: r.metadata.file,
+        language: r.metadata.language,
+        symbol: r.metadata.symbol,
+        kind: r.metadata.kind,
+        startLine: r.metadata.start_line,
+        endLine: r.metadata.end_line,
+        content: r.metadata.imports + '\n' + r.metadata.exports,
+        score: r.score,
+        matchType,
+      });
+    }
+
+    return results.slice(0, maxResults);
+  }
+}

package/src/security.ts

@@ -0,0 +1,46 @@
+import { realpathSync } from 'node:fs';
+import { resolve, relative, normalize, sep } from 'node:path';
+
+export function isSubPath(parent: string, child: string): boolean {
+  const rel = relative(parent, child);
+  return !rel.startsWith('..') && !isAbsolutePath(rel);
+}
+
+function isAbsolutePath(p: string): boolean {
+  return sep === '/' ? p.startsWith('/') : /^[a-zA-Z]:[/\\]/.test(p);
+}
+
+export function sanitizePath(root: string, requestedPath: string): string | null {
+  const resolved = resolve(root, requestedPath);
+  const normalized = normalize(resolved);
+
+  if (!isSubPath(root, normalized)) {
+    return null;
+  }
+
+  return normalized;
+}
+
+export function resolveSafePath(root: string, requestedPath: string): string | null {
+  const sanitized = sanitizePath(root, requestedPath);
+  if (!sanitized) return null;
+
+  try {
+    const real = realpathSync(sanitized);
+    if (!isSubPath(root, real)) {
+      return null;
+    }
+    return real;
+  } catch {
+    return sanitized;
+  }
+}
+
+export function checkSymlinkSafety(root: string, targetPath: string): boolean {
+  try {
+    const real = realpathSync(targetPath);
+    return isSubPath(root, real);
+  } catch {
+    return true;
+  }
+}

package/src/storage/fts.ts

@@ -0,0 +1,130 @@
+import Database from 'better-sqlite3';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { logger } from '../logging.js';
+
+export interface FTSResult {
+  id: string;
+  rank: number;
+  content: string;
+  file: string;
+  symbol: string;
+  kind: string | null;
+  start_line: number;
+  end_line: number;
+  language: string;
+}
+
+export class FTSStore {
+  private db: Database.Database;
+  private path: string;
+
+  constructor(indexDir: string) {
+    this.path = join(indexDir, 'fts.db');
+    this.db = new Database(this.path);
+    this.createSchema();
+  }
+
+  private createSchema() {
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        content,
+        file UNINDEXED,
+        symbol,
+        language UNINDEXED,
+        content_id UNINDEXED,
+        tokenize='porter unicode61'
+      );
+
+      PRAGMA journal_mode = WAL;
+      PRAGMA synchronous = NORMAL;
+    `);
+  }
+
+  beginTransaction() {
+    this.db.exec('BEGIN TRANSACTION');
+  }
+
+  commit() {
+    this.db.exec('COMMIT');
+  }
+
+  rollback() {
+    this.db.exec('ROLLBACK');
+  }
+
+  insertChunk(id: string, content: string, file: string, symbol: string, language: string) {
+    try {
+      this.db.prepare(`
+        INSERT INTO chunks_fts (content, file, symbol, language, content_id)
+        VALUES (?, ?, ?, ?, ?)
+      `).run(content, file, symbol, language, id);
+    } catch (e: any) {
+      if (e?.message?.includes('already exists')) {
+        this.db.prepare(`
+          UPDATE chunks_fts SET content = ?, file = ?, symbol = ?, language = ?
+          WHERE content_id = ?
+        `).run(content, file, symbol, language, id);
+      }
+    }
+  }
+
+  deleteChunk(id: string) {
+    this.db.prepare('DELETE FROM chunks_fts WHERE content_id = ?').run(id);
+  }
+
+  deleteChunksForFile(filePath: string) {
+    this.db.prepare('DELETE FROM chunks_fts WHERE file = ?').run(filePath);
+  }
+
+  search(query: string, limit: number): FTSResult[] {
+    const cleaned = query.replace(/[^\w\s]/g, ' ').trim();
+    if (!cleaned) return [];
+
+    try {
+      const results = this.db.prepare(`
+        SELECT
+          content_id as id,
+          rank,
+          content,
+          file,
+          symbol,
+          language
+        FROM chunks_fts
+        WHERE chunks_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(cleaned, limit) as FTSResult[];
+
+      return results.map(r => ({
+        ...r,
+        kind: null,
+        start_line: 0,
+        end_line: 0,
+      }));
+    } catch {
+      return [];
+    }
+  }
+
+  searchWithRank(query: string, limit: number): Array<{ id: string; rank: number }> {
+    const cleaned = query.replace(/[^\w\s]/g, ' ').trim();
+    if (!cleaned) return [];
+
+    try {
+      return this.db.prepare(`
+        SELECT content_id as id, rank
+        FROM chunks_fts
+        WHERE chunks_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(cleaned, limit) as Array<{ id: string; rank: number }>;
+    } catch {
+      return [];
+    }
+  }
+
+  close() {
+    this.db.close();
+  }
+}

package/src/storage/metadata.ts

@@ -0,0 +1,162 @@
+import Database from 'better-sqlite3';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { logger } from '../logging.js';
+
+export interface ChunkMetadata {
+  id: string;
+  file: string;
+  language: string;
+  symbol: string;
+  kind: string | null;
+  parent: string | null;
+  hash: string;
+  git_commit: string | null;
+  start_line: number;
+  end_line: number;
+  imports: string;
+  exports: string;
+  embedding_model: string;
+  last_indexed: string;
+  checksum: string;
+}
+
+export class MetadataStore {
+  private db: Database.Database;
+  private path: string;
+
+  constructor(indexDir: string) {
+    this.path = join(indexDir, 'metadata.db');
+    const exists = existsSync(this.path);
+    this.db = new Database(this.path);
+
+    if (!exists) {
+      this.createSchema();
+    }
+  }
+
+  private createSchema() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id TEXT PRIMARY KEY,
+        file TEXT NOT NULL,
+        language TEXT NOT NULL,
+        symbol TEXT NOT NULL DEFAULT '',
+        kind TEXT,
+        parent TEXT,
+        hash TEXT NOT NULL,
+        git_commit TEXT,
+        start_line INTEGER NOT NULL,
+        end_line INTEGER NOT NULL,
+        imports TEXT DEFAULT '',
+        exports TEXT DEFAULT '',
+        embedding_model TEXT NOT NULL DEFAULT '',
+        last_indexed TEXT NOT NULL,
+        checksum TEXT NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file);
+      CREATE INDEX IF NOT EXISTS idx_chunks_language ON chunks(language);
+      CREATE INDEX IF NOT EXISTS idx_chunks_symbol ON chunks(symbol);
+      CREATE INDEX IF NOT EXISTS idx_chunks_kind ON chunks(kind);
+      CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(hash);
+
+      CREATE TABLE IF NOT EXISTS file_hashes (
+        path TEXT PRIMARY KEY,
+        hash TEXT NOT NULL,
+        last_indexed TEXT NOT NULL
+      );
+
+      PRAGMA journal_mode = WAL;
+      PRAGMA synchronous = NORMAL;
+    `);
+  }
+
+  beginTransaction() {
+    this.db.exec('BEGIN TRANSACTION');
+  }
+
+  commit() {
+    this.db.exec('COMMIT');
+  }
+
+  rollback() {
+    this.db.exec('ROLLBACK');
+  }
+
+  upsertChunk(chunk: ChunkMetadata) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO chunks
+        (id, file, language, symbol, kind, parent, hash, git_commit,
+         start_line, end_line, imports, exports, embedding_model, last_indexed, checksum)
+      VALUES
+        (@id, @file, @language, @symbol, @kind, @parent, @hash, @git_commit,
+         @start_line, @end_line, @imports, @exports, @embedding_model, @last_indexed, @checksum)
+    `);
+    stmt.run(chunk);
+  }
+
+  deleteChunks(chunkIds: string[]) {
+    if (chunkIds.length === 0) return;
+    const stmt = this.db.prepare('DELETE FROM chunks WHERE id = ?');
+    for (const id of chunkIds) {
+      stmt.run(id);
+    }
+  }
+
+  deleteChunksForFile(filePath: string) {
+    this.db.prepare('DELETE FROM chunks WHERE file = ?').run(filePath);
+  }
+
+  getChunk(id: string): ChunkMetadata | undefined {
+    return this.db.prepare('SELECT * FROM chunks WHERE id = ?').get(id) as ChunkMetadata | undefined;
+  }
+
+  getAllChunkHashes(): Map<string, string> {
+    const rows = this.db.prepare('SELECT id, hash FROM chunks').all() as { id: string; hash: string }[];
+    const map = new Map<string, string>();
+    for (const row of rows) {
+      map.set(row.id, row.hash);
+    }
+    return map;
+  }
+
+  getAllFileHashes(): Map<string, string> {
+    const rows = this.db.prepare('SELECT path, hash FROM file_hashes').all() as { path: string; hash: string }[];
+    const map = new Map<string, string>();
+    for (const row of rows) {
+      map.set(row.path, row.hash);
+    }
+    return map;
+  }
+
+  upsertFileHash(path: string, hash: string) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO file_hashes (path, hash, last_indexed)
+      VALUES (?, ?, ?)
+    `);
+    stmt.run(path, hash, new Date().toISOString());
+  }
+
+  deleteFileHash(path: string) {
+    this.db.prepare('DELETE FROM file_hashes WHERE path = ?').run(path);
+  }
+
+  getChunksForFile(file: string): ChunkMetadata[] {
+    return this.db.prepare('SELECT * FROM chunks WHERE file = ?').all() as ChunkMetadata[];
+  }
+
+  getChunkCount(): number {
+    const row = this.db.prepare('SELECT COUNT(*) as count FROM chunks').get() as { count: number };
+    return row.count;
+  }
+
+  getFileCount(): number {
+    const row = this.db.prepare('SELECT COUNT(DISTINCT file) as count FROM chunks').get() as { count: number };
+    return row.count;
+  }
+
+  close() {
+    this.db.close();
+  }
+}

package/src/vector/lancedb.ts

@@ -0,0 +1,95 @@
+import { VectorStore } from './store.js';
+import { logger } from '../logging.js';
+
+export class LanceDBStore implements VectorStore {
+  private db: any = null;
+  private table: any = null;
+  private uri: string;
+  private dims: number;
+  private tableName = 'chunks';
+  private initialized = false;
+
+  constructor(indexDir: string, dimensions: number) {
+    this.uri = indexDir;
+    this.dims = dimensions;
+  }
+
+  async init(): Promise<void> {
+    if (this.initialized) return;
+    const mod = await import('@lancedb/lancedb');
+    this.db = await mod.connect(this.uri);
+    const tableNames = await this.db.tableNames();
+
+    if (tableNames.includes(this.tableName)) {
+      this.table = await this.db.openTable(this.tableName);
+    } else {
+      this.table = await this.db.createTable(this.tableName, [
+        { id: '', vector: new Array(this.dims).fill(0), metadata: {} },
+      ]);
+    }
+    this.initialized = true;
+  }
+
+  async insert(vectors: Array<{ id: string; values: number[]; metadata: Record<string, any> }>): Promise<void> {
+    await this.init();
+    const data = vectors.map(v => ({
+      id: v.id,
+      vector: v.values,
+      metadata: { ...v.metadata, text: v.metadata.text || '' },
+    }));
+    await this.table.add(data);
+  }
+
+  async delete(ids: string[]): Promise<void> {
+    await this.init();
+    for (const id of ids) {
+      try {
+        await this.table.delete(`id = '${id.replace(/'/g, "''")}'`);
+      } catch (e) {
+        logger.debug(`Failed to delete vector ${id}:`, e);
+      }
+    }
+  }
+
+  async search(query: number[], topK: number): Promise<Array<{ id: string; score: number }>> {
+    await this.init();
+    try {
+      const results = await this.table
+        .vectorSearch(query)
+        .limit(topK)
+        .toArray();
+
+      return results.map((r: any) => ({
+        id: r.id as string,
+        score: r._distance !== undefined ? 1 / (1 + r._distance) : 1,
+      }));
+    } catch (e) {
+      logger.debug('Vector search failed:', e);
+      return [];
+    }
+  }
+
+  async compact(): Promise<void> {
+    await this.init();
+    try {
+      await this.table.compact();
+    } catch (e) {
+      logger.debug('Compact failed:', e);
+    }
+  }
+
+  async count(): Promise<number> {
+    await this.init();
+    try {
+      return await this.table.countRows();
+    } catch {
+      return 0;
+    }
+  }
+
+  async close(): Promise<void> {
+    if (this.db) {
+      try { await this.db.close(); } catch { /* ignore */ }
+    }
+  }
+}

package/src/vector/store.ts

@@ -0,0 +1,8 @@
+export interface VectorStore {
+  insert(vectors: Array<{ id: string; values: number[]; metadata: Record<string, any> }>): Promise<void>;
+  delete(ids: string[]): Promise<void>;
+  search(query: number[], topK: number): Promise<Array<{ id: string; score: number }>>;
+  compact(): Promise<void>;
+  count(): Promise<number>;
+  close(): Promise<void>;
+}

package/src/version.ts

@@ -0,0 +1,55 @@
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { logger } from './logging.js';
+
+export interface IndexVersion {
+  schema: number;
+  embeddingModel: string;
+  chunkStrategy: number;
+  createdAt: string;
+  lastIndexed: string;
+}
+
+const CURRENT_SCHEMA = 1;
+const CURRENT_CHUNK_STRATEGY = 1;
+
+export function getExpectedVersion(embeddingModelName: string): IndexVersion {
+  return {
+    schema: CURRENT_SCHEMA,
+    embeddingModel: embeddingModelName,
+    chunkStrategy: CURRENT_CHUNK_STRATEGY,
+    createdAt: new Date().toISOString(),
+    lastIndexed: new Date().toISOString(),
+  };
+}
+
+export function loadVersion(indexDir: string): IndexVersion | null {
+  const path = join(indexDir, 'version.json');
+  if (!existsSync(path)) return null;
+  try {
+    const raw = readFileSync(path, 'utf-8');
+    return JSON.parse(raw) as IndexVersion;
+  } catch (e) {
+    logger.debug('Failed to load version.json:', e);
+    return null;
+  }
+}
+
+export function saveVersion(indexDir: string, version: IndexVersion) {
+  const path = join(indexDir, 'version.json');
+  if (!existsSync(indexDir)) {
+    mkdirSync(indexDir, { recursive: true });
+  }
+  writeFileSync(path, JSON.stringify(version, null, 2), 'utf-8');
+}
+
+export function needsRebuild(
+  existing: IndexVersion | null,
+  embeddingModel: string,
+): boolean {
+  if (!existing) return true;
+  if (existing.schema !== CURRENT_SCHEMA) return true;
+  if (existing.embeddingModel !== embeddingModel) return true;
+  if (existing.chunkStrategy !== CURRENT_CHUNK_STRATEGY) return true;
+  return false;
+}