edhindex 1.0.0

package/src/indexer/indexer.ts

@@ -0,0 +1,181 @@
+import { readFileSync, existsSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { logger } from '../logging.js';
+import { emitProgress } from '../progress.js';
+import { Config, MODEL_CONFIGS } from '../config/index.js';
+import { scanFiles, computeFileHash } from './file-utils.js';
+import { createIgnoreRules } from './ignore.js';
+import { initParser, loadLanguage, parseContentAsync, extractSymbols, extractImports, extractExports } from './parser.js';
+import { Chunk, chunkFile } from './chunker.js';
+import { computeFileDelta, computeChunkDelta } from './incremental.js';
+
+export interface IndexFileResult {
+  chunks: Chunk[];
+  filePath: string;
+  relativePath: string;
+  hash: string;
+}
+
+export type IndexCallback = (result: IndexFileResult) => Promise<void>;
+export type DeleteCallback = (chunkIds: string[]) => Promise<void>;
+
+export interface IndexerOptions {
+  config: Config;
+  rootPath: string;
+  indexDir: string;
+  onIndex: IndexCallback;
+  onDelete?: DeleteCallback;
+  signal?: AbortSignal;
+}
+
+export class Indexer {
+  private config: Config;
+  private rootPath: string;
+  private indexDir: string;
+  private onIndex: IndexCallback;
+  private onDelete?: DeleteCallback;
+  private signal: AbortSignal;
+  private extToLang: Map<string, string>;
+
+  constructor(opts: IndexerOptions) {
+    this.config = opts.config;
+    this.rootPath = opts.rootPath;
+    this.indexDir = opts.indexDir;
+    this.onIndex = opts.onIndex;
+    this.onDelete = opts.onDelete;
+    this.signal = opts.signal || new AbortController().signal;
+
+    this.extToLang = new Map();
+    for (const lang of this.config.languages) {
+      const exts = LANG_EXTENSIONS[lang] || [];
+      for (const ext of exts) {
+        this.extToLang.set(ext, lang);
+      }
+    }
+  }
+
+  async initialize() {
+    await initParser();
+    for (const lang of this.config.languages) {
+      try {
+        await loadLanguage(lang);
+        logger.verbose(`Loaded tree-sitter grammar for ${lang}`);
+      } catch (e) {
+        logger.error(`Failed to load tree-sitter grammar for ${lang}:`, e);
+      }
+    }
+  }
+
+  async runFullIndex(): Promise<Chunk[]> {
+    emitProgress({ phase: 'scanning', current: 0, total: 0 });
+
+    const files = scanFiles(this.rootPath, this.config.languages);
+    const allChunks: Chunk[] = [];
+    let processed = 0;
+
+    emitProgress({ phase: 'scanning', current: files.length, total: files.length });
+
+    for (const file of files) {
+      if (this.signal.aborted) break;
+
+      emitProgress({ phase: 'embedding', percent: Math.round((processed / files.length) * 100) });
+
+      try {
+        const chunks = await this.processFile(file.path, file.relativePath, file.language);
+        allChunks.push(...chunks);
+        await this.onIndex({ chunks, filePath: file.path, relativePath: file.relativePath, hash: file.hash });
+      } catch (e) {
+        logger.debug(`Failed to process ${file.relativePath}:`, e);
+      }
+
+      processed++;
+    }
+
+    emitProgress({ phase: 'building_fts' });
+    emitProgress({ phase: 'ready' });
+
+    return allChunks;
+  }
+
+  async runIncremental(
+    existingFileHashes: Map<string, string>,
+    existingChunkHashes: Map<string, string>,
+  ): Promise<{ chunks: Chunk[]; removed: string[] }> {
+    emitProgress({ phase: 'scanning', current: 0, total: 0 });
+
+    const scannedFiles = scanFiles(this.rootPath, this.config.languages);
+    const scannedMap = new Map(scannedFiles.map(f => [f.relativePath, f]));
+    const delta = computeFileDelta(scannedMap, existingFileHashes);
+
+    emitProgress({ phase: 'scanning', current: scannedFiles.length, total: scannedFiles.length });
+
+    const changedFiles = [...delta.added, ...delta.modified];
+    const allChunks: Chunk[] = [];
+    let processed = 0;
+
+    for (const file of changedFiles) {
+      if (this.signal.aborted) break;
+
+      emitProgress({ phase: 'embedding', percent: Math.round((processed / changedFiles.length) * 100) });
+
+      try {
+        const chunks = await this.processFile(file.path, file.relativePath, file.language);
+        allChunks.push(...chunks);
+        await this.onIndex({ chunks, filePath: file.path, relativePath: file.relativePath, hash: file.hash });
+      } catch (e) {
+        logger.debug(`Failed to process ${file.relativePath}:`, e);
+      }
+
+      processed++;
+    }
+
+    if (this.onDelete && delta.removed.length > 0) {
+      const removedChunkIds: string[] = [];
+      for (const relPath of delta.removed) {
+        for (const [chunkId] of existingChunkHashes) {
+          if (chunkId.startsWith(`${relPath}:`)) {
+            removedChunkIds.push(chunkId);
+          }
+        }
+      }
+      await this.onDelete(removedChunkIds);
+    }
+
+    emitProgress({ phase: 'building_fts' });
+    emitProgress({ phase: 'ready' });
+
+    return { chunks: allChunks, removed: delta.removed };
+  }
+
+  async processFile(
+    filePath: string,
+    relativePath: string,
+    language: string,
+  ): Promise<Chunk[]> {
+    const content = readFileSync(filePath, 'utf-8');
+    const lines = content.split('\n');
+
+    const { tree } = await parseContentAsync(content, language);
+    const symbols = extractSymbols(tree, language);
+    const imports = extractImports(tree, language);
+    const exports = extractExports(tree, language);
+
+    return chunkFile(
+      filePath,
+      relativePath,
+      language,
+      symbols,
+      content,
+      lines,
+      imports,
+      exports,
+    );
+  }
+}
+
+const LANG_EXTENSIONS: Record<string, string[]> = {
+  ts: ['.ts', '.tsx', '.mts', '.cts'],
+  js: ['.js', '.jsx', '.mjs', '.cjs'],
+  py: ['.py', '.pyw'],
+  go: ['.go'],
+};

package/src/indexer/parser.ts

@@ -0,0 +1,267 @@
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { createRequire } from 'node:module';
+import { logger } from '../logging.js';
+
+const require = createRequire(import.meta.url);
+
+import { Parser, Language } from 'web-tree-sitter';
+
+let initialized = false;
+const loadedLanguages = new Map<string, any>();
+
+export interface ASTNode {
+  type: string;
+  startLine: number;
+  endLine: number;
+  startCol: number;
+  endCol: number;
+  text: string;
+  children: ASTNode[];
+}
+
+export async function initParser() {
+  if (initialized) return;
+  await Parser.init();
+  initialized = true;
+}
+
+export async function loadLanguage(language: string): Promise<any> {
+  if (loadedLanguages.has(language)) {
+    return loadedLanguages.get(language)!;
+  }
+
+  let wasmPath: string;
+
+  switch (language) {
+    case 'ts':
+    case 'js':
+      wasmPath = require.resolve('tree-sitter-typescript/tree-sitter-typescript.wasm');
+      break;
+    case 'py':
+      wasmPath = require.resolve('tree-sitter-python/tree-sitter-python.wasm');
+      break;
+    case 'go':
+      wasmPath = require.resolve('tree-sitter-go/tree-sitter-go.wasm');
+      break;
+    default:
+      throw new Error(`Unsupported language: ${language}`);
+  }
+
+  const lang = await Language.load(wasmPath);
+  loadedLanguages.set(language, lang);
+
+  return lang;
+}
+
+function langKey(language: string): string {
+  return language === 'ts' || language === 'js' ? 'ts' : language;
+}
+
+export function parseFile(filePath: string, language: string): any {
+  const content = readFileSync(filePath, 'utf-8');
+  return parseContent(content, language);
+}
+
+function parseContent(content: string, _language?: string): never {
+  throw new Error('Use parseContentAsync for async parsing');
+}
+
+export async function parseContentAsync(content: string, language: string): Promise<{ tree: any; rootNode: ASTNode }> {
+  const lk = langKey(language);
+  const lang = loadedLanguages.get(language) || loadedLanguages.get(lk);
+  if (!lang) {
+    throw new Error(`Language not loaded: ${language}`);
+  }
+  const parser = new Parser();
+  parser.setLanguage(lang);
+  const tree = parser.parse(content);
+  if (!tree) {
+    throw new Error('Failed to parse content');
+  }
+  const rootNode = convertNode(tree.rootNode);
+  return { tree, rootNode };
+}
+
+export function getNodeText(node: any): string {
+  return node.text;
+}
+
+export function getChildCount(node: any): number {
+  return node.childCount;
+}
+
+export function getChild(node: any, index: number): any {
+  return node.child(index);
+}
+
+function convertNode(node: any): ASTNode {
+  return {
+    type: node.type,
+    startLine: node.startPosition.row + 1,
+    endLine: node.endPosition.row + 1,
+    startCol: node.startPosition.column,
+    endCol: node.endPosition.column,
+    text: node.text || '',
+    children: [],
+  };
+}
+
+export interface SymbolInfo {
+  name: string;
+  kind: 'function' | 'method' | 'class' | 'interface' | 'enum' | 'module';
+  startLine: number;
+  endLine: number;
+  text: string;
+}
+
+const QUERY_PATTERNS: Record<string, Record<string, string>> = {
+  ts: {
+    function: '(function_declaration name: (identifier) @name)',
+    method: '(method_definition name: (property_identifier) @name)',
+    class: '(class_declaration name: (type_identifier) @name)',
+    interface: '(interface_declaration name: (type_identifier) @name)',
+    enum: '(enum_declaration name: (type_identifier) @name)',
+  },
+  js: {
+    function: '(function_declaration name: (identifier) @name)',
+    method: '(method_definition name: (property_identifier) @name)',
+    class: '(class_declaration name: (identifier) @name)',
+  },
+  py: {
+    function: '(function_definition name: (identifier) @name)',
+    class: '(class_definition name: (identifier) @name)',
+  },
+  go: {
+    function: '(function_declaration name: (identifier) @name)',
+    method: '(method_declaration name: (field_identifier) @name)',
+  },
+};
+
+export function extractSymbols(tree: any, language: string): SymbolInfo[] {
+  const symbols: SymbolInfo[] = [];
+  const patterns = QUERY_PATTERNS[language];
+  if (!patterns) return symbols;
+
+  const rootNode = tree.rootNode;
+  walkTree(rootNode, symbols, language);
+  return symbols;
+}
+
+function walkTree(node: any, symbols: SymbolInfo[], language: string) {
+  const kind = nodeTypeToKind(node.type, language);
+  if (kind) {
+    const name = extractName(node, language);
+    symbols.push({
+      name: name || `anonymous_${kind}_${node.startPosition.row + 1}`,
+      kind,
+      startLine: node.startPosition.row + 1,
+      endLine: node.endPosition.row + 1,
+      text: node.text || '',
+    });
+  }
+
+  for (let i = 0; i < node.childCount; i++) {
+    walkTree(node.child(i), symbols, language);
+  }
+}
+
+function nodeTypeToKind(type: string, language: string): SymbolInfo['kind'] | null {
+  switch (language) {
+    case 'ts':
+    case 'js':
+      switch (type) {
+        case 'function_declaration': return 'function';
+        case 'arrow_function': return 'function';
+        case 'method_definition': return 'method';
+        case 'class_declaration': return 'class';
+        case 'interface_declaration': return 'interface';
+        case 'enum_declaration': return 'enum';
+        default: return null;
+      }
+    case 'py':
+      switch (type) {
+        case 'function_definition': return 'function';
+        case 'class_definition': return 'class';
+        default: return null;
+      }
+    case 'go':
+      switch (type) {
+        case 'function_declaration': return 'function';
+        case 'method_declaration': return 'method';
+        default: return null;
+      }
+    default:
+      return null;
+  }
+}
+
+function extractName(node: any, language: string): string | null {
+  try {
+    for (let i = 0; i < node.childCount; i++) {
+      const child = node.child(i);
+      if (language === 'ts' || language === 'js') {
+        if (child.type === 'identifier' || child.type === 'property_identifier' || child.type === 'type_identifier') {
+          return child.text;
+        }
+      } else if (language === 'py') {
+        if (child.type === 'identifier') {
+          return child.text;
+        }
+      } else if (language === 'go') {
+        if (child.type === 'identifier' || child.type === 'field_identifier') {
+          return child.text;
+        }
+      }
+    }
+  } catch {
+    // ignore
+  }
+  return null;
+}
+
+export function extractImports(tree: any, language: string): string[] {
+  const imports: string[] = [];
+  const rootNode = tree.rootNode;
+  collectImports(rootNode, imports, language);
+  return imports;
+}
+
+function collectImports(node: any, imports: string[], language: string) {
+  if (node.type === 'import_statement' || node.type === 'import_declaration' ||
+      node.type === 'import_from_statement' || node.type === 'require_statement' ||
+      (language === 'py' && node.type === 'import_statement') ||
+      (language === 'py' && node.type === 'import_from_statement') ||
+      (language === 'go' && node.type === 'import_declaration') ||
+      (language === 'go' && node.type === 'import_spec')) {
+    imports.push(node.text || '');
+  }
+  for (let i = 0; i < node.childCount; i++) {
+    collectImports(node.child(i), imports, language);
+  }
+}
+
+export function extractExports(tree: any, language: string): string[] {
+  const exports: string[] = [];
+  const rootNode = tree.rootNode;
+  collectExports(rootNode, exports, language);
+  return exports;
+}
+
+function collectExports(node: any, exports: string[], language: string) {
+  if (language === 'ts' || language === 'js') {
+    if (node.type === 'export_statement') {
+      for (let i = 0; i < node.childCount; i++) {
+        const child = node.child(i);
+        if (child.type === 'function_declaration' || child.type === 'class_declaration' ||
+            child.type === 'variable_declaration' || child.type === 'interface_declaration' ||
+            child.type === 'type_alias_declaration') {
+          exports.push(child.text || '');
+        }
+      }
+    }
+  }
+  for (let i = 0; i < node.childCount; i++) {
+    collectExports(node.child(i), exports, language);
+  }
+}

package/src/logging.ts

@@ -0,0 +1,44 @@
+export enum LogLevel {
+  Silent = 0,
+  Info = 1,
+  Verbose = 2,
+  Debug = 3,
+}
+
+const levelNames: Record<LogLevel, string> = {
+  [LogLevel.Silent]: 'SILENT',
+  [LogLevel.Info]: 'INFO',
+  [LogLevel.Verbose]: 'VERBOSE',
+  [LogLevel.Debug]: 'DEBUG',
+};
+
+let currentLevel = LogLevel.Info;
+
+export function setLogLevel(level: LogLevel) {
+  currentLevel = level;
+}
+
+export function getLogLevel(): LogLevel {
+  return currentLevel;
+}
+
+function log(level: LogLevel, ...args: unknown[]) {
+  if (level <= currentLevel) {
+    const prefix = levelNames[level];
+    const timestamp = new Date().toISOString();
+    if (level === LogLevel.Debug) {
+      console.error(`[${timestamp}] [${prefix}]`, ...args);
+    } else if (level === LogLevel.Info) {
+      console.error(...args);
+    } else {
+      console.error(`[${prefix}]`, ...args);
+    }
+  }
+}
+
+export const logger = {
+  info: (...args: unknown[]) => log(LogLevel.Info, ...args),
+  verbose: (...args: unknown[]) => log(LogLevel.Verbose, ...args),
+  debug: (...args: unknown[]) => log(LogLevel.Debug, ...args),
+  error: (...args: unknown[]) => console.error(...args),
+};

package/src/mcp/server.ts

@@ -0,0 +1,126 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import {
+  ListToolsRequestSchema,
+  CallToolRequestSchema,
+  McpError,
+  ErrorCode,
+} from '@modelcontextprotocol/sdk/types.js';
+import { SearchEngine, SearchResult } from '../retrieval/search.js';
+import { logger } from '../logging.js';
+
+export function createMCPServer(searchEngine: SearchEngine, rootPath: string): Server {
+  const server = new Server(
+    {
+      name: 'edhindex',
+      version: '1.0.0',
+    },
+    {
+      capabilities: {
+        tools: {},
+      },
+    },
+  );
+
+  server.setRequestHandler(ListToolsRequestSchema, async () => {
+    return {
+      tools: [
+        {
+          name: 'search_codebase',
+          description: `Search the indexed codebase using hybrid (keyword + semantic + reranked) search. Workspace: ${rootPath}. Searches file contents, symbols, and code structure.`,
+          inputSchema: {
+            type: 'object',
+            properties: {
+              query: {
+                type: 'string',
+                description: 'Search query - natural language or code terms',
+              },
+              maxResults: {
+                type: 'number',
+                description: 'Maximum results to return (default 10)',
+                default: 10,
+              },
+              fileFilter: {
+                type: 'string',
+                description: 'Optional file path filter (glob pattern)',
+              },
+              languageFilter: {
+                type: 'string',
+                description: 'Optional language filter (ts, js, py, go)',
+              },
+            },
+            required: ['query'],
+          },
+        },
+      ],
+    };
+  });
+
+  server.setRequestHandler(CallToolRequestSchema, async (request: any) => {
+    if (request.params.name !== 'search_codebase') {
+      throw new McpError(ErrorCode.MethodNotFound, `Unknown tool: ${request.params.name}`);
+    }
+
+    const args = request.params.arguments || {};
+    const query = String(args.query || '').trim();
+
+    if (!query) {
+      throw new McpError(ErrorCode.InvalidParams, 'query is required');
+    }
+
+    const maxResults = Math.min(Math.max(Number(args.maxResults) || 10, 1), 100);
+    const fileFilter = args.fileFilter ? String(args.fileFilter) : undefined;
+    const languageFilter = args.languageFilter ? String(args.languageFilter) : undefined;
+
+    // Security: sanitize and restrict to workspace
+    if (fileFilter) {
+      const { sanitizePath } = await import('../security.js');
+      const safe = sanitizePath(rootPath, fileFilter);
+      if (!safe) {
+        throw new McpError(ErrorCode.InvalidParams, 'File filter is outside the workspace');
+      }
+    }
+
+    try {
+      const results = await searchEngine.search({ query, maxResults, fileFilter, languageFilter });
+      return formatResults(results);
+    } catch (e: any) {
+      logger.error('Search failed:', e);
+      throw new McpError(ErrorCode.InternalError, `Search failed: ${e.message}`);
+    }
+  });
+
+  // Error handler
+  server.onerror = (error: any) => {
+    logger.error('MCP Server error:', error);
+  };
+
+  return server;
+}
+
+function formatResults(results: SearchResult[]) {
+  const textParts: string[] = [];
+
+  if (results.length === 0) {
+    textParts.push('No results found.');
+  } else {
+    textParts.push(`Found ${results.length} result(s):\n`);
+    for (let i = 0; i < results.length; i++) {
+      const r = results[i];
+      const lines: string[] = [];
+      lines.push(`[${i + 1}] ${r.file}:${r.startLine}-${r.endLine}`);
+      lines.push(`    Language: ${r.language}  |  Type: ${r.matchType}  |  Score: ${r.score.toFixed(3)}`);
+      if (r.symbol) lines.push(`    Symbol: ${r.symbol}${r.kind ? ` (${r.kind})` : ''}`);
+      lines.push(`    ${'─'.repeat(60)}`);
+      textParts.push(lines.join('\n'));
+    }
+  }
+
+  return {
+    content: [
+      {
+        type: 'text',
+        text: textParts.join('\n'),
+      },
+    ],
+  };
+}

package/src/progress.ts

@@ -0,0 +1,28 @@
+export type ProgressPhase = 'scanning' | 'embedding' | 'building_fts' | 'ready';
+
+export interface ProgressEvent {
+  phase: ProgressPhase;
+  current?: number;
+  total?: number;
+  percent?: number;
+  message?: string;
+}
+
+export type ProgressCallback = (event: ProgressEvent) => void;
+
+const listeners: Set<ProgressCallback> = new Set();
+
+export function onProgress(cb: ProgressCallback): () => void {
+  listeners.add(cb);
+  return () => listeners.delete(cb);
+}
+
+export function emitProgress(event: ProgressEvent) {
+  for (const cb of listeners) {
+    try {
+      cb(event);
+    } catch {
+      // swallow
+    }
+  }
+}

package/src/retrieval/bm25.ts

@@ -0,0 +1,22 @@
+import { FTSStore, FTSResult } from '../storage/fts.js';
+
+export function searchBM25(fts: FTSStore, query: string, topK: number): Array<{ id: string; score: number }> {
+  const results = fts.searchWithRank(query, topK * 2);
+  if (results.length === 0) return [];
+
+  if (results.length === 1) {
+    return [{ id: results[0].id, score: 1.0 }];
+  }
+
+  const negRanks = results.map(r => -r.rank);
+  const minNeg = Math.min(...negRanks);
+  const maxNeg = Math.max(...negRanks);
+  const range = maxNeg - minNeg;
+  if (range === 0) {
+    return results.map(r => ({ id: r.id, score: 1.0 }));
+  }
+  return results.map((r, i) => ({
+    id: r.id,
+    score: (negRanks[i] - minNeg) / range,
+  }));
+}