edhindex 1.0.0

package/src/cli/commands/start.ts

@@ -0,0 +1,253 @@
+import { join } from 'node:path';
+import { existsSync, mkdirSync } from 'node:fs';
+import { loadConfig, MODEL_CONFIGS } from '../../config/index.js';
+import { logger, setLogLevel, LogLevel, getLogLevel } from '../../logging.js';
+import { emitProgress, onProgress } from '../../progress.js';
+import { loadVersion, saveVersion, getExpectedVersion, needsRebuild } from '../../version.js';
+import { MetadataStore } from '../../storage/metadata.js';
+import { FTSStore } from '../../storage/fts.js';
+import { TransformersEmbeddingProvider, TransformersReranker } from '../../embeddings/transformers.js';
+import { LanceDBStore } from '../../vector/lancedb.js';
+import { Indexer } from '../../indexer/indexer.js';
+import { SearchEngine } from '../../retrieval/search.js';
+import { createMCPServer } from '../../mcp/server.js';
+import { FileWatcher } from '../../watcher/index.js';
+import cliProgress from 'cli-progress';
+import chalk from 'chalk';
+
+export async function startCommand(rootPath: string) {
+  const indexDir = join(rootPath, '.edhindex');
+  if (!existsSync(indexDir)) {
+    mkdirSync(indexDir, { recursive: true });
+  }
+
+  const config = loadConfig(indexDir);
+  const modelCfg = MODEL_CONFIGS[config.model];
+
+  const logLevelMap: Record<string, LogLevel> = {
+    silent: LogLevel.Silent,
+    info: LogLevel.Info,
+    verbose: LogLevel.Verbose,
+    debug: LogLevel.Debug,
+  };
+  setLogLevel(logLevelMap[config.logLevel] || LogLevel.Info);
+
+  console.log(chalk.bold('EDHIndex v1.0.0'));
+  console.log(`Workspace: ${rootPath}`);
+  console.log(`Model: ${modelCfg.displayName} (${modelCfg.dimensions}d)`);
+  console.log(`Languages: ${config.languages.join(', ')}`);
+  console.log('');
+
+  // Check index version
+  const version = loadVersion(indexDir);
+  if (needsRebuild(version, config.model)) {
+    if (version) {
+      console.log(chalk.yellow('Index version mismatch — full rebuild required.'));
+      console.log(chalk.dim(`  Expected model: ${config.model}, existing: ${version.embeddingModel}`));
+    } else {
+      console.log(chalk.dim('No existing index found — building from scratch.'));
+    }
+  } else {
+    console.log(chalk.dim('Existing index is compatible — incremental update.'));
+  }
+
+  // Setup progress bar
+  let progressBar: cliProgress.SingleBar | null = null;
+  const unsub = onProgress((event) => {
+    if (progressBar) {
+      if (event.phase === 'scanning' && event.total) {
+        progressBar.setTotal(event.total);
+        progressBar.update(event.current || 0);
+      } else if (event.phase === 'embedding' && event.percent !== undefined) {
+        progressBar.setTotal(100);
+        progressBar.update(event.percent);
+      }
+    }
+  });
+
+  // Initialize stores
+  console.log(chalk.dim('Initializing storage...'));
+  const metadataStore = new MetadataStore(indexDir);
+  const ftsStore = new FTSStore(indexDir);
+
+  // Initialize vector store
+  const vectorStore = new LanceDBStore(indexDir, modelCfg.dimensions);
+  await vectorStore.init();
+
+  // Initialize embedding provider
+  const embedder = new TransformersEmbeddingProvider(config.model);
+  console.log(chalk.dim(`Loading embedding model (${modelCfg.displayName})...`));
+  await embedder.load();
+
+  // Initialize reranker if enabled
+  let reranker: TransformersReranker | null = null;
+  if (config.rerank) {
+    console.log(chalk.dim('Loading reranker model...'));
+    reranker = new TransformersReranker();
+    await reranker.load();
+  }
+
+  // Initialize indexer
+  const indexer = new Indexer({
+    config,
+    rootPath,
+    indexDir,
+    onIndex: async (result) => {
+      for (const chunk of result.chunks) {
+        metadataStore.upsertChunk({
+          id: chunk.id,
+          file: chunk.file,
+          language: chunk.language,
+          symbol: chunk.symbol,
+          kind: chunk.kind,
+          parent: chunk.parent,
+          hash: chunk.hash,
+          git_commit: null,
+          start_line: chunk.startLine,
+          end_line: chunk.endLine,
+          imports: chunk.imports.join('\n'),
+          exports: chunk.exports.join('\n'),
+          embedding_model: embedder.name(),
+          last_indexed: new Date().toISOString(),
+          checksum: chunk.hash,
+        });
+
+        ftsStore.insertChunk(chunk.id, chunk.content, chunk.file, chunk.symbol, chunk.language);
+      }
+
+      if (result.chunks.length > 0) {
+        const texts = result.chunks.map(c => c.content);
+        const embeddings = await embedder.embed(texts);
+        const vectors = result.chunks.map((c, i) => ({
+          id: c.id,
+          values: embeddings[i],
+          metadata: {
+            file: c.file,
+            symbol: c.symbol,
+            kind: c.kind,
+            text: c.content,
+            language: c.language,
+          },
+        }));
+        await vectorStore.insert(vectors);
+      }
+
+      metadataStore.upsertFileHash(result.relativePath, result.hash);
+    },
+    onDelete: async (chunkIds) => {
+      for (const id of chunkIds) {
+        ftsStore.deleteChunk(id);
+      }
+      await vectorStore.delete(chunkIds);
+    },
+  });
+
+  await indexer.initialize();
+
+  // Determine if full or incremental
+  const existingFileHashes = metadataStore.getAllFileHashes();
+  const existingChunkHashes = metadataStore.getAllChunkHashes();
+  const doFullRebuild = needsRebuild(version, config.model) || existingFileHashes.size === 0;
+
+  // Index
+  console.log('');
+  progressBar = new cliProgress.SingleBar({
+    format: '{phase} | {bar} | {percentage}% | {value}/{total}',
+    barCompleteChar: '\u2588',
+    barIncompleteChar: '\u2591',
+    hideCursor: true,
+  });
+
+  if (doFullRebuild) {
+    progressBar.start(100, 0, { phase: 'Scanning' });
+    console.log(chalk.dim('Running full index...'));
+    const chunks = await indexer.runFullIndex();
+    progressBar.stop();
+    console.log(chalk.green(`Indexed ${chunks.length} chunks.`));
+
+    // Save version
+    const newVersion = getExpectedVersion(config.model);
+    saveVersion(indexDir, newVersion);
+  } else {
+    progressBar.start(100, 0, { phase: 'Scanning' });
+    console.log(chalk.dim('Running incremental index...'));
+    const result = await indexer.runIncremental(existingFileHashes, existingChunkHashes);
+    progressBar.stop();
+    console.log(chalk.green(`Indexed ${result.chunks.length} chunks, removed ${result.removed.length} files.`));
+
+    // Update version
+    const existingVersion = version!;
+    existingVersion.lastIndexed = new Date().toISOString();
+    saveVersion(indexDir, existingVersion);
+  }
+
+  unsub();
+
+  // Build search engine
+  const searchEngine = new SearchEngine(ftsStore, metadataStore, vectorStore, embedder, reranker, config.rerank);
+
+  // Start file watcher if enabled
+  let watcher: FileWatcher | null = null;
+  if (config.watch) {
+    watcher = new FileWatcher(rootPath, async (filePath) => {
+      logger.info(`File changed: ${filePath}`);
+      const { scanFiles } = await import('../../indexer/file-utils.js');
+      const files = scanFiles(rootPath, config.languages);
+      const file = files.find(f => f.relativePath === filePath);
+      if (file) {
+        const oldChunks = metadataStore.getChunksForFile(filePath);
+        const oldIds = oldChunks.map(c => c.id);
+        if (oldIds.length > 0) {
+          for (const id of oldIds) ftsStore.deleteChunk(id);
+          await vectorStore.delete(oldIds);
+        }
+        metadataStore.deleteChunksForFile(filePath);
+        metadataStore.deleteFileHash(filePath);
+
+        const chunks = await indexer.processFile(file.path, file.relativePath, file.language);
+        for (const chunk of chunks) {
+          metadataStore.upsertChunk({
+            id: chunk.id, file: chunk.file, language: chunk.language, symbol: chunk.symbol,
+            kind: chunk.kind, parent: chunk.parent, hash: chunk.hash, git_commit: null,
+            start_line: chunk.startLine, end_line: chunk.endLine,
+            imports: chunk.imports.join('\n'), exports: chunk.exports.join('\n'),
+            embedding_model: embedder.name(), last_indexed: new Date().toISOString(), checksum: chunk.hash,
+          });
+          ftsStore.insertChunk(chunk.id, chunk.content, chunk.file, chunk.symbol, chunk.language);
+        }
+        if (chunks.length > 0) {
+          const embeddings = await embedder.embed(chunks.map(c => c.content));
+          await vectorStore.insert(chunks.map((c, i) => ({
+            id: c.id, values: embeddings[i],
+            metadata: { file: c.file, symbol: c.symbol, kind: c.kind, text: c.content, language: c.language },
+          })));
+        }
+        metadataStore.upsertFileHash(filePath, file.hash);
+        logger.info(`Re-indexed: ${filePath}`);
+      }
+    });
+    await watcher.start();
+    console.log(chalk.green('File watcher started.'));
+  }
+
+  // Start MCP server
+  console.log(chalk.green('Starting MCP server on stdio...'));
+  const mcpServer = createMCPServer(searchEngine, rootPath);
+
+  const transport = await import('@modelcontextprotocol/sdk/server/stdio.js');
+  const stdioTransport = new transport.StdioServerTransport();
+  await mcpServer.connect(stdioTransport);
+
+  // Handle shutdown
+  const shutdown = async () => {
+    console.log(chalk.yellow('\nShutting down...'));
+    if (watcher) await watcher.stop();
+    vectorStore.close();
+    metadataStore.close();
+    ftsStore.close();
+    process.exit(0);
+  };
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+}

package/src/cli/commands/status.ts

@@ -0,0 +1,54 @@
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { loadConfig, MODEL_CONFIGS } from '../../config/index.js';
+import { loadVersion } from '../../version.js';
+import { MetadataStore } from '../../storage/metadata.js';
+import { FTSStore } from '../../storage/fts.js';
+import { LanceDBStore } from '../../vector/lancedb.js';
+import chalk from 'chalk';
+
+export async function statusCommand(rootPath: string) {
+  const indexDir = join(rootPath, '.edhindex');
+
+  if (!existsSync(indexDir)) {
+    console.log(chalk.yellow('No .edhindex directory found. Run "edhindex init" first.'));
+    return;
+  }
+
+  const config = loadConfig(indexDir);
+  const version = loadVersion(indexDir);
+
+  console.log(chalk.bold('EDHIndex Status'));
+  console.log('');
+
+  if (version) {
+    console.log(`  Index:        ${chalk.green('Present')}`);
+    console.log(`  Schema:       v${version.schema}`);
+    console.log(`  Model tier:   ${version.embeddingModel}`);
+    console.log(`  Created:      ${version.createdAt}`);
+    console.log(`  Last indexed: ${version.lastIndexed}`);
+  } else {
+    console.log(`  Index:        ${chalk.yellow('No version file')}`);
+  }
+
+  console.log(`  Config:       ${chalk.cyan(JSON.stringify(config))}`);
+
+  const metadataStore = new MetadataStore(indexDir);
+  const chunkCount = metadataStore.getChunkCount();
+  const fileCount = metadataStore.getFileCount();
+  metadataStore.close();
+
+  console.log(`  Chunks:       ${chunkCount}`);
+  console.log(`  Files:        ${fileCount}`);
+  console.log(`  Languages:    ${config.languages.join(', ')}`);
+
+  try {
+    const vectorStore = new LanceDBStore(indexDir, MODEL_CONFIGS[config.model].dimensions);
+    await vectorStore.init();
+    const vecCount = await vectorStore.count();
+    console.log(`  Vectors:      ${vecCount}`);
+    await vectorStore.close();
+  } catch {
+    console.log(`  Vectors:      ${chalk.yellow('unavailable')}`);
+  }
+}

package/src/cli/index.ts

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { join } from 'node:path';
+import { initCommand } from './commands/init.js';
+import { startCommand } from './commands/start.js';
+import { indexCommand } from './commands/index_cmd.js';
+import { searchCommand } from './commands/search.js';
+import { statusCommand } from './commands/status.js';
+import { configCommand } from './commands/config_cmd.js';
+import { modelsCommand } from './commands/models.js';
+import { doctorCommand } from './commands/doctor_cmd.js';
+import { setLogLevel, LogLevel } from '../logging.js';
+
+const program = new Command();
+
+program
+  .name('edhindex')
+  .description('Local-first hybrid code search engine with MCP')
+  .version('1.0.0')
+  .option('-v, --verbose', 'Enable verbose logging')
+  .option('--debug', 'Enable debug logging')
+  .hook('preAction', (thisCommand) => {
+    if (thisCommand.opts().debug) setLogLevel(LogLevel.Debug);
+    else if (thisCommand.opts().verbose) setLogLevel(LogLevel.Verbose);
+  });
+
+program
+  .command('init')
+  .description('Initialize EDHIndex in the current directory')
+  .action(async () => {
+    const rootPath = process.cwd();
+    await initCommand(rootPath);
+  });
+
+program
+  .command('start')
+  .description('Build/update index and start the MCP server')
+  .action(async () => {
+    const rootPath = process.cwd();
+    await startCommand(rootPath);
+  });
+
+program
+  .command('index')
+  .description('Build or update the search index')
+  .action(async () => {
+    const rootPath = process.cwd();
+    await indexCommand(rootPath);
+  });
+
+program
+  .command('search')
+  .description('Search the indexed codebase from the CLI')
+  .argument('<query>', 'Search query')
+  .option('-n, --max-results <number>', 'Max results', '10')
+  .action(async (query: string, opts: { maxResults: string }) => {
+    const rootPath = process.cwd();
+    await searchCommand(rootPath, query, parseInt(opts.maxResults, 10) || 10);
+  });
+
+program
+  .command('status')
+  .description('Show index status')
+  .action(async () => {
+    const rootPath = process.cwd();
+    await statusCommand(rootPath);
+  });
+
+program
+  .command('config')
+  .description('View or update configuration')
+  .argument('[key]', 'Config key')
+  .argument('[value]', 'Config value')
+  .action((key?: string, value?: string) => {
+    const rootPath = process.cwd();
+    const changes: Record<string, string> = {};
+    if (key && value !== undefined) {
+      changes[key] = value;
+    }
+    configCommand(rootPath, changes);
+  });
+
+program
+  .command('models')
+  .description('List available embedding models')
+  .action(() => {
+    modelsCommand();
+  });
+
+program
+  .command('doctor')
+  .description('Run diagnostics')
+  .action(async () => {
+    const rootPath = process.cwd();
+    const indexDir = join(rootPath, '.edhindex');
+    await doctorCommand(rootPath, indexDir);
+  });
+
+program.parse(process.argv);

package/src/config/index.ts

@@ -0,0 +1,75 @@
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { logger } from '../logging.js';
+
+export type ModelTier = 'light' | 'balanced' | 'max';
+export type LogLevelConfig = 'silent' | 'info' | 'verbose' | 'debug';
+
+export interface Config {
+  model: ModelTier;
+  languages: string[];
+  watch: boolean;
+  rerank: boolean;
+  maxResults: number;
+  logLevel: LogLevelConfig;
+}
+
+const DEFAULT_CONFIG: Config = {
+  model: 'balanced',
+  languages: ['ts', 'js', 'go', 'py'],
+  watch: true,
+  rerank: true,
+  maxResults: 10,
+  logLevel: 'info',
+};
+
+export function defaultConfig(): Config {
+  return { ...DEFAULT_CONFIG };
+}
+
+export function loadConfig(indexDir: string): Config {
+  const path = join(indexDir, 'config.json');
+  if (!existsSync(path)) {
+    const cfg = defaultConfig();
+    saveConfig(indexDir, cfg);
+    return cfg;
+  }
+  try {
+    const raw = readFileSync(path, 'utf-8');
+    const parsed = JSON.parse(raw) as Partial<Config>;
+    return { ...defaultConfig(), ...parsed };
+  } catch (e) {
+    logger.error('Failed to load config, using defaults:', e);
+    return defaultConfig();
+  }
+}
+
+export function saveConfig(indexDir: string, config: Config) {
+  if (!existsSync(indexDir)) {
+    mkdirSync(indexDir, { recursive: true });
+  }
+  writeFileSync(join(indexDir, 'config.json'), JSON.stringify(config, null, 2), 'utf-8');
+}
+
+export const MODEL_CONFIGS: Record<ModelTier, { modelId: string; dimensions: number; displayName: string; size: string }> = {
+  light: {
+    modelId: 'Xenova/all-MiniLM-L6-v2',
+    dimensions: 384,
+    displayName: 'all-MiniLM-L6-v2',
+    size: '~46MB',
+  },
+  balanced: {
+    modelId: 'Xenova/bge-base-en-v1.5',
+    dimensions: 768,
+    displayName: 'bge-base-en-v1.5',
+    size: '~109MB',
+  },
+  max: {
+    modelId: 'Xenova/bge-m3',
+    dimensions: 1024,
+    displayName: 'BGE-M3',
+    size: '~1.1GB',
+  },
+};
+
+export const RERANKER_MODEL = 'Xenova/ms-marco-MiniLM-L-6-v2';

package/src/doctor/index.ts

@@ -0,0 +1,131 @@
+import { existsSync, statSync, accessSync, constants, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { logger } from '../logging.js';
+import { MetadataStore } from '../storage/metadata.js';
+import { FTSStore } from '../storage/fts.js';
+import { EmbeddingProvider } from '../embeddings/provider.js';
+import { VectorStore } from '../vector/store.js';
+import { loadVersion } from '../version.js';
+import { loadConfig, MODEL_CONFIGS, ModelTier } from '../config/index.js';
+
+export interface DoctorResult {
+  category: string;
+  status: 'ok' | 'warn' | 'error';
+  message: string;
+  detail?: string;
+}
+
+export async function runDoctor(rootPath: string, indexDir: string): Promise<DoctorResult[]> {
+  const results: DoctorResult[] = [];
+
+  // 1. Workspace check
+  results.push(checkPath('Workspace', rootPath, true));
+
+  // 2. Index directory
+  results.push(checkPath('Index directory', indexDir, false));
+
+  // 3. Config
+  try {
+    const config = loadConfig(indexDir);
+    results.push({ category: 'Config', status: 'ok', message: `Loaded (model: ${config.model})` });
+  } catch (e: any) {
+    results.push({ category: 'Config', status: 'error', message: `Failed: ${e.message}` });
+  }
+
+  // 4. Version
+  const version = loadVersion(indexDir);
+  if (version) {
+    results.push({ category: 'Version', status: 'ok', message: `Schema v${version.schema}, ${version.embeddingModel}` });
+  } else {
+    results.push({ category: 'Version', status: 'warn', message: 'No version file (not indexed yet)' });
+  }
+
+  // 5. SQLite metadata
+  try {
+    const meta = new MetadataStore(indexDir);
+    const chunkCount = meta.getChunkCount();
+    const fileCount = meta.getFileCount();
+    meta.close();
+    results.push({ category: 'SQLite (metadata)', status: 'ok', message: `${chunkCount} chunks from ${fileCount} files` });
+  } catch (e: any) {
+    results.push({ category: 'SQLite (metadata)', status: 'error', message: `Failed: ${e.message}` });
+  }
+
+  // 6. SQLite FTS
+  try {
+    const fts = new FTSStore(indexDir);
+    fts.close();
+    results.push({ category: 'SQLite (FTS)', status: 'ok', message: 'Accessible' });
+  } catch (e: any) {
+    results.push({ category: 'SQLite (FTS)', status: 'error', message: `Failed: ${e.message}` });
+  }
+
+  // 7. LanceDB
+  try {
+    const mod = await import('@lancedb/lancedb');
+    const db = await mod.connect(indexDir);
+    const names = await db.tableNames();
+    await db.close();
+    results.push({ category: 'LanceDB', status: 'ok', message: `Tables: ${names.join(', ') || 'none'}` });
+  } catch (e: any) {
+    results.push({ category: 'LanceDB', status: 'warn', message: `Check failed: ${e.message}` });
+  }
+
+  // 8. Embedding model
+  try {
+    const config = loadConfig(indexDir);
+    const modelTier: ModelTier = config.model;
+    const modelCfg = MODEL_CONFIGS[modelTier];
+    const mod = await import('@huggingface/transformers');
+    await mod.pipeline('feature-extraction', modelCfg.modelId, { local_files_only: true });
+    results.push({ category: 'Embedding model', status: 'ok', message: `${modelCfg.displayName} (cached)` });
+  } catch {
+    try {
+      const config = loadConfig(indexDir);
+      const modelTier: ModelTier = config.model;
+      const modelCfg = MODEL_CONFIGS[modelTier];
+      results.push({ category: 'Embedding model', status: 'warn', message: `${modelCfg.displayName} (not cached, will download on first use)` });
+    } catch {
+      results.push({ category: 'Embedding model', status: 'warn', message: 'Unknown (config issue)' });
+    }
+  }
+
+  // 9. Disk space
+  try {
+    const stat = statSync(indexDir);
+    const free = stat.size > 0 ? 'has data' : 'empty';
+    results.push({ category: 'Disk space', status: 'ok', message: `Index dir ${free}` });
+  } catch {
+    results.push({ category: 'Disk space', status: 'warn', message: 'Index dir not created yet' });
+  }
+
+  // 10. Permissions
+  try {
+    accessSync(rootPath, constants.R_OK | constants.X_OK);
+    results.push({ category: 'Permissions', status: 'ok', message: 'Readable' });
+  } catch (e: any) {
+    results.push({ category: 'Permissions', status: 'error', message: `Cannot read workspace: ${e.message}` });
+  }
+
+  // 11. MCP
+  results.push({ category: 'MCP Server', status: 'ok', message: 'Available on stdio transport' });
+
+  return results;
+}
+
+function checkPath(category: string, path: string, mustExist: boolean): DoctorResult {
+  const exists = existsSync(path);
+  if (mustExist && !exists) {
+    return { category, status: 'error', message: `Not found: ${path}` };
+  }
+  if (!mustExist && !exists) {
+    return { category, status: 'warn', message: `Not found: ${path} (will create on first index)` };
+  }
+  try {
+    const s = statSync(path);
+    const type = s.isDirectory() ? 'directory' : 'file';
+    return { category, status: 'ok', message: `${type}: ${path}` };
+  } catch (e: any) {
+    return { category, status: 'error', message: `Cannot access: ${e.message}` };
+  }
+}

package/src/embeddings/provider.ts

@@ -0,0 +1,9 @@
+export interface EmbeddingProvider {
+  embed(texts: string[]): Promise<number[][]>;
+  dimensions(): number;
+  name(): string;
+  load(): Promise<void>;
+  dispose(): void;
+}
+
+export type EmbeddingModelId = 'light' | 'balanced' | 'max';