@nomos-arc/arc 0.1.0

package/src/search/indexer.ts

@@ -0,0 +1,483 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { Logger } from 'winston';
+import { NomosError } from '../core/errors.js';
+import type { IndexMetadata, NomosConfig, ProjectMap, TextChunk, VectorRecord } from '../types/index.js';
+import { ChunkExtractor } from './chunk-extractor.js';
+import { Embedder } from './embedder.js';
+import { VectorStore } from './vector-store.js';
+import { AuthManager } from '../core/auth/manager.js';
+
+// ─── SearchIndexer ─────────────────────────────────────────────────────────────
+
+export class SearchIndexer {
+  private readonly extractor: ChunkExtractor;
+  private _embedder: Embedder | null = null; // lazy — not created until first embedding call
+  private readonly store: VectorStore;
+  private readonly metaPath: string;
+
+  constructor(
+    private readonly projectRoot: string,
+    private readonly config: NomosConfig,
+    private readonly logger: Logger,
+    private readonly authManager?: AuthManager | null,
+  ) {
+    this.extractor = new ChunkExtractor(projectRoot, logger);
+    this.store = new VectorStore(
+      config.search.vector_store_path,
+      logger,
+      config.search.embedding_dimensions,
+    );
+    this.metaPath = path.join(config.search.vector_store_path, 'index-meta.json');
+  }
+
+  /** Lazy Embedder accessor — throws at construction time only if actually needed. */
+  private async getEmbedder(): Promise<Embedder> {
+    if (!this._embedder) {
+      this._embedder = await Embedder.create(
+        this.config.search, this.logger, this.authManager,
+      );
+    }
+    return this._embedder;
+  }
+
+  // ─── Public API ─────────────────────────────────────────────────────────────
+
+  /**
+   * Full index: extract all chunks, embed in streaming batches, upsert to staging,
+   * then atomic table-swap to live [BLOCKER-1].
+   * Writes metadata with status tracking [BLOCKER-2].
+   */
+  async fullIndex(cancellationFlag?: { cancelled: boolean }): Promise<IndexMetadata> {
+    // Pre-check: access embedder once to validate API key before any mutations [AC-11]
+    await this.getEmbedder();
+
+    // Step 1: Load project_map.json
+    const projectMap = await this.loadProjectMap();
+
+    // Step 2: Write IndexMetadata with status: "in_progress" BEFORE any mutation [BLOCKER-2]
+    const startedAt = new Date().toISOString();
+    const inProgressMeta: IndexMetadata = {
+      status: 'in_progress',
+      last_full_index: startedAt,
+      last_incremental_index: null,
+      total_files_indexed: 0,
+      total_symbols_indexed: 0,
+      total_chunks: 0,
+      embedding_model: this.config.search.embedding_model,
+      vector_dimensions: this.config.search.embedding_dimensions,
+      failed_files: [],
+      files: {},
+    };
+    await this.writeMeta(inProgressMeta);
+
+    // Step 3: Extract all chunks
+    const allChunks = this.extractor.extract(projectMap);
+    const fileChunks = allChunks.filter(c => c.type === 'file');
+    const symbolChunks = allChunks.filter(c => c.type === 'symbol');
+    this.logger.info(
+      `[nomos:search:info] Extracted ${allChunks.length} chunks (${fileChunks.length} file-level, ${symbolChunks.length} symbol-level)`,
+    );
+
+    // Step 4: Init store + cleanup orphaned staging table from prior crash [GAP-3]
+    await this.store.init();
+    await this.store.cleanupStaging();
+
+    // Step 5: Streaming batch loop [BLOCKER-4]
+    const batches = chunk(allChunks, this.config.search.batch_size);
+    const failedFiles: string[] = [];
+    let batchesDone = 0;
+    let cancelled = false;
+
+    for (let i = 0; i < batches.length; i++) {
+      // 5a: Check cancellation flag at TOP of each iteration
+      if (cancellationFlag?.cancelled) {
+        cancelled = true;
+        break;
+      }
+
+      const batch = batches[i]!;
+
+      // 5b: Embed batch — on failure, record file paths and continue [GAP-1]
+      let vectors: Float32Array[];
+      try {
+        vectors = await (await this.getEmbedder()).embedBatch(batch.map(c => c.text));
+      } catch (err) {
+        this.logger.error(
+          `[nomos:search:error] Batch ${i + 1}/${batches.length} embedding failed: ${(err as Error).message}`,
+        );
+        for (const c of batch) {
+          if (!failedFiles.includes(c.file_path)) {
+            failedFiles.push(c.file_path);
+          }
+        }
+        continue;
+      }
+
+      // 5c: Compose VectorRecord[] for THIS BATCH ONLY
+      const records = this.composeRecords(batch, vectors, projectMap);
+
+      // 5d: Upsert batch to staging immediately, release references
+      await this.store.upsertToStaging(records);
+      batchesDone++;
+
+      this.logger.info(
+        `[nomos:search:info] Embedded batch ${i + 1}/${batches.length} (${batch.length} chunks)`,
+      );
+    }
+
+    if (cancelled) {
+      // Partial run: metadata written as "in_progress"; next run cleans staging + forces full re-index
+      const partialMeta: IndexMetadata = {
+        ...inProgressMeta,
+        status: 'in_progress',
+        failed_files: failedFiles,
+        total_chunks: batchesDone * this.config.search.batch_size,
+      };
+      await this.writeMeta(partialMeta);
+      return partialMeta;
+    }
+
+    // Step 6: Atomic table-swap [BLOCKER-1]
+    await this.store.promoteStagingToLive();
+
+    // Step 7: Write IndexMetadata with status: "complete" [BLOCKER-2]
+    const finalMeta = this.buildFinalMeta(
+      allChunks,
+      startedAt,
+      null,
+      failedFiles,
+      projectMap,
+    );
+    await this.writeMeta(finalMeta);
+
+    // Step 8: Return IndexMetadata
+    return finalMeta;
+  }
+
+  /**
+   * Incremental index: validates dimensions [BLOCKER-3], re-indexes changed + failed files [GAP-1].
+   */
+  async incrementalIndex(cancellationFlag?: { cancelled: boolean }): Promise<IndexMetadata> {
+    // Pre-check: access embedder once to validate API key before any mutations [AC-11]
+    await this.getEmbedder();
+
+    // Step 1: Load project_map.json
+    const projectMap = await this.loadProjectMap();
+
+    // Step 2: Load existing IndexMetadata
+    let existingMeta: IndexMetadata | null = null;
+    try {
+      const raw = await fs.readFile(this.metaPath, 'utf-8');
+      existingMeta = JSON.parse(raw) as IndexMetadata;
+    } catch {
+      // Not found → fall back to fullIndex
+      this.logger.info('[nomos:search:info] No existing index metadata. Running full index.');
+      return this.fullIndex(cancellationFlag);
+    }
+
+    // If previous run was interrupted, force full re-index [BLOCKER-2]
+    if (existingMeta.status === 'in_progress') {
+      this.logger.warn(
+        '[nomos:search:warn] Previous index incomplete. Running full re-index.',
+      );
+      return this.fullIndex(cancellationFlag);
+    }
+
+    // Step 3: Dimension validation [BLOCKER-3]
+    const modelChanged = existingMeta.embedding_model !== this.config.search.embedding_model;
+    const dimsChanged = existingMeta.vector_dimensions !== this.config.search.embedding_dimensions;
+    if (modelChanged || dimsChanged) {
+      const oldModel = `${existingMeta.embedding_model}/${existingMeta.vector_dimensions}`;
+      const newModel = `${this.config.search.embedding_model}/${this.config.search.embedding_dimensions}`;
+      this.logger.warn(
+        `[nomos:search:warn] Embedding model/dimensions changed (${oldModel} → ${newModel}). Forcing full re-index.`,
+      );
+      return this.fullIndex(cancellationFlag);
+    }
+
+    // Step 4: Extract all chunks
+    const allChunks = this.extractor.extract(projectMap);
+
+    // Step 5: Compute diff — decisions are made at file-path granularity.
+    // The metadata stores one content_hash per file path (from the file-level chunk).
+    // Symbol chunks share the same file_path and are always re-indexed alongside their parent.
+    const chunksByFile = groupByFilePath(allChunks);
+    const currentFilePaths = new Set(Object.keys(chunksByFile));
+
+    const toReindexPaths = new Set<string>();
+
+    for (const [fp, chunks] of Object.entries(chunksByFile)) {
+      const existing = existingMeta.files[fp];
+      if (!existing) {
+        // new file
+        toReindexPaths.add(fp);
+        continue;
+      }
+      // Use the file-level chunk's hash for comparison (matches what buildFinalMeta stores)
+      const fileChunk = chunks.find(c => c.type === 'file');
+      if (fileChunk && existing.content_hash !== fileChunk.content_hash) {
+        toReindexPaths.add(fp);
+        continue;
+      }
+      if (existingMeta.failed_files.includes(fp)) {
+        toReindexPaths.add(fp);
+      }
+    }
+
+    // Removed: files in metadata not in current projectMap
+    const removed = Object.keys(existingMeta.files).filter(fp => !currentFilePaths.has(fp));
+
+    const toReindex = allChunks.filter(c => toReindexPaths.has(c.file_path));
+
+    // Step 6: Nothing to do
+    if (toReindex.length === 0 && removed.length === 0) {
+      this.logger.info('[nomos:search:info] Index is up-to-date. No changes detected.');
+      return existingMeta;
+    }
+
+    // Step 7: Mark in_progress [BLOCKER-2]
+    await this.writeMeta({ ...existingMeta, status: 'in_progress' });
+
+    // Step 8: Init store and delete stale records
+    await this.store.init();
+    const toDeletePaths = unique([...removed, ...toReindex.map(c => c.file_path)]);
+    await this.store.deleteByFilePaths(toDeletePaths);
+
+    // Step 9: Streaming batch loop for toReindex [BLOCKER-4]
+    const batches = chunk(toReindex, this.config.search.batch_size);
+    const newFailedFiles: string[] = [];
+
+    for (let i = 0; i < batches.length; i++) {
+      // Check cancellation at TOP of each iteration
+      if (cancellationFlag?.cancelled) {
+        const partialMeta: IndexMetadata = {
+          ...existingMeta,
+          status: 'in_progress',
+          failed_files: newFailedFiles,
+        };
+        await this.writeMeta(partialMeta);
+        return partialMeta;
+      }
+
+      const batch = batches[i]!;
+
+      let vectors: Float32Array[];
+      try {
+        vectors = await (await this.getEmbedder()).embedBatch(batch.map(c => c.text));
+      } catch (err) {
+        this.logger.error(
+          `[nomos:search:error] Incremental batch ${i + 1}/${batches.length} failed: ${(err as Error).message}`,
+        );
+        for (const c of batch) {
+          if (!newFailedFiles.includes(c.file_path)) {
+            newFailedFiles.push(c.file_path);
+          }
+        }
+        continue;
+      }
+
+      const records = this.composeRecords(batch, vectors, projectMap);
+      await this.store.upsert(records);
+
+      this.logger.info(
+        `[nomos:search:info] Embedded batch ${i + 1}/${batches.length} (${batch.length} chunks)`,
+      );
+    }
+
+    // Step 10: Update metadata
+    const now = new Date().toISOString();
+    const updatedFiles: IndexMetadata['files'] = { ...existingMeta.files };
+
+    // Remove deleted file entries
+    for (const fp of removed) {
+      delete updatedFiles[fp];
+    }
+
+    // Update/add re-indexed file entries
+    for (const c of toReindex) {
+      if (!newFailedFiles.includes(c.file_path)) {
+        const existingEntry = updatedFiles[c.file_path];
+        updatedFiles[c.file_path] = {
+          last_indexed: now,
+          content_hash: c.content_hash,
+          chunk_count: (existingEntry?.chunk_count ?? 0),
+        };
+      }
+    }
+
+    // Recount from updated files entries
+    const totalFileChunks = allChunks.filter(c => c.type === 'file' && updatedFiles[c.file_path]).length;
+    const totalSymbolChunks = allChunks.filter(c => c.type === 'symbol' && updatedFiles[c.file_path]).length;
+
+    // Retain failed_files that were not successfully re-embedded; clear those that succeeded
+    const persistedFailedFiles = [
+      ...existingMeta.failed_files.filter(fp => newFailedFiles.includes(fp)),
+      ...newFailedFiles.filter(fp => !existingMeta.failed_files.includes(fp)),
+    ];
+
+    const updatedMeta: IndexMetadata = {
+      status: 'complete',
+      last_full_index: existingMeta.last_full_index,
+      last_incremental_index: now,
+      total_files_indexed: totalFileChunks,
+      total_symbols_indexed: totalSymbolChunks,
+      total_chunks: totalFileChunks + totalSymbolChunks,
+      embedding_model: this.config.search.embedding_model,
+      vector_dimensions: this.config.search.embedding_dimensions,
+      failed_files: persistedFailedFiles,
+      files: updatedFiles,
+    };
+
+    // Step 11: Write metadata atomically [BLOCKER-2]
+    await this.writeMeta(updatedMeta);
+
+    // Step 12: Return updated metadata
+    return updatedMeta;
+  }
+
+  /**
+   * Dry-run: extract and count chunks without embedding or writing [S-2].
+   */
+  async dryRun(): Promise<{ fileChunks: number; symbolChunks: number; totalChunks: number }> {
+    const projectMap = await this.loadProjectMap();
+    const allChunks = this.extractor.extract(projectMap);
+    const fileChunks = allChunks.filter(c => c.type === 'file').length;
+    const symbolChunks = allChunks.filter(c => c.type === 'symbol').length;
+    return { fileChunks, symbolChunks, totalChunks: allChunks.length };
+  }
+
+  // ─── Private helpers ─────────────────────────────────────────────────────────
+
+  private async loadProjectMap(): Promise<ProjectMap> {
+    const mapPath = path.join(this.config.graph.output_dir, 'project_map.json');
+    try {
+      const raw = await fs.readFile(mapPath, 'utf-8');
+      return JSON.parse(raw) as ProjectMap;
+    } catch {
+      throw new NomosError(
+        'search_index_failed',
+        `project_map.json not found at ${mapPath}. Run: arc map`,
+      );
+    }
+  }
+
+  private composeRecords(
+    chunks: TextChunk[],
+    vectors: Float32Array[],
+    projectMap: ProjectMap,
+  ): VectorRecord[] {
+    const now = new Date().toISOString();
+    return chunks.map((c, idx) => {
+      const fileNode = projectMap.files[c.file_path];
+      return {
+        id: c.id,
+        type: c.type,
+        vector: vectors[idx]!,
+        file_path: c.file_path,
+        module: path.dirname(c.file_path),
+        purpose: fileNode?.semantic?.purpose ?? c.file_path,
+        symbol_name: c.symbol_name,
+        symbol_type: c.symbol_type,
+        line_start: c.line_start,
+        line_end: c.line_end,
+        parent_file_id: c.parent_file_id,
+        graph_depth: fileNode?.depth ?? 0,
+        dependents_count: fileNode?.dependents?.length ?? 0,
+        last_indexed: now,
+        content_hash: c.content_hash,
+      };
+    });
+  }
+
+  private buildFinalMeta(
+    allChunks: TextChunk[],
+    startedAt: string,
+    lastIncremental: string | null,
+    failedFiles: string[],
+    projectMap: ProjectMap,
+  ): IndexMetadata {
+    const now = new Date().toISOString();
+    const successfulChunks = allChunks.filter(c => !failedFiles.includes(c.file_path));
+    const fileChunks = successfulChunks.filter(c => c.type === 'file');
+    const symbolChunks = successfulChunks.filter(c => c.type === 'symbol');
+
+    // Build per-file entry: group chunks by file_path
+    const fileEntries: IndexMetadata['files'] = {};
+    for (const c of successfulChunks) {
+      if (!fileEntries[c.file_path]) {
+        fileEntries[c.file_path] = {
+          last_indexed: now,
+          content_hash: c.content_hash,
+          chunk_count: 0,
+        };
+      }
+      fileEntries[c.file_path]!.chunk_count++;
+    }
+
+    // Resolve content_hash per file from the ProjectMap-derived chunk [S-6]
+    for (const [fp, entry] of Object.entries(fileEntries)) {
+      const fileChunk = allChunks.find(c => c.file_path === fp && c.type === 'file');
+      if (fileChunk) {
+        entry.content_hash = fileChunk.content_hash;
+      }
+    }
+
+    return {
+      status: 'complete',
+      last_full_index: startedAt,
+      last_incremental_index: lastIncremental,
+      total_files_indexed: fileChunks.length,
+      total_symbols_indexed: symbolChunks.length,
+      total_chunks: successfulChunks.length,
+      embedding_model: this.config.search.embedding_model,
+      vector_dimensions: this.config.search.embedding_dimensions,
+      failed_files: failedFiles,
+      files: fileEntries,
+    };
+  }
+
+  /**
+   * Atomically write IndexMetadata: write to .tmp file then rename [BLOCKER-2].
+   */
+  private async writeMeta(meta: IndexMetadata): Promise<void> {
+    await fs.mkdir(this.config.search.vector_store_path, { recursive: true });
+    const tmpPath = `${this.metaPath}.tmp`;
+    await fs.writeFile(tmpPath, JSON.stringify(meta, null, 2), 'utf-8');
+    await fs.rename(tmpPath, this.metaPath);
+  }
+}
+
+// ─── Utilities ────────────────────────────────────────────────────────────────
+
+function chunk<T>(arr: T[], size: number): T[][] {
+  const result: T[][] = [];
+  for (let i = 0; i < arr.length; i += size) {
+    result.push(arr.slice(i, i + size));
+  }
+  return result;
+}
+
+/** Deduplicate TextChunks by id, preserving first occurrence. */
+function dedup(chunks: TextChunk[]): TextChunk[] {
+  const seen = new Set<string>();
+  return chunks.filter(c => {
+    if (seen.has(c.id)) return false;
+    seen.add(c.id);
+    return true;
+  });
+}
+
+/** Return unique strings, preserving order. */
+function unique(arr: string[]): string[] {
+  return [...new Set(arr)];
+}
+
+/** Group TextChunks by file_path. */
+function groupByFilePath(chunks: TextChunk[]): Record<string, TextChunk[]> {
+  const map: Record<string, TextChunk[]> = {};
+  for (const c of chunks) {
+    (map[c.file_path] ??= []).push(c);
+  }
+  return map;
+}

package/src/search/lexical-searcher.ts

@@ -0,0 +1,190 @@
+import type { ProjectMap, SearchResult } from '../types/index.js';
+
+// ─── LexicalSearcher ─────────────────────────────────────────────────────────
+
+/**
+ * Keyword-based search for files that are not yet vector-indexed.
+ * Scores files based on term matches in file path, symbol names,
+ * symbol signatures, and import sources.
+ *
+ * Used as a fallback when the vector index is unavailable or incomplete,
+ * enabling `arc search` to work immediately after `arc map --no-ai`.
+ */
+export class LexicalSearcher {
+
+  /**
+   * Search a subset of files from the project map using keyword matching.
+   *
+   * @param query - The user's search query string
+   * @param projectMap - The full project map
+   * @param filePaths - Set of file paths to search (e.g., files not in vector index)
+   * @param topK - Maximum results to return
+   * @param threshold - Minimum score (0–1) to include a result
+   */
+  search(
+    query: string,
+    projectMap: ProjectMap,
+    filePaths: Set<string>,
+    topK: number,
+    threshold: number,
+  ): SearchResult[] {
+    const terms = this.tokenize(query);
+    if (terms.length === 0) return [];
+
+    const results: SearchResult[] = [];
+    const coreModuleSet = new Set(projectMap.stats.core_modules);
+
+    for (const fp of filePaths) {
+      const fileNode = projectMap.files[fp];
+      if (!fileNode) continue;
+
+      const score = this.scoreFile(terms, fp, fileNode);
+      if (score < threshold) continue;
+
+      results.push({
+        id: fp,
+        type: 'file',
+        file_path: fp,
+        symbol_name: null,
+        symbol_type: null,
+        line_start: null,
+        line_end: null,
+        purpose: fileNode.semantic?.purpose ?? fp,
+        similarity_score: score,
+        graph_depth: fileNode.depth,
+        dependents_count: fileNode.dependents.length,
+        is_core_module: coreModuleSet.has(fp),
+        is_stale: false,
+      });
+
+      // Also score individual symbols
+      for (const symbol of fileNode.symbols) {
+        if (!symbol.exported && symbol.kind !== 'class' && symbol.kind !== 'function') continue;
+
+        const symbolScore = this.scoreSymbol(terms, fp, symbol.name, symbol.kind, symbol.signature);
+        if (symbolScore < threshold) continue;
+
+        results.push({
+          id: `${fp}::${symbol.name}`,
+          type: 'symbol',
+          file_path: fp,
+          symbol_name: symbol.name,
+          symbol_type: symbol.kind,
+          line_start: symbol.line,
+          line_end: symbol.end_line,
+          purpose: fileNode.semantic?.purpose ?? fp,
+          similarity_score: symbolScore,
+          graph_depth: fileNode.depth,
+          dependents_count: fileNode.dependents.length,
+          is_core_module: coreModuleSet.has(fp),
+          is_stale: false,
+        });
+      }
+    }
+
+    // Sort by score descending, return top-K
+    results.sort((a, b) => b.similarity_score - a.similarity_score);
+    return results.slice(0, topK);
+  }
+
+  // ─── Private ───────────────────────────────────────────────────────────────
+
+  /**
+   * Tokenize a query into lowercase terms, splitting on whitespace and common
+   * code separators (camelCase, underscores, hyphens, dots, slashes).
+   */
+  private tokenize(query: string): string[] {
+    // Split on whitespace first
+    const words = query.toLowerCase().split(/\s+/).filter(Boolean);
+
+    // Further split camelCase/PascalCase and separators
+    const terms: string[] = [];
+    for (const word of words) {
+      // Split on separators: _ - . /
+      const parts = word.split(/[_\-./]+/).filter(Boolean);
+      for (const part of parts) {
+        // Split camelCase: "getAccessToken" → ["get", "access", "token"]
+        const camelParts = part.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' ');
+        terms.push(...camelParts.filter(Boolean));
+      }
+    }
+
+    return [...new Set(terms)]; // deduplicate
+  }
+
+  /**
+   * Score a file based on how many query terms appear in its searchable text.
+   * Returns a value in [0, 1].
+   */
+  private scoreFile(
+    terms: string[],
+    filePath: string,
+    fileNode: { symbols: Array<{ name: string }>; imports: Array<{ source: string }> },
+  ): number {
+    // Build searchable corpus from file metadata
+    const corpus = this.buildCorpus(filePath, fileNode);
+    return this.computeScore(terms, corpus);
+  }
+
+  /**
+   * Score a symbol based on how many query terms match its name/signature.
+   */
+  private scoreSymbol(
+    terms: string[],
+    filePath: string,
+    symbolName: string,
+    kind: string,
+    signature: string | null,
+  ): number {
+    const parts = [
+      filePath.toLowerCase(),
+      symbolName.toLowerCase(),
+      kind.toLowerCase(),
+      signature?.toLowerCase() ?? '',
+    ];
+    const corpus = parts.join(' ');
+    // Boost symbol score if the symbol name directly matches a term
+    const baseScore = this.computeScore(terms, corpus);
+    const nameTokens = this.tokenize(symbolName);
+    const nameOverlap = terms.filter(t => nameTokens.includes(t)).length;
+    const nameBoost = nameOverlap > 0 ? 0.1 * Math.min(nameOverlap / terms.length, 1) : 0;
+    return Math.min(baseScore + nameBoost, 1);
+  }
+
+  /**
+   * Build a searchable text corpus from a file's metadata.
+   */
+  private buildCorpus(
+    filePath: string,
+    fileNode: { symbols: Array<{ name: string }>; imports: Array<{ source: string }> },
+  ): string {
+    const parts = [
+      filePath.toLowerCase(),
+      ...fileNode.symbols.map(s => s.name.toLowerCase()),
+      ...fileNode.imports.map(i => i.source.toLowerCase()),
+    ];
+    return parts.join(' ');
+  }
+
+  /**
+   * Compute a [0, 1] score based on term coverage.
+   * Exact matches score higher than substring matches.
+   */
+  private computeScore(terms: string[], corpus: string): number {
+    if (terms.length === 0) return 0;
+
+    let matched = 0;
+    for (const term of terms) {
+      if (corpus.includes(term)) {
+        matched++;
+      }
+    }
+
+    // Base score: fraction of terms that matched
+    const coverage = matched / terms.length;
+
+    // Scale to a reasonable similarity range [0, 0.85]
+    // Lexical results are capped below 0.85 so vector results naturally rank higher
+    return coverage * 0.85;
+  }
+}