@nomos-arc/arc 0.1.0

package/src/search/__tests__/indexer.test.ts

@@ -0,0 +1,518 @@
+import * as os from 'node:os';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import type { Logger } from 'winston';
+import { SearchIndexer } from '../indexer.js';
+import { NomosError } from '../../core/errors.js';
+import type { IndexMetadata, NomosConfig, ProjectMap } from '../../types/index.js';
+
+// ─── Mocks ────────────────────────────────────────────────────────────────────
+
+// Mock Embedder: deterministic fake vectors, never calls Gemini
+const mockEmbedBatch = vi.fn<(texts: string[]) => Promise<Float32Array[]>>();
+const mockEmbedOne = vi.fn<(text: string) => Promise<Float32Array>>();
+
+vi.mock('../embedder.js', () => {
+  const EmbedderMock = Object.assign(
+    vi.fn().mockImplementation(function (this: Record<string, unknown>) {
+      this['embedBatch'] = mockEmbedBatch;
+      this['embedOne'] = mockEmbedOne;
+      this['dimensions'] = 4;
+    }),
+    {
+      create: vi.fn().mockImplementation(async () => ({
+        embedBatch: mockEmbedBatch,
+        embedOne: mockEmbedOne,
+        dimensions: 4,
+      })),
+    },
+  );
+  return { Embedder: EmbedderMock };
+});
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+const VECTOR_DIMS = 4;
+
+function makeLogger(): Logger {
+  return {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    debug: vi.fn(),
+  } as unknown as Logger;
+}
+
+function makeConfig(vectorStorePath: string, graphOutputDir: string): NomosConfig {
+  return {
+    execution: { default_mode: 'supervised', shadow_branch_prefix: 'nomos/', worktree_base: '/tmp', supervised_heartbeat_timeout_ms: 30000 },
+    binaries: {
+      planner: { cmd: 'claude', args: [], pty: true, total_timeout_ms: 60000, heartbeat_timeout_ms: 30000, max_output_bytes: 1_000_000, usage_pattern: null },
+      reviewer: { cmd: 'openai', args: [], pty: false, total_timeout_ms: 60000, heartbeat_timeout_ms: 30000, max_output_bytes: 1_000_000, usage_pattern: null },
+    },
+    convergence: { score_threshold: 8, max_iterations: 3 },
+    budget: { max_tokens_per_task: 100_000, warn_at_percent: 80, cost_per_1k_tokens: {} },
+    security: { sanitize_patterns: [], entropy_threshold: 4.5, sanitize_on: ['input', 'output'], safe_commands: [], redaction_label: '[REDACTED]' },
+    git: { auto_commit: false, include_logs: true, commit_prefix: 'arc:', sign_commits: false },
+    review: { max_context_files: 10 },
+    graph: {
+      exclude_patterns: [],
+      ai_enrichment: false,
+      ai_model: 'gemini-pro',
+      ai_concurrency: 1,
+      ai_requests_per_minute: 60,
+      max_file_chars: 50_000,
+      core_modules_count: 5,
+      output_dir: graphOutputDir,
+    },
+    logging: { level: 'info', retain_days: 7 },
+    search: {
+      embedding_model: 'text-embedding-004',
+      embedding_dimensions: VECTOR_DIMS,
+      vector_store_path: vectorStorePath,
+      default_top_k: 10,
+      default_threshold: 0.7,
+      batch_size: 2,  // small for test control
+      embedding_requests_per_minute: 60,
+      request_timeout_ms: 30_000,
+    },
+  } as unknown as NomosConfig;
+}
+
+function makeProjectMap(graphDir: string): ProjectMap {
+  return {
+    schema_version: 1,
+    generated_at: '2024-01-01T00:00:00.000Z',
+    root: '/project',
+    files: {
+      'src/foo.ts': {
+        file: 'src/foo.ts',
+        hash: 'abc',
+        language: 'typescript',
+        symbols: [
+          { name: 'Foo', kind: 'class', line: 1, end_line: 10, signature: 'class Foo', exported: true },
+        ],
+        imports: [],
+        dependents: ['src/bar.ts'],
+        dependencies: [],
+        depth: 1,
+        last_parsed_at: null,
+        semantic: {
+          overview: 'Foo module',
+          purpose: 'Does foo things',
+          key_logic: ['key logic'],
+          usage_context: ['used everywhere'],
+          source_hash: 'h1',
+          enriched_at: '2024-01-01T00:00:00.000Z',
+          model: 'gemini-pro',
+        },
+        enrichment_status: 'semantic',
+      },
+      'src/bar.ts': {
+        file: 'src/bar.ts',
+        hash: 'def',
+        language: 'typescript',
+        symbols: [
+          { name: 'Bar', kind: 'function', line: 1, end_line: 5, signature: 'function Bar()', exported: true },
+        ],
+        imports: [{ source: 'src/foo.ts', resolved: 'src/foo.ts', symbols: ['Foo'], is_external: false }],
+        dependents: [],
+        dependencies: ['src/foo.ts'],
+        depth: 2,
+        last_parsed_at: null,
+        semantic: null,
+        enrichment_status: 'structural',
+      },
+    },
+    stats: { total_files: 2, total_symbols: 2, total_edges: 1, core_modules: [], structural_only: 1, semantically_enriched: 1, indexed: 0 },
+  };
+}
+
+/** Fake embed: returns VECTOR_DIMS-length Float32Array per text, deterministic. */
+function makeFakeVectors(count: number): Float32Array[] {
+  return Array.from({ length: count }, (_, i) => {
+    const v = new Float32Array(VECTOR_DIMS);
+    v.fill((i + 1) * 0.1);
+    return v;
+  });
+}
+
+// ─── Test setup ───────────────────────────────────────────────────────────────
+
+let tmpDir: string;
+let vectorStorePath: string;
+let graphDir: string;
+let logger: Logger;
+let config: NomosConfig;
+
+beforeEach(async () => {
+  tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'nomos-idx-'));
+  vectorStorePath = path.join(tmpDir, 'vectors');
+  graphDir = path.join(tmpDir, 'graph');
+  await fs.mkdir(graphDir, { recursive: true });
+
+  // Write project_map.json
+  const projectMap = makeProjectMap(graphDir);
+  await fs.writeFile(
+    path.join(graphDir, 'project_map.json'),
+    JSON.stringify(projectMap),
+    'utf-8',
+  );
+
+  logger = makeLogger();
+  config = makeConfig(vectorStorePath, graphDir);
+
+  // Default: embedBatch returns fake vectors matching batch length
+  mockEmbedBatch.mockImplementation(async (texts: string[]) => makeFakeVectors(texts.length));
+  mockEmbedOne.mockImplementation(async () => makeFakeVectors(1)[0]!);
+});
+
+afterEach(async () => {
+  vi.clearAllMocks();
+  await fs.rm(tmpDir, { recursive: true, force: true });
+});
+
+// ─── Tests ────────────────────────────────────────────────────────────────────
+
+describe('SearchIndexer', () => {
+
+  // ── Test 1: fullIndex() runs the full pipeline ────────────────────────────
+  it('fullIndex() loads project map, extracts chunks, embeds, and writes metadata', async () => {
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+
+    const meta = await indexer.fullIndex();
+
+    expect(meta.status).toBe('complete');
+    expect(meta.embedding_model).toBe(config.search.embedding_model);
+    expect(meta.total_chunks).toBeGreaterThan(0);
+    expect(mockEmbedBatch).toHaveBeenCalled();
+
+    // Verify metadata file was written
+    const metaRaw = await fs.readFile(
+      path.join(vectorStorePath, 'index-meta.json'),
+      'utf-8',
+    );
+    const writtenMeta = JSON.parse(metaRaw) as IndexMetadata;
+    expect(writtenMeta.status).toBe('complete');
+  });
+
+  // ── Test 2: fullIndex() uses table-swap NOT reset()+upsert() [BLOCKER-1] ──
+  it('fullIndex() writes to staging then promotes — never calls upsert() directly [BLOCKER-1]', async () => {
+    const { VectorStore } = await import('../vector-store.js');
+    const upsertSpy = vi.spyOn(VectorStore.prototype, 'upsert');
+    const upsertToStagingSpy = vi.spyOn(VectorStore.prototype, 'upsertToStaging');
+    const promoteSpy = vi.spyOn(VectorStore.prototype, 'promoteStagingToLive');
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.fullIndex();
+
+    // Must use staging path, not direct upsert
+    expect(upsertToStagingSpy).toHaveBeenCalled();
+    expect(promoteSpy).toHaveBeenCalled();
+    expect(upsertSpy).not.toHaveBeenCalled();
+  });
+
+  // ── Test 3: fullIndex() writes "in_progress" BEFORE embedding [BLOCKER-2] ──
+  it('fullIndex() writes status: "in_progress" before embedding, "complete" after [BLOCKER-2]', async () => {
+    const metaPath = path.join(vectorStorePath, 'index-meta.json');
+    const statusLog: string[] = [];
+
+    mockEmbedBatch.mockImplementation(async (texts: string[]) => {
+      // Read meta file during embedding — should be "in_progress"
+      try {
+        const raw = await fs.readFile(metaPath, 'utf-8');
+        const m = JSON.parse(raw) as IndexMetadata;
+        statusLog.push(m.status);
+      } catch {
+        statusLog.push('missing');
+      }
+      return makeFakeVectors(texts.length);
+    });
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    const finalMeta = await indexer.fullIndex();
+
+    // During embedding, status must have been "in_progress"
+    expect(statusLog.some(s => s === 'in_progress')).toBe(true);
+    // After completion, status is "complete"
+    expect(finalMeta.status).toBe('complete');
+  });
+
+  // ── Test 4: fullIndex() upserts per-batch, not all-at-once [BLOCKER-4] ────
+  it('fullIndex() calls upsertToStaging once per batch, not once for all chunks [BLOCKER-4]', async () => {
+    const { VectorStore } = await import('../vector-store.js');
+    const upsertToStagingSpy = vi.spyOn(VectorStore.prototype, 'upsertToStaging');
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.fullIndex();
+
+    // batch_size = 2; project map has 2 files × ~2 chunks each → at least 2 batches
+    expect(upsertToStagingSpy.mock.calls.length).toBeGreaterThan(1);
+    // Each call must be to a partial array, not all chunks at once
+    for (const [records] of upsertToStagingSpy.mock.calls) {
+      expect((records as unknown[]).length).toBeLessThanOrEqual(config.search.batch_size);
+    }
+  });
+
+  // ── Test 5: incrementalIndex() only re-embeds changed files ───────────────
+  it('incrementalIndex() only re-embeds changed files (changed hash)', async () => {
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.fullIndex();
+
+    const firstCallCount = mockEmbedBatch.mock.calls.length;
+    mockEmbedBatch.mockClear();
+
+    // Modify one file's hash in project_map.json to trigger incremental re-index
+    const projectMap = makeProjectMap(graphDir);
+    projectMap.files['src/foo.ts']!.hash = 'CHANGED-HASH';
+    if (projectMap.files['src/foo.ts']!.semantic) {
+      projectMap.files['src/foo.ts']!.semantic!.overview = 'Changed overview';
+    }
+    await fs.writeFile(
+      path.join(graphDir, 'project_map.json'),
+      JSON.stringify(projectMap),
+      'utf-8',
+    );
+
+    const meta = await indexer.incrementalIndex();
+    expect(meta.status).toBe('complete');
+
+    // embedBatch should be called fewer times than full index
+    expect(mockEmbedBatch.mock.calls.length).toBeLessThan(firstCallCount);
+  });
+
+  // ── Test 6: incrementalIndex() re-embeds files in failed_files [GAP-1] ────
+  it('incrementalIndex() re-embeds files in failed_files even if hash unchanged [GAP-1]', async () => {
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.fullIndex();
+
+    // Manually write metadata marking foo.ts as failed
+    const metaPath = path.join(vectorStorePath, 'index-meta.json');
+    const existing = JSON.parse(await fs.readFile(metaPath, 'utf-8')) as IndexMetadata;
+    const manipulated: IndexMetadata = {
+      ...existing,
+      failed_files: ['src/foo.ts'],
+    };
+    await fs.writeFile(metaPath, JSON.stringify(manipulated), 'utf-8');
+
+    mockEmbedBatch.mockClear();
+
+    const meta = await indexer.incrementalIndex();
+    expect(meta.status).toBe('complete');
+
+    // Should have called embedBatch for src/foo.ts chunks
+    expect(mockEmbedBatch).toHaveBeenCalled();
+    const allEmbeddedTexts = mockEmbedBatch.mock.calls.flatMap(([texts]) => texts);
+    expect(allEmbeddedTexts.some((t: string) => t.includes('src/foo.ts'))).toBe(true);
+  });
+
+  // ── Test 7: incrementalIndex() deletes records for removed files ──────────
+  it('incrementalIndex() deletes records for files removed from project map', async () => {
+    const { VectorStore } = await import('../vector-store.js');
+    const deleteSpy = vi.spyOn(VectorStore.prototype, 'deleteByFilePaths');
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.fullIndex();
+
+    // Remove src/bar.ts from project map
+    const projectMap = makeProjectMap(graphDir);
+    delete projectMap.files['src/bar.ts'];
+    projectMap.stats.total_files = 1;
+    await fs.writeFile(
+      path.join(graphDir, 'project_map.json'),
+      JSON.stringify(projectMap),
+      'utf-8',
+    );
+
+    deleteSpy.mockClear();
+    await indexer.incrementalIndex();
+
+    // deleteByFilePaths must have been called with src/bar.ts
+    const deletedPaths = deleteSpy.mock.calls.flatMap(([paths]) => paths);
+    expect(deletedPaths).toContain('src/bar.ts');
+  });
+
+  // ── Test 8: incrementalIndex() falls back to fullIndex() when no metadata ──
+  it('incrementalIndex() falls back to fullIndex() when no index-meta.json exists', async () => {
+    // No prior fullIndex — metadata file doesn't exist
+    const { VectorStore } = await import('../vector-store.js');
+    const promoteSpy = vi.spyOn(VectorStore.prototype, 'promoteStagingToLive');
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    const meta = await indexer.incrementalIndex();
+
+    // Should have performed a full index (staging → live swap)
+    expect(promoteSpy).toHaveBeenCalled();
+    expect(meta.status).toBe('complete');
+  });
+
+  // ── Test 9: incrementalIndex() falls back when status === "in_progress" [BLOCKER-2]
+  it('incrementalIndex() falls back to fullIndex() when metadata status is "in_progress" [BLOCKER-2]', async () => {
+    const metaPath = path.join(vectorStorePath, 'index-meta.json');
+    await fs.mkdir(vectorStorePath, { recursive: true });
+    const stale: IndexMetadata = {
+      status: 'in_progress',
+      last_full_index: new Date().toISOString(),
+      last_incremental_index: null,
+      total_files_indexed: 0,
+      total_symbols_indexed: 0,
+      total_chunks: 0,
+      embedding_model: config.search.embedding_model,
+      vector_dimensions: config.search.embedding_dimensions,
+      failed_files: [],
+      files: {},
+    };
+    await fs.writeFile(metaPath, JSON.stringify(stale), 'utf-8');
+
+    const { VectorStore } = await import('../vector-store.js');
+    const promoteSpy = vi.spyOn(VectorStore.prototype, 'promoteStagingToLive');
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.incrementalIndex();
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining('Previous index incomplete'),
+    );
+    expect(promoteSpy).toHaveBeenCalled();
+  });
+
+  // ── Test 10: incrementalIndex() falls back on embedding model mismatch [BLOCKER-3]
+  it('incrementalIndex() falls back to fullIndex() on embedding model mismatch [BLOCKER-3]', async () => {
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.fullIndex();
+
+    // Overwrite metadata with a different embedding_model
+    const metaPath = path.join(vectorStorePath, 'index-meta.json');
+    const existing = JSON.parse(await fs.readFile(metaPath, 'utf-8')) as IndexMetadata;
+    await fs.writeFile(
+      metaPath,
+      JSON.stringify({ ...existing, embedding_model: 'old-model-v1' }),
+      'utf-8',
+    );
+
+    const { VectorStore } = await import('../vector-store.js');
+    const promoteSpy = vi.spyOn(VectorStore.prototype, 'promoteStagingToLive');
+    promoteSpy.mockClear();
+
+    await indexer.incrementalIndex();
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining('Embedding model/dimensions changed'),
+    );
+    expect(promoteSpy).toHaveBeenCalled();
+  });
+
+  // ── Test 11: incrementalIndex() falls back on vector dimension mismatch [BLOCKER-3]
+  it('incrementalIndex() falls back to fullIndex() on vector dimension mismatch [BLOCKER-3]', async () => {
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    await indexer.fullIndex();
+
+    const metaPath = path.join(vectorStorePath, 'index-meta.json');
+    const existing = JSON.parse(await fs.readFile(metaPath, 'utf-8')) as IndexMetadata;
+    await fs.writeFile(
+      metaPath,
+      JSON.stringify({ ...existing, vector_dimensions: 9999 }),
+      'utf-8',
+    );
+
+    const { VectorStore } = await import('../vector-store.js');
+    const promoteSpy = vi.spyOn(VectorStore.prototype, 'promoteStagingToLive');
+    promoteSpy.mockClear();
+
+    await indexer.incrementalIndex();
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining('Embedding model/dimensions changed'),
+    );
+    expect(promoteSpy).toHaveBeenCalled();
+  });
+
+  // ── Test 12: metadata contains correct totals, hashes, and failed_files ───
+  it('fullIndex() metadata contains correct totals, per-file hashes, and failed_files', async () => {
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    const meta = await indexer.fullIndex();
+
+    expect(meta.status).toBe('complete');
+    expect(meta.total_chunks).toBe(meta.total_files_indexed + meta.total_symbols_indexed);
+    expect(meta.failed_files).toEqual([]);
+    expect(typeof meta.last_full_index).toBe('string');
+
+    // Per-file entries should include hashes
+    for (const [, entry] of Object.entries(meta.files)) {
+      expect(typeof entry.content_hash).toBe('string');
+      expect(entry.content_hash.length).toBeGreaterThan(0);
+    }
+  });
+
+  // ── Test 13: Cancellation flag stops processing, writes "in_progress" ─────
+  it('cancellation flag stops processing between batches; metadata written as "in_progress"', async () => {
+    const cancellationFlag = { cancelled: false };
+
+    // Cancel after first batch
+    let batchCount = 0;
+    mockEmbedBatch.mockImplementation(async (texts: string[]) => {
+      batchCount++;
+      if (batchCount >= 1) {
+        cancellationFlag.cancelled = true;
+      }
+      return makeFakeVectors(texts.length);
+    });
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    const meta = await indexer.fullIndex(cancellationFlag);
+
+    expect(meta.status).toBe('in_progress');
+  });
+
+  // ── Test 14: Missing project_map.json throws NomosError ───────────────────
+  it('fullIndex() throws NomosError("search_index_failed") when project_map.json is missing', async () => {
+    // Remove project_map.json
+    await fs.rm(path.join(graphDir, 'project_map.json'));
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+
+    await expect(indexer.fullIndex()).rejects.toThrow(NomosError);
+    await expect(indexer.fullIndex()).rejects.toMatchObject({ code: 'search_index_failed' });
+  });
+
+  // ── Test 15: Partial embedding failure — failed files recorded [GAP-1] ────
+  it('partial embedding failure: failed files recorded in failed_files, rest indexed [GAP-1]', async () => {
+    let callCount = 0;
+    mockEmbedBatch.mockImplementation(async (texts: string[]) => {
+      callCount++;
+      if (callCount === 1) {
+        // First batch fails
+        throw new Error('Simulated embedding API failure');
+      }
+      return makeFakeVectors(texts.length);
+    });
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    const meta = await indexer.fullIndex();
+
+    // Some files failed, rest succeeded
+    expect(meta.failed_files.length).toBeGreaterThan(0);
+    // Index is still marked complete (partial success)
+    expect(meta.status).toBe('complete');
+    // Total chunks should reflect only successful batches
+    expect(meta.total_chunks).toBeGreaterThanOrEqual(0);
+  });
+
+  // ── Test 16: dryRun() returns chunk counts without calling Embedder or VectorStore [S-2]
+  it('dryRun() returns chunk counts without calling Embedder or VectorStore [S-2]', async () => {
+    const { VectorStore } = await import('../vector-store.js');
+    const initSpy = vi.spyOn(VectorStore.prototype, 'init');
+
+    const indexer = new SearchIndexer(tmpDir, config, logger);
+    const result = await indexer.dryRun();
+
+    expect(result.totalChunks).toBe(result.fileChunks + result.symbolChunks);
+    expect(result.totalChunks).toBeGreaterThan(0);
+
+    // Must NOT call Embedder or VectorStore
+    expect(mockEmbedBatch).not.toHaveBeenCalled();
+    expect(mockEmbedOne).not.toHaveBeenCalled();
+    expect(initSpy).not.toHaveBeenCalled();
+  });
+});