@softerist/heuristic-mcp 2.1.47 → 3.0.0

package/test/index-codebase-lines.test.js

@@ -0,0 +1,151 @@
+import { describe, it, expect, vi } from 'vitest';
+import { CodebaseIndexer } from '../features/index-codebase.js';
+import path from 'path';
+
+// Helper to access the private/internal function if it was exported,
+// but since it's not, we have to test it through public methods that use it.
+// buildExcludeMatchers uses globToRegExp.
+// isExcluded uses matchesExcludePatterns which uses buildExcludeMatchers.
+
+describe('CodebaseIndexer Glob Coverage', () => {
+  it('should handle single star glob patterns correctly', () => {
+    // This targets the branch in globToRegExp:
+    // if (char === "*") { ... } else { regex += "[^/]*"; ... }
+
+    // We need a pattern with a single '*' that is NOT followed by '*'
+    // e.g., "*.log" or "src/*.js"
+
+    const config = {
+      excludePatterns: ['*.log', 'src/*.js'],
+      fileExtensions: ['js'],
+      searchDirectory: '/test',
+      verbose: true,
+    };
+
+    // Mock dependencies
+    const embedder = vi.fn();
+    const cache = { load: vi.fn() };
+
+    const indexer = new CodebaseIndexer(embedder, cache, config);
+
+    // These calls should trigger the regex generation and matching logic
+    expect(indexer.isExcluded('error.log')).toBe(true);
+    expect(indexer.isExcluded('src/utils.js')).toBe(true);
+    expect(indexer.isExcluded('src/utils.test.js')).toBe(true);
+    expect(indexer.isExcluded('src/sub/utils.js')).toBe(false); // Single * shouldn't match across dirs usually if it's not **
+    expect(indexer.isExcluded('other.js')).toBe(false);
+  });
+
+  it('should handle question mark glob patterns', () => {
+    // Targets: if (char === "?")
+
+    const config = {
+      excludePatterns: ['test?.js'],
+
+      fileExtensions: ['js'],
+
+      searchDirectory: '/test',
+    };
+
+    const indexer = new CodebaseIndexer(vi.fn(), {}, config);
+
+    expect(indexer.isExcluded('test1.js')).toBe(true);
+
+    expect(indexer.isExcluded('testA.js')).toBe(true);
+
+    expect(indexer.isExcluded('test10.js')).toBe(false);
+  });
+
+  it('should handle double star not followed by slash', () => {
+    // Targets: if (pattern[i + 1] === "*") ... else { regex += ".*"; ... }
+
+    // Use a pattern with "/" to ensure matchBase is false, testing the full regex against path
+
+    const config = {
+      excludePatterns: ['dir/foo**bar'],
+
+      fileExtensions: ['js'],
+
+      searchDirectory: '/test',
+    };
+
+    const indexer = new CodebaseIndexer(vi.fn(), {}, config);
+
+    // Pattern "dir/foo**bar" becomes "^dir/foo.*bar$"
+
+    expect(indexer.isExcluded('dir/fooxyzbar')).toBe(true);
+
+    expect(indexer.isExcluded('dir/foobar')).toBe(true);
+
+    // ".*" matches "/" so this should match
+
+    expect(indexer.isExcluded('dir/foo/nested/bar')).toBe(true);
+  });
+});
+
+describe('CodebaseIndexer Worker Chunking', () => {
+  it('should handle fewer chunks than workers (Line 222 coverage)', async () => {
+    // config.workerThreads = 2
+
+    // chunks = [1]
+
+    // Worker 0 gets 1, Worker 1 gets 0 -> continues
+
+    // Mock os.cpus() first
+
+    vi.mock('os', async () => {
+      const actual = await vi.importActual('os');
+
+      return {
+        ...actual,
+
+        cpus: () => [{}, {}, {}, {}],
+      };
+    });
+
+    // Mock worker_threads
+
+    vi.mock('worker_threads', async () => {
+      const { EventEmitter } = await import('events');
+
+      class Worker extends EventEmitter {
+        constructor() {
+          super();
+          setTimeout(() => this.emit('message', { type: 'ready' }), 1);
+        }
+
+        terminate() {
+          return Promise.resolve();
+        }
+
+        postMessage(msg) {
+          if (msg.type === 'process') {
+            this.emit('message', { type: 'results', results: [], batchId: msg.batchId });
+          }
+        }
+      }
+
+      return { Worker };
+    });
+
+    const { CodebaseIndexer } = await import('../features/index-codebase.js');
+
+    const config = { workerThreads: 2, verbose: true }; // Verbose to hit logging branches too
+
+    const indexer = new CodebaseIndexer(
+      vi.fn(),
+      { save: vi.fn(), getVectorStore: () => [] },
+      config
+    );
+
+    await indexer.initializeWorkers();
+
+    // 1 chunk, 2 workers
+
+    await indexer.processChunksWithWorkers([{ text: 'abc', file: 'f.js' }]);
+
+    await indexer.terminateWorkers();
+
+    vi.restoreAllMocks();
+  });
+});

package/test/index-codebase-watcher.test.js

@@ -0,0 +1,259 @@
+/**
+ * Tests for CodebaseIndexer file watcher behavior
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+import path from 'path';
+import fs from 'fs/promises';
+import os from 'os';
+
+vi.mock('chokidar', () => {
+  const { EventEmitter } = require('events');
+  const watch = vi.fn(() => {
+    const emitter = new EventEmitter();
+    emitter.close = vi.fn().mockResolvedValue();
+    const on = emitter.on.bind(emitter);
+    emitter.on = (event, handler) => {
+      on(event, handler);
+      return emitter;
+    };
+    globalThis.__heuristicWatcher = emitter;
+    return emitter;
+  });
+
+  return {
+    __esModule: true,
+    default: {
+      watch,
+    },
+  };
+});
+
+import { CodebaseIndexer } from '../features/index-codebase.js';
+
+async function withTempDir(testFn) {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'heuristic-watcher-'));
+  try {
+    await testFn(dir);
+  } finally {
+    await fs.rm(dir, { recursive: true, force: true });
+  }
+}
+
+function flushPromises() {
+  return new Promise((resolve) => setImmediate(resolve));
+}
+
+describe('CodebaseIndexer watcher', () => {
+  it('wires watcher events to indexing and cache updates', async () => {
+    await withTempDir(async (dir) => {
+      const config = {
+        fileExtensions: ['js'],
+        fileNames: [],
+        searchDirectory: dir,
+        excludePatterns: [],
+        watchFiles: true,
+        enableCache: true,
+        callGraphEnabled: false,
+        embeddingModel: 'test',
+        verbose: false,
+      };
+
+      const cache = {
+        save: vi.fn().mockResolvedValue(),
+        removeFileFromStore: vi.fn(),
+        deleteFileHash: vi.fn(),
+      };
+
+      const server = {
+        hybridSearch: {
+          clearFileModTime: vi.fn(),
+        },
+      };
+
+      const indexer = new CodebaseIndexer(async () => ({ data: [] }), cache, config, server);
+      indexer.indexFile = vi.fn().mockResolvedValue();
+      indexer.isIndexing = false;
+      indexer.processingWatchEvents = false;
+
+      await indexer.setupFileWatcher();
+
+      const relPath = path.join('src', 'file.js');
+      indexer.watcher.emit('add', relPath);
+      await flushPromises();
+
+      expect(indexer.indexFile).toHaveBeenCalledWith(path.join(dir, relPath));
+      expect(cache.save).toHaveBeenCalled();
+      expect(server.hybridSearch.clearFileModTime).toHaveBeenCalledWith(path.join(dir, relPath));
+
+      indexer.watcher.emit('change', relPath);
+      await flushPromises();
+      expect(indexer.indexFile).toHaveBeenCalledTimes(2);
+
+      indexer.watcher.emit('unlink', relPath);
+      await flushPromises();
+      expect(cache.removeFileFromStore).toHaveBeenCalledWith(path.join(dir, relPath));
+      expect(cache.deleteFileHash).toHaveBeenCalledWith(path.join(dir, relPath));
+    });
+  });
+
+  it('closes existing watcher before reinitializing', async () => {
+    await withTempDir(async (dir) => {
+      const config = {
+        fileExtensions: ['js'],
+        fileNames: [],
+        searchDirectory: dir,
+        excludePatterns: [],
+        watchFiles: true,
+        enableCache: true,
+        callGraphEnabled: false,
+        embeddingModel: 'test',
+        verbose: false,
+      };
+
+      const cache = {
+        save: vi.fn().mockResolvedValue(),
+        removeFileFromStore: vi.fn(),
+        deleteFileHash: vi.fn(),
+      };
+
+      const indexer = new CodebaseIndexer(async () => ({ data: [] }), cache, config, null);
+      indexer.indexFile = vi.fn().mockResolvedValue();
+
+      await indexer.setupFileWatcher();
+      const firstWatcher = globalThis.__heuristicWatcher;
+
+      await indexer.setupFileWatcher();
+
+      expect(firstWatcher.close).toHaveBeenCalledTimes(1);
+      expect(globalThis.__heuristicWatcher).not.toBe(firstWatcher);
+    });
+  });
+
+  it('queues change and unlink events during active indexing', async () => {
+    await withTempDir(async (dir) => {
+      const config = {
+        fileExtensions: ['js'],
+        fileNames: [],
+        searchDirectory: dir,
+        excludePatterns: [],
+        watchFiles: true,
+        enableCache: true,
+        callGraphEnabled: false,
+        embeddingModel: 'test',
+        verbose: true,
+      };
+
+      const cache = {
+        save: vi.fn().mockResolvedValue(),
+        removeFileFromStore: vi.fn(),
+        deleteFileHash: vi.fn(),
+      };
+
+      const indexer = new CodebaseIndexer(async () => ({ data: [] }), cache, config, null);
+      indexer.indexFile = vi.fn().mockResolvedValue();
+      indexer.isIndexing = true;
+
+      const enqueueSpy = vi.spyOn(indexer, 'enqueueWatchEvent');
+
+      await indexer.setupFileWatcher();
+
+      const relPath = path.join('src', 'file.js');
+      globalThis.__heuristicWatcher.emit('add', relPath);
+      globalThis.__heuristicWatcher.emit('change', relPath);
+      globalThis.__heuristicWatcher.emit('unlink', relPath);
+      await flushPromises();
+
+      expect(enqueueSpy).toHaveBeenCalledWith('add', path.join(dir, relPath));
+      expect(enqueueSpy).toHaveBeenCalledWith('change', path.join(dir, relPath));
+      expect(enqueueSpy).toHaveBeenCalledWith('unlink', path.join(dir, relPath));
+      expect(indexer.indexFile).not.toHaveBeenCalled();
+      expect(cache.save).not.toHaveBeenCalled();
+    });
+  });
+
+  it('queues events without verbose logging when indexing', async () => {
+    await withTempDir(async (dir) => {
+      const config = {
+        fileExtensions: ['js'],
+        fileNames: [],
+        searchDirectory: dir,
+        excludePatterns: [],
+        watchFiles: true,
+        enableCache: true,
+        callGraphEnabled: false,
+        embeddingModel: 'test',
+        verbose: false,
+      };
+
+      const cache = {
+        save: vi.fn().mockResolvedValue(),
+        removeFileFromStore: vi.fn(),
+        deleteFileHash: vi.fn(),
+      };
+
+      const indexer = new CodebaseIndexer(async () => ({ data: [] }), cache, config, null);
+      indexer.indexFile = vi.fn().mockResolvedValue();
+      indexer.isIndexing = true;
+
+      const enqueueSpy = vi.spyOn(indexer, 'enqueueWatchEvent');
+
+      await indexer.setupFileWatcher();
+
+      const relPath = path.join('src', 'file.js');
+      globalThis.__heuristicWatcher.emit('add', relPath);
+      globalThis.__heuristicWatcher.emit('change', relPath);
+      globalThis.__heuristicWatcher.emit('unlink', relPath);
+      await flushPromises();
+
+      expect(enqueueSpy).toHaveBeenCalledWith('add', path.join(dir, relPath));
+      expect(enqueueSpy).toHaveBeenCalledWith('change', path.join(dir, relPath));
+      expect(enqueueSpy).toHaveBeenCalledWith('unlink', path.join(dir, relPath));
+    });
+  });
+
+  it('processes pending watch events and clears hybrid search cache', async () => {
+    await withTempDir(async (dir) => {
+      const config = {
+        fileExtensions: ['js'],
+        fileNames: [],
+        searchDirectory: dir,
+        excludePatterns: [],
+        watchFiles: true,
+        enableCache: true,
+        callGraphEnabled: false,
+        embeddingModel: 'test',
+        verbose: false,
+      };
+
+      const cache = {
+        save: vi.fn().mockResolvedValue(),
+        removeFileFromStore: vi.fn(),
+        deleteFileHash: vi.fn(),
+      };
+
+      const server = {
+        hybridSearch: {
+          clearFileModTime: vi.fn(),
+        },
+      };
+
+      const indexer = new CodebaseIndexer(async () => ({ data: [] }), cache, config, server);
+      indexer.indexFile = vi.fn().mockResolvedValue();
+
+      const changePath = path.join(dir, 'change.js');
+      const unlinkPath = path.join(dir, 'unlink.js');
+      indexer.pendingWatchEvents.set(changePath, 'change');
+      indexer.pendingWatchEvents.set(unlinkPath, 'unlink');
+
+      await indexer.processPendingWatchEvents();
+
+      expect(server.hybridSearch.clearFileModTime).toHaveBeenCalledWith(changePath);
+      expect(server.hybridSearch.clearFileModTime).toHaveBeenCalledWith(unlinkPath);
+      expect(indexer.indexFile).toHaveBeenCalledWith(changePath);
+      expect(cache.removeFileFromStore).toHaveBeenCalledWith(unlinkPath);
+      expect(cache.deleteFileHash).toHaveBeenCalledWith(unlinkPath);
+      expect(cache.save).toHaveBeenCalled();
+    });
+  });
+});

package/test/index-codebase-zone.test.js

@@ -0,0 +1,259 @@
+
+import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
+import {
+  createTestFixtures,
+  cleanupFixtures,
+  clearTestCache,
+} from './helpers.js';
+import fs from 'fs/promises';
+import path from 'path';
+
+describe('CodebaseIndexer Phase 2 Coverage', () => {
+  let fixtures;
+
+  beforeAll(async () => {
+    fixtures = await createTestFixtures({ workerThreads: 1 });
+  });
+
+  afterAll(async () => {
+    await cleanupFixtures(fixtures);
+  });
+
+  beforeEach(async () => {
+    fixtures.indexer.isIndexing = false;
+    await fixtures.indexer.terminateWorkers();
+    await clearTestCache(fixtures.config);
+    fixtures.cache.setVectorStore([]);
+    fixtures.cache.clearFileHashes();
+    fixtures.config.verbose = true;
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should handle read errors in pre-filter batch processing (lines 553-554)', async () => {
+    const subDir = path.join(fixtures.config.searchDirectory, 'p2_read_data');
+    await fs.mkdir(subDir, { recursive: true });
+    
+    const fileGood = path.join(subDir, 'good.js');
+    const fileBad = path.join(subDir, 'bad.js');
+    
+    await fs.writeFile(fileGood, 'good content');
+    await fs.writeFile(fileBad, 'bad content');
+    
+    const realStat = fs.stat;
+    const statSpy = vi.spyOn(fs, 'stat').mockImplementation(async (filePath) => {
+        if (filePath.toString().includes('bad.js')) {
+            throw new Error('Simulated stat error');
+        }
+        return realStat.call(fs, filePath);
+    });
+    
+    try {
+        // Call preFilterFiles directly to bypass discovery issues
+        const result = await fixtures.indexer.preFilterFiles([fileGood, fileBad]);
+        
+        console.error('PreFilter Result:', result);
+        
+        // Good file should be included
+        const hasGood = result.some(r => r.file.includes('good.js'));
+        // Bad file should be excluded (due to error)
+        const hasBad = result.some(r => r.file.includes('bad.js'));
+        
+        expect(hasGood).toBe(true);
+        expect(hasBad).toBe(false);
+        
+    } finally {
+        statSpy.mockRestore();
+        await fs.rm(subDir, { recursive: true, force: true });
+    }
+  });
+
+  it('should flush read batch when size limit exceeded (lines 571-573)', async () => {
+    const subDir = path.join(fixtures.config.searchDirectory, 'p2_flush_data');
+    await fs.mkdir(subDir, { recursive: true });
+    
+    const file1 = path.join(subDir, 'f1.js');
+    const file2 = path.join(subDir, 'f2.js');
+    const file3 = path.join(subDir, 'f3.js');
+    
+    await fs.writeFile(file1, 'c1');
+    await fs.writeFile(file2, 'c2');
+    await fs.writeFile(file3, 'c3');
+    
+    const realStat = fs.stat;
+    const statSpy = vi.spyOn(fs, 'stat').mockImplementation(async (filePath) => {
+        if (filePath.toString().includes('p2_flush_data')) {
+            const s = await realStat.call(fs, filePath);
+            // Modify directly
+            s.size = 20 * 1024 * 1024; // 20MB
+            return s;
+        }
+        return realStat.call(fs, filePath);
+    });
+    
+    const oldMax = fixtures.config.maxFileSize;
+    fixtures.config.maxFileSize = 100 * 1024 * 1024; // 100MB
+    
+    try {
+        // Pass 3 files. Total 60MB. Batch limit 50MB.
+        // Should trigger intermediate flush.
+        const result = await fixtures.indexer.preFilterFiles([file1, file2, file3]);
+        
+        console.error('Batch flush result:', result);
+
+        expect(result.some(r => r.file.includes('f1.js'))).toBe(true);
+        expect(result.some(r => r.file.includes('f2.js'))).toBe(true);
+        expect(result.some(r => r.file.includes('f3.js'))).toBe(true);
+        
+    } finally {
+        statSpy.mockRestore();
+        fixtures.config.maxFileSize = oldMax;
+        await fs.rm(subDir, { recursive: true, force: true });
+    }
+  });
+
+  it('should handle invalid stats in call graph recovery (line 701)', async () => {
+    const subDir = path.join(fixtures.config.searchDirectory, 'p2_recovery_data');
+    await fs.mkdir(subDir, { recursive: true });
+    
+    const fileRecover = path.join(subDir, 'recover.js');
+    await fs.writeFile(fileRecover, 'content');
+    
+    fixtures.cache.addToStore({
+        file: fileRecover,
+        startLine: 1, endLine: 1, content: 'content', vector: []
+    });
+    
+    const realStat = fs.stat;
+    const statSpy = vi.spyOn(fs, 'stat').mockImplementation(async (filePath) => {
+        if (filePath.toString().includes('recover.js')) {
+            return { isDirectory: 'not-a-function' };
+        }
+        return realStat.call(fs, filePath);
+    });
+    
+    fixtures.config.callGraphEnabled = true;
+    
+    try {
+        await fixtures.indexer.indexAll(false);
+    } finally {
+        statSpy.mockRestore();
+        await fs.rm(subDir, { recursive: true, force: true });
+    }
+  });
+
+  it('should skip large files provided with content (L825)', async () => {
+    const largeContent = 'x'.repeat(fixtures.config.maxFileSize + 1024);
+    fixtures.indexer.discoverFiles = vi.fn().mockResolvedValue(['large.js']);
+    fixtures.indexer.preFilterFiles = vi
+      .fn()
+      .mockResolvedValue([{ file: 'large.js', content: largeContent, hash: 'h', force: true }]);
+
+    await fixtures.indexer.indexAll();
+  });
+
+  it('coerces non-string content and skips when too large (L825)', async () => {
+    fixtures.indexer.discoverFiles = vi.fn().mockResolvedValue(['large2.js']);
+    fixtures.indexer.preFilterFiles = vi
+      .fn()
+      .mockResolvedValue([{ file: 'large2.js', content: 42, hash: null, force: true }]);
+    const oldMax = fixtures.config.maxFileSize;
+    fixtures.config.maxFileSize = 1;
+
+    try {
+      await fixtures.indexer.indexAll();
+    } finally {
+      fixtures.config.maxFileSize = oldMax;
+    }
+  });
+
+  it('should log error when stat fails (L837)', async () => {
+    fixtures.indexer.discoverFiles = vi.fn().mockResolvedValue(['stat-error.js']);
+    fixtures.indexer.preFilterFiles = vi
+      .fn()
+      .mockResolvedValue([{ file: 'stat-error.js', hash: 'h', force: true }]);
+
+    vi.spyOn(fs, 'stat').mockRejectedValue(new Error('Stat failed'));
+
+    await fixtures.indexer.indexAll();
+  });
+
+  it('should handle read failures in main loop (L870)', async () => {
+    fixtures.indexer.discoverFiles = vi.fn().mockResolvedValue(['fail.js']);
+    fixtures.indexer.preFilterFiles = vi
+      .fn()
+      .mockResolvedValue([{ file: 'fail.js', hash: 'h', force: true }]);
+
+    vi.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => false, size: 100 });
+    vi.spyOn(fs, 'readFile').mockRejectedValue(new Error('Read fail'));
+
+    await fixtures.indexer.indexAll();
+  });
+
+  it('should skip unchanged files in loop (L882)', async () => {
+    fixtures.indexer.discoverFiles = vi.fn().mockResolvedValue(['same.js']);
+    fixtures.indexer.preFilterFiles = vi
+      .fn()
+      .mockResolvedValue([{ file: 'same.js', hash: 'h', force: false }]);
+
+    vi.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => false, size: 100 });
+    vi.spyOn(fs, 'readFile').mockResolvedValue('content');
+    vi.spyOn(fixtures.indexer.cache, 'getFileHash').mockReturnValue('h');
+
+    await fixtures.indexer.indexAll();
+  });
+
+  it('handles invalid stats in main loop (L846)', async () => {
+    fixtures.indexer.discoverFiles = vi.fn().mockResolvedValue(['invalid.js']);
+    fixtures.indexer.preFilterFiles = vi
+      .fn()
+      .mockResolvedValue([{ file: 'invalid.js', hash: 'h', force: true }]);
+
+    vi.spyOn(fs, 'stat').mockResolvedValue({});
+
+    await fixtures.indexer.indexAll();
+  });
+
+  it('skips large file during stat pass (L859)', async () => {
+    fixtures.indexer.discoverFiles = vi.fn().mockResolvedValue(['big.js']);
+    fixtures.indexer.preFilterFiles = vi
+      .fn()
+      .mockResolvedValue([{ file: 'big.js', hash: 'h', force: true }]);
+
+    vi.spyOn(fs, 'stat').mockResolvedValue({
+      isDirectory: () => false,
+      size: fixtures.config.maxFileSize + 1,
+    });
+
+    await fixtures.indexer.indexAll();
+  });
+
+  it('should queue watcher events during indexing (L1106, L1126, L1146)', async () => {
+      // 1. Setup watcher
+      fixtures.config.watchFiles = true;
+      await fixtures.indexer.setupFileWatcher();
+      
+      // 2. Set indexing flag
+      fixtures.indexer.isIndexing = true;
+      
+      // 3. Emit events
+      const watcher = fixtures.indexer.watcher;
+      if (watcher) {
+          watcher.emit('add', 'new.js');
+          watcher.emit('change', 'changed.js');
+          watcher.emit('unlink', 'deleted.js');
+      }
+
+      // Check queue
+      expect(fixtures.indexer.pendingWatchEvents.has(path.join(fixtures.config.searchDirectory, 'new.js'))).toBe(true);
+      expect(fixtures.indexer.pendingWatchEvents.has(path.join(fixtures.config.searchDirectory, 'changed.js'))).toBe(true);
+      // unlink might use absolute path logic
+      const delPath = path.join(fixtures.config.searchDirectory, 'deleted.js');
+      expect(fixtures.indexer.pendingWatchEvents.get(delPath)).toBe('unlink');
+
+      // Reset
+      fixtures.indexer.isIndexing = false;
+  });
+});