@softerist/heuristic-mcp 2.1.47 → 3.0.0

package/test/worker_throttling.test.js

@@ -0,0 +1,93 @@
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { CodebaseIndexer } from '../features/index-codebase.js';
+import os from 'os';
+
+vi.mock('os');
+vi.mock('../lib/embedding-worker.js', () => {
+  return {
+    default: class MockWorker {
+        constructor() {
+            this.on = vi.fn();
+            this.once = vi.fn();
+            this.postMessage = vi.fn();
+            this.terminate = vi.fn();
+            // Simulate ready immediately
+            setTimeout(() => {
+                const calls = this.once.mock.calls;
+                const readyHandler = calls.find(c => c[0] === 'message')?.[1];
+                if (readyHandler) readyHandler({ type: 'ready' });
+            }, 10);
+        }
+    }
+  };
+});
+
+// Mock Worker from worker_threads
+vi.mock('worker_threads', () => {
+    return {
+        Worker: class {
+            constructor() {
+                this.on = vi.fn();
+                this.once = vi.fn();
+                this.postMessage = vi.fn();
+                this.terminate = vi.fn();
+                setTimeout(() => {
+                    const calls = this.once.mock.calls;
+                    const readyHandler = calls.find(c => c[0] === 'message')?.[1];
+                    if (readyHandler) readyHandler({ type: 'ready' });
+                }, 10);
+            }
+        }
+    };
+});
+
+
+describe('CodebaseIndexer RAM Throttling', () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    vi.resetModules();
+    process.env = { ...originalEnv };
+    // Trick the module into thinking we are NOT in a test env so throttling runs
+    delete process.env.VITEST;
+    delete process.env.NODE_ENV;
+  });
+
+  afterEach(() => {
+    process.env = originalEnv;
+    vi.clearAllMocks();
+  });
+
+  it('should throttle workers when RAM is low', async () => {
+    // Mock 16 cores
+    os.cpus.mockReturnValue(Array(16).fill({}));
+    // Mock 2GB free memory (enough for ~1 worker @ 1.5GB/worker)
+    os.freemem.mockReturnValue(2 * 1024 * 1024 * 1024);
+
+    const config = {
+      workerThreads: 10, // Simulated "resolved" auto count or user explicit
+      embeddingModel: 'jinaai/jina-embeddings-v2-base-code',
+      verbose: true,
+      searchDirectory: '/tmp'
+    };
+
+    const indexer = new CodebaseIndexer({}, {}, config);
+    
+    // We expect initializeWorkers to NOT throttle effectively because it checks for 'auto'
+    // but we are passing a number (10).
+    // Current buggy behavior: It uses 10 workers.
+    // Desired behavior: It sees 2GB RAM and throttles to ~1 worker.
+
+    await indexer.initializeWorkers();
+
+    const activeWorkers = indexer.workers.length;
+    console.log(`Initialized ${activeWorkers} workers`);
+    
+    // CURRENTLY BROKEN: This expectation represents the BUG.
+    // It SHOULD be 1, but it WILL be 10.
+    // We write the test to expect 1 (the correct behavior), so it fails now.
+    expect(activeWorkers).toBeLessThan(10); 
+    expect(activeWorkers).toBe(1); // 2GB / ~1.5GB = 1 worker
+  });
+});

package/tools/scripts/benchmark-search.js

@@ -0,0 +1,95 @@
+import { loadConfig } from '../../lib/config.js';
+import { EmbeddingsCache } from '../../lib/cache.js';
+import { HybridSearch } from '../../features/hybrid-search.js';
+import { pipeline, env } from '@xenova/transformers';
+
+// Force same thread config as server
+env.backends.onnx.numThreads = 2;
+env.backends.onnx.wasm.numThreads = 2;
+
+function parseArgs(argv) {
+  const args = argv.slice(2);
+  let query = null;
+  let runs = 5;
+  let maxResults = 5;
+
+  for (let i = 0; i < args.length; i += 1) {
+    const arg = args[i];
+    if (arg === '--query' && args[i + 1]) {
+      query = args[i + 1];
+      i += 1;
+    } else if (arg === '--runs' && args[i + 1]) {
+      runs = parseInt(args[i + 1], 10);
+      i += 1;
+    } else if (arg === '--max-results' && args[i + 1]) {
+      maxResults = parseInt(args[i + 1], 10);
+      i += 1;
+    } else if (!arg.startsWith('-') && !query) {
+      query = arg;
+    }
+  }
+
+  return {
+    query: query || 'database implementation',
+    runs: Number.isFinite(runs) && runs > 0 ? runs : 5,
+    maxResults: Number.isFinite(maxResults) && maxResults > 0 ? maxResults : 5,
+  };
+}
+
+function percentile(sorted, p) {
+  if (sorted.length === 0) return 0;
+  const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
+  return sorted[idx];
+}
+
+async function main() {
+  const { query, runs, maxResults } = parseArgs(process.argv);
+  const config = await loadConfig(process.cwd());
+  const cache = new EmbeddingsCache(config);
+  await cache.load();
+
+  const embedder = async (text) => {
+    if (!embedder._pipeline) {
+      embedder._pipeline = await pipeline('feature-extraction', config.embeddingModel, {
+        session_options: { numThreads: 2 },
+      });
+    }
+    return embedder._pipeline(text, { pooling: 'mean', normalize: true });
+  };
+
+  const searcher = new HybridSearch(embedder, cache, config);
+
+  console.info(`Benchmark query: "${query}"`);
+  console.info(`Runs: ${runs}, maxResults: ${maxResults}`);
+  console.info(
+    `Vector load mode: ${config.vectorStoreLoadMode}, format: ${config.vectorStoreFormat}`
+  );
+
+  // Warm-up
+  await searcher.search(query, maxResults);
+
+  const durations = [];
+  const memBefore = process.memoryUsage().rss;
+
+  for (let i = 0; i < runs; i += 1) {
+    const start = process.hrtime.bigint();
+    await searcher.search(query, maxResults);
+    const end = process.hrtime.bigint();
+    const ms = Number(end - start) / 1e6;
+    durations.push(ms);
+    console.info(`Run ${i + 1}: ${ms.toFixed(2)}ms`);
+  }
+
+  const memAfter = process.memoryUsage().rss;
+  const sorted = [...durations].sort((a, b) => a - b);
+  const avg = durations.reduce((a, b) => a + b, 0) / durations.length;
+  const p95 = percentile(sorted, 95);
+
+  console.info(`Avg: ${avg.toFixed(2)}ms, p95: ${p95.toFixed(2)}ms`);
+  console.info(`RSS change: ${((memAfter - memBefore) / 1024 / 1024).toFixed(1)}MB`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/tools/scripts/cache-stats.js

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+import fs from 'fs/promises';
+import path from 'path';
+import { loadConfig } from '../../lib/config.js';
+import { BinaryVectorStore } from '../../lib/vector-store-binary.js';
+
+async function main() {
+  const args = process.argv.slice(2);
+  const workspaceIndex = args.indexOf('--workspace');
+  const workspaceDir = workspaceIndex !== -1 ? args[workspaceIndex + 1] : null;
+
+  const config = await loadConfig(workspaceDir || process.cwd());
+  const cacheDir = config.cacheDirectory;
+
+  const metaPath = path.join(cacheDir, 'meta.json');
+  const hashPath = path.join(cacheDir, 'file-hashes.json');
+  const jsonPath = path.join(cacheDir, 'embeddings.json');
+
+  const stats = {
+    cacheDir,
+    vectorStoreFormat: config.vectorStoreFormat,
+    hasMeta: false,
+    hasJson: false,
+    hasBinary: false,
+    vectorCount: 0,
+    fileHashCount: 0,
+  };
+
+  try {
+    const metaRaw = await fs.readFile(metaPath, 'utf-8');
+    stats.hasMeta = true;
+    stats.meta = JSON.parse(metaRaw);
+  } catch {
+    // ignore
+  }
+
+  try {
+    const hashRaw = await fs.readFile(hashPath, 'utf-8');
+    const hashes = JSON.parse(hashRaw);
+    stats.fileHashCount = Object.keys(hashes).length;
+  } catch {
+    // ignore
+  }
+
+  try {
+    const jsonRaw = await fs.readFile(jsonPath, 'utf-8');
+    stats.hasJson = true;
+    const parsed = JSON.parse(jsonRaw);
+    if (Array.isArray(parsed)) {
+      stats.jsonVectorCount = parsed.length;
+    }
+  } catch {
+    // ignore
+  }
+
+  try {
+    const store = await BinaryVectorStore.load(cacheDir);
+    stats.hasBinary = true;
+    stats.vectorCount = store.length;
+    stats.binaryDim = store.dim;
+  } catch {
+    // ignore
+  }
+
+  console.info(JSON.stringify(stats, null, 2));
+}
+
+main().catch((err) => {
+  console.error(`[cache-stats] ${err.message}`);
+  process.exit(1);
+});

package/tools/scripts/manual-search.js

@@ -0,0 +1,34 @@
+
+import { loadConfig } from '../../lib/config.js';
+import { EmbeddingsCache } from '../../lib/cache.js';
+import { HybridSearch } from '../../features/hybrid-search.js';
+import { pipeline, env } from '@xenova/transformers';
+
+// Force same thread config as server
+env.backends.onnx.numThreads = 2;
+env.backends.onnx.wasm.numThreads = 2;
+
+async function runSearch(query) {
+  const config = await loadConfig(process.cwd());
+  const cache = new EmbeddingsCache(config);
+  await cache.load();
+
+  const embedder = async (text) => {
+      const pipe = await pipeline('feature-extraction', config.embeddingModel, {
+        session_options: { numThreads: 2 } 
+      });
+      return pipe(text, { pooling: 'mean', normalize: true });
+  };
+
+  const searcher = new HybridSearch(embedder, cache, config);
+  console.info(`\n--- Searching for: "${query}" ---`);
+  const { results } = await searcher.search(query, 5);
+
+  results.forEach((r, i) => {
+    console.info(`[${i+1}] ${r.file}:${r.startLine}-${r.endLine} (Score: ${r.score.toFixed(4)})`);
+    console.info(`    Content: ${r.content.substring(0, 200).replace(/\n/g, ' ')}...`);
+  });
+}
+
+const query = process.argv[2] || 'database implementation';
+runSearch(query).catch(err => console.error(err));

package/vitest.config.js

@@ -4,26 +4,36 @@ export default defineConfig({
   test: {
     // Test files pattern
     include: ['test/**/*.test.js'],
-    
+
     // Global test timeout (embedding models can be slow)
     testTimeout: 180000,
-    
+
     // Hook timeout for setup/teardown
     hookTimeout: 180000,
-    
+
     // Run test files sequentially to avoid resource conflicts
     // Each file loads the embedding model which uses significant memory
     fileParallelism: false,
-    
+
     // Run tests within a file sequentially
     sequence: {
-      concurrent: false
+      concurrent: false,
     },
-    
+
     // Verbose output
     reporters: ['verbose'],
-    
+
     // Isolate tests to prevent memory leaks between test files
-    isolate: true
-  }
+    isolate: true,
+
+    coverage: {
+      provider: 'v8',
+      all: true,
+      include: ['features/**/*.js', 'lib/**/*.js', 'index.js'],
+      exclude: ['**/test/**'],
+      reporter: ['text', 'html', 'json'],
+      reportsDirectory: '.vitest-coverage',
+      clean: true,
+    },
+  },
 });