@softerist/heuristic-mcp 2.0.0

package/index.js

@@ -0,0 +1,208 @@
+#!/usr/bin/env node
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import { pipeline } from "@xenova/transformers";
+import fs from "fs/promises";
+import { createRequire } from "module";
+
+// Import package.json for version
+const require = createRequire(import.meta.url);
+const packageJson = require("./package.json");
+
+import { loadConfig } from "./lib/config.js";
+import { EmbeddingsCache } from "./lib/cache.js";
+import { CodebaseIndexer } from "./features/index-codebase.js";
+import { HybridSearch } from "./features/hybrid-search.js";
+
+import * as IndexCodebaseFeature from "./features/index-codebase.js";
+import * as HybridSearchFeature from "./features/hybrid-search.js";
+import * as ClearCacheFeature from "./features/clear-cache.js";
+import * as FindSimilarCodeFeature from "./features/find-similar-code.js";
+
+// Parse workspace from command line arguments
+const args = process.argv.slice(2);
+const workspaceIndex = args.findIndex(arg => arg.startsWith('--workspace'));
+let workspaceDir = null;
+
+if (workspaceIndex !== -1) {
+  const arg = args[workspaceIndex];
+  let rawWorkspace = null;
+
+  if (arg.includes('=')) {
+    rawWorkspace = arg.split('=')[1];
+  } else if (workspaceIndex + 1 < args.length) {
+    rawWorkspace = args[workspaceIndex + 1];
+  }
+
+  // Check if IDE variable wasn't expanded (contains ${})
+  if (rawWorkspace && rawWorkspace.includes('${')) {
+    console.error(`[Server] IDE variable not expanded: ${rawWorkspace}, using current directory`);
+    workspaceDir = process.cwd();
+  } else if (rawWorkspace) {
+    workspaceDir = rawWorkspace;
+  }
+
+  if (workspaceDir) {
+    console.error(`[Server] Workspace mode: ${workspaceDir}`);
+  }
+}
+
+// Global state
+let embedder = null;
+let cache = null;
+let indexer = null;
+let hybridSearch = null;
+let config = null;
+
+// Feature registry - ordered by priority (semantic_search first as primary tool)
+const features = [
+  {
+    module: HybridSearchFeature,
+    instance: null,
+    handler: HybridSearchFeature.handleToolCall
+  },
+  {
+    module: IndexCodebaseFeature,
+    instance: null,
+    handler: IndexCodebaseFeature.handleToolCall
+  },
+  {
+    module: ClearCacheFeature,
+    instance: null,
+    handler: ClearCacheFeature.handleToolCall
+  },
+  {
+    module: FindSimilarCodeFeature,
+    instance: null,
+    handler: FindSimilarCodeFeature.handleToolCall
+  }
+];
+
+// Initialize application
+async function initialize() {
+  // Load configuration with workspace support
+  config = await loadConfig(workspaceDir);
+
+  // Ensure search directory exists
+  try {
+    await fs.access(config.searchDirectory);
+  } catch {
+    console.error(`[Server] Error: Search directory "${config.searchDirectory}" does not exist`);
+    process.exit(1);
+  }
+
+  // Load AI model
+  console.error("[Server] Loading AI embedding model (this may take time on first run)...");
+  embedder = await pipeline("feature-extraction", config.embeddingModel);
+
+  // Initialize cache
+  cache = new EmbeddingsCache(config);
+  await cache.load();
+
+  // Initialize features
+  indexer = new CodebaseIndexer(embedder, cache, config, server);
+  hybridSearch = new HybridSearch(embedder, cache, config);
+  const cacheClearer = new ClearCacheFeature.CacheClearer(embedder, cache, config, indexer);
+  const findSimilarCode = new FindSimilarCodeFeature.FindSimilarCode(embedder, cache, config);
+
+  // Store feature instances (matches features array order)
+  features[0].instance = hybridSearch;
+  features[1].instance = indexer;
+  features[2].instance = cacheClearer;
+  features[3].instance = findSimilarCode;
+
+  // Attach hybridSearch to server for cross-feature access (e.g. cache invalidation)
+  server.hybridSearch = hybridSearch;
+
+  // Start indexing in background (non-blocking)
+  console.error("[Server] Starting background indexing...");
+  indexer.indexAll().then(() => {
+    // Only start file watcher if explicitly enabled in config
+    if (config.watchFiles) {
+      indexer.setupFileWatcher();
+    }
+  }).catch(err => {
+    console.error("[Server] Background indexing error:", err.message);
+  });
+}
+
+// Setup MCP server
+const server = new Server(
+  {
+    name: "heuristic-mcp",
+    version: packageJson.version
+  },
+  {
+    capabilities: {
+      tools: {}
+    }
+  }
+);
+
+// Register tools from all features
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  const tools = [];
+
+  for (const feature of features) {
+    const toolDef = feature.module.getToolDefinition(config);
+    tools.push(toolDef);
+  }
+
+  return { tools };
+});
+
+// Handle tool calls
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  for (const feature of features) {
+    const toolDef = feature.module.getToolDefinition(config);
+
+    if (request.params.name === toolDef.name) {
+      return await feature.handler(request, feature.instance);
+    }
+  }
+
+  return {
+    content: [{
+      type: "text",
+      text: `Unknown tool: ${request.params.name}`
+    }]
+  };
+});
+
+// Main entry point
+async function main() {
+  await initialize();
+
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+
+  console.error("[Server] Smart Coding MCP server ready!");
+}
+
+// Graceful shutdown
+process.on('SIGINT', async () => {
+  console.error("\n[Server] Shutting down gracefully...");
+
+  // Stop file watcher
+  if (indexer && indexer.watcher) {
+    await indexer.watcher.close();
+    console.error("[Server] File watcher stopped");
+  }
+
+  // Save cache
+  if (cache) {
+    await cache.save();
+    console.error("[Server] Cache saved");
+  }
+
+  console.error("[Server] Goodbye!");
+  process.exit(0);
+});
+
+process.on('SIGTERM', async () => {
+  console.error("\n[Server] Received SIGTERM, shutting down...");
+  process.exit(0);
+});
+
+main().catch(console.error);

package/lib/cache.js

@@ -0,0 +1,163 @@
+import fs from "fs/promises";
+import path from "path";
+
+const CACHE_META_VERSION = 1;
+const CACHE_META_FILE = "meta.json";
+
+export class EmbeddingsCache {
+  constructor(config) {
+    this.config = config;
+    this.vectorStore = [];
+    this.fileHashes = new Map();
+    this.isSaving = false;
+    this.cacheMeta = {
+      version: CACHE_META_VERSION,
+      embeddingModel: config.embeddingModel
+    };
+  }
+
+  async load() {
+    if (!this.config.enableCache) return;
+    
+    try {
+      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
+      const cacheFile = path.join(this.config.cacheDirectory, "embeddings.json");
+      const hashFile = path.join(this.config.cacheDirectory, "file-hashes.json");
+      const metaFile = path.join(this.config.cacheDirectory, CACHE_META_FILE);
+      
+      const [metaData, cacheData, hashData] = await Promise.all([
+        fs.readFile(metaFile, "utf-8").catch(() => null),
+        fs.readFile(cacheFile, "utf-8").catch(() => null),
+        fs.readFile(hashFile, "utf-8").catch(() => null)
+      ]);
+
+      if (!metaData && !cacheData && !hashData) {
+        return;
+      }
+
+      if (!metaData) {
+        console.error("[Cache] Missing cache metadata, ignoring cache");
+        return;
+      }
+
+      let meta = null;
+      try {
+        meta = JSON.parse(metaData);
+      } catch {
+        console.error("[Cache] Invalid cache metadata, ignoring cache");
+        return;
+      }
+
+      if (meta?.version !== CACHE_META_VERSION) {
+        console.error(`[Cache] Cache version mismatch (${meta?.version}), ignoring cache`);
+        return;
+      }
+
+      if (meta?.embeddingModel !== this.config.embeddingModel) {
+        console.error(`[Cache] Embedding model changed, ignoring cache (${meta?.embeddingModel} -> ${this.config.embeddingModel})`);
+        return;
+      }
+
+      this.cacheMeta = meta;
+
+      if (cacheData && hashData) {
+        const rawVectorStore = JSON.parse(cacheData);
+        const rawHashes = new Map(Object.entries(JSON.parse(hashData)));
+        
+        // Filter cache to only include files matching current extensions
+        const allowedExtensions = this.config.fileExtensions.map(ext => `.${ext}`);
+        
+        this.vectorStore = rawVectorStore.filter(chunk => {
+          const ext = path.extname(chunk.file);
+          return allowedExtensions.includes(ext);
+        });
+        
+        // Only keep hashes for files matching current extensions
+        for (const [file, hash] of rawHashes) {
+          const ext = path.extname(file);
+          if (allowedExtensions.includes(ext)) {
+            this.fileHashes.set(file, hash);
+          }
+        }
+        
+        const filtered = rawVectorStore.length - this.vectorStore.length;
+        if (filtered > 0) {
+          console.error(`[Cache] Filtered ${filtered} outdated cache entries`);
+        }
+        console.error(`[Cache] Loaded ${this.vectorStore.length} cached embeddings`);
+      }
+    } catch (error) {
+      console.error("[Cache] Failed to load cache:", error.message);
+    }
+  }
+
+  async save() {
+    if (!this.config.enableCache) return;
+    
+    this.isSaving = true;
+    
+    try {
+      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
+      const cacheFile = path.join(this.config.cacheDirectory, "embeddings.json");
+      const hashFile = path.join(this.config.cacheDirectory, "file-hashes.json");
+      const metaFile = path.join(this.config.cacheDirectory, CACHE_META_FILE);
+      this.cacheMeta = {
+        version: CACHE_META_VERSION,
+        embeddingModel: this.config.embeddingModel
+      };
+      
+      await Promise.all([
+        fs.writeFile(cacheFile, JSON.stringify(this.vectorStore, null, 2)),
+        fs.writeFile(hashFile, JSON.stringify(Object.fromEntries(this.fileHashes), null, 2)),
+        fs.writeFile(metaFile, JSON.stringify(this.cacheMeta, null, 2))
+      ]);
+    } catch (error) {
+      console.error("[Cache] Failed to save cache:", error.message);
+    } finally {
+      this.isSaving = false;
+    }
+  }
+
+  getVectorStore() {
+    return this.vectorStore;
+  }
+
+  setVectorStore(store) {
+    this.vectorStore = store;
+  }
+
+  getFileHash(file) {
+    return this.fileHashes.get(file);
+  }
+
+  setFileHash(file, hash) {
+    this.fileHashes.set(file, hash);
+  }
+
+  deleteFileHash(file) {
+    this.fileHashes.delete(file);
+  }
+
+  removeFileFromStore(file) {
+    this.vectorStore = this.vectorStore.filter(chunk => chunk.file !== file);
+  }
+
+
+  addToStore(chunk) {
+    this.vectorStore.push(chunk);
+  }
+
+  async clear() {
+    if (!this.config.enableCache) return;
+    
+    try {
+      await fs.rm(this.config.cacheDirectory, { recursive: true, force: true });
+      this.vectorStore = [];
+      this.fileHashes = new Map();
+      console.error(`[Cache] Cache cleared successfully: ${this.config.cacheDirectory}`);
+    } catch (error) {
+      console.error("[Cache] Failed to clear cache:", error.message);
+      throw error;
+    }
+  }
+}

package/lib/config.js

@@ -0,0 +1,257 @@
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
+import { ProjectDetector } from "./project-detector.js";
+
+const DEFAULT_CONFIG = {
+  searchDirectory: ".",
+  fileExtensions: [
+    // JavaScript/TypeScript
+    "js", "ts", "jsx", "tsx", "mjs", "cjs",
+    // Styles
+    "css", "scss", "sass", "less", "styl",
+    // Markup
+    "html", "htm", "xml", "svg",
+    // Python
+    "py", "pyw", "pyx",
+    // Java/Kotlin/Scala
+    "java", "kt", "kts", "scala",
+    // C/C++
+    "c", "cpp", "cc", "cxx", "h", "hpp", "hxx",
+    // C#
+    "cs", "csx",
+    // Go
+    "go",
+    // Rust
+    "rs",
+    // Ruby
+    "rb", "rake",
+    // PHP
+    "php", "phtml",
+    // Swift
+    "swift",
+    // Shell scripts
+    "sh", "bash", "zsh", "fish",
+    // Config & Data
+    "json", "yaml", "yml", "toml", "ini", "env",
+    // Documentation
+    "md", "mdx", "txt", "rst",
+    // Database
+    "sql",
+    // Other
+    "r", "R", "lua", "vim", "pl", "pm"
+  ],
+  excludePatterns: [
+    "**/node_modules/**",
+    "**/dist/**",
+    "**/build/**",
+    "**/.git/**",
+    "**/coverage/**",
+    "**/.next/**",
+    "**/target/**",
+    "**/vendor/**",
+    "**/.smart-coding-cache/**"
+  ],
+  chunkSize: 25, // Lines per chunk (larger = fewer embeddings = faster indexing)
+  chunkOverlap: 5, // Overlap between chunks for context continuity
+  batchSize: 100,
+  maxFileSize: 1048576, // 1MB - skip files larger than this
+  maxResults: 5,
+  enableCache: true,
+  cacheDirectory: "./.smart-coding-cache",
+  watchFiles: true,
+  verbose: false,
+  workerThreads: "auto", // "auto" = CPU cores - 1, or set a number
+  embeddingModel: "Xenova/all-MiniLM-L6-v2",
+  semanticWeight: 0.7,
+  exactMatchBoost: 1.5,
+  recencyBoost: 0.1, // Boost for recently modified files (max 0.1 added to score)
+  recencyDecayDays: 30, // After this many days, recency boost is 0
+  smartIndexing: true
+};
+
+let config = { ...DEFAULT_CONFIG };
+
+export async function loadConfig(workspaceDir = null) {
+  try {
+    // Determine the base directory for configuration
+    let baseDir;
+    let configPath;
+
+    if (workspaceDir) {
+      // Workspace mode: load config from workspace root
+      baseDir = path.resolve(workspaceDir);
+      configPath = path.join(baseDir, "config.json");
+      console.error(`[Config] Workspace mode: ${baseDir}`);
+    } else {
+      // Server mode: load config from server directory
+      const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+      baseDir = path.resolve(scriptDir, '..');
+      configPath = path.join(baseDir, "config.json");
+    }
+
+    let userConfig = {};
+    try {
+      const configData = await fs.readFile(configPath, "utf-8");
+      userConfig = JSON.parse(configData);
+    } catch (configError) {
+      if (workspaceDir) {
+        console.error(`[Config] No config.json in workspace, using defaults`);
+      } else {
+        console.error(`[Config] No config.json found: ${configError.message}`);
+      }
+    }
+
+    config = { ...DEFAULT_CONFIG, ...userConfig };
+
+    // Set workspace-specific directories
+    if (workspaceDir) {
+      config.searchDirectory = baseDir;
+      config.cacheDirectory = path.join(baseDir, ".smart-coding-cache");
+    } else {
+      config.searchDirectory = path.resolve(baseDir, config.searchDirectory);
+      config.cacheDirectory = path.resolve(baseDir, config.cacheDirectory);
+    }
+
+    // Smart project detection
+    if (config.smartIndexing !== false) {
+      const detector = new ProjectDetector(config.searchDirectory);
+      const detectedTypes = await detector.detectProjectTypes();
+
+      if (detectedTypes.length > 0) {
+        const smartPatterns = detector.getSmartIgnorePatterns();
+
+        // Merge smart patterns with user patterns (user patterns take precedence)
+        const userPatterns = userConfig.excludePatterns || [];
+        config.excludePatterns = [
+          ...smartPatterns,
+          ...userPatterns
+        ];
+
+        console.error(`[Config] Smart indexing: ${detectedTypes.join(', ')}`);
+        console.error(`[Config] Applied ${smartPatterns.length} smart ignore patterns`);
+      } else {
+        console.error("[Config] No project markers detected, using default patterns");
+      }
+    }
+
+    console.error("[Config] Loaded configuration from config.json");
+  } catch (error) {
+    console.error("[Config] Using default configuration (config.json not found or invalid)");
+    console.error(`[Config] Error: ${error.message}`);
+  }
+
+  // Apply environment variable overrides (prefix: SMART_CODING_) with validation
+  if (process.env.SMART_CODING_VERBOSE !== undefined) {
+    const value = process.env.SMART_CODING_VERBOSE;
+    if (value === 'true' || value === 'false') {
+      config.verbose = value === 'true';
+    }
+  }
+
+  if (process.env.SMART_CODING_BATCH_SIZE !== undefined) {
+    const value = parseInt(process.env.SMART_CODING_BATCH_SIZE, 10);
+    if (!isNaN(value) && value > 0 && value <= 1000) {
+      config.batchSize = value;
+    } else {
+      console.error(`[Config] Invalid SMART_CODING_BATCH_SIZE: ${process.env.SMART_CODING_BATCH_SIZE}, using default`);
+    }
+  }
+
+  if (process.env.SMART_CODING_MAX_FILE_SIZE !== undefined) {
+    const value = parseInt(process.env.SMART_CODING_MAX_FILE_SIZE, 10);
+    if (!isNaN(value) && value > 0) {
+      config.maxFileSize = value;
+    } else {
+      console.error(`[Config] Invalid SMART_CODING_MAX_FILE_SIZE: ${process.env.SMART_CODING_MAX_FILE_SIZE}, using default`);
+    }
+  }
+
+  if (process.env.SMART_CODING_CHUNK_SIZE !== undefined) {
+    const value = parseInt(process.env.SMART_CODING_CHUNK_SIZE, 10);
+    if (!isNaN(value) && value > 0 && value <= 100) {
+      config.chunkSize = value;
+    } else {
+      console.error(`[Config] Invalid SMART_CODING_CHUNK_SIZE: ${process.env.SMART_CODING_CHUNK_SIZE}, using default`);
+    }
+  }
+
+  if (process.env.SMART_CODING_MAX_RESULTS !== undefined) {
+    const value = parseInt(process.env.SMART_CODING_MAX_RESULTS, 10);
+    if (!isNaN(value) && value > 0 && value <= 100) {
+      config.maxResults = value;
+    } else {
+      console.error(`[Config] Invalid SMART_CODING_MAX_RESULTS: ${process.env.SMART_CODING_MAX_RESULTS}, using default`);
+    }
+  }
+
+  if (process.env.SMART_CODING_SMART_INDEXING !== undefined) {
+    const value = process.env.SMART_CODING_SMART_INDEXING;
+    if (value === 'true' || value === 'false') {
+      config.smartIndexing = value === 'true';
+    }
+  }
+
+  if (process.env.SMART_CODING_WATCH_FILES !== undefined) {
+    const value = process.env.SMART_CODING_WATCH_FILES;
+    if (value === 'true' || value === 'false') {
+      config.watchFiles = value === 'true';
+    }
+  }
+
+  if (process.env.SMART_CODING_SEMANTIC_WEIGHT !== undefined) {
+    const value = parseFloat(process.env.SMART_CODING_SEMANTIC_WEIGHT);
+    if (!isNaN(value) && value >= 0 && value <= 1) {
+      config.semanticWeight = value;
+    } else {
+      console.error(`[Config] Invalid SMART_CODING_SEMANTIC_WEIGHT: ${process.env.SMART_CODING_SEMANTIC_WEIGHT}, using default (must be 0-1)`);
+    }
+  }
+
+  if (process.env.SMART_CODING_EXACT_MATCH_BOOST !== undefined) {
+    const value = parseFloat(process.env.SMART_CODING_EXACT_MATCH_BOOST);
+    if (!isNaN(value) && value >= 0) {
+      config.exactMatchBoost = value;
+    } else {
+      console.error(`[Config] Invalid SMART_CODING_EXACT_MATCH_BOOST: ${process.env.SMART_CODING_EXACT_MATCH_BOOST}, using default`);
+    }
+  }
+
+  if (process.env.SMART_CODING_EMBEDDING_MODEL !== undefined) {
+    const value = process.env.SMART_CODING_EMBEDDING_MODEL.trim();
+    if (value.length > 0) {
+      config.embeddingModel = value;
+      console.error(`[Config] Using custom embedding model: ${value}`);
+    }
+  }
+
+  if (process.env.SMART_CODING_WORKER_THREADS !== undefined) {
+    const value = process.env.SMART_CODING_WORKER_THREADS.trim().toLowerCase();
+    if (value === 'auto') {
+      config.workerThreads = 'auto';
+    } else {
+      const numValue = parseInt(value, 10);
+      if (!isNaN(numValue) && numValue >= 1 && numValue <= 32) {
+        config.workerThreads = numValue;
+      } else {
+        console.error(`[Config] Invalid SMART_CODING_WORKER_THREADS: ${value}, using default (must be 'auto' or 1-32)`);
+      }
+    }
+  }
+
+  // Safety cap for auto workers
+  if (config.workerThreads === 'auto') {
+    // Cap at 4 workers max by default to prevent OOM (each model ~150MB)
+    // Users can override this by setting a specific number
+    const cpuCount = process.env.UV_THREADPOOL_SIZE || 4; // Node doesn't expose os.cpus() in some envs
+    // Actual logic happens in index-codebase.js, but we document the intent here
+  }
+
+  return config;
+}
+
+export function getConfig() {
+  return config;
+}
+
+export { DEFAULT_CONFIG };

package/lib/embedding-worker.js

@@ -0,0 +1,67 @@
+import { parentPort, workerData } from "worker_threads";
+import { pipeline } from "@xenova/transformers";
+
+let embedder = null;
+
+// Initialize the embedding model once when worker starts
+async function initializeEmbedder() {
+  if (!embedder) {
+    embedder = await pipeline("feature-extraction", workerData.embeddingModel);
+  }
+  return embedder;
+}
+
+/**
+ * Process chunks with optimized single-text embedding
+ * Note: Batch processing with transformers.js WASM backend doesn't improve speed
+ * because it loops internally. Single calls are actually faster.
+ */
+async function processChunks(chunks) {
+  const embedder = await initializeEmbedder();
+  const results = [];
+
+  for (const chunk of chunks) {
+    try {
+      const output = await embedder(chunk.text, { pooling: "mean", normalize: true });
+      results.push({
+        file: chunk.file,
+        startLine: chunk.startLine,
+        endLine: chunk.endLine,
+        content: chunk.text,
+        vector: Array.from(output.data),
+        success: true
+      });
+    } catch (error) {
+      results.push({
+        file: chunk.file,
+        startLine: chunk.startLine,
+        endLine: chunk.endLine,
+        error: error.message,
+        success: false
+      });
+    }
+  }
+
+  return results;
+}
+
+// Listen for messages from main thread
+parentPort.on("message", async (message) => {
+  if (message.type === "process") {
+    try {
+      const results = await processChunks(message.chunks);
+      parentPort.postMessage({ type: "results", results, batchId: message.batchId });
+    } catch (error) {
+      parentPort.postMessage({ type: "error", error: error.message, batchId: message.batchId });
+    }
+  } else if (message.type === "shutdown") {
+    process.exit(0);
+  }
+});
+
+// Signal that worker is ready
+initializeEmbedder().then(() => {
+  parentPort.postMessage({ type: "ready" });
+}).catch((error) => {
+  parentPort.postMessage({ type: "error", error: error.message });
+});