@esankhan3/code-search-mcp 1.0.5 → 1.1.0

package/dist/server.js

@@ -7,16 +7,25 @@ import { registerSearchTools, handleSearchTool } from "./tools/search.js";
 import { registerGraphTools, handleGraphTool } from "./tools/graph.js";
 import { registerProfileTools, handleProfileTool } from "./tools/profile.js";
 import { registerIndexTools, handleIndexTool } from "./tools/index-tools.js";
-import { registerResources, handleResource } from "./resources/resources.js";
-import { getBlobStore, readVectorIndexHealth } from "@esankhan3/anvil-knowledge-core";
-import { indexFromPath, invalidateRetriever } from "@esankhan3/anvil-knowledge-core";
-import { loadServerConfig } from "./core/env-config.js";
-import { toKnowledgeConfig } from "./core/config.js";
-import { startHttpTransport } from "./transports/http-transport.js";
+import { registerResources, registerResourceTemplates, handleResource } from "./resources/resources.js";
+import {
+  getBlobStore,
+  readVectorIndexHealth,
+  indexFromPath,
+  invalidateRetriever,
+  KnowledgeIndexer
+} from "@esankhan3/anvil-knowledge-core";
+import { loadServerConfig, resolvedKnowledgeConfig } from "./core/env-config.js";
+import { PACKAGE_VERSION } from "./core/version.js";
+import { startHttpTransport, HttpError } from "./transports/http-transport.js";
+import { localRateHit, localRateLimiter } from "./middleware/rate-limit.js";
+import { ProjectRegistry, projectAccessAllowed, projectQuotaKey } from "./projects/registry.js";
 import { metrics } from "./observability/metrics.js";
+import { getComponentLogger } from "./observability/logger.js";
+const log = getComponentLogger("code-search-mcp");
 function createMcpServerInstance(ctx) {
   const server = new Server(
-    { name: "code-search-mcp", version: "1.0.0" },
+    { name: "code-search-mcp", version: PACKAGE_VERSION },
     { capabilities: { tools: {}, resources: {} } }
   );
   const allTools = [
@@ -45,13 +54,17 @@ function createMcpServerInstance(ctx) {
   server.setRequestHandler("resources/list", async () => ({
     resources: allResources
   }));
+  server.setRequestHandler("resources/templates/list", async () => ({
+    resourceTemplates: registerResourceTemplates()
+  }));
   server.setRequestHandler("resources/read", async (request) => {
     return handleResource(request.params.uri, ctx);
   });
   return server;
 }
-async function startServer(projectName, directoryPath) {
+async function startServer(projectName, directoryPath, opts = {}) {
   const config = loadServerConfig();
+  const reindexIntervalMs = config.__unified.indexing.reindexIntervalMs;
   const ctx = {
     projectName,
     directoryPath,
@@ -70,153 +83,212 @@ async function startServer(projectName, directoryPath) {
     }
   };
   const llmInfo = config.llmMode === "none" ? "disabled (profiling + service mesh skipped)" : config.llmMode === "api" ? `api \u2192 ${config.llmProvider}/${config.llmModel}${config.llmApiKey ? "" : " (WARNING: no API key!)"}` : `cli \u2192 ${config.claudeBin}`;
-  console.error(`[code-search-mcp] LLM: ${llmInfo}`);
-  await autoIndex(ctx, { readOnly: cluster.isWorker });
+  log.info(`LLM: ${llmInfo}`);
   if (config.transport === "stdio") {
+    await autoIndex(ctx, { force: opts.force });
     const server = createMcpServerInstance(ctx);
     const transport = new StdioServerTransport();
     await server.connect(transport);
-    console.error(`[code-search-mcp] Server running for "${projectName}" (stdio)`);
-  } else {
-    const workerCount = parseWorkerCount();
-    if (workerCount > 1 && cluster.isPrimary) {
-      armScheduledReindex(ctx, () => broadcastInvalidate(ctx));
-      runClusterPrimary(ctx, workerCount);
-      return;
-    }
-    if (config.authMode === "none" && config.host !== "127.0.0.1" && config.host !== "localhost") {
-      console.error(`[code-search-mcp] WARNING: Auth is disabled but server binds to ${config.host}. Any machine on the network can access your code search API.`);
-      console.error(`[code-search-mcp] Set CODE_SEARCH_AUTH_MODE=api-key or CODE_SEARCH_HOST=127.0.0.1 for security.`);
-    }
-    const isReadOnlyWorker = cluster.isWorker;
-    if (isReadOnlyWorker) registerWorkerIpc(ctx);
-    await startHttpTransport({
-      config,
-      createMcpServer: async () => ({
-        server: createMcpServerInstance(ctx)
-      }),
-      onReady: (url) => {
-        console.error(`[code-search-mcp] Server running for "${projectName}" at ${url}/mcp`);
-        console.error(`[code-search-mcp] Health:  GET  ${url}/health`);
-        console.error(`[code-search-mcp] Status:  GET  ${url}/status`);
-        console.error(`[code-search-mcp] Index:   POST ${url}/index`);
-        console.error(`[code-search-mcp] Auth: ${config.authMode}`);
-      },
-      getHealth: () => ({
-        project: ctx.projectName,
-        indexReady: ctx.indexReady,
-        indexing: ctx.indexing.status,
-        uptime: Math.floor((Date.now() - ctx.startedAt) / 1e3),
-        transport: config.transport,
-        authMode: config.authMode
-      }),
-      getStatus: () => ({
-        project: ctx.projectName,
-        directoryPath: ctx.directoryPath,
-        indexReady: ctx.indexReady,
-        uptime: Math.floor((Date.now() - ctx.startedAt) / 1e3),
-        indexing: {
-          ...ctx.indexing,
-          elapsedMs: ctx.indexing.startedAt ? Date.now() - ctx.indexing.startedAt : null
-        }
-      }),
-      // From the writer's blob record, not process state: in cluster mode the
-      // workers serving HTTP are not the writer.
-      collectMetrics: async () => {
-        const blobs = getBlobStore(ctx.projectName, toKnowledgeConfig(loadServerConfig().__unified));
-        const health = await readVectorIndexHealth(blobs);
-        if (!health) return;
-        const labels = { project: ctx.projectName };
-        metrics.vectorIndexUnindexedRows.set(health.numUnindexedRows ?? 0, labels);
-        if (health.trainedRows) metrics.vectorIndexGrowthRatio.set(health.rowsAddedSinceTrain / health.trainedRows, labels);
-      },
-      onIndex: (body) => isReadOnlyWorker ? forwardIndexToPrimary(body) : runAdminIndex(ctx, body)
-    });
+    log.info(`Server running for "${projectName}" (stdio)`);
+    armScheduledReindex(ctx, reindexIntervalMs);
+    return;
+  }
+  const workerCount = parseWorkerCount();
+  if (workerCount > 1 && cluster.isPrimary) {
+    runClusterPrimary(ctx, workerCount);
+    armScheduledReindex(ctx, reindexIntervalMs, () => broadcastInvalidate(ctx.projectName));
+    if (await autoIndex(ctx, { force: opts.force })) broadcastInvalidate(ctx.projectName);
+    return;
+  }
+  if (config.authMode === "none" && config.host !== "127.0.0.1" && config.host !== "localhost") {
+    log.warn(`WARNING: Auth is disabled but server binds to ${config.host}. Any machine on the network can access your code search API.`);
+    log.warn(`Set CODE_SEARCH_AUTH_MODE=api-key or CODE_SEARCH_HOST=127.0.0.1 for security.`);
   }
-  if (!cluster.isWorker) armScheduledReindex(ctx);
+  const isReadOnlyWorker = cluster.isWorker;
+  if (isReadOnlyWorker) registerWorkerIpc(ctx);
+  const rateLimiter = isReadOnlyWorker ? primaryRateLimiter : localRateLimiter;
+  await startHttpTransport({
+    config,
+    createMcpServer: async () => ({
+      server: createMcpServerInstance(ctx)
+    }),
+    onReady: (url) => {
+      log.info(`Server running for "${projectName}" at ${url}/mcp`);
+      log.info(`Health:  GET  ${url}/health`);
+      log.info(`Status:  GET  ${url}/status`);
+      log.info(`Index:   POST ${url}/index`);
+      log.info(`Auth: ${config.authMode}`);
+    },
+    getHealth: () => ({
+      project: ctx.projectName,
+      indexReady: ctx.indexReady,
+      indexing: ctx.indexing.status,
+      uptime: Math.floor((Date.now() - ctx.startedAt) / 1e3),
+      transport: config.transport,
+      authMode: config.authMode
+    }),
+    getStatus: () => ({
+      project: ctx.projectName,
+      directoryPath: ctx.directoryPath,
+      indexReady: ctx.indexReady,
+      uptime: Math.floor((Date.now() - ctx.startedAt) / 1e3),
+      indexing: {
+        ...ctx.indexing,
+        elapsedMs: ctx.indexing.startedAt ? Date.now() - ctx.indexing.startedAt : null
+      }
+    }),
+    collectMetrics: () => collectIndexMetrics(ctx.projectName, resolvedKnowledgeConfig(), ctx.indexing.lastSuccess),
+    onIndex: (body) => isReadOnlyWorker ? forwardIndexToPrimary(body) : runAdminIndex(ctx, body),
+    authorize: createProjectAuthorizer(ctx, config.dataDir, rateLimiter),
+    rateLimiter
+  });
+  if (!isReadOnlyWorker) armScheduledReindex(ctx, reindexIntervalMs);
+  await autoIndex(ctx, { readOnly: isReadOnlyWorker, force: opts.force });
 }
-function armScheduledReindex(ctx, onSuccess) {
-  const reindexIntervalMs = parseReindexInterval();
-  if (reindexIntervalMs > 0 && ctx.directoryPath) {
-    console.error(`[code-search-mcp] Auto-reindex every ${Math.round(reindexIntervalMs / 6e4)}m`);
+async function collectIndexMetrics(project, knowledgeConfig, lastSuccess = null) {
+  const owned = [metrics.indexAge, metrics.indexChunks, metrics.vectorIndexUnindexedRows, metrics.vectorIndexGrowthRatio];
+  let stats;
+  let health;
+  try {
+    const blobs = getBlobStore(project, knowledgeConfig);
+    [stats, health] = await Promise.all([
+      blobs.getJson("index_stats.json"),
+      readVectorIndexHealth(blobs)
+    ]);
+  } catch (err) {
+    for (const g of owned) g.clear();
+    throw err;
+  }
+  for (const g of owned) g.clear();
+  const labels = { project };
+  const lastMs = [stats?.lastIndexed, lastSuccess].map((t) => t ? Date.parse(t) : NaN).filter((t) => Number.isFinite(t));
+  if (lastMs.length) metrics.indexAge.set(Math.max(0, Math.round((Date.now() - Math.max(...lastMs)) / 1e3)), labels);
+  for (const r of stats?.repos ?? []) metrics.indexChunks.set(r.chunkCount, { project, repo: r.name });
+  if (health) {
+    metrics.vectorIndexUnindexedRows.set(health.numUnindexedRows ?? 0, labels);
+    if (health.trainedRows) metrics.vectorIndexGrowthRatio.set(health.rowsAddedSinceTrain / health.trainedRows, labels);
+  }
+}
+function armScheduledReindex(ctx, intervalMs, onSuccess) {
+  if (intervalMs > 0 && ctx.directoryPath) {
+    log.info(`Auto-reindex every ${Math.round(intervalMs / 6e4)}m`);
     setInterval(async () => {
       if (!ctx.directoryPath || ctx.indexing.status === "indexing") return;
       try {
         await trackedIndex(ctx, ctx.projectName, ctx.directoryPath, { label: "auto-reindex" });
         onSuccess?.();
       } catch (err) {
-        console.error(`[auto-reindex] Failed:`, err);
+        getComponentLogger("auto-reindex").error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
       }
-    }, reindexIntervalMs).unref();
+    }, intervalMs).unref();
   }
 }
+function createProjectAuthorizer(ctx, dataDir, rateLimiter) {
+  const projects = new ProjectRegistry(dataDir);
+  projects.reload();
+  return async (identity) => {
+    const entry = projects.get(ctx.projectName);
+    if (!entry) return { ok: true };
+    if (!projectAccessAllowed(entry, identity.scopes)) {
+      return { ok: false, status: 403, error: `Forbidden: project "${entry.name}" requires one of the scopes: ${entry.scopes.join(", ")}` };
+    }
+    const d = await rateLimiter(projectQuotaKey(entry, identity.subject), entry.quotas.maxQueriesPerMinute);
+    if (!d.ok) {
+      return {
+        ok: false,
+        status: 429,
+        error: `Quota exceeded for project "${entry.name}": ${entry.quotas.maxQueriesPerMinute} requests/min`,
+        retryAfterSec: d.retryAfterSec
+      };
+    }
+    return { ok: true };
+  };
+}
 async function runAdminIndex(ctx, body) {
+  const project = body.project ?? ctx.projectName;
+  if (project !== ctx.projectName) {
+    throw new HttpError(400, `This server serves project "${ctx.projectName}"; POST /index only (re)indexes it. Index "${project}" with the code-search CLI or another server.`);
+  }
   const dirPath = resolve(body.path);
   if (!existsSync(dirPath)) {
-    throw new Error(`Path does not exist: ${dirPath}`);
+    throw new HttpError(400, `Path does not exist: ${dirPath}`);
   }
   if (ctx.indexing.status === "indexing") {
-    throw new Error(`Indexing already in progress (phase: ${ctx.indexing.phase}). Wait for it to complete or check GET /status.`);
+    throw new HttpError(409, `Indexing already in progress (phase: ${ctx.indexing.phase}). Wait for it to complete or check GET /status.`);
   }
-  const project = body.project || dirPath.split("/").filter(Boolean).pop() || "project";
   const stats = await trackedIndex(ctx, project, dirPath, {
     force: body.force,
     label: "admin-index"
   });
-  ctx.projectName = project;
   ctx.directoryPath = dirPath;
+  const rollup = await new KnowledgeIndexer().getStats(project, resolvedKnowledgeConfig());
   return {
     status: "ok",
     project,
     path: dirPath,
     chunks: stats.totalChunks,
     repos: stats.repos.length,
-    crossRepoEdges: stats.crossRepoEdges,
+    crossRepoEdges: rollup.crossRepoEdges,
     durationMs: stats.indexDurationMs
   };
 }
-function broadcastInvalidate(ctx) {
-  const msg = { type: "csm:invalidate", project: ctx.projectName, directoryPath: ctx.directoryPath };
-  for (const w of Object.values(cluster.workers ?? {})) {
-    try {
-      w?.send(msg);
-    } catch {
-    }
+function sendToWorker(w, msg) {
+  try {
+    w?.send(msg);
+  } catch {
   }
 }
+function broadcastInvalidate(project) {
+  const msg = { type: "csm:invalidate", project };
+  for (const w of Object.values(cluster.workers ?? {})) sendToWorker(w, msg);
+}
 function parseWorkerCount() {
   const n = parseInt(process.env.CODE_SEARCH_WORKERS ?? "", 10);
   return Number.isFinite(n) && n > 1 ? Math.min(n, 16) : 1;
 }
 function runClusterPrimary(ctx, workerCount) {
-  console.error(`[code-search-mcp] Cluster primary ${process.pid}: forking ${workerCount} read-only workers (stateless MCP)`);
+  log.info(`Cluster primary ${process.pid}: forking ${workerCount} read-only workers (stateless MCP)`);
   let shuttingDown = false;
-  const send = (w, msg) => {
-    try {
-      w?.send(msg);
-    } catch {
-    }
-  };
+  const stateMsg = () => ({
+    type: "csm:state",
+    indexReady: ctx.indexReady,
+    directoryPath: ctx.directoryPath,
+    indexing: ctx.indexing
+  });
+  let lastState = "";
+  setInterval(() => {
+    const msg = stateMsg();
+    const json = JSON.stringify(msg);
+    if (json === lastState) return;
+    lastState = json;
+    for (const w of Object.values(cluster.workers ?? {})) sendToWorker(w, msg);
+  }, 250).unref();
+  cluster.on("online", (worker) => sendToWorker(worker, stateMsg()));
   cluster.on("message", (worker, msg) => {
+    if (msg?.type === "csm:rate") {
+      sendToWorker(worker, { type: "csm:rate-result", id: msg.id, ...localRateHit(msg.key, msg.limit) });
+      return;
+    }
     if (msg?.type !== "csm:index") return;
     void (async () => {
       try {
         const result = await runAdminIndex(ctx, msg.body);
-        broadcastInvalidate(ctx);
-        send(worker, { type: "csm:index-result", id: msg.id, ok: true, result });
+        broadcastInvalidate(ctx.projectName);
+        sendToWorker(worker, { type: "csm:index-result", id: msg.id, ok: true, result });
       } catch (err) {
-        send(worker, {
+        sendToWorker(worker, {
           type: "csm:index-result",
           id: msg.id,
           ok: false,
-          error: err instanceof Error ? err.message : String(err)
+          error: err instanceof Error ? err.message : String(err),
+          status: err instanceof HttpError ? err.status : void 0
         });
       }
     })();
   });
   cluster.on("exit", (worker, code, signal) => {
     if (shuttingDown) return;
-    console.error(`[code-search-mcp] Worker ${worker.process.pid} exited (${signal ?? code}) \u2014 respawning`);
+    log.warn(`Worker ${worker.process.pid} exited (${signal ?? code}) \u2014 respawning`);
     cluster.fork();
   });
   const shutdown = (sig) => {
@@ -235,6 +307,7 @@ function runClusterPrimary(ctx, workerCount) {
 }
 let ipcSeq = 0;
 const pendingIndexRequests = /* @__PURE__ */ new Map();
+const pendingRateRequests = /* @__PURE__ */ new Map();
 function forwardIndexToPrimary(body) {
   return new Promise((resolvePromise, rejectPromise) => {
     if (!process.send) {
@@ -247,6 +320,23 @@ function forwardIndexToPrimary(body) {
     process.send(msg);
   });
 }
+const primaryRateLimiter = (key, limit) => new Promise((resolveP) => {
+  if (!process.send || limit <= 0) {
+    resolveP(localRateHit(key, limit));
+    return;
+  }
+  const id = ++ipcSeq;
+  const timer = setTimeout(() => {
+    pendingRateRequests.delete(id);
+    resolveP(localRateHit(key, limit));
+  }, 2e3);
+  pendingRateRequests.set(id, (d) => {
+    clearTimeout(timer);
+    resolveP(d);
+  });
+  const msg = { type: "csm:rate", id, key, limit };
+  process.send(msg);
+});
 function registerWorkerIpc(ctx) {
   process.on("message", (msg) => {
     if (msg?.type === "csm:index-result") {
@@ -254,29 +344,23 @@ function registerWorkerIpc(ctx) {
       if (!pending) return;
       pendingIndexRequests.delete(msg.id);
       if (msg.ok) pending.resolve(msg.result ?? {});
-      else pending.reject(new Error(msg.error ?? "Index failed"));
+      else pending.reject(new HttpError(msg.status ?? 500, msg.error ?? "Index failed"));
+    } else if (msg?.type === "csm:rate-result") {
+      const pending = pendingRateRequests.get(msg.id);
+      pendingRateRequests.delete(msg.id);
+      pending?.({ ok: msg.ok, retryAfterSec: msg.retryAfterSec });
     } else if (msg?.type === "csm:invalidate") {
       void invalidateRetriever(msg.project).catch(() => {
       });
-      ctx.projectName = msg.project;
-      if (msg.directoryPath) ctx.directoryPath = msg.directoryPath;
-      ctx.indexReady = true;
+      if (msg.project === ctx.projectName) ctx.indexReady = true;
+    } else if (msg?.type === "csm:state") {
+      ctx.indexing = msg.indexing;
+      ctx.directoryPath = msg.directoryPath;
+      ctx.indexReady = ctx.indexReady || msg.indexReady;
     }
   });
   process.on("disconnect", () => process.exit(0));
 }
-function parseReindexInterval() {
-  const raw = process.env.CODE_SEARCH_REINDEX_INTERVAL?.trim();
-  if (!raw || raw === "0" || raw === "none") return 0;
-  const match = raw.match(/^(\d+)(m|h)$/);
-  if (!match) {
-    console.error(`[code-search-mcp] Invalid REINDEX_INTERVAL "${raw}" \u2014 use "30m", "1h", etc. Disabling.`);
-    return 0;
-  }
-  const value = parseInt(match[1], 10);
-  const unit = match[2];
-  return unit === "h" ? value * 60 * 6e4 : value * 6e4;
-}
 const MAX_HISTORY = 50;
 function pushHistory(ctx, type, message) {
   ctx.indexing.history.push({ timestamp: (/* @__PURE__ */ new Date()).toISOString(), type, message });
@@ -286,6 +370,7 @@ function pushHistory(ctx, type, message) {
 }
 async function trackedIndex(ctx, project, dirPath, opts) {
   const label = opts?.label ?? "index";
+  const progressLog = getComponentLogger(label);
   ctx.indexing.status = "indexing";
   ctx.indexing.phase = "starting";
   ctx.indexing.message = `Starting ${label}...`;
@@ -293,15 +378,13 @@ async function trackedIndex(ctx, project, dirPath, opts) {
   ctx.indexing.startedAt = Date.now();
   ctx.indexing.error = null;
   pushHistory(ctx, "start", `${label}: started for "${project}" at ${dirPath}`);
-  const unified = loadServerConfig().__unified;
-  const knowledgeConfig = toKnowledgeConfig(unified);
   try {
     const stats = await indexFromPath(project, dirPath, {
       force: opts?.force,
-      config: knowledgeConfig,
+      config: resolvedKnowledgeConfig(),
       onProgress: (m) => {
         ctx.indexing.message = m;
-        console.error(`[${label}] ${m}`);
+        progressLog.info(m);
       },
       onDetailedProgress: (p) => {
         ctx.indexing.phase = p.phase;
@@ -333,31 +416,35 @@ async function trackedIndex(ctx, project, dirPath, opts) {
 }
 async function autoIndex(ctx, opts) {
   try {
-    const blobs = getBlobStore(ctx.projectName, toKnowledgeConfig(loadServerConfig().__unified));
+    const rebuild = !!opts?.force && !opts.readOnly && !!ctx.directoryPath;
+    const blobs = getBlobStore(ctx.projectName, resolvedKnowledgeConfig());
     const stats = await blobs.getJson("index_stats.json");
     const statsReady = !!stats && stats.embeddingProvider !== "pending";
     const hasLanceDB = await blobs.exists("lancedb");
     const hasGraph = await blobs.exists("system_graph.sqlite") || await blobs.exists("system_graph_v2.json");
-    if (statsReady || hasLanceDB && hasGraph) {
+    if ((statsReady || hasLanceDB && hasGraph) && !rebuild) {
       ctx.indexReady = true;
-      console.error(`[code-search-mcp] Index loaded for "${ctx.projectName}"`);
-      return;
+      log.info(`Index loaded for "${ctx.projectName}"`);
+      return false;
     }
     if (opts?.readOnly) {
-      console.error(`[code-search-mcp] No index yet \u2014 read-only worker waiting for the writer`);
-      return;
+      log.info(`No index yet \u2014 read-only worker waiting for the writer`);
+      return false;
     }
     if (!ctx.directoryPath) {
-      console.error(`[code-search-mcp] No index found and no directory path \u2014 tools will return empty results`);
-      return;
+      log.warn(`No index found and no directory path \u2014 tools will return empty results`);
+      return false;
     }
-    console.error(`[code-search-mcp] No index found \u2014 building from ${ctx.directoryPath}...`);
-    await trackedIndex(ctx, ctx.projectName, ctx.directoryPath, { label: "auto-index" });
-    console.error(`[code-search-mcp] Index ready.`);
+    log.info(`${rebuild ? "Forced re-index" : "No index found"} \u2014 building from ${ctx.directoryPath}...`);
+    await trackedIndex(ctx, ctx.projectName, ctx.directoryPath, { label: "auto-index", force: rebuild });
+    log.info(`Index ready.`);
+    return true;
   } catch (err) {
-    console.error(`[code-search-mcp] Auto-index failed:`, err);
+    log.error(`Auto-index failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
+    return false;
   }
 }
 export {
+  collectIndexMetrics,
   startServer
 };