@maxgfr/codeindex 2.7.0

package/src/mcp.ts

@@ -0,0 +1,512 @@
+// MCP (Model Context Protocol) server over stdio — hand-rolled JSON-RPC 2.0 so
+// the engine stays zero-dependency. Newline-delimited JSON messages, protocol
+// 2024-11-05 (compatible with later revisions' initialize handshake). Exposes
+// the engine's read-only indexing capabilities as MCP tools; every tool takes a
+// `repo` path and returns JSON text content.
+//
+// Register in an MCP client as:  node scripts/engine.mjs mcp
+import { createInterface } from "node:readline";
+import { ENGINE_VERSION } from "./types.js";
+import { ensureGrammars, allGrammarKeys } from "./ast/loader.js";
+import { buildIndexArtifacts } from "./pipeline.js";
+import { renderGraphJson } from "./render/graph-json.js";
+import { scanRepo } from "./scan.js";
+import { buildCallerIndex } from "./callers.js";
+import { detectWorkspaces } from "./workspaces.js";
+import { gitChurn } from "./git.js";
+import { grepRepo } from "./grep.js";
+import { changeCoupling, rankHotspots } from "./coupling.js";
+import { renderRepoMap } from "./repomap.js";
+import { findDeadCode } from "./deadcode.js";
+import { symbolComplexity, riskHotspots } from "./complexity.js";
+import { renderMermaid } from "./viz.js";
+import { symbolsOverview, findSymbol, findReferences } from "./query.js";
+import { replaceSymbolBody, insertAfterSymbol, insertBeforeSymbol } from "./edit.js";
+import { writeMemory, readMemory, deleteMemory, listMemories } from "./memory.js";
+import { searchIndex } from "./bm25.js";
+import { checkRules, parseRules } from "./rules.js";
+
+interface RpcRequest {
+  jsonrpc: "2.0";
+  id?: number | string | null;
+  method: string;
+  params?: Record<string, unknown>;
+}
+
+const repoProp = { repo: { type: "string", description: "Absolute path to the repository root" } };
+const scopeProps = {
+  scope: { type: "string", description: "Restrict to one directory (repo-relative)" },
+  include: { type: "array", items: { type: "string" }, description: "Include globs" },
+  exclude: { type: "array", items: { type: "string" }, description: "Exclude globs" },
+};
+
+const TOOLS = [
+  {
+    name: "scan_summary",
+    description:
+      "Deterministically scan a repository: file count, per-language file histogram, HEAD commit, and whether the walk was capped. Fast first look at any codebase.",
+    inputSchema: { type: "object", properties: { ...repoProp, ...scopeProps }, required: ["repo"] },
+  },
+  {
+    name: "graph",
+    description:
+      "Build the full typed cross-file link-graph (import/call/use/doc-link/mention edges, module grouping, PageRank centrality, Louvain communities, tests-map). Returns graph.json. Large on big repos — prefer scan_summary/symbols/callers for targeted questions.",
+    inputSchema: { type: "object", properties: { ...repoProp, ...scopeProps }, required: ["repo"] },
+  },
+  {
+    name: "symbols",
+    description:
+      "Where is a symbol defined and which files reference it? Returns the definition sites (file, line, kind, exported) and referencing files. Omit `name` for the full symbol index.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, name: { type: "string", description: "Symbol name to look up" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "callers",
+    description:
+      "Who calls a function? Per-symbol caller index: each defined symbol with the exact (file, line) call sites that bind to it. Omit `name` for the full index.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, name: { type: "string", description: "Symbol name to look up" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "workspaces",
+    description:
+      "Detect monorepo packages (npm/pnpm/yarn/lerna/nx/cargo/go.work/maven) with the workspace dependency graph, one cycle if present, and a topological build order.",
+    inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
+  },
+  {
+    name: "churn",
+    description: "Per-file git commit counts (whole history, or since a ref) — the churn half of hotspot analysis.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, since: { type: "string", description: "Only count commits after this ref" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "symbols_overview",
+    description:
+      "All symbols declared in ONE file (name, kind, line span, exported, parent), in declaration order — the fastest way to understand a file without reading it.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, file: { type: "string", description: "Repo-relative file path" } },
+      required: ["repo", "file"],
+    },
+  },
+  {
+    name: "find_symbol",
+    description:
+      "Find symbol declarations by name or name path ('Class/method' matches a method inside Class). Options: substring matching, includeBody to return the declaration's source. Exact-name matches rank first.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...repoProp,
+        namePath: { type: "string", description: "Symbol name or Parent/child path" },
+        substring: { type: "boolean" },
+        includeBody: { type: "boolean" },
+      },
+      required: ["repo", "namePath"],
+    },
+  },
+  {
+    name: "find_references",
+    description:
+      "Who references a symbol? Three labeled tiers: defs (declarations), callSites (line-precise, import-corroborated call bindings), referencingFiles (file-level identifier/doc mentions — may include homonyms). Confidence decreases across tiers; the labels let you decide what to trust.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, name: { type: "string", description: "Symbol name" } },
+      required: ["repo", "name"],
+    },
+  },
+  {
+    name: "repo_map",
+    description:
+      "Token-budgeted map of the repository: the highest-PageRank files with their key exported signatures, deterministically rendered to fit `budgetTokens` (default 1024). The densest single read to understand an unfamiliar codebase.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, budgetTokens: { type: "number", description: "Approximate token budget (default 1024)" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "hotspots",
+    description:
+      "Where does work concentrate? Files ranked by git churn × size (commits × log2 lines). High-scoring files are where changes and defects cluster.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, since: { type: "string", description: "Only count commits after this ref" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "coupling",
+    description:
+      "Change coupling: pairs of files that repeatedly change in the same commits — hidden dependencies no import shows. strength 1.0 = every change to one touched the other.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, since: { type: "string", description: "Only mine commits after this ref" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "replace_symbol_body",
+    description:
+      "WRITE: replace a symbol's whole declaration with `body` (verbatim, supply full indentation). The symbol is resolved by name path ('Class/method'); ambiguity errors list the candidates — qualify with `file`. Line spans come from the AST index.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...repoProp,
+        namePath: { type: "string" },
+        body: { type: "string" },
+        file: { type: "string", description: "Disambiguate: repo-relative file containing the symbol" },
+      },
+      required: ["repo", "namePath", "body"],
+    },
+  },
+  {
+    name: "insert_after_symbol",
+    description:
+      "WRITE: insert `body` after a symbol's declaration (blank-line separation preserved for definition-like kinds). Resolved like replace_symbol_body.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, namePath: { type: "string" }, body: { type: "string" }, file: { type: "string" } },
+      required: ["repo", "namePath", "body"],
+    },
+  },
+  {
+    name: "insert_before_symbol",
+    description:
+      "WRITE: insert `body` before a symbol's declaration (blank-line separation preserved). Resolved like replace_symbol_body.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, namePath: { type: "string" }, body: { type: "string" }, file: { type: "string" } },
+      required: ["repo", "namePath", "body"],
+    },
+  },
+  {
+    name: "write_memory",
+    description:
+      "Persist a named markdown note under <repo>/.codeindex/memories/ (names may use topic/name form). Write small, focused notes: project map, build commands, conventions.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, name: { type: "string" }, content: { type: "string" } },
+      required: ["repo", "name", "content"],
+    },
+  },
+  {
+    name: "read_memory",
+    description: "Read one persisted memory by name.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, name: { type: "string" } },
+      required: ["repo", "name"],
+    },
+  },
+  {
+    name: "list_memories",
+    description: "List persisted memory names — load this first, then read individual memories on relevance.",
+    inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
+  },
+  {
+    name: "delete_memory",
+    description: "Delete one persisted memory by name.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, name: { type: "string" } },
+      required: ["repo", "name"],
+    },
+  },
+  {
+    name: "dead_code",
+    description:
+      "Dead-code candidates in two labeled tiers: 'unreferenced' (no call site binds AND nothing references the name) and 'uncalled' (referenced somewhere — re-export, type position — but never called). Exported symbols only; test files and entrypoint-looking files excluded as roots.",
+    inputSchema: { type: "object", properties: { ...repoProp, ...scopeProps }, required: ["repo"] },
+  },
+  {
+    name: "complexity",
+    description:
+      "Cyclomatic-complexity estimates (branch-token counting over AST line spans), most-complex first. Pass `file` for one file's symbols, omit for the repo-wide top. Combine with hotspots: the `risk` field of this tool's sibling ranks complexity × churn.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, file: { type: "string" }, risk: { type: "boolean", description: "Return complexity × git-churn risk ranking instead" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "mermaid",
+    description:
+      "Mermaid diagram of the module graph (renders inline in Claude/GitHub — no graph database). Optionally scoped to one module's neighborhood.",
+    inputSchema: {
+      type: "object",
+      properties: { ...repoProp, module: { type: "string", description: "Module slug to focus on" } },
+      required: ["repo"],
+    },
+  },
+  {
+    name: "grep",
+    description:
+      "Search file contents (ripgrep when available, deterministic JS fallback otherwise). Returns sorted (file, line, text) hits.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...repoProp,
+        pattern: { type: "string", description: "Regular expression to search for" },
+        globs: { type: "array", items: { type: "string" }, description: "Restrict to matching paths" },
+        ignoreCase: { type: "boolean" },
+        maxHits: { type: "number" },
+      },
+      required: ["repo", "pattern"],
+    },
+  },
+  {
+    name: "search",
+    description:
+      'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols.',
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...repoProp,
+        ...scopeProps,
+        query: { type: "string", description: "Natural-language or identifier query" },
+        limit: { type: "number", description: "Max results (default 20)" },
+      },
+      required: ["repo", "query"],
+    },
+  },
+  {
+    name: "check_rules",
+    description:
+      'Validate dependency-cruiser-style architecture rules against the link-graph. Rules (inline JSON array): forbidden edges {name, from, to, kind?, severity?, comment?} with glob paths, plus builtins {name, builtin: "cycles"|"orphans"} (module-level import cycles; edge-less code files). Returns deterministic violations with severity error|warn — a CI gate.',
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...repoProp,
+        ...scopeProps,
+        rules: { type: "array", description: "Rules array (inline JSON — see description)" },
+      },
+      required: ["repo", "rules"],
+    },
+  },
+] as const;
+
+function str(v: unknown): string | undefined {
+  return typeof v === "string" && v ? v : undefined;
+}
+function strArray(v: unknown): string[] | undefined {
+  return Array.isArray(v) && v.every((x) => typeof x === "string") && v.length ? (v as string[]) : undefined;
+}
+
+function callTool(name: string, args: Record<string, unknown>): string {
+  const repo = str(args.repo);
+  if (!repo) throw new Error("`repo` is required (absolute path to the repository root)");
+  const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) };
+
+  if (name === "scan_summary") {
+    const scan = scanRepo(repo, scanOpts);
+    return JSON.stringify(
+      { engineVersion: ENGINE_VERSION, commit: scan.commit, fileCount: scan.files.length, languages: scan.languages, capped: scan.capped },
+      null,
+      2,
+    );
+  }
+  if (name === "graph") {
+    return renderGraphJson(buildIndexArtifacts(repo, scanOpts).graph);
+  }
+  if (name === "symbols") {
+    const { symbols } = buildIndexArtifacts(repo, scanOpts);
+    const lookup = str(args.name);
+    if (lookup) {
+      return JSON.stringify({ name: lookup, defs: symbols.defs[lookup] ?? [], refs: symbols.refs[lookup] ?? [] }, null, 2);
+    }
+    return JSON.stringify(symbols, null, 2);
+  }
+  if (name === "callers") {
+    const index = buildCallerIndex(scanRepo(repo, scanOpts));
+    const lookup = str(args.name);
+    if (lookup) {
+      const entry = index.get(lookup);
+      return JSON.stringify(entry ?? { error: `no tracked callers for "${lookup}"` }, null, 2);
+    }
+    const obj: Record<string, unknown> = {};
+    for (const [k, v] of index) obj[k] = v;
+    return JSON.stringify(obj, null, 2);
+  }
+  if (name === "workspaces") {
+    const info = detectWorkspaces(repo);
+    return JSON.stringify({ packages: info.packages, cycle: info.cycle ?? null, topoOrder: info.topoOrder }, null, 2);
+  }
+  if (name === "churn") {
+    const { churn, ok } = gitChurn(repo, { since: str(args.since) });
+    const sorted: Record<string, number> = {};
+    for (const k of [...churn.keys()].sort()) sorted[k] = churn.get(k)!;
+    return JSON.stringify({ ok, churn: sorted }, null, 2);
+  }
+  if (name === "symbols_overview") {
+    const file = str(args.file);
+    if (!file) throw new Error("`file` is required");
+    return JSON.stringify(symbolsOverview(scanRepo(repo, scanOpts), file), null, 2);
+  }
+  if (name === "find_symbol") {
+    const namePath = str(args.namePath);
+    if (!namePath) throw new Error("`namePath` is required");
+    const matches = findSymbol(scanRepo(repo, scanOpts), namePath, {
+      substring: args.substring === true,
+      includeBody: args.includeBody === true,
+    });
+    return JSON.stringify(matches, null, 2);
+  }
+  if (name === "find_references") {
+    const symName = str(args.name);
+    if (!symName) throw new Error("`name` is required");
+    return JSON.stringify(findReferences(scanRepo(repo, scanOpts), symName), null, 2);
+  }
+  if (name === "replace_symbol_body" || name === "insert_after_symbol" || name === "insert_before_symbol") {
+    const namePath = str(args.namePath);
+    const body = typeof args.body === "string" ? args.body : undefined;
+    if (!namePath || body === undefined) throw new Error("`namePath` and `body` are required");
+    const scan = scanRepo(repo, scanOpts);
+    const fn = name === "replace_symbol_body" ? replaceSymbolBody : name === "insert_after_symbol" ? insertAfterSymbol : insertBeforeSymbol;
+    return JSON.stringify(fn(scan, namePath, body, str(args.file)), null, 2);
+  }
+  if (name === "write_memory") {
+    const memName = str(args.name);
+    const content = typeof args.content === "string" ? args.content : undefined;
+    if (!memName || content === undefined) throw new Error("`name` and `content` are required");
+    return JSON.stringify({ written: writeMemory(repo, memName, content) }, null, 2);
+  }
+  if (name === "read_memory") {
+    const memName = str(args.name);
+    if (!memName) throw new Error("`name` is required");
+    const content = readMemory(repo, memName);
+    if (content === undefined) throw new Error(`no memory named "${memName}" — see list_memories`);
+    return content;
+  }
+  if (name === "list_memories") {
+    return JSON.stringify(listMemories(repo), null, 2);
+  }
+  if (name === "delete_memory") {
+    const memName = str(args.name);
+    if (!memName) throw new Error("`name` is required");
+    return JSON.stringify({ deleted: deleteMemory(repo, memName) }, null, 2);
+  }
+  if (name === "dead_code") {
+    return JSON.stringify(findDeadCode(scanRepo(repo, scanOpts)), null, 2);
+  }
+  if (name === "complexity") {
+    const scan = scanRepo(repo, scanOpts);
+    if (args.risk === true) {
+      const { churn, ok } = gitChurn(repo);
+      return JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn) }, null, 2);
+    }
+    return JSON.stringify(symbolComplexity(scan, str(args.file)), null, 2);
+  }
+  if (name === "mermaid") {
+    const { graph } = buildIndexArtifacts(repo, scanOpts);
+    return renderMermaid(graph, { module: str(args.module) });
+  }
+  if (name === "repo_map") {
+    const { scan, graph } = buildIndexArtifacts(repo, scanOpts);
+    return renderRepoMap(scan, graph, { budgetTokens: typeof args.budgetTokens === "number" ? args.budgetTokens : undefined });
+  }
+  if (name === "hotspots") {
+    const scan = scanRepo(repo, scanOpts);
+    const { churn, ok } = gitChurn(repo, { since: str(args.since) });
+    return JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2);
+  }
+  if (name === "coupling") {
+    const { ok, couplings } = changeCoupling(repo, { since: str(args.since) });
+    return JSON.stringify({ ok, couplings }, null, 2);
+  }
+  if (name === "grep") {
+    const pattern = str(args.pattern);
+    if (!pattern) throw new Error("`pattern` is required");
+    const hits = grepRepo(repo, pattern, {
+      globs: strArray(args.globs),
+      ignoreCase: args.ignoreCase === true,
+      maxHits: typeof args.maxHits === "number" ? args.maxHits : undefined,
+    });
+    return JSON.stringify(hits, null, 2);
+  }
+  if (name === "search") {
+    const query = str(args.query);
+    if (!query) throw new Error("`query` is required");
+    const results = searchIndex(scanRepo(repo, scanOpts), query, {
+      limit: typeof args.limit === "number" ? args.limit : undefined,
+    });
+    return JSON.stringify(results, null, 2);
+  }
+  if (name === "check_rules") {
+    const rules = parseRules(args.rules); // throws a descriptive error on a malformed payload
+    const { graph } = buildIndexArtifacts(repo, scanOpts);
+    return JSON.stringify(checkRules(graph, rules), null, 2);
+  }
+  throw new Error(`unknown tool: ${name}`);
+}
+
+export async function runMcpServer(): Promise<void> {
+  await ensureGrammars(allGrammarKeys()); // AST tier when the sidecar is present
+
+  const send = (msg: Record<string, unknown>): void => {
+    process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...msg }) + "\n");
+  };
+
+  const rl = createInterface({ input: process.stdin, terminal: false });
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      send({ id: null, error: { code: -32700, message: "parse error" } });
+      continue;
+    }
+    // JSON-RPC 2.0 batch: answer each member (a batching client would
+    // otherwise hang forever on a silently dropped array).
+    const requests = Array.isArray(parsed) ? (parsed as RpcRequest[]) : [parsed as RpcRequest];
+    for (const req of requests) handle(req);
+  }
+
+  function handle(req: RpcRequest): void {
+    if (req.id === undefined || req.id === null) return; // notification — no response
+
+    try {
+      if (req.method === "initialize") {
+        send({
+          id: req.id,
+          result: {
+            protocolVersion: "2024-11-05",
+            capabilities: { tools: {} },
+            serverInfo: { name: "codeindex", version: ENGINE_VERSION },
+          },
+        });
+      } else if (req.method === "ping") {
+        send({ id: req.id, result: {} });
+      } else if (req.method === "tools/list") {
+        send({ id: req.id, result: { tools: TOOLS } });
+      } else if (req.method === "tools/call") {
+        const params = req.params ?? {};
+        const name = str(params.name) ?? "";
+        const args = (params.arguments ?? {}) as Record<string, unknown>;
+        try {
+          const text = callTool(name, args);
+          send({ id: req.id, result: { content: [{ type: "text", text }] } });
+        } catch (e) {
+          send({
+            id: req.id,
+            result: { content: [{ type: "text", text: e instanceof Error ? e.message : String(e) }], isError: true },
+          });
+        }
+      } else {
+        send({ id: req.id, error: { code: -32601, message: `method not found: ${req.method}` } });
+      }
+    } catch (e) {
+      send({ id: req.id, error: { code: -32603, message: e instanceof Error ? e.message : String(e) } });
+    }
+  }
+}

package/src/memory.ts

@@ -0,0 +1,75 @@
+// Project memories (Serena-parity): named markdown notes an agent persists
+// across sessions — project map, build commands, conventions. Stored under
+// <repo>/.codeindex/memories/<name>.md; names may contain `/` for topic
+// subdirectories. Plain files, no daemon; the value is the discipline (small
+// named notes read on relevance) rather than any machinery.
+import { mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+const MEMORY_DIR = [".codeindex", "memories"];
+
+// Reject anything that could escape the memories directory: names become
+// filesystem paths.
+function sanitize(name: string): string {
+  const clean = name.replace(/^mem:/, "").replace(/\.md$/, "");
+  if (!clean) throw new Error("memory name is empty");
+  const segments = clean.split("/");
+  for (const seg of segments) {
+    if (!seg || seg === "." || seg === ".." || seg.includes("\\")) {
+      throw new Error(`invalid memory name: "${name}"`);
+    }
+    if (!/^[\w][\w.-]*$/.test(seg)) throw new Error(`invalid memory name segment: "${seg}"`);
+  }
+  return clean;
+}
+
+function memoryPath(repo: string, name: string): string {
+  return join(repo, ...MEMORY_DIR, `${sanitize(name)}.md`);
+}
+
+export function writeMemory(repo: string, name: string, content: string): string {
+  const path = memoryPath(repo, name);
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, content.endsWith("\n") ? content : content + "\n");
+  return sanitize(name);
+}
+
+export function readMemory(repo: string, name: string): string | undefined {
+  try {
+    return readFileSync(memoryPath(repo, name), "utf8");
+  } catch {
+    return undefined;
+  }
+}
+
+export function deleteMemory(repo: string, name: string): boolean {
+  const path = memoryPath(repo, name);
+  try {
+    statSync(path);
+  } catch {
+    return false;
+  }
+  rmSync(path);
+  return true;
+}
+
+// Sorted list of memory names (topic/name form) — agents load the LIST first
+// and read individual memories on relevance.
+export function listMemories(repo: string): string[] {
+  const root = join(repo, ...MEMORY_DIR);
+  const out: string[] = [];
+  const walk = (dir: string, prefix: string): void => {
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      if (e.isDirectory()) walk(join(dir, e.name), prefix ? `${prefix}/${e.name}` : e.name);
+      else if (e.name.endsWith(".md")) out.push(prefix ? `${prefix}/${e.name.slice(0, -3)}` : e.name.slice(0, -3));
+    }
+  };
+  walk(root, "");
+  return out.sort();
+}