@danielblomma/cortex-mcp 1.3.2 → 1.4.0

package/scaffold/scripts/parsers/ruby-treesitter.mjs

@@ -0,0 +1,271 @@
+/**
+ * Tree-sitter Ruby parser for Cortex.
+ *
+ * Extracts class, module, method (instance `def`), and singleton_method
+ * (`def self.foo`) as chunks. Methods are qualified by enclosing
+ * class/module path, using Ruby-standard notation:
+ *
+ *   top-level method:         name = "foo"
+ *   class instance method:    name = "Foo#bar"
+ *   class singleton method:   name = "Foo.baz"   (called as Foo.baz)
+ *   nested module/class:      name = "Outer::Inner"
+ *   method in nested class:   name = "Outer::Inner#run"
+ *   singleton in nested:      name = "Outer::Inner.load"
+ *
+ * The `#` vs `.` distinction is the long-standing Ruby documentation
+ * convention (e.g. "String#length" vs "String.new") and lets the
+ * call-graph distinguish overloads that share a bare method name.
+ *
+ * Imports: require / require_relative / load / autoload are captured
+ * as `call` nodes at program scope; the adapter filters by method
+ * name and extracts the string path argument (autoload takes the
+ * path as second arg).
+ *
+ * parseCode is async; the WASM grammar is lazily loaded on first call
+ * and cached for subsequent calls.
+ */
+
+import path from "node:path";
+import fs from "node:fs";
+import { fileURLToPath } from "node:url";
+import {
+  dedupe,
+  initTreeSitter,
+  lineRangeOf,
+  loadGrammar,
+  parseSource,
+  runQuery
+} from "./tree-sitter/base.mjs";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const QUERY_DIR = path.join(__dirname, "tree-sitter", "queries");
+
+let RUBY_LANG = null;
+let langPromise = null;
+
+async function ensureLanguage() {
+  if (RUBY_LANG) return RUBY_LANG;
+  if (!langPromise) {
+    langPromise = (async () => {
+      await initTreeSitter();
+      RUBY_LANG = await loadGrammar("ruby");
+      return RUBY_LANG;
+    })();
+  }
+  await langPromise;
+  return RUBY_LANG;
+}
+
+export async function isAvailable() {
+  try {
+    await ensureLanguage();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const CHUNK_QUERY = fs.readFileSync(path.join(QUERY_DIR, "ruby.chunks.scm"), "utf8");
+const CALL_QUERY = fs.readFileSync(path.join(QUERY_DIR, "ruby.calls.scm"), "utf8");
+const IMPORT_QUERY = fs.readFileSync(path.join(QUERY_DIR, "ruby.imports.scm"), "utf8");
+
+const LOADER_NAMES = new Set(["require", "require_relative", "load", "autoload"]);
+
+// Keywords that parse as call nodes but aren't real call edges.
+// `puts`, `print`, `p` are stdlib IO helpers — keep them out so the
+// graph isn't dominated by debug/logging noise.
+const CALL_FILTER = new Set([
+  "puts", "print", "p", "pp",
+  "require", "require_relative", "load", "autoload",
+  "attr_reader", "attr_writer", "attr_accessor",
+  "private", "protected", "public",
+  "raise", "throw", "catch",
+  "lambda", "proc"
+]);
+
+function normalizeWhitespace(value) {
+  return String(value).replace(/\s+/g, " ").trim();
+}
+
+function signatureOfDecl(node) {
+  const firstNewline = node.text.indexOf("\n");
+  if (firstNewline === -1) return normalizeWhitespace(node.text);
+  return normalizeWhitespace(node.text.slice(0, firstNewline));
+}
+
+function unquoteString(node) {
+  // tree-sitter-ruby string nodes contain a string_content child.
+  for (let i = 0; i < node.namedChildCount; i += 1) {
+    const c = node.namedChild(i);
+    if (c.type === "string_content") return c.text;
+  }
+  // Fallback: strip surrounding quotes.
+  const text = node.text;
+  if (text.length >= 2 && (text.startsWith("'") || text.startsWith('"'))) {
+    return text.slice(1, -1);
+  }
+  return text;
+}
+
+function enclosingModulePath(node) {
+  const parts = [];
+  let cur = node.parent;
+  while (cur && cur.type !== "program") {
+    if (cur.type === "class" || cur.type === "module") {
+      const nameNode = cur.childForFieldName("name");
+      if (nameNode) parts.unshift(nameNode.text);
+    }
+    cur = cur.parent;
+  }
+  return parts;
+}
+
+function collectImports(rootNode) {
+  const captures = runQuery(RUBY_LANG, IMPORT_QUERY, rootNode);
+  const imports = [];
+
+  // Only top-level require calls count as imports (not calls made
+  // inside methods that happen to be named `require`).
+  const callNodes = new Map();
+  for (const cap of captures) {
+    if (cap.name === "import.call") {
+      callNodes.set(cap.node.id, cap.node);
+    }
+  }
+
+  for (const [, callNode] of callNodes) {
+    // Walk up: an import call's ancestors should be only program /
+    // body_statement, never a method body.
+    let ancestor = callNode.parent;
+    let isTopLevel = true;
+    while (ancestor) {
+      if (
+        ancestor.type === "method" ||
+        ancestor.type === "singleton_method" ||
+        ancestor.type === "block" ||
+        ancestor.type === "do_block"
+      ) {
+        isTopLevel = false;
+        break;
+      }
+      ancestor = ancestor.parent;
+    }
+    if (!isTopLevel) continue;
+
+    const methodNode = callNode.childForFieldName("method");
+    const argumentsNode = callNode.childForFieldName("arguments");
+    if (!methodNode || !argumentsNode) continue;
+    if (!LOADER_NAMES.has(methodNode.text)) continue;
+
+    const isAutoload = methodNode.text === "autoload";
+    let targetArgIndex = 0;
+    if (isAutoload) targetArgIndex = 1;
+    const stringArgs = [];
+    for (let i = 0; i < argumentsNode.namedChildCount; i += 1) {
+      const arg = argumentsNode.namedChild(i);
+      if (arg.type === "string") stringArgs.push(arg);
+    }
+    const target = stringArgs[targetArgIndex] ?? stringArgs[0];
+    if (target) imports.push(unquoteString(target));
+  }
+
+  return dedupe(imports);
+}
+
+function collectCallsInNode(node) {
+  const captures = runQuery(RUBY_LANG, CALL_QUERY, node);
+  const names = captures
+    .filter((c) => c.name === "call.name")
+    .map((c) => c.node.text)
+    .filter((name) => name && !CALL_FILTER.has(name));
+  return dedupe(names);
+}
+
+function buildTypeChunk(node, kind, language) {
+  const nameNode = node.childForFieldName("name");
+  if (!nameNode) return null;
+  const baseName = nameNode.text;
+  const parentPath = enclosingModulePath(node);
+  const qualifiedName = parentPath.length > 0
+    ? `${parentPath.join("::")}::${baseName}`
+    : baseName;
+  const { startLine, endLine } = lineRangeOf(node);
+  return {
+    name: qualifiedName,
+    kind,
+    signature: signatureOfDecl(node),
+    body: node.text,
+    startLine,
+    endLine,
+    language,
+    exported: true,
+    calls: [],
+    imports: []
+  };
+}
+
+function buildMethodChunk(node, imports, language, isSingleton) {
+  const nameNode = node.childForFieldName("name");
+  if (!nameNode) return null;
+  const methodName = nameNode.text;
+  const parentPath = enclosingModulePath(node);
+  const owner = parentPath.length > 0 ? parentPath.join("::") : "";
+  const separator = isSingleton ? "." : "#";
+  const qualifiedName = owner ? `${owner}${separator}${methodName}` : methodName;
+  const { startLine, endLine } = lineRangeOf(node);
+  return {
+    name: qualifiedName,
+    kind: isSingleton ? "class_method" : "method",
+    signature: signatureOfDecl(node),
+    body: node.text,
+    startLine,
+    endLine,
+    language,
+    exported: !methodName.startsWith("_"),
+    calls: collectCallsInNode(node),
+    imports
+  };
+}
+
+export async function parseCode(code, filePath, language = "ruby") {
+  await ensureLanguage();
+  const { tree } = parseSource(RUBY_LANG, code);
+  const root = tree.rootNode;
+  const imports = collectImports(root);
+
+  const captures = runQuery(RUBY_LANG, CHUNK_QUERY, root);
+  const declCaptures = captures.filter((c) => c.name.endsWith(".decl"));
+
+  const chunks = [];
+  for (const cap of declCaptures) {
+    const kind = cap.name.split(".")[0];
+    let chunk = null;
+    if (kind === "class") chunk = buildTypeChunk(cap.node, "class", language);
+    else if (kind === "module") chunk = buildTypeChunk(cap.node, "module", language);
+    else if (kind === "method") chunk = buildMethodChunk(cap.node, imports, language, false);
+    else if (kind === "singleton") chunk = buildMethodChunk(cap.node, imports, language, true);
+    if (chunk) chunks.push(chunk);
+  }
+
+  const seen = new Set();
+  const deduped = chunks.filter((chunk) => {
+    const key = `${chunk.kind}|${chunk.name}|${chunk.startLine}|${chunk.endLine}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
+  return { chunks: deduped, errors: [] };
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const target = process.argv[2];
+  if (!target) {
+    console.error("Usage: ruby-treesitter.mjs <file.rb>");
+    process.exit(1);
+  }
+  const code = fs.readFileSync(target, "utf8");
+  const result = await parseCode(code, target, "ruby");
+  console.log(JSON.stringify(result, null, 2));
+}

package/scaffold/scripts/parsers/rust-dispatch.mjs

@@ -0,0 +1,43 @@
+/**
+ * Rust parser dispatcher.
+ *
+ * Selects between the tree-sitter parser (default, richer output) and
+ * the regex parser (fallback, zero deps) based on the CORTEX_RUST_PARSER
+ * environment variable. Selection is deferred until the first parseCode
+ * call so no WASM is loaded if the project contains no .rs files.
+ *
+ *   CORTEX_RUST_PARSER=regex       → always use regex parser
+ *   CORTEX_RUST_PARSER=tree-sitter → force tree-sitter (error if unavailable)
+ *   unset / other                  → tree-sitter with regex auto-fallback
+ */
+
+const choice = process.env.CORTEX_RUST_PARSER;
+
+let activeParser = null;
+let resolvePromise = null;
+
+async function resolveParser() {
+  if (activeParser) return activeParser;
+  if (resolvePromise) return resolvePromise;
+  resolvePromise = (async () => {
+    if (choice === "regex") {
+      activeParser = await import("./rust.mjs");
+    } else if (choice === "tree-sitter") {
+      activeParser = await import("./rust-treesitter.mjs");
+    } else {
+      const ts = await import("./rust-treesitter.mjs");
+      if (await ts.isAvailable()) {
+        activeParser = ts;
+      } else {
+        activeParser = await import("./rust.mjs");
+      }
+    }
+    return activeParser;
+  })();
+  return resolvePromise;
+}
+
+export async function parseCode(code, filePath, language = "rust") {
+  const parser = await resolveParser();
+  return parser.parseCode(code, filePath, language);
+}

package/scaffold/scripts/parsers/rust-treesitter.mjs

@@ -0,0 +1,291 @@
+/**
+ * Tree-sitter Rust parser for Cortex.
+ *
+ * Produces the same chunk shape as scripts/parsers/rust.mjs (the regex
+ * parser) but uses tree-sitter-rust via web-tree-sitter WASM. This is
+ * the pilot language for the tree-sitter infrastructure; behavioral
+ * parity with the regex parser is verified by the existing rust-parser
+ * test suite run against this module.
+ *
+ * parseCode is async; the WASM grammar is lazily loaded on first call
+ * and cached for subsequent calls. Callers must `await parseCode(...)`.
+ */
+
+import path from "node:path";
+import fs from "node:fs";
+import { fileURLToPath } from "node:url";
+import {
+  dedupe,
+  initTreeSitter,
+  lineRangeOf,
+  loadGrammar,
+  parseSource,
+  runQuery
+} from "./tree-sitter/base.mjs";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const QUERY_DIR = path.join(__dirname, "tree-sitter", "queries");
+
+let RUST_LANG = null;
+let langPromise = null;
+
+async function ensureLanguage() {
+  if (RUST_LANG) return RUST_LANG;
+  if (!langPromise) {
+    langPromise = (async () => {
+      await initTreeSitter();
+      RUST_LANG = await loadGrammar("rust");
+      return RUST_LANG;
+    })();
+  }
+  await langPromise;
+  return RUST_LANG;
+}
+
+export async function isAvailable() {
+  try {
+    await ensureLanguage();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const CHUNK_QUERY = fs.readFileSync(path.join(QUERY_DIR, "rust.chunks.scm"), "utf8");
+const CALL_QUERY = fs.readFileSync(path.join(QUERY_DIR, "rust.calls.scm"), "utf8");
+const IMPORT_QUERY = fs.readFileSync(path.join(QUERY_DIR, "rust.imports.scm"), "utf8");
+
+const CALL_KEYWORDS = new Set([
+  "if", "for", "while", "loop", "match", "return",
+  "Some", "None", "Ok", "Err", "Box", "Vec", "String",
+  "println", "eprintln", "format", "write", "writeln",
+  "panic", "todo", "unimplemented", "unreachable",
+  "assert", "assert_eq", "assert_ne", "debug_assert",
+  "debug_assert_eq", "debug_assert_ne",
+  "cfg", "derive", "allow", "warn", "deny"
+]);
+
+function normalizeWhitespace(value) {
+  return String(value).replace(/\s+/g, " ").trim();
+}
+
+function buildSignature(bodyText) {
+  const braceIndex = bodyText.indexOf("{");
+  if (braceIndex === -1) return normalizeWhitespace(bodyText);
+  return normalizeWhitespace(bodyText.slice(0, braceIndex));
+}
+
+function collectImports(rootNode) {
+  const captures = runQuery(RUST_LANG, IMPORT_QUERY, rootNode);
+  const imports = captures
+    .filter((c) => c.name === "use.path")
+    .map((c) => normalizeWhitespace(c.node.text));
+  return dedupe(imports);
+}
+
+const MACRO_INNER_CALL_PATTERN = /\b([A-Za-z_][A-Za-z0-9_]*)\s*\(/g;
+
+function collectCallsFromMacroBodies(node) {
+  const names = [];
+  const visit = (n) => {
+    if (n.type === "macro_invocation") {
+      for (let i = 0; i < n.namedChildCount; i += 1) {
+        const child = n.namedChild(i);
+        if (child.type !== "token_tree") continue;
+        const text = child.text;
+        MACRO_INNER_CALL_PATTERN.lastIndex = 0;
+        let m;
+        while ((m = MACRO_INNER_CALL_PATTERN.exec(text)) !== null) {
+          names.push(m[1]);
+        }
+      }
+      return;
+    }
+    for (let i = 0; i < n.namedChildCount; i += 1) visit(n.namedChild(i));
+  };
+  visit(node);
+  return names.filter((n) => !CALL_KEYWORDS.has(n));
+}
+
+function collectCallsInNode(node) {
+  const captures = runQuery(RUST_LANG, CALL_QUERY, node);
+  const astNames = captures
+    .filter((c) => c.name === "call.name")
+    .map((c) => c.node.text)
+    .filter((name) => name && !CALL_KEYWORDS.has(name));
+  const macroInnerNames = collectCallsFromMacroBodies(node);
+  return dedupe([...astNames, ...macroInnerNames]);
+}
+
+/**
+ * Walk captures from CHUNK_QUERY and group companion captures with
+ * their decl anchor. Returns [{ kind, decl, name, typeNode }] entries
+ * in document order.
+ */
+function groupDeclarations(rootNode) {
+  const captures = runQuery(RUST_LANG, CHUNK_QUERY, rootNode);
+
+  const declCaptures = captures.filter((c) => c.name.endsWith(".decl"));
+  declCaptures.sort((a, b) => a.node.startIndex - b.node.startIndex);
+
+  const entries = declCaptures.map((c) => ({
+    kind: c.name.split(".")[0],
+    node: c.node,
+    meta: {}
+  }));
+
+  for (const cap of captures) {
+    if (cap.name.endsWith(".decl")) continue;
+    const [kind, field] = cap.name.split(".");
+    let closest = null;
+    let closestSize = Infinity;
+    for (const entry of entries) {
+      if (entry.kind !== kind) continue;
+      if (cap.node.startIndex < entry.node.startIndex) continue;
+      if (cap.node.endIndex > entry.node.endIndex) continue;
+      const size = entry.node.endIndex - entry.node.startIndex;
+      if (size < closestSize) {
+        closest = entry;
+        closestSize = size;
+      }
+    }
+    if (!closest) continue;
+    if (!(field in closest.meta)) {
+      closest.meta[field] = cap.node;
+    }
+  }
+
+  return entries;
+}
+
+function chunkFrom(kind, node, name, signatureOverride, calls, imports, language) {
+  const { startLine, endLine } = lineRangeOf(node);
+  return {
+    name,
+    kind,
+    signature: signatureOverride ?? buildSignature(node.text),
+    body: node.text,
+    startLine,
+    endLine,
+    language,
+    calls,
+    imports
+  };
+}
+
+function isInsideAny(node, others) {
+  return others.some(
+    (o) =>
+      node.startIndex >= o.startIndex &&
+      node.endIndex <= o.endIndex &&
+      node !== o
+  );
+}
+
+function extractFunctionCalls(functionNode) {
+  const bodyNode = functionNode.childForFieldName("body");
+  if (!bodyNode) return [];
+  return collectCallsInNode(bodyNode);
+}
+
+export async function parseCode(code, filePath, language = "rust") {
+  await ensureLanguage();
+  const { tree } = parseSource(RUST_LANG, code);
+  const root = tree.rootNode;
+
+  const imports = collectImports(root);
+  const decls = groupDeclarations(root);
+
+  const implNodes = decls.filter((d) => d.kind === "impl").map((d) => d.node);
+
+  const chunks = [];
+
+  for (const entry of decls) {
+    const { kind, node, meta } = entry;
+
+    if (kind === "fn") {
+      if (isInsideAny(node, implNodes)) continue;
+      const name = meta.name?.text;
+      if (!name) continue;
+      chunks.push(
+        chunkFrom("function", node, name, null, extractFunctionCalls(node), imports, language)
+      );
+    } else if (kind === "struct") {
+      const name = meta.name?.text;
+      if (!name) continue;
+      const isUnit = !node.text.includes("{");
+      const signature = isUnit ? normalizeWhitespace(node.text) : buildSignature(node.text);
+      chunks.push(chunkFrom("struct", node, name, signature, [], [], language));
+    } else if (kind === "enum") {
+      const name = meta.name?.text;
+      if (!name) continue;
+      chunks.push(chunkFrom("enum", node, name, null, [], [], language));
+    } else if (kind === "trait") {
+      const name = meta.name?.text;
+      if (!name) continue;
+      chunks.push(chunkFrom("trait", node, name, null, [], [], language));
+    } else if (kind === "mod") {
+      const name = meta.name?.text;
+      if (!name) continue;
+      chunks.push(chunkFrom("module", node, name, null, [], [], language));
+    } else if (kind === "macro") {
+      const name = meta.name?.text;
+      if (!name) continue;
+      chunks.push(
+        chunkFrom("macro", node, name, `macro_rules! ${name}`, [], [], language)
+      );
+    } else if (kind === "impl") {
+      const typeName = meta.type?.text;
+      if (!typeName) continue;
+      const traitNode = node.childForFieldName("trait");
+      const implName = traitNode ? `${traitNode.text} for ${typeName}` : typeName;
+      chunks.push(chunkFrom("impl", node, implName, null, [], [], language));
+
+      const bodyNode = node.childForFieldName("body");
+      if (!bodyNode) continue;
+      for (let i = 0; i < bodyNode.namedChildCount; i += 1) {
+        const child = bodyNode.namedChild(i);
+        if (child.type !== "function_item") continue;
+        const fnName = child.childForFieldName("name")?.text;
+        if (!fnName) continue;
+        const hasBody = child.childForFieldName("body");
+        if (!hasBody) continue;
+        const qualifiedName = `${typeName}::${fnName}`;
+        const { startLine, endLine } = lineRangeOf(child);
+        chunks.push({
+          name: qualifiedName,
+          kind: "method",
+          signature: buildSignature(child.text),
+          body: child.text,
+          startLine,
+          endLine,
+          language,
+          calls: extractFunctionCalls(child),
+          imports
+        });
+      }
+    }
+  }
+
+  const seen = new Set();
+  const deduped = chunks.filter((chunk) => {
+    const key = `${chunk.kind}|${chunk.name}|${chunk.startLine}|${chunk.endLine}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
+  return { chunks: deduped, errors: [] };
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const filePath = process.argv[2];
+  if (!filePath) {
+    console.error("Usage: rust-treesitter.mjs <file.rs>");
+    process.exit(1);
+  }
+  const code = fs.readFileSync(filePath, "utf8");
+  const result = await parseCode(code, filePath, "rust");
+  console.log(JSON.stringify(result, null, 2));
+}