@danielblomma/cortex-mcp 2.1.1 → 2.2.0

package/scaffold/scripts/bootstrap.sh

@@ -3,7 +3,8 @@ set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-MCP_DIR="$REPO_ROOT/.context/mcp"
+CONTEXT_RUNTIME_DIR="$REPO_ROOT/.context/mcp"
+MCP_DIR="$CONTEXT_RUNTIME_DIR"
 TOTAL_STEPS=6
 STEP_INDEX=0
 
@@ -23,10 +24,42 @@ info "pipeline: deps -> ingest -> embeddings -> graph -> status"
 
 mkdir -p "$MCP_DIR/.npm-cache"
 
-step "Installing MCP dependencies"
+# Runs `npm install` for a prefix only when its package-lock.json changed
+# since the last successful install (or node_modules is missing). A warm
+# no-op install still costs npm tens of seconds per prefix; the lockfile-hash
+# marker makes repeat bootstraps and updates skip it entirely.
+install_deps_if_changed() {
+  local prefix="$1" cache="$2"
+  local lock="$prefix/package-lock.json"
+  local marker="$prefix/node_modules/.cortex-lock-hash"
+  local current=""
+  if [ -f "$lock" ]; then
+    current=$(node -e '
+      const crypto = require("node:crypto");
+      const fs = require("node:fs");
+      // Node version is part of the key: native deps must reinstall after a
+      // runtime switch even when the lockfile is unchanged.
+      const hash = crypto.createHash("sha256");
+      hash.update(process.version);
+      hash.update(fs.readFileSync(process.argv[1]));
+      console.log(hash.digest("hex"));
+    ' "$lock" 2>/dev/null || true)
+  fi
+  if [ -n "$current" ] && [ -d "$prefix/node_modules" ] && [ -f "$marker" ] \
+    && [ "$(cat "$marker" 2>/dev/null)" = "$current" ]; then
+    info "dependencies up to date in $prefix (lockfile unchanged)"
+    return 0
+  fi
+  NPM_CONFIG_CACHE="$cache" npm --prefix "$prefix" install --no-fund --no-update-notifier --loglevel=warn
+  if [ -n "$current" ]; then
+    printf '%s' "$current" > "$marker" || true
+  fi
+}
+
+step "Installing context runtime dependencies"
 info "note: upstream RyuGraph dependencies may print deprecation warnings during install"
-NPM_CONFIG_CACHE="$MCP_DIR/.npm-cache" npm --prefix "$MCP_DIR" install --no-fund --no-update-notifier --loglevel=warn
-NPM_CONFIG_CACHE="$REPO_ROOT/.context/scripts/parsers/.npm-cache" npm --prefix "$REPO_ROOT/.context/scripts/parsers" install --no-fund --no-update-notifier --loglevel=warn
+install_deps_if_changed "$MCP_DIR" "$MCP_DIR/.npm-cache"
+install_deps_if_changed "$REPO_ROOT/.context/scripts/parsers" "$REPO_ROOT/.context/scripts/parsers/.npm-cache"
 
 source "$SCRIPT_DIR/lib/enterprise-check.sh"
 

package/scaffold/scripts/dashboard.mjs

@@ -17,7 +17,7 @@ const CONFIG_PATH = path.join(CONTEXT_DIR, "config.yaml");
 const SUPPORTED_TEXT_EXTENSIONS = new Set([
   ".md", ".mdx", ".txt", ".adoc", ".rst",
   ".yaml", ".yml", ".json", ".toml", ".csv",
-  ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
+  ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs",
   ".py", ".go", ".java", ".cs", ".rb", ".rs", ".php", ".swift", ".kt",
   ".sql", ".sh", ".bash", ".zsh", ".ps1",
   ".c", ".h", ".cpp", ".hpp", ".cc", ".hh"

package/scaffold/scripts/doctor.sh

@@ -4,7 +4,8 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 CONTEXT_DIR="$REPO_ROOT/.context"
-MCP_DIR="$CONTEXT_DIR/mcp"
+CONTEXT_RUNTIME_DIR="$CONTEXT_DIR/mcp"
+MCP_DIR="$CONTEXT_RUNTIME_DIR"
 
 PASS=0
 FAIL=0
@@ -160,24 +161,24 @@ if [[ -f "$INGEST_MANIFEST" ]] && command -v git &>/dev/null && git -C "$REPO_RO
   fi
 fi
 
-# ── MCP Server ──────────────────────────────────────────
+# ── Context Runtime / MCP Server ─────────────────────────
 
 echo ""
-echo "  MCP Server"
+echo "  Context Runtime"
 
 if [[ -f "$MCP_DIR/dist/server.js" ]]; then
-  pass ".context/mcp/dist/server.js exists"
+  pass "runtime dist present (.context/mcp/dist/server.js)"
 else
-  fail ".context/mcp/dist/server.js missing — run: cd .context/mcp && npm run build"
+  fail "runtime dist missing — run: cd .context/mcp && npm run build"
 fi
 
 if [[ -d "$MCP_DIR/node_modules" ]]; then
-  pass ".context/mcp/node_modules present"
+  pass "runtime dependencies present (.context/mcp/node_modules)"
 else
-  fail ".context/mcp/node_modules missing — run: cd .context/mcp && npm install"
+  fail "runtime dependencies missing — run: cd .context/mcp && npm install"
 fi
 
-# Quick MCP import check
+# Quick runtime import check
 if [[ -f "$MCP_DIR/dist/server.js" ]] && [[ -d "$MCP_DIR/node_modules" ]]; then
   MCP_CHECK=$(cd "$REPO_ROOT" && timeout 10 node -e '
     const start = Date.now();

package/scaffold/scripts/embed.sh

@@ -3,7 +3,8 @@ set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-MCP_DIR="$REPO_ROOT/.context/mcp"
+CONTEXT_RUNTIME_DIR="$REPO_ROOT/.context/mcp"
+MCP_DIR="$CONTEXT_RUNTIME_DIR"
 
 if ! command -v npm >/dev/null 2>&1; then
   echo "[embed] npm is required but not found on PATH"
@@ -12,5 +13,5 @@ fi
 
 mkdir -p "$MCP_DIR/.npm-cache"
 
-echo "[embed] generating embeddings via .context/mcp/embed"
+echo "[embed] generating embeddings via context runtime"
 CORTEX_PROJECT_ROOT="$REPO_ROOT" NPM_CONFIG_CACHE="$MCP_DIR/.npm-cache" npm --prefix "$MCP_DIR" run embed --silent -- "$@"

package/scaffold/scripts/ingest-parsers.mjs

@@ -0,0 +1,387 @@
+/**
+ * Shared chunk-parser registry.
+ *
+ * Single source of truth for "which parser handles which extension", used by
+ * both the main ingest process and the worker-thread pool. Keeping one table
+ * guarantees a file parses identically whether it runs inline or in a worker.
+ *
+ * Parsers self-initialize lazily (tree-sitter grammars load WASM on first
+ * parse and cache per module instance), so a worker amortizes that cost over
+ * its lifetime.
+ */
+import { parseCode } from "./parsers/javascript.mjs";
+
+const parseJavaScriptCode = parseCode;
+let parseVbNetCode = null;
+let parseCSharpCode = null;
+let parseCSharpProjectImpl = null;
+let parseCppCode = null;
+let parseConfigCode = null;
+let parseResourcesCode = null;
+let parseSqlCode = null;
+let parseRustCode = null;
+let parsePythonCode = null;
+let parseGoCode = null;
+let parseJavaCode = null;
+let parseRubyCode = null;
+let parseBashCode = null;
+let parseVb6Code = null;
+let parseMarkdownCode = null;
+let isVbNetParserAvailable = () => false;
+let isCSharpParserAvailableImpl = () => false;
+let isCppParserAvailable = () => false;
+let getCSharpParserRuntimeImpl = () => ({ available: false, reason: "parser module not loaded" });
+
+let loadPromise = null;
+
+// Languages whose parser runs purely in-process (JS or WASM) with no
+// subprocess and no cross-file global state, so they are safe to run in a
+// worker thread. csharp (Roslyn subprocess + project-wide batch parse in the
+// main process), vbnet (subprocess), and cpp (clang-bridge fallback
+// subprocess) stay on the main thread.
+export const PARALLEL_SAFE_LANGUAGES = new Set([
+  "javascript",
+  "jsx",
+  "typescript",
+  "tsx",
+  "python",
+  "go",
+  "java",
+  "ruby",
+  "bash",
+  "sql",
+  "config",
+  "resource",
+  "settings",
+  "markdown",
+  "rust",
+  "vb6"
+]);
+
+export async function loadParsers() {
+  if (loadPromise) {
+    return loadPromise;
+  }
+  const loaders = [
+    import("./parsers/vbnet.mjs").then((module) => {
+      parseVbNetCode = module.parseCode;
+      isVbNetParserAvailable =
+        typeof module.isVbNetParserAvailable === "function"
+          ? module.isVbNetParserAvailable
+          : () => typeof module.parseCode === "function";
+    }),
+    import("./parsers/csharp.mjs").then((module) => {
+      parseCSharpCode = module.parseCode;
+      parseCSharpProjectImpl = module.parseProject ?? null;
+      getCSharpParserRuntimeImpl =
+        typeof module.getCSharpParserRuntime === "function"
+          ? module.getCSharpParserRuntime
+          : () => ({ available: typeof module.parseCode === "function", reason: "runtime details unavailable" });
+      isCSharpParserAvailableImpl =
+        typeof module.isCSharpParserAvailable === "function"
+          ? module.isCSharpParserAvailable
+          : () => typeof module.parseCode === "function";
+    }),
+    import("./parsers/cpp-dispatch.mjs").then((module) => {
+      parseCppCode = module.parseCode;
+      isCppParserAvailable =
+        typeof module.isCppParserAvailable === "function"
+          ? module.isCppParserAvailable
+          : () => typeof module.parseCode === "function";
+    }),
+    import("./parsers/config.mjs").then((module) => {
+      parseConfigCode = module.parseCode;
+    }),
+    import("./parsers/resources.mjs").then((module) => {
+      parseResourcesCode = module.parseCode;
+    }),
+    import("./parsers/sql.mjs").then((module) => {
+      parseSqlCode = module.parseCode;
+    }),
+    import("./parsers/rust-dispatch.mjs").then((module) => {
+      parseRustCode = module.parseCode;
+    }),
+    import("./parsers/python-treesitter.mjs").then((module) => {
+      parsePythonCode = module.parseCode;
+    }),
+    import("./parsers/go-treesitter.mjs").then((module) => {
+      parseGoCode = module.parseCode;
+    }),
+    import("./parsers/java-treesitter.mjs").then((module) => {
+      parseJavaCode = module.parseCode;
+    }),
+    import("./parsers/ruby-treesitter.mjs").then((module) => {
+      parseRubyCode = module.parseCode;
+    }),
+    import("./parsers/bash-treesitter.mjs").then((module) => {
+      parseBashCode = module.parseCode;
+    }),
+    import("./parsers/vb6.mjs").then((module) => {
+      parseVb6Code = module.parseCode;
+    }),
+    import("./parsers/markdown.mjs").then((module) => {
+      parseMarkdownCode = module.parseCode;
+    })
+  ];
+
+  loadPromise = Promise.allSettled(loaders).then(() => undefined);
+  return loadPromise;
+}
+
+const CHUNK_PARSERS = new Map([
+  [".js", { language: "javascript", parse: parseJavaScriptCode }],
+  [".jsx", { language: "jsx", parse: parseJavaScriptCode }],
+  [".mjs", { language: "javascript", parse: parseJavaScriptCode }],
+  [".cjs", { language: "javascript", parse: parseJavaScriptCode }],
+  [".ts", { language: "typescript", parse: parseJavaScriptCode }],
+  [".tsx", { language: "tsx", parse: parseJavaScriptCode }],
+  [".mts", { language: "typescript", parse: parseJavaScriptCode }],
+  [".cts", { language: "typescript", parse: parseJavaScriptCode }],
+  [
+    ".vb",
+    {
+      language: "vbnet",
+      parse: (...args) => parseVbNetCode(...args),
+      isAvailable: () => typeof parseVbNetCode === "function" && isVbNetParserAvailable()
+    }
+  ],
+  [
+    ".cs",
+    {
+      language: "csharp",
+      parse: (...args) => parseCSharpCode(...args),
+      isAvailable: () => typeof parseCSharpCode === "function" && isCSharpParserAvailableImpl()
+    }
+  ],
+  [
+    ".sql",
+    {
+      language: "sql",
+      parse: (...args) => parseSqlCode(...args),
+      isAvailable: () => typeof parseSqlCode === "function"
+    }
+  ],
+  [
+    ".md",
+    {
+      language: "markdown",
+      parse: (...args) => parseMarkdownCode(...args),
+      isAvailable: () => typeof parseMarkdownCode === "function"
+    }
+  ],
+  [
+    ".mdx",
+    {
+      language: "markdown",
+      parse: (...args) => parseMarkdownCode(...args),
+      isAvailable: () => typeof parseMarkdownCode === "function"
+    }
+  ],
+  [
+    ".config",
+    {
+      language: "config",
+      parse: (...args) => parseConfigCode(...args),
+      isAvailable: () => typeof parseConfigCode === "function"
+    }
+  ],
+  [
+    ".resx",
+    {
+      language: "resource",
+      parse: (...args) => parseResourcesCode(...args),
+      isAvailable: () => typeof parseResourcesCode === "function"
+    }
+  ],
+  [
+    ".settings",
+    {
+      language: "settings",
+      parse: (...args) => parseResourcesCode(...args),
+      isAvailable: () => typeof parseResourcesCode === "function"
+    }
+  ],
+  [
+    ".c",
+    {
+      language: "c",
+      parse: (...args) => parseCppCode(...args),
+      isAvailable: () => typeof parseCppCode === "function" && isCppParserAvailable()
+    }
+  ],
+  [
+    ".h",
+    {
+      language: "c",
+      parse: (...args) => parseCppCode(...args),
+      isAvailable: () => typeof parseCppCode === "function" && isCppParserAvailable()
+    }
+  ],
+  [
+    ".cpp",
+    {
+      language: "cpp",
+      parse: (...args) => parseCppCode(...args),
+      isAvailable: () => typeof parseCppCode === "function" && isCppParserAvailable()
+    }
+  ],
+  [
+    ".cc",
+    {
+      language: "cpp",
+      parse: (...args) => parseCppCode(...args),
+      isAvailable: () => typeof parseCppCode === "function" && isCppParserAvailable()
+    }
+  ],
+  [
+    ".hpp",
+    {
+      language: "cpp",
+      parse: (...args) => parseCppCode(...args),
+      isAvailable: () => typeof parseCppCode === "function" && isCppParserAvailable()
+    }
+  ],
+  [
+    ".hh",
+    {
+      language: "cpp",
+      parse: (...args) => parseCppCode(...args),
+      isAvailable: () => typeof parseCppCode === "function" && isCppParserAvailable()
+    }
+  ],
+  [
+    ".rs",
+    {
+      language: "rust",
+      parse: (...args) => parseRustCode(...args),
+      isAvailable: () => typeof parseRustCode === "function"
+    }
+  ],
+  [
+    ".py",
+    {
+      language: "python",
+      parse: (...args) => parsePythonCode(...args),
+      isAvailable: () => typeof parsePythonCode === "function"
+    }
+  ],
+  [
+    ".go",
+    {
+      language: "go",
+      parse: (...args) => parseGoCode(...args),
+      isAvailable: () => typeof parseGoCode === "function"
+    }
+  ],
+  [
+    ".java",
+    {
+      language: "java",
+      parse: (...args) => parseJavaCode(...args),
+      isAvailable: () => typeof parseJavaCode === "function"
+    }
+  ],
+  [
+    ".rb",
+    {
+      language: "ruby",
+      parse: (...args) => parseRubyCode(...args),
+      isAvailable: () => typeof parseRubyCode === "function"
+    }
+  ],
+  [
+    ".sh",
+    {
+      language: "bash",
+      parse: (...args) => parseBashCode(...args),
+      isAvailable: () => typeof parseBashCode === "function"
+    }
+  ],
+  [
+    ".bash",
+    {
+      language: "bash",
+      parse: (...args) => parseBashCode(...args),
+      isAvailable: () => typeof parseBashCode === "function"
+    }
+  ],
+  [
+    ".zsh",
+    {
+      language: "bash",
+      parse: (...args) => parseBashCode(...args),
+      isAvailable: () => typeof parseBashCode === "function"
+    }
+  ],
+  [
+    ".bas",
+    {
+      language: "vb6",
+      parse: (...args) => parseVb6Code(...args),
+      isAvailable: () => typeof parseVb6Code === "function"
+    }
+  ],
+  [
+    ".cls",
+    {
+      language: "vb6",
+      parse: (...args) => parseVb6Code(...args),
+      isAvailable: () => typeof parseVb6Code === "function"
+    }
+  ],
+  [
+    ".frm",
+    {
+      language: "vb6",
+      parse: (...args) => parseVb6Code(...args),
+      isAvailable: () => typeof parseVb6Code === "function"
+    }
+  ],
+  [
+    ".ctl",
+    {
+      language: "vb6",
+      parse: (...args) => parseVb6Code(...args),
+      isAvailable: () => typeof parseVb6Code === "function"
+    }
+  ]
+]);
+
+export function getChunkParserForExtension(ext) {
+  return CHUNK_PARSERS.get(ext) ?? null;
+}
+
+// Stable wrappers so callers can import these once; they delegate to whatever
+// the dynamic import populated.
+export function isCSharpParserAvailable() {
+  return isCSharpParserAvailableImpl();
+}
+
+export function getCSharpParserRuntime() {
+  return getCSharpParserRuntimeImpl();
+}
+
+export function parseCSharpProject(inputs) {
+  if (typeof parseCSharpProjectImpl !== "function") {
+    return null;
+  }
+  return parseCSharpProjectImpl(inputs);
+}
+
+export function hasCSharpProjectParser() {
+  return typeof parseCSharpProjectImpl === "function";
+}
+
+// Parse a single file with the registered parser for its extension. Used by
+// the worker; returns the parser's { chunks, errors } result, or null if no
+// parser is registered or it is unavailable in this process.
+export async function parseFileContent(ext, content, filePath) {
+  const parser = getChunkParserForExtension(ext);
+  if (!parser) {
+    return null;
+  }
+  if (typeof parser.isAvailable === "function" && !(await parser.isAvailable())) {
+    return null;
+  }
+  return { language: parser.language, result: await parser.parse(content, filePath, parser.language) };
+}

package/scaffold/scripts/ingest-worker.mjs

@@ -0,0 +1,52 @@
+/**
+ * Ingest worker thread.
+ *
+ * Runs the pure, parallel-safe parse step (tree-sitter / acorn / regex
+ * parsers) for one file at a time off the main thread. It does nothing
+ * stateful: no id allocation, windowing, checksums, or relation building —
+ * all of that stays on the main thread in deterministic order. The worker
+ * only turns (ext, file path) into a parse result.
+ *
+ * Parsers initialize lazily on first use and cache per module instance, so a
+ * long-lived worker pays each grammar's WASM init once.
+ */
+import fs from "node:fs";
+import { parentPort } from "node:worker_threads";
+import { loadParsers, parseFileContent } from "./ingest-parsers.mjs";
+
+if (!parentPort) {
+  throw new Error("ingest-worker.mjs must be run as a worker thread");
+}
+
+const ready = loadParsers();
+
+parentPort.on("message", async (message) => {
+  if (message && message.type === "shutdown") {
+    process.exit(0);
+  }
+
+  const { taskId, ext, filePath } = message;
+  try {
+    await ready;
+    let content = typeof message.content === "string" ? message.content : null;
+    if (content === null) {
+      const limit = Number.isFinite(message.contentLimit) ? Math.max(0, Math.floor(message.contentLimit)) : null;
+      content = fs.readFileSync(message.absolutePath, "utf8");
+      if (limit !== null) {
+        content = content.slice(0, limit);
+      }
+    }
+    const parsed = await parseFileContent(ext, content, filePath);
+    if (!parsed) {
+      parentPort.postMessage({ taskId, ok: false, reason: "no parser available" });
+      return;
+    }
+    parentPort.postMessage({ taskId, ok: true, result: parsed.result });
+  } catch (error) {
+    parentPort.postMessage({
+      taskId,
+      ok: false,
+      reason: error instanceof Error ? error.message : String(error)
+    });
+  }
+});