@danielblomma/cortex-mcp 2.1.1 → 2.1.4

package/README.md

@@ -9,6 +9,7 @@
 [![npm version](https://img.shields.io/npm/v/%40danielblomma%2Fcortex-mcp)](https://www.npmjs.com/package/@danielblomma/cortex-mcp)
 [![npm downloads](https://img.shields.io/npm/dw/%40danielblomma%2Fcortex-mcp)](https://www.npmjs.com/package/@danielblomma/cortex-mcp)
 [![license](https://img.shields.io/npm/l/%40danielblomma%2Fcortex-mcp)](./LICENSE)
+[![website](https://img.shields.io/badge/website-cortex-2563eb)](https://danielblomma.github.io/cortex/)
 
 ---
 
@@ -380,6 +381,16 @@ Required npm configuration:
 - Use GitHub Actions publisher `DanielBlomma/cortex`
 - Workflow filename must match `release-publish.yml`
 
+## Embedding performance
+
+Embedding generation tunes itself to the machine: the number of parallel
+workers, memory limits for long files, and skip-work caching are all derived
+from the available CPU cores and RAM at run time (container memory limits
+included). No configuration is needed — on a laptop or a CI runner, cortex
+picks safe, fast settings by itself. The one exception worth knowing:
+when several cortex instances share one machine, set `CORTEX_EMBED_THREADS`
+to give each its fair share of cores.
+
 ## Limitations
 
 - Requires repo initialization (`cortex init --bootstrap`).
@@ -400,6 +411,14 @@ Required npm configuration:
 - MCP tools return stale context:
   Run `cortex update`, then reconnect MCP or call `context.reload` from your MCP client.
 
+## Website and Benchmarks
+
+- `frontend/` hosts the cortex website (GitHub Pages, deployed on push to `main`):
+  product overview plus bootstrap evaluation metrics.
+- `benchmark/bootstrapbench/` runs `cortex bootstrap` against 69 pinned
+  real-world repositories in isolated containers and extracts chunk, embedding
+  and graph statistics. See [benchmark/bootstrapbench/README.md](benchmark/bootstrapbench/README.md).
+
 ## Support
 
 - Issues: https://github.com/DanielBlomma/cortex/issues

package/bin/cortex.mjs

@@ -229,6 +229,8 @@ const INIT_SOURCE_EXTENSIONS = new Set([
   ".csv",
   ".ts",
   ".tsx",
+  ".mts",
+  ".cts",
   ".js",
   ".jsx",
   ".mjs",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@danielblomma/cortex-mcp",
   "mcpName": "io.github.DanielBlomma/cortex",
-  "version": "2.1.1",
+  "version": "2.1.4",
   "description": "Local, repo-scoped context platform for coding assistants. Semantic search, graph relationships, and architectural rule context.",
   "type": "module",
   "author": "Daniel Blomma",
@@ -42,6 +42,7 @@
     "scaffold/scripts",
     "scaffold/mcp/src",
     "scaffold/mcp/tests",
+    "scaffold/mcp/build.mjs",
     "scaffold/mcp/package.json",
     "scaffold/mcp/package-lock.json",
     "scaffold/mcp/tsconfig.json",
@@ -50,7 +51,7 @@
   ],
   "scripts": {
     "pretest": "test -d scaffold/scripts/parsers/node_modules || npm --prefix scaffold/scripts/parsers install --no-fund --no-update-notifier --silent",
-    "test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/javascript-parser.test.mjs tests/markdown-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs",
+    "test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/ingest-parallel.test.mjs tests/ingest-worker-crash.test.mjs tests/javascript-parser.test.mjs tests/markdown-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs tests/bootstrapbench-stats.test.mjs tests/bootstrapbench-aggregate.test.mjs",
     "release:sync-version": "node scripts/sync-release-version.mjs",
     "release:check-version-sync": "node scripts/sync-release-version.mjs --check",
     "prepublishOnly": "echo 'Ready to publish to npm'"

package/scaffold/mcp/build.mjs

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+/**
+ * Builds the MCP server only when its TypeScript inputs changed.
+ *
+ * Every embed/graph-load/test invocation used to run a full `tsc` compile
+ * even with untouched sources — twice per bootstrap, on every `cortex
+ * update`. This guard hashes the inputs (src/**, tsconfig.json,
+ * package-lock.json) and skips the compiler when the previous build is
+ * current. `--force` rebuilds unconditionally.
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const ROOT = path.dirname(fileURLToPath(import.meta.url));
+const DIST = path.join(ROOT, "dist");
+const MARKER = path.join(DIST, ".cortex-build-hash");
+// Every runtime entrypoint must exist for a build to count as current;
+// checking only one would let a partially deleted dist pass as fresh.
+const ENTRIES = ["server.js", "embed.js", "loadGraph.js"].map((name) => path.join(DIST, name));
+
+function collectSources(dir, files = []) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+    const absolute = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      collectSources(absolute, files);
+    } else if (entry.isFile() && entry.name.endsWith(".ts")) {
+      files.push(absolute);
+    }
+  }
+  return files;
+}
+
+function inputHash() {
+  const hash = crypto.createHash("sha256");
+  const inputs = [
+    path.join(ROOT, "tsconfig.json"),
+    path.join(ROOT, "package.json"),
+    path.join(ROOT, "package-lock.json")
+  ];
+  const srcDir = path.join(ROOT, "src");
+  if (fs.existsSync(srcDir)) {
+    inputs.push(...collectSources(srcDir));
+  }
+  for (const file of inputs) {
+    if (!fs.existsSync(file)) {
+      continue;
+    }
+    hash.update(path.relative(ROOT, file));
+    hash.update("\0");
+    hash.update(fs.readFileSync(file));
+    hash.update("\0");
+  }
+  return hash.digest("hex");
+}
+
+const force = process.argv.includes("--force");
+const current = inputHash();
+
+if (!force && ENTRIES.every((entry) => fs.existsSync(entry)) && fs.existsSync(MARKER)) {
+  try {
+    if (fs.readFileSync(MARKER, "utf8").trim() === current) {
+      console.log("[build] dist is up to date (sources unchanged)");
+      process.exit(0);
+    }
+  } catch {
+    // unreadable marker -> rebuild
+  }
+}
+
+const tscLocal = path.join(ROOT, "node_modules", ".bin", "tsc");
+if (!fs.existsSync(tscLocal)) {
+  console.error("[build] TypeScript not installed; run npm install in this directory first");
+  process.exit(1);
+}
+const result = spawnSync(tscLocal, ["-p", "tsconfig.json"], { cwd: ROOT, stdio: "inherit" });
+if (result.error) {
+  console.error(`[build] failed to launch tsc: ${result.error.message}`);
+  process.exit(1);
+}
+if (result.status !== 0) {
+  process.exit(result.status ?? 1);
+}
+
+try {
+  fs.mkdirSync(DIST, { recursive: true });
+  fs.writeFileSync(MARKER, `${current}\n`);
+} catch {
+  // A missing marker only means the next run rebuilds; never fail the build.
+}

package/scaffold/mcp/package.json

@@ -5,12 +5,12 @@
   "type": "module",
   "scripts": {
     "build": "tsc -p tsconfig.json",
-    "embed": "npm run build --silent && node dist/embed.js",
-    "graph:load": "npm run build --silent && node dist/loadGraph.js",
+    "embed": "node build.mjs && node dist/embed.js",
+    "graph:load": "node build.mjs && node dist/loadGraph.js",
     "dev": "node --loader ts-node/esm src/server.ts",
     "start": "node dist/server.js",
-    "test": "npm run build --silent && node --test tests/*.test.mjs",
-    "test:ci": "npm run build --silent && node --test --test-reporter=spec tests/*.test.mjs"
+    "test": "node build.mjs && node --test tests/*.test.mjs",
+    "test:ci": "node build.mjs && node --test --test-reporter=spec tests/*.test.mjs"
   },
   "dependencies": {
     "@huggingface/transformers": "^4.1.0",

package/scaffold/mcp/src/embed.ts

@@ -1,10 +1,25 @@
 import crypto from "node:crypto";
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 import { env, pipeline } from "@huggingface/transformers";
-import { readJsonl, asString, asNumber, asBoolean } from "./jsonl.js";
+import { readJsonl, readJsonlRecords, writeJsonlRecords, asString, asNumber, asBoolean } from "./jsonl.js";
 import { CACHE_DIR, PATHS } from "./paths.js";
+import {
+  createTokenCounter,
+  DEFAULT_SCHEDULER_OPTIONS,
+  groupDuplicates,
+  packWorkUnits,
+  resolveInFlightTokens,
+  resolveMemoryHeadroom,
+  resolveModelMaxTokens,
+  resolvePoolConfig,
+  runWorkUnits,
+  type EmbedExtractor,
+  type MeasuredText,
+  type PendingText
+} from "./embedScheduler.js";
 import type { JsonObject, JsonValue } from "./types.js";
 
 const EMBEDDINGS_PATH = PATHS.embeddingsEntities;
@@ -137,11 +152,6 @@ function normalizeText(value: string): string {
   return value.replace(/\s+/g, " ").trim();
 }
 
-function writeJsonl(filePath: string, records: EmbeddingRecord[]): void {
-  const body = records.map((record) => JSON.stringify(record)).join("\n");
-  fs.writeFileSync(filePath, body ? `${body}\n` : "", "utf8");
-}
-
 function ensureRequiredFiles(): void {
   const required = [
     path.join(CACHE_DIR, "documents.jsonl"),
@@ -350,7 +360,7 @@ function parseProjectEntities(raw: JsonObject[]): ProjectEntity[] {
     .filter((value): value is ProjectEntity => value !== null);
 }
 
-function parseExistingEmbeddings(raw: JsonObject[], modelId: string): Map<string, EmbeddingRecord> {
+function parseExistingEmbeddings(raw: Iterable<JsonObject>, modelId: string): Map<string, EmbeddingRecord> {
   const index = new Map<string, EmbeddingRecord>();
 
   for (const item of raw) {
@@ -360,9 +370,12 @@ function parseExistingEmbeddings(raw: JsonObject[], modelId: string): Map<string
     const vectorRaw = item.vector;
     if (!Array.isArray(vectorRaw)) continue;
 
-    const vector = vectorRaw
-      .map((value) => (typeof value === "number" && Number.isFinite(value) ? value : null))
-      .filter((value): value is number => value !== null);
+    const vector: number[] = [];
+    for (const value of vectorRaw) {
+      if (typeof value === "number" && Number.isFinite(value)) {
+        vector.push(value);
+      }
+    }
 
     if (vector.length === 0) continue;
     const model = asString(item.model);
@@ -388,23 +401,20 @@ function parseExistingEmbeddings(raw: JsonObject[], modelId: string): Map<string
   return index;
 }
 
-function toEmbeddingVector(output: unknown): number[] {
-  if (!output || typeof output !== "object") {
-    throw new Error("Invalid embedding output type");
-  }
-
-  const data = (output as { data?: unknown }).data;
-  if (!data || typeof (data as ArrayLike<number>).length !== "number") {
-    throw new Error("Missing embedding data");
-  }
-
-  return Array.from(data as ArrayLike<number>).map((value) => Number(value));
-}
-
 function roundVector(values: number[]): number[] {
   return values.map((value) => Number(value.toFixed(6)));
 }
 
+function* presentEmbeddingRecords(
+  slots: Array<EmbeddingRecord | null>
+): Generator<EmbeddingRecord> {
+  for (const record of slots) {
+    if (record) {
+      yield record;
+    }
+  }
+}
+
 async function main(): Promise<void> {
   const { mode } = parseArgs(process.argv);
   ensureRequiredFiles();
@@ -429,36 +439,28 @@ async function main(): Promise<void> {
 
   const entities: SearchEntity[] = [...documents, ...rules, ...adrs, ...modules, ...projects, ...chunks].sort((a, b) => a.id.localeCompare(b.id));
 
-  const existing = parseExistingEmbeddings(readJsonl(EMBEDDINGS_PATH), modelId);
+  const existing = parseExistingEmbeddings(readJsonlRecords(EMBEDDINGS_PATH), modelId);
 
   env.cacheDir = MODEL_CACHE_DIR;
-  // Optional intra-op thread cap so co-located embedders (e.g. parallel CI
-  // jobs or eval containers) do not oversubscribe shared cores. Unset =
-  // library default (all cores), which is correct for a single instance.
+  // Total thread budget for embedding. CORTEX_EMBED_THREADS caps it so
+  // co-located embedders (parallel CI jobs, eval containers) do not
+  // oversubscribe shared cores; unset = all cores.
   const threadsRaw = Number(process.env.CORTEX_EMBED_THREADS);
-  const sessionOptions =
-    Number.isFinite(threadsRaw) && threadsRaw >= 1
-      ? { session_options: { intraOpNumThreads: Math.floor(threadsRaw), interOpNumThreads: 1 } }
-      : undefined;
-  const extractor = await pipeline(
-    "feature-extraction",
-    modelId,
-    sessionOptions as Parameters<typeof pipeline>[2]
-  );
+  const threadBudget =
+    Number.isFinite(threadsRaw) && threadsRaw >= 1 ? Math.floor(threadsRaw) : os.cpus().length;
 
   let reused = 0;
-  let embedded = 0;
-  let failed = 0;
-  const failures: string[] = [];
-  const output: EmbeddingRecord[] = [];
+  // Slot per entity keeps output in entity order; failed slots stay null.
+  const slots: Array<EmbeddingRecord | null> = entities.map(() => null);
+  const pending: PendingText[] = [];
   let dimensions = 0;
 
-  for (const entity of entities) {
+  entities.forEach((entity, index) => {
     const previous = existing.get(entity.id);
     if (previous && previous.signature === entity.signature && previous.vector.length > 0) {
       reused += 1;
       dimensions = dimensions || previous.vector.length;
-      output.push({
+      slots[index] = {
         ...previous,
         entity_type: entity.type,
         kind: entity.kind,
@@ -471,46 +473,146 @@ async function main(): Promise<void> {
         signature: entity.signature,
         model: modelId,
         dimensions: previous.vector.length
-      });
-      continue;
+      };
+      return;
     }
+    pending.push({ index, text: normalizeText(entity.text) });
+  });
 
-    try {
-      const embeddingOutput = await extractor(normalizeText(entity.text), {
-        pooling: "mean",
-        normalize: true
-      });
-      const vector = roundVector(toEmbeddingVector(embeddingOutput));
-      if (vector.length === 0) {
-        throw new Error("Empty embedding vector");
-      }
+  // Deduplicate identical texts (lossless: identical input -> identical
+  // vector), then measure token lengths for routing and batch packing.
+  const unique = groupDuplicates(pending);
+  // Memory headroom drives pool size and the concurrency gate; everything
+  // adapts to the machine so no tuning is expected from users. Container
+  // limits (cgroups) and platforms that under-report free memory are both
+  // handled inside resolveMemoryHeadroom.
+  const readHeadroom = () =>
+    resolveMemoryHeadroom({
+      freeMemory: os.freemem(),
+      totalMemory: os.totalmem(),
+      constrainedMemory: process.constrainedMemory?.() ?? null,
+      availableMemory: process.availableMemory?.() ?? null
+    });
+  const memoryHeadroom = readHeadroom();
+  const poolConfig = resolvePoolConfig({
+    threadBudget,
+    poolOverride: Number(process.env.CORTEX_EMBED_POOL) || null,
+    uniqueCount: unique.length,
+    memoryBytes: memoryHeadroom
+  });
 
-      embedded += 1;
-      dimensions = dimensions || vector.length;
-      output.push({
-        id: entity.id,
-        entity_type: entity.type,
-        kind: entity.kind,
-        label: entity.label,
-        path: entity.path,
-        status: entity.status,
-        source_of_truth: entity.source_of_truth,
-        trust_level: entity.trust_level,
-        updated_at: entity.updated_at,
-        signature: entity.signature,
-        model: modelId,
-        dimensions: vector.length,
-        vector
-      });
-    } catch (error) {
-      failed += 1;
-      failures.push(
-        `${entity.id}: ${error instanceof Error ? error.message : "embedding generation failed"}`
+  const batchSizeRaw = Number(process.env.CORTEX_EMBED_BATCH_SIZE);
+  const batchTokensRaw = Number(process.env.CORTEX_EMBED_BATCH_TOKENS);
+  const shortTokensRaw = Number(process.env.CORTEX_EMBED_SHORT_TOKENS);
+  const schedulerOptions = {
+    ...DEFAULT_SCHEDULER_OPTIONS,
+    // 0 (or any value below 1) disables micro-batching rather than silently
+    // meaning "use the default".
+    ...(Number.isFinite(batchSizeRaw) && batchSizeRaw >= 0
+      ? { batchMaxItems: Math.max(1, Math.floor(batchSizeRaw)) }
+      : {}),
+    ...(Number.isFinite(batchTokensRaw) && batchTokensRaw >= 16
+      ? { batchTokenBudget: Math.floor(batchTokensRaw) }
+      : {}),
+    ...(Number.isFinite(shortTokensRaw) && shortTokensRaw >= 1
+      ? { shortMaxTokens: Math.floor(shortTokensRaw) }
+      : {})
+  };
+
+  // Fully warm cache: nothing to embed, so skip model loading entirely —
+  // this is the common repeat-bootstrap / small-update path.
+  let embedded = 0;
+  let result: { vectors: Map<number, number[]>; failures: Array<{ index: number; message: string }> } = {
+    vectors: new Map(),
+    failures: []
+  };
+
+  if (unique.length > 0) {
+    const makeExtractor = async (threads: number) =>
+      (await pipeline("feature-extraction", modelId, {
+        session_options: { intraOpNumThreads: threads, interOpNumThreads: 1 }
+      } as Parameters<typeof pipeline>[2])) as unknown as EmbedExtractor & {
+        tokenizer?: ((text: string) => { input_ids?: { dims?: number[] } }) & {
+          model_max_length?: number;
+        };
+      };
+
+    // First session loads (and caches) the model; the rest load in parallel.
+    // Extra sessions failing (e.g. memory pressure) degrades the pool instead
+    // of aborting the run — one session can always finish the work.
+    const first = await makeExtractor(poolConfig.threadsPerSession);
+    const extraSessions = await Promise.allSettled(
+      Array.from({ length: poolConfig.sessions - 1 }, () =>
+        makeExtractor(poolConfig.threadsPerSession)
+      )
+    );
+    const extractors: EmbedExtractor[] = [
+      first,
+      ...extraSessions
+        .filter(
+          (settled): settled is PromiseFulfilledResult<Awaited<ReturnType<typeof makeExtractor>>> =>
+            settled.status === "fulfilled"
+        )
+        .map((settled) => settled.value)
+    ];
+    const failedSessions = extraSessions.length + 1 - extractors.length;
+    if (failedSessions > 0) {
+      const firstFailure = extraSessions.find(
+        (settled): settled is PromiseRejectedResult => settled.status === "rejected"
+      );
+      const reason =
+        firstFailure?.reason instanceof Error ? firstFailure.reason.message : String(firstFailure?.reason ?? "unknown");
+      console.warn(
+        `[embed] ${failedSessions} pool session(s) failed to load (${reason}); continuing with ${extractors.length}`
       );
     }
+
+    // Inference truncates at the model max; token counts must too, or one
+    // giant file inflates scheduling cost and gate mass far beyond reality.
+    const modelMaxTokens = resolveModelMaxTokens(first.tokenizer?.model_max_length);
+    const countTokens = createTokenCounter(first.tokenizer, modelMaxTokens);
+
+    const measured: MeasuredText[] = unique.map((item) => ({ ...item, tokens: countTokens(item.text) }));
+    const units = packWorkUnits(measured, schedulerOptions);
+    // Recompute headroom after the model copies are resident so the gate
+    // reflects what is actually left for inference activations.
+    const inFlightRaw = Number(process.env.CORTEX_EMBED_INFLIGHT_TOKENS);
+    const maxInFlightTokens =
+      Number.isFinite(inFlightRaw) && inFlightRaw >= 1024
+        ? Math.floor(inFlightRaw)
+        : resolveInFlightTokens({ memoryBytes: readHeadroom(), modelMaxTokens });
+    result = await runWorkUnits(units, extractors, {
+      maxInFlightTokens,
+      onVector(index, rawVector) {
+        const entity = entities[index];
+        const vector = roundVector(rawVector);
+        embedded += 1;
+        dimensions = dimensions || vector.length;
+        slots[index] = {
+          id: entity.id,
+          entity_type: entity.type,
+          kind: entity.kind,
+          label: entity.label,
+          path: entity.path,
+          status: entity.status,
+          source_of_truth: entity.source_of_truth,
+          trust_level: entity.trust_level,
+          updated_at: entity.updated_at,
+          signature: entity.signature,
+          model: modelId,
+          dimensions: vector.length,
+          vector
+        };
+      }
+    });
   }
 
-  writeJsonl(EMBEDDINGS_PATH, output);
+  const failures = result.failures.map(
+    (failure) => `${entities[failure.index].id}: ${failure.message}`
+  );
+  const failed = result.failures.length;
+
+  const outputCount = writeJsonlRecords(EMBEDDINGS_PATH, presentEmbeddingRecords(slots));
 
   const manifest = {
     generated_at: new Date().toISOString(),
@@ -519,7 +621,7 @@ async function main(): Promise<void> {
     dimensions,
     counts: {
       entities: entities.length,
-      output: output.length,
+      output: outputCount,
       embedded,
       reused,
       failed
@@ -529,7 +631,9 @@ async function main(): Promise<void> {
 
   fs.writeFileSync(EMBEDDINGS_MANIFEST_PATH, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
 
-  console.log(`[embed] mode=${mode} model=${modelId} dim=${dimensions}`);
+  console.log(
+    `[embed] mode=${mode} model=${modelId} dim=${dimensions} pool=${poolConfig.sessions}x${poolConfig.threadsPerSession} batch<=${schedulerOptions.batchMaxItems}`
+  );
   console.log(
     `[embed] entities=${entities.length} embedded=${embedded} reused=${reused} failed=${failed}`
   );