@danielblomma/cortex-mcp 2.2.2 → 2.2.4

package/README.md

@@ -413,12 +413,19 @@ Required npm configuration:
 ## Embedding performance
 
 Embedding generation tunes itself to the machine: the number of parallel
-workers, memory limits for long files, and skip-work caching are all derived
-from the available CPU cores and RAM at run time (container memory limits
-included). No configuration is needed — on a laptop or a CI runner, cortex
-picks safe, fast settings by itself. The one exception worth knowing:
-when several cortex instances share one machine, set `CORTEX_EMBED_THREADS`
-to give each its fair share of cores.
+workers, memory limits for long files, token budget, and skip-work caching are
+all derived from the available CPU cores, RAM, and repository size at run time
+(container memory limits included). No configuration is needed — on a laptop or
+a CI runner, cortex picks safe settings by itself.
+
+The embedding token budget defaults to `auto`, which preserves the embedding
+model's own maximum context. Cortex does not lower the default token budget
+just because a repository is large; lower caps can discard semantically useful
+text and are reserved for explicit benchmark or quality experiments. To force a
+specific capped run, set `CORTEX_EMBED_MAX_TOKENS` to a number such as `2048`;
+set it to `model` or `full` to make the full-model baseline explicit. When
+several cortex instances share one machine, set `CORTEX_EMBED_THREADS` to give
+each its fair share of cores.
 
 ## Limitations
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@danielblomma/cortex-mcp",
   "mcpName": "io.github.DanielBlomma/cortex",
-  "version": "2.2.2",
+  "version": "2.2.4",
   "description": "Local, repo-scoped context platform for coding assistants. Semantic search, graph relationships, and architectural rule context.",
   "type": "module",
   "author": "Daniel Blomma",
@@ -64,7 +64,7 @@
   ],
   "scripts": {
     "pretest": "test -d scaffold/scripts/parsers/node_modules || npm --prefix scaffold/scripts/parsers install --no-fund --no-update-notifier --silent",
-    "test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/ingest-parallel.test.mjs tests/ingest-worker-crash.test.mjs tests/javascript-parser.test.mjs tests/markdown-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs tests/bootstrapbench-stats.test.mjs tests/bootstrapbench-aggregate.test.mjs tests/bootstrapbench-cleanup.test.mjs tests/query-cli-shim.test.mjs",
+    "test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/ingest-parallel.test.mjs tests/ingest-worker-crash.test.mjs tests/javascript-parser.test.mjs tests/markdown-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs tests/bootstrapbench-run.test.mjs tests/bootstrapbench-stats.test.mjs tests/bootstrapbench-aggregate.test.mjs tests/bootstrapbench-cleanup.test.mjs tests/query-cli-shim.test.mjs",
     "release:sync-version": "node scripts/sync-release-version.mjs",
     "release:check-version-sync": "node scripts/sync-release-version.mjs --check",
     "prepublishOnly": "echo 'Ready to publish to npm'"

package/scaffold/mcp/src/embed.ts

@@ -15,7 +15,9 @@ import {
   resolveMemoryHeadroom,
   resolveModelMaxTokens,
   resolvePoolConfig,
+  resolveTokenBudgetChoice,
   runWorkUnits,
+  truncateTextToTokenBudget,
   type EmbedExtractor,
   type MeasuredText,
   type PendingText
@@ -405,6 +407,14 @@ function roundVector(values: number[]): number[] {
   return values.map((value) => Number(value.toFixed(6)));
 }
 
+function resolveSignatureProfile(maxTokenCap: number | null): string {
+  return maxTokenCap ? `embed|max_tokens=${maxTokenCap}` : "";
+}
+
+function embeddingSignature(entitySignature: string, profile: string): string {
+  return profile ? hashText(`${profile}|${entitySignature}`) : entitySignature;
+}
+
 function* presentEmbeddingRecords(
   slots: Array<EmbeddingRecord | null>
 ): Generator<EmbeddingRecord> {
@@ -438,6 +448,14 @@ async function main(): Promise<void> {
   const chunks = parseChunkEntities(readJsonl(path.join(CACHE_DIR, "entities.chunk.jsonl")), filePathById);
 
   const entities: SearchEntity[] = [...documents, ...rules, ...adrs, ...modules, ...projects, ...chunks].sort((a, b) => a.id.localeCompare(b.id));
+  const uniqueSignatures = new Set<string>();
+  for (const entity of entities) {
+    uniqueSignatures.add(entity.signature);
+  }
+  const uniqueTextCount = uniqueSignatures.size;
+  const tokenBudget = resolveTokenBudgetChoice(process.env.CORTEX_EMBED_MAX_TOKENS, uniqueTextCount);
+  uniqueSignatures.clear();
+  const signatureProfile = resolveSignatureProfile(tokenBudget.cap);
 
   const existing = parseExistingEmbeddings(readJsonlRecords(EMBEDDINGS_PATH), modelId);
 
@@ -456,8 +474,9 @@ async function main(): Promise<void> {
   let dimensions = 0;
 
   entities.forEach((entity, index) => {
+    const signature = embeddingSignature(entity.signature, signatureProfile);
     const previous = existing.get(entity.id);
-    if (previous && previous.signature === entity.signature && previous.vector.length > 0) {
+    if (previous && previous.signature === signature && previous.vector.length > 0) {
       reused += 1;
       dimensions = dimensions || previous.vector.length;
       slots[index] = {
@@ -470,7 +489,7 @@ async function main(): Promise<void> {
         source_of_truth: entity.source_of_truth,
         trust_level: entity.trust_level,
         updated_at: entity.updated_at,
-        signature: entity.signature,
+        signature,
         model: modelId,
         dimensions: previous.vector.length
       };
@@ -522,6 +541,7 @@ async function main(): Promise<void> {
   // Fully warm cache: nothing to embed, so skip model loading entirely —
   // this is the common repeat-bootstrap / small-update path.
   let embedded = 0;
+  let modelMaxTokensUsed = 0;
   let result: { vectors: Map<number, number[]>; failures: Array<{ index: number; message: string }> } = {
     vectors: new Map(),
     failures: []
@@ -569,11 +589,22 @@ async function main(): Promise<void> {
 
     // Inference truncates at the model max; token counts must too, or one
     // giant file inflates scheduling cost and gate mass far beyond reality.
-    const modelMaxTokens = resolveModelMaxTokens(first.tokenizer?.model_max_length);
+    const modelMaxTokens = resolveModelMaxTokens(
+      first.tokenizer?.model_max_length,
+      tokenBudget.cap ?? undefined
+    );
+    modelMaxTokensUsed = modelMaxTokens;
     const countTokens = createTokenCounter(first.tokenizer, modelMaxTokens);
+    const countRawTokens = createTokenCounter(first.tokenizer, 131072);
 
-    const measured: MeasuredText[] = unique.map((item) => ({ ...item, tokens: countTokens(item.text) }));
+    const measured: MeasuredText[] = unique.map((item) => {
+      const text = truncateTextToTokenBudget(item.text, countRawTokens, modelMaxTokens);
+      return { ...item, text, tokens: countTokens(text) };
+    });
     const units = packWorkUnits(measured, schedulerOptions);
+    console.log(
+      `[embed] scheduler unique=${unique.length} units=${units.length} max_tokens<=${modelMaxTokens} token_budget=${tokenBudget.mode} reason=${tokenBudget.reason}`
+    );
     // Recompute headroom after the model copies are resident so the gate
     // reflects what is actually left for inference activations.
     const inFlightRaw = Number(process.env.CORTEX_EMBED_INFLIGHT_TOKENS);
@@ -598,7 +629,7 @@ async function main(): Promise<void> {
           source_of_truth: entity.source_of_truth,
           trust_level: entity.trust_level,
           updated_at: entity.updated_at,
-          signature: entity.signature,
+          signature: embeddingSignature(entity.signature, signatureProfile),
           model: modelId,
           dimensions: vector.length,
           vector
@@ -632,7 +663,7 @@ async function main(): Promise<void> {
   fs.writeFileSync(EMBEDDINGS_MANIFEST_PATH, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
 
   console.log(
-    `[embed] mode=${mode} model=${modelId} dim=${dimensions} pool=${poolConfig.sessions}x${poolConfig.threadsPerSession} batch<=${schedulerOptions.batchMaxItems}`
+    `[embed] mode=${mode} model=${modelId} dim=${dimensions} pool=${poolConfig.sessions}x${poolConfig.threadsPerSession} batch<=${schedulerOptions.batchMaxItems} max_tokens<=${modelMaxTokensUsed || tokenBudget.cap || "model"} token_budget=${tokenBudget.mode}`
   );
   console.log(
     `[embed] entities=${entities.length} embedded=${embedded} reused=${reused} failed=${failed}`

package/scaffold/mcp/src/embedScheduler.ts

@@ -183,9 +183,46 @@ export function resolveMemoryHeadroom({
 /** Sanitizes a tokenizer-reported maximum sequence length. Sentinel and
  * absurd values (some configs report 1e30) fall back to 8192; anything
  * beyond 128k tokens is treated as absurd. */
-export function resolveModelMaxTokens(raw: unknown): number {
+export function resolveModelMaxTokens(raw: unknown, overrideRaw?: unknown): number {
   const value = Number(raw);
-  return Number.isFinite(value) && value >= 1 && value <= 131072 ? Math.floor(value) : 8192;
+  const modelMax = Number.isFinite(value) && value >= 1 && value <= 131072 ? Math.floor(value) : 8192;
+  const override = Number(overrideRaw);
+  if (Number.isFinite(override) && override >= 16 && override <= 131072) {
+    return Math.min(modelMax, Math.floor(override));
+  }
+  return modelMax;
+}
+
+export type TokenBudgetChoice = {
+  cap: number | null;
+  mode: "auto" | "explicit" | "model";
+  reason: string;
+};
+
+/**
+ * Chooses the requested embedding token budget before model load, so cache
+ * signatures can include the same cap that inference will later enforce.
+ * Auto mode is quality-preserving: it uses the model's own maximum by default
+ * and only truncates when the user explicitly requests a numeric cap.
+ */
+export function resolveTokenBudgetChoice(overrideRaw: unknown, uniqueCount: number): TokenBudgetChoice {
+  const raw = String(overrideRaw ?? "").trim().toLowerCase();
+  const override = Number(raw);
+  if (Number.isFinite(override) && override >= 16 && override <= 131072) {
+    const cap = Math.floor(override);
+    return { cap, mode: "explicit", reason: "env_override" };
+  }
+
+  if (raw === "model" || raw === "none" || raw === "off" || raw === "full") {
+    return { cap: null, mode: "model", reason: "env_full_model" };
+  }
+
+  const count = Number.isFinite(uniqueCount) && uniqueCount > 0 ? Math.floor(uniqueCount) : 0;
+  return {
+    cap: null,
+    mode: "auto",
+    reason: `quality_preserving_model_max:unique=${count}`
+  };
 }
 
 /**
@@ -216,6 +253,29 @@ export function createTokenCounter(
   };
 }
 
+export function truncateTextToTokenBudget(
+  text: string,
+  countTokens: (text: string) => number,
+  maxTokens: number
+): string {
+  if (!Number.isFinite(maxTokens) || maxTokens < 1 || countTokens(text) <= maxTokens) {
+    return text;
+  }
+
+  let low = 0;
+  let high = text.length;
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2);
+    if (countTokens(text.slice(0, mid)) <= maxTokens) {
+      low = mid;
+    } else {
+      high = mid - 1;
+    }
+  }
+
+  return text.slice(0, low).trimEnd();
+}
+
 /** Estimated working memory one full pool session needs (model copy plus
  * inference activations for a base-size model). Used only as a derating
  * heuristic; deliberately conservative. */

package/scaffold/mcp/src/loadGraph.ts

@@ -1,7 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import ryugraph, { type Connection, type PreparedStatement, type QueryResult, type RyuValue } from "ryugraph";
-import { readJsonl, asString, asNumber, asBoolean } from "./jsonl.js";
+import { readJsonl, readJsonlRecords, asString, asNumber, asBoolean } from "./jsonl.js";
 import { CACHE_DIR, CONTEXT_DIR, DB_PATH } from "./paths.js";
 import { CSV_COPY_OPTIONS, writeCsv, toCopyPathLiteral, type CsvValue } from "./graphCsv.js";
 import type { JsonObject } from "./types.js";
@@ -127,232 +127,244 @@ function readEntityFile(fileName: string): JsonObject[] {
   return readJsonl(path.join(CACHE_DIR, fileName));
 }
 
-function parseFiles(raw: JsonObject[]): FileEntity[] {
-  return raw
-    .map((item) => {
-      const id = asString(item.id);
-      const filePath = asString(item.path);
-      if (!id || !filePath) {
-        return null;
-      }
+function readEntityRecords(fileName: string): Iterable<JsonObject> {
+  return readJsonlRecords(path.join(CACHE_DIR, fileName));
+}
+
+function* parseFiles(raw: Iterable<JsonObject>): Generator<FileEntity> {
+  for (const item of raw) {
+    const id = asString(item.id);
+    const filePath = asString(item.path);
+    if (!id || !filePath) {
+      continue;
+    }
 
-      return {
-        id,
-        path: filePath,
-        kind: asString(item.kind, "DOC"),
-        excerpt: asString(item.excerpt),
-        checksum: asString(item.checksum),
-        updated_at: asString(item.updated_at),
-        source_of_truth: asBoolean(item.source_of_truth, false),
-        trust_level: asNumber(item.trust_level, 50),
-        status: asString(item.status, "active")
-      };
-    })
-    .filter((value): value is FileEntity => value !== null);
+    yield {
+      id,
+      path: filePath,
+      kind: asString(item.kind, "DOC"),
+      excerpt: asString(item.excerpt),
+      checksum: asString(item.checksum),
+      updated_at: asString(item.updated_at),
+      source_of_truth: asBoolean(item.source_of_truth, false),
+      trust_level: asNumber(item.trust_level, 50),
+      status: asString(item.status, "active")
+    };
+  }
 }
 
-function parseRules(raw: JsonObject[]): RuleEntity[] {
-  return raw
-    .map((item) => {
-      const id = asString(item.id);
-      if (!id) {
-        return null;
-      }
+function* parseRules(raw: Iterable<JsonObject>): Generator<RuleEntity> {
+  for (const item of raw) {
+    const id = asString(item.id);
+    if (!id) {
+      continue;
+    }
 
-      return {
-        id,
-        title: asString(item.title, id),
-        body: asString(item.body),
-        scope: asString(item.scope, "global"),
-        updated_at: asString(item.updated_at),
-        source_of_truth: asBoolean(item.source_of_truth, true),
-        trust_level: asNumber(item.trust_level, 95),
-        status: asString(item.status, "active"),
-        priority: asNumber(item.priority, 0)
-      };
-    })
-    .filter((value): value is RuleEntity => value !== null);
+    yield {
+      id,
+      title: asString(item.title, id),
+      body: asString(item.body),
+      scope: asString(item.scope, "global"),
+      updated_at: asString(item.updated_at),
+      source_of_truth: asBoolean(item.source_of_truth, true),
+      trust_level: asNumber(item.trust_level, 95),
+      status: asString(item.status, "active"),
+      priority: asNumber(item.priority, 0)
+    };
+  }
 }
 
-function parseAdrs(raw: JsonObject[]): AdrEntity[] {
-  return raw
-    .map((item) => {
-      const id = asString(item.id);
-      if (!id) {
-        return null;
-      }
+function* parseAdrs(raw: Iterable<JsonObject>): Generator<AdrEntity> {
+  for (const item of raw) {
+    const id = asString(item.id);
+    if (!id) {
+      continue;
+    }
 
-      return {
-        id,
-        path: asString(item.path),
-        title: asString(item.title, id),
-        body: asString(item.body),
-        decision_date: asString(item.decision_date),
-        supersedes_id: asString(item.supersedes_id),
-        source_of_truth: asBoolean(item.source_of_truth, true),
-        trust_level: asNumber(item.trust_level, 95),
-        status: asString(item.status, "active")
-      };
-    })
-    .filter((value): value is AdrEntity => value !== null);
+    yield {
+      id,
+      path: asString(item.path),
+      title: asString(item.title, id),
+      body: asString(item.body),
+      decision_date: asString(item.decision_date),
+      supersedes_id: asString(item.supersedes_id),
+      source_of_truth: asBoolean(item.source_of_truth, true),
+      trust_level: asNumber(item.trust_level, 95),
+      status: asString(item.status, "active")
+    };
+  }
 }
 
-function parseChunks(raw: JsonObject[]): ChunkEntity[] {
-  return raw
-    .map((item) => {
-      const id = asString(item.id);
-      const file_id = asString(item.file_id);
-      const name = asString(item.name);
-      if (!id || !file_id || !name) {
-        return null;
-      }
+function* parseChunks(raw: Iterable<JsonObject>): Generator<ChunkEntity> {
+  for (const item of raw) {
+    const id = asString(item.id);
+    const file_id = asString(item.file_id);
+    const name = asString(item.name);
+    if (!id || !file_id || !name) {
+      continue;
+    }
 
-      return {
-        id,
-        file_id,
-        name,
-        kind: asString(item.kind, "function"),
-        signature: asString(item.signature),
-        body: asString(item.body),
-        description: asString(item.description),
-        start_line: asNumber(item.start_line, 0),
-        end_line: asNumber(item.end_line, 0),
-        language: asString(item.language, "javascript"),
-        exported: asBoolean(item.exported, false),
-        checksum: asString(item.checksum),
-        updated_at: asString(item.updated_at),
-        source_of_truth: asBoolean(item.source_of_truth, true),
-        trust_level: asNumber(item.trust_level, 80),
-        status: asString(item.status, "active")
-      };
-    })
-    .filter((value): value is ChunkEntity => value !== null);
+    yield {
+      id,
+      file_id,
+      name,
+      kind: asString(item.kind, "function"),
+      signature: asString(item.signature),
+      body: asString(item.body),
+      description: asString(item.description),
+      start_line: asNumber(item.start_line, 0),
+      end_line: asNumber(item.end_line, 0),
+      language: asString(item.language, "javascript"),
+      exported: asBoolean(item.exported, false),
+      checksum: asString(item.checksum),
+      updated_at: asString(item.updated_at),
+      source_of_truth: asBoolean(item.source_of_truth, true),
+      trust_level: asNumber(item.trust_level, 80),
+      status: asString(item.status, "active")
+    };
+  }
 }
 
-function parseModules(raw: JsonObject[]): ModuleEntity[] {
-  return raw
-    .map((item) => {
-      const id = asString(item.id);
-      if (!id) {
-        return null;
-      }
+function* parseModules(raw: Iterable<JsonObject>): Generator<ModuleEntity> {
+  for (const item of raw) {
+    const id = asString(item.id);
+    if (!id) {
+      continue;
+    }
 
-      return {
-        id,
-        path: asString(item.path),
-        name: asString(item.name),
-        summary: asString(item.summary),
-        file_count: asNumber(item.file_count, 0),
-        exported_symbols: asString(item.exported_symbols),
-        updated_at: asString(item.updated_at),
-        source_of_truth: asBoolean(item.source_of_truth, false),
-        trust_level: asNumber(item.trust_level, 75),
-        status: asString(item.status, "active")
-      };
-    })
-    .filter((value): value is ModuleEntity => value !== null);
+    yield {
+      id,
+      path: asString(item.path),
+      name: asString(item.name),
+      summary: asString(item.summary),
+      file_count: asNumber(item.file_count, 0),
+      exported_symbols: asString(item.exported_symbols),
+      updated_at: asString(item.updated_at),
+      source_of_truth: asBoolean(item.source_of_truth, false),
+      trust_level: asNumber(item.trust_level, 75),
+      status: asString(item.status, "active")
+    };
+  }
 }
 
-function parseProjects(raw: JsonObject[]): ProjectEntity[] {
-  return raw
-    .map((item) => {
-      const id = asString(item.id);
-      if (!id) {
-        return null;
-      }
+function* parseProjects(raw: Iterable<JsonObject>): Generator<ProjectEntity> {
+  for (const item of raw) {
+    const id = asString(item.id);
+    if (!id) {
+      continue;
+    }
 
-      return {
-        id,
-        path: asString(item.path),
-        name: asString(item.name),
-        kind: asString(item.kind, "project"),
-        language: asString(item.language, "dotnet"),
-        target_framework: asString(item.target_framework),
-        summary: asString(item.summary),
-        file_count: asNumber(item.file_count, 0),
-        updated_at: asString(item.updated_at),
-        source_of_truth: asBoolean(item.source_of_truth, false),
-        trust_level: asNumber(item.trust_level, 80),
-        status: asString(item.status, "active")
-      };
-    })
-    .filter((value): value is ProjectEntity => value !== null);
+    yield {
+      id,
+      path: asString(item.path),
+      name: asString(item.name),
+      kind: asString(item.kind, "project"),
+      language: asString(item.language, "dotnet"),
+      target_framework: asString(item.target_framework),
+      summary: asString(item.summary),
+      file_count: asNumber(item.file_count, 0),
+      updated_at: asString(item.updated_at),
+      source_of_truth: asBoolean(item.source_of_truth, false),
+      trust_level: asNumber(item.trust_level, 80),
+      status: asString(item.status, "active")
+    };
+  }
+}
+
+function* parseRelationRecords(raw: Iterable<JsonObject>, noteField: string): Generator<Relation> {
+  for (const item of raw) {
+    const from = asString(item.from);
+    const to = asString(item.to);
+    if (!from || !to) {
+      continue;
+    }
+
+    yield {
+      from,
+      to,
+      note: asString(item[noteField as keyof JsonObject])
+    };
+  }
 }
 
 function parseRelations(fileName: string, noteField: string): Relation[] {
-  const raw = readEntityFile(fileName);
-  return raw
-    .map((item) => {
-      const from = asString(item.from);
-      const to = asString(item.to);
-      if (!from || !to) {
-        return null;
-      }
+  return Array.from(parseRelationRecords(readEntityRecords(fileName), noteField));
+}
+
+function* relationRecords(fileName: string, noteField: string): Generator<Relation> {
+  yield* parseRelationRecords(readEntityRecords(fileName), noteField);
+}
+
+function* parseCallRelationRecords(raw: Iterable<JsonObject>): Generator<CallRelation> {
+  for (const item of raw) {
+    const from = asString(item.from);
+    const to = asString(item.to);
+    if (!from || !to) {
+      continue;
+    }
 
-      return {
-        from,
-        to,
-        note: asString(item[noteField as keyof JsonObject])
-      };
-    })
-    .filter((value): value is Relation => value !== null);
+    yield {
+      from,
+      to,
+      call_type: asString(item.call_type, "direct")
+    };
+  }
 }
 
 function parseCallRelations(fileName: string): CallRelation[] {
-  const raw = readEntityFile(fileName);
-  return raw
-    .map((item) => {
-      const from = asString(item.from);
-      const to = asString(item.to);
-      if (!from || !to) {
-        return null;
-      }
+  return Array.from(parseCallRelationRecords(readEntityRecords(fileName)));
+}
+
+function* callRelationRecords(fileName: string): Generator<CallRelation> {
+  yield* parseCallRelationRecords(readEntityRecords(fileName));
+}
+
+function* parseImportRelationRecords(raw: Iterable<JsonObject>): Generator<ImportRelation> {
+  for (const item of raw) {
+    const from = asString(item.from);
+    const to = asString(item.to);
+    if (!from || !to) {
+      continue;
+    }
 
-      return {
-        from,
-        to,
-        call_type: asString(item.call_type, "direct")
-      };
-    })
-    .filter((value): value is CallRelation => value !== null);
+    yield {
+      from,
+      to,
+      import_name: asString(item.import_name, "")
+    };
+  }
 }
 
 function parseImportRelations(fileName: string): ImportRelation[] {
-  const raw = readEntityFile(fileName);
-  return raw
-    .map((item) => {
-      const from = asString(item.from);
-      const to = asString(item.to);
-      if (!from || !to) {
-        return null;
-      }
+  return Array.from(parseImportRelationRecords(readEntityRecords(fileName)));
+}
+
+function* importRelationRecords(fileName: string): Generator<ImportRelation> {
+  yield* parseImportRelationRecords(readEntityRecords(fileName));
+}
 
-      return {
-        from,
-        to,
-        import_name: asString(item.import_name, "")
-      };
-    })
-    .filter((value): value is ImportRelation => value !== null);
+function* parseSimpleRelationRecords(raw: Iterable<JsonObject>): Generator<Relation> {
+  for (const item of raw) {
+    const from = asString(item.from);
+    const to = asString(item.to);
+    if (!from || !to) {
+      continue;
+    }
+
+    yield {
+      from,
+      to,
+      note: ""
+    };
+  }
 }
 
 function parseSimpleRelations(fileName: string): Relation[] {
-  const raw = readEntityFile(fileName);
-  return raw
-    .map((item) => {
-      const from = asString(item.from);
-      const to = asString(item.to);
-      if (!from || !to) {
-        return null;
-      }
+  return Array.from(parseSimpleRelationRecords(readEntityRecords(fileName)));
+}
 
-      return {
-        from,
-        to,
-        note: ""
-      };
-    })
-    .filter((value): value is Relation => value !== null);
+function* simpleRelationRecords(fileName: string): Generator<Relation> {
+  yield* parseSimpleRelationRecords(readEntityRecords(fileName));
 }
 
 async function rows(result: QueryResult | QueryResult[]): Promise<Record<string, unknown>[]> {
@@ -460,12 +472,12 @@ type GraphData = {
 
 function parseGraphData(): GraphData {
   return {
-    fileEntities: parseFiles(readEntityFile("entities.file.jsonl")),
-    ruleEntities: parseRules(readEntityFile("entities.rule.jsonl")),
-    adrEntities: parseAdrs(readEntityFile("entities.adr.jsonl")),
-    chunkEntities: parseChunks(readEntityFile("entities.chunk.jsonl")),
-    moduleEntities: parseModules(readEntityFile("entities.module.jsonl")),
-    projectEntities: parseProjects(readEntityFile("entities.project.jsonl")),
+    fileEntities: Array.from(parseFiles(readEntityFile("entities.file.jsonl"))),
+    ruleEntities: Array.from(parseRules(readEntityFile("entities.rule.jsonl"))),
+    adrEntities: Array.from(parseAdrs(readEntityFile("entities.adr.jsonl"))),
+    chunkEntities: Array.from(parseChunks(readEntityFile("entities.chunk.jsonl"))),
+    moduleEntities: Array.from(parseModules(readEntityFile("entities.module.jsonl"))),
+    projectEntities: Array.from(parseProjects(readEntityFile("entities.project.jsonl"))),
     constrains: parseRelations("relations.constrains.jsonl", "note"),
     implementsEdges: parseRelations("relations.implements.jsonl", "note"),
     supersedes: parseRelations("relations.supersedes.jsonl", "reason"),
@@ -506,9 +518,10 @@ function filterEdges<T extends { from: string; to: string }>(
   })();
 }
 
-function* ids<T extends { id: string }>(items: Iterable<T>): Iterable<string> {
+function* collectIds<T extends { id: string }>(items: Iterable<T>, collected: Set<string>): Iterable<T> {
   for (const item of items) {
-    yield item.id;
+    collected.add(item.id);
+    yield item;
   }
 }
 
@@ -537,28 +550,28 @@ async function copyTable(
   await conn.query(`COPY ${table} FROM "${csvForCopy}" ${CSV_COPY_OPTIONS};`);
 }
 
-// Bulk path: one COPY per table instead of a prepared statement per row.
-// Only used on the reset path, where the freshly created tables are empty.
-// Produces a byte-identical graph to rowByRowLoad (same parsed arrays, same
-// node-then-edge order, same dangling-edge filtering).
-async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
+// Bulk path: one COPY per table instead of a prepared statement per row. Only
+// used on the reset path, where the freshly created tables are empty. It reads
+// node and relation JSONL streams directly into CSVs so the hot path retains
+// only endpoint id sets rather than a full parsed GraphData object.
+async function bulkLoad(conn: Connection): Promise<void> {
   fs.rmSync(GRAPH_IMPORT_DIR, { recursive: true, force: true });
   fs.mkdirSync(GRAPH_IMPORT_DIR, { recursive: true });
 
   try {
-    const fileIds = new Set(ids(data.fileEntities));
-    const ruleIds = new Set(ids(data.ruleEntities));
-    const adrIds = new Set(ids(data.adrEntities));
-    const chunkIds = new Set(ids(data.chunkEntities));
-    const moduleIds = new Set(ids(data.moduleEntities));
-    const projectIds = new Set(ids(data.projectEntities));
+    const fileIds = new Set<string>();
+    const ruleIds = new Set<string>();
+    const adrIds = new Set<string>();
+    const chunkIds = new Set<string>();
+    const moduleIds = new Set<string>();
+    const projectIds = new Set<string>();
 
   // Nodes first — edges reference them by primary key.
   await copyTable(
     conn,
     "File",
     ["id", "path", "kind", "excerpt", "checksum", "updated_at", "source_of_truth", "trust_level", "status"],
-    csvRows(data.fileEntities, (e) => [
+    csvRows(collectIds(parseFiles(readEntityRecords("entities.file.jsonl")), fileIds), (e) => [
       e.id, e.path, e.kind, e.excerpt, e.checksum, e.updated_at, e.source_of_truth, e.trust_level, e.status
     ])
   );
@@ -566,7 +579,7 @@ async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
     conn,
     "Rule",
     ["id", "title", "body", "scope", "priority", "updated_at", "source_of_truth", "trust_level", "status"],
-    csvRows(data.ruleEntities, (e) => [
+    csvRows(collectIds(parseRules(readEntityRecords("entities.rule.jsonl")), ruleIds), (e) => [
       e.id, e.title, e.body, e.scope, e.priority, e.updated_at, e.source_of_truth, e.trust_level, e.status
     ])
   );
@@ -574,7 +587,7 @@ async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
     conn,
     "ADR",
     ["id", "path", "title", "body", "decision_date", "supersedes_id", "source_of_truth", "trust_level", "status"],
-    csvRows(data.adrEntities, (e) => [
+    csvRows(collectIds(parseAdrs(readEntityRecords("entities.adr.jsonl")), adrIds), (e) => [
       e.id, e.path, e.title, e.body, e.decision_date, e.supersedes_id, e.source_of_truth, e.trust_level, e.status
     ])
   );
@@ -585,7 +598,7 @@ async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
       "id", "file_id", "name", "kind", "signature", "body", "description", "start_line", "end_line",
       "language", "exported", "checksum", "updated_at", "source_of_truth", "trust_level", "status"
     ],
-    csvRows(data.chunkEntities, (e) => [
+    csvRows(collectIds(parseChunks(readEntityRecords("entities.chunk.jsonl")), chunkIds), (e) => [
       e.id, e.file_id, e.name, e.kind, e.signature, e.body, e.description, e.start_line, e.end_line,
       e.language, e.exported, e.checksum, e.updated_at, e.source_of_truth, e.trust_level, e.status
     ])
@@ -594,7 +607,7 @@ async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
     conn,
     "Module",
     ["id", "path", "name", "summary", "file_count", "exported_symbols", "updated_at", "source_of_truth", "trust_level", "status"],
-    csvRows(data.moduleEntities, (e) => [
+    csvRows(collectIds(parseModules(readEntityRecords("entities.module.jsonl")), moduleIds), (e) => [
       e.id, e.path, e.name, e.summary, e.file_count, e.exported_symbols, e.updated_at, e.source_of_truth, e.trust_level, e.status
     ])
   );
@@ -605,7 +618,7 @@ async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
       "id", "path", "name", "kind", "language", "target_framework", "summary", "file_count",
       "updated_at", "source_of_truth", "trust_level", "status"
     ],
-    csvRows(data.projectEntities, (e) => [
+    csvRows(collectIds(parseProjects(readEntityRecords("entities.project.jsonl")), projectIds), (e) => [
       e.id, e.path, e.name, e.kind, e.language, e.target_framework, e.summary, e.file_count,
       e.updated_at, e.source_of_truth, e.trust_level, e.status
     ])
@@ -615,43 +628,43 @@ async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
   // (COPY into a rel table reads src key, dst key, then properties in schema
   // order), but emitting real names keeps the CSVs self-describing.
   await copyTable(conn, "CONSTRAINS", ["from", "to", "note"],
-    csvRows(filterEdges(data.constrains, ruleIds, fileIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.constrains.jsonl", "note"), ruleIds, fileIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "IMPLEMENTS", ["from", "to", "note"],
-    csvRows(filterEdges(data.implementsEdges, fileIds, ruleIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.implements.jsonl", "note"), fileIds, ruleIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "SUPERSEDES", ["from", "to", "reason"],
-    csvRows(filterEdges(data.supersedes, adrIds, adrIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.supersedes.jsonl", "reason"), adrIds, adrIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "DEFINES", ["from", "to"],
-    csvRows(filterEdges(data.defines, fileIds, chunkIds), (e) => [e.from, e.to]));
+    csvRows(filterEdges(simpleRelationRecords("relations.defines.jsonl"), fileIds, chunkIds), (e) => [e.from, e.to]));
   await copyTable(conn, "CALLS", ["from", "to", "call_type"],
-    csvRows(filterEdges(data.calls, chunkIds, chunkIds), (e) => [e.from, e.to, e.call_type]));
+    csvRows(filterEdges(callRelationRecords("relations.calls.jsonl"), chunkIds, chunkIds), (e) => [e.from, e.to, e.call_type]));
   await copyTable(conn, "IMPORTS", ["from", "to", "import_name"],
-    csvRows(filterEdges(data.imports, chunkIds, fileIds), (e) => [e.from, e.to, e.import_name]));
+    csvRows(filterEdges(importRelationRecords("relations.imports.jsonl"), chunkIds, fileIds), (e) => [e.from, e.to, e.import_name]));
   await copyTable(conn, "CALLS_SQL", ["from", "to", "note"],
-    csvRows(filterEdges(data.callsSql, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.calls_sql.jsonl", "note"), fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "USES_CONFIG_KEY", ["from", "to", "note"],
-    csvRows(filterEdges(data.usesConfigKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.uses_config_key.jsonl", "note"), fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "USES_RESOURCE_KEY", ["from", "to", "note"],
-    csvRows(filterEdges(data.usesResourceKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.uses_resource_key.jsonl", "note"), fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "USES_SETTING_KEY", ["from", "to", "note"],
-    csvRows(filterEdges(data.usesSettingKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.uses_setting_key.jsonl", "note"), fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "CONTAINS", ["from", "to"],
-    csvRows(filterEdges(data.contains, moduleIds, fileIds), (e) => [e.from, e.to]));
+    csvRows(filterEdges(simpleRelationRecords("relations.contains.jsonl"), moduleIds, fileIds), (e) => [e.from, e.to]));
   await copyTable(conn, "CONTAINS_MODULE", ["from", "to"],
-    csvRows(filterEdges(data.containsModule, moduleIds, moduleIds), (e) => [e.from, e.to]));
+    csvRows(filterEdges(simpleRelationRecords("relations.contains_module.jsonl"), moduleIds, moduleIds), (e) => [e.from, e.to]));
   await copyTable(conn, "EXPORTS", ["from", "to"],
-    csvRows(filterEdges(data.exports, moduleIds, chunkIds), (e) => [e.from, e.to]));
+    csvRows(filterEdges(simpleRelationRecords("relations.exports.jsonl"), moduleIds, chunkIds), (e) => [e.from, e.to]));
   await copyTable(conn, "INCLUDES_FILE", ["from", "to"],
-    csvRows(filterEdges(data.includesFile, projectIds, fileIds), (e) => [e.from, e.to]));
+    csvRows(filterEdges(simpleRelationRecords("relations.includes_file.jsonl"), projectIds, fileIds), (e) => [e.from, e.to]));
   await copyTable(conn, "REFERENCES_PROJECT", ["from", "to", "note"],
-    csvRows(filterEdges(data.referencesProject, projectIds, projectIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.references_project.jsonl", "note"), projectIds, projectIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "USES_RESOURCE", ["from", "to", "note"],
-    csvRows(filterEdges(data.usesResource, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.uses_resource.jsonl", "note"), fileIds, fileIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "USES_SETTING", ["from", "to", "note"],
-    csvRows(filterEdges(data.usesSetting, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.uses_setting.jsonl", "note"), fileIds, fileIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "USES_CONFIG", ["from", "to", "note"],
-    csvRows(filterEdges(data.usesConfig, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.uses_config.jsonl", "note"), fileIds, fileIds), (e) => [e.from, e.to, e.note]));
   await copyTable(conn, "TRANSFORMS_CONFIG", ["from", "to", "note"],
-    csvRows(filterEdges(data.transformsConfig, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
+    csvRows(filterEdges(relationRecords("relations.transforms_config.jsonl", "note"), fileIds, fileIds), (e) => [e.from, e.to, e.note]));
   } finally {
     fs.rmSync(GRAPH_IMPORT_DIR, { recursive: true, force: true });
   }
@@ -942,13 +955,11 @@ async function main(): Promise<void> {
   const ontologyStatements = parseOntologyStatements(fs.readFileSync(ONTOLOGY_PATH, "utf8"));
   await executeStatements(conn, ontologyStatements);
 
-  const data = parseGraphData();
-
   const bulkEnabled = reset && process.env.CORTEX_GRAPH_BULK_LOAD !== "never";
   let usedBulk = false;
   if (bulkEnabled) {
     try {
-      await bulkLoad(conn, data);
+      await bulkLoad(conn);
       usedBulk = true;
       console.log("[graph-load] loaded via COPY bulk import");
     } catch (error) {
@@ -959,6 +970,7 @@ async function main(): Promise<void> {
     }
   }
   if (!usedBulk) {
+    const data = parseGraphData();
     await rowByRowLoad(conn, data);
   }
 

package/scaffold/mcp/tests/embed-scheduler.test.mjs

@@ -448,6 +448,48 @@ test("resolveModelMaxTokens: sane values pass, sentinels and absurdities fall ba
   assert.equal(resolveModelMaxTokens(0), 8192);
 });
 
+test("resolveModelMaxTokens: explicit override caps but never raises model max", async () => {
+  const { resolveModelMaxTokens } = await import("../dist/embedScheduler.js");
+  assert.equal(resolveModelMaxTokens(8192, 2048), 2048);
+  assert.equal(resolveModelMaxTokens(512, 2048), 512);
+  assert.equal(resolveModelMaxTokens(8192, "1024"), 1024);
+  assert.equal(resolveModelMaxTokens(8192, 4), 8192);
+  assert.equal(resolveModelMaxTokens(8192, "nope"), 8192);
+});
+
+test("resolveTokenBudgetChoice: auto preserves model max regardless of repo size", async () => {
+  const { resolveTokenBudgetChoice } = await import("../dist/embedScheduler.js");
+  assert.deepEqual(resolveTokenBudgetChoice(undefined, 180), {
+    cap: null,
+    mode: "auto",
+    reason: "quality_preserving_model_max:unique=180"
+  });
+  assert.deepEqual(resolveTokenBudgetChoice("auto", 20000), {
+    cap: null,
+    mode: "auto",
+    reason: "quality_preserving_model_max:unique=20000"
+  });
+  assert.deepEqual(resolveTokenBudgetChoice(undefined, 48533), {
+    cap: null,
+    mode: "auto",
+    reason: "quality_preserving_model_max:unique=48533"
+  });
+});
+
+test("resolveTokenBudgetChoice: explicit caps and full-model opt-out win over auto", async () => {
+  const { resolveTokenBudgetChoice } = await import("../dist/embedScheduler.js");
+  assert.deepEqual(resolveTokenBudgetChoice("4096", 48533), {
+    cap: 4096,
+    mode: "explicit",
+    reason: "env_override"
+  });
+  assert.deepEqual(resolveTokenBudgetChoice("model", 48533), {
+    cap: null,
+    mode: "model",
+    reason: "env_full_model"
+  });
+});
+
 test("createTokenCounter: uses the tokenizer, clamps at model max, survives failures", async () => {
   const { createTokenCounter } = await import("../dist/embedScheduler.js");
   const tokenizer = (text) => ({ input_ids: { dims: [1, text.length * 2] } });
@@ -462,6 +504,13 @@ test("createTokenCounter: uses the tokenizer, clamps at model max, survives fail
   assert.equal(missing("x".repeat(40)), 10);
 });
 
+test("truncateTextToTokenBudget: trims a prefix to the requested token cap", async () => {
+  const { truncateTextToTokenBudget } = await import("../dist/embedScheduler.js");
+  const countTokens = (text) => text.length;
+  assert.equal(truncateTextToTokenBudget("abcdef", countTokens, 3), "abc");
+  assert.equal(truncateTextToTokenBudget("abc", countTokens, 3), "abc");
+});
+
 test("resolveMemoryHeadroom: constrainedMemory 0 means unconstrained (Node sentinel)", async () => {
   const { resolveMemoryHeadroom } = await import("../dist/embedScheduler.js");
   const unconstrained = resolveMemoryHeadroom({ freeMemory: 50e9, totalMemory: 64e9 });