@danielblomma/cortex-mcp 2.0.19 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -87,6 +87,32 @@ The result is an assistant that behaves as if it already knows your codebase, be
87
87
  npm i -g @danielblomma/cortex-mcp
88
88
  ```
89
89
 
90
+ ## Upgrading
91
+
92
+ To upgrade an already-scaffolded project to a new Cortex version:
93
+
94
+ ```bash
95
+ npm i -g @danielblomma/cortex-mcp
96
+ cortex init --force # re-scaffolds .context/mcp + .context/scripts
97
+ cortex bootstrap
98
+ cortex update
99
+ ```
100
+
101
+ `cortex init --force` preserves per-project files: `.context/config.yaml`,
102
+ `.context/rules.yaml`, and your notes/decisions.
103
+
104
+ Version-specific notes (see [CHANGELOG.md](CHANGELOG.md) for details):
105
+
106
+ - **2.1.0**: the default embedding model changed, so the first
107
+ `cortex update` after upgrading triggers a full re-embed automatically
108
+ (~2 min per 1000 entities plus a one-time model download). The
109
+ `CORTEX_EMBED_MAX_CHARS` env var is removed and silently ignored.
110
+ Existing projects keep their old ranking weights in `config.yaml`; the
111
+ recommended block is now `semantic: 0.55, graph: 0.10, trust: 0.20,
112
+ recency: 0.15`. After re-embedding, restart the MCP server, and note
113
+ that the first search after a re-embed can hit a stale embeddings
114
+ cache — re-run the query.
115
+
90
116
  ## Quick Start
91
117
 
92
118
  From the repository you want to index:
package/bin/cortex.mjs CHANGED
@@ -191,7 +191,7 @@ function ensureScaffoldExists() {
191
191
 
192
192
  // Files that should never be overwritten if they already exist in the target.
193
193
  // These contain user-specific configuration that would be lost on re-init.
194
- const PRESERVE_FILES = new Set(["config.yaml", "enterprise.yml", "enterprise.yaml", "CLAUDE.md", "AGENTS.md"]);
194
+ const PRESERVE_FILES = new Set(["config.yaml", "rules.yaml", "enterprise.yml", "enterprise.yaml", "CLAUDE.md", "AGENTS.md"]);
195
195
  const DEFAULT_SOURCE_PATHS = [
196
196
  "src",
197
197
  "docs",
@@ -410,9 +410,12 @@ function buildInitialConfig(targetDir) {
410
410
  " - RULE",
411
411
  " - CODE",
412
412
  " - WIKI",
413
+ "# graph weight is low because graph degree mostly measures how many rules",
414
+ "# constrain an entity (docs are hubs, leaf code is not). Tuned as a pair with",
415
+ "# the midrank-percentile graph_score.",
413
416
  "ranking:",
414
- " semantic: 0.40",
415
- " graph: 0.25",
417
+ " semantic: 0.55",
418
+ " graph: 0.10",
416
419
  " trust: 0.20",
417
420
  " recency: 0.15",
418
421
  "runtime:",
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@danielblomma/cortex-mcp",
3
3
  "mcpName": "io.github.DanielBlomma/cortex",
4
- "version": "2.0.19",
4
+ "version": "2.1.1",
5
5
  "description": "Local, repo-scoped context platform for coding assistants. Semantic search, graph relationships, and architectural rule context.",
6
6
  "type": "module",
7
7
  "author": "Daniel Blomma",
@@ -50,7 +50,7 @@
50
50
  ],
51
51
  "scripts": {
52
52
  "pretest": "test -d scaffold/scripts/parsers/node_modules || npm --prefix scaffold/scripts/parsers install --no-fund --no-update-notifier --silent",
53
- "test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/javascript-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs",
53
+ "test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/javascript-parser.test.mjs tests/markdown-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs",
54
54
  "release:sync-version": "node scripts/sync-release-version.mjs",
55
55
  "release:check-version-sync": "node scripts/sync-release-version.mjs --check",
56
56
  "prepublishOnly": "echo 'Ready to publish to npm'"
@@ -10,9 +10,12 @@ truth_order:
10
10
  - RULE
11
11
  - CODE
12
12
  - WIKI
13
+ # graph weight is low because graph degree mostly measures how many rules
14
+ # constrain an entity (docs are hubs, leaf code is not). Tuned as a pair with
15
+ # the midrank-percentile graph_score.
13
16
  ranking:
14
- semantic: 0.40
15
- graph: 0.25
17
+ semantic: 0.55
18
+ graph: 0.10
16
19
  trust: 0.20
17
20
  recency: 0.15
18
21
  runtime:
@@ -1760,9 +1760,9 @@
1760
1760
  }
1761
1761
  },
1762
1762
  "node_modules/hono": {
1763
- "version": "4.12.18",
1764
- "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.18.tgz",
1765
- "integrity": "sha512-RWzP96k/yv0PQfyXnWjs6zot20TqfpfsNXhOnev8d1InAxubW93L11/oNUc3tQqn2G0bSdAOBpX+2uDFHV7kdQ==",
1763
+ "version": "4.12.25",
1764
+ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz",
1765
+ "integrity": "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==",
1766
1766
  "license": "MIT",
1767
1767
  "engines": {
1768
1768
  "node": ">=16.9.0"
@@ -2165,9 +2165,9 @@
2165
2165
  }
2166
2166
  },
2167
2167
  "node_modules/qs": {
2168
- "version": "6.15.0",
2169
- "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz",
2170
- "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==",
2168
+ "version": "6.15.2",
2169
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz",
2170
+ "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==",
2171
2171
  "license": "BSD-3-Clause",
2172
2172
  "dependencies": {
2173
2173
  "side-channel": "^1.1.0"
@@ -24,7 +24,7 @@
24
24
  "overrides": {
25
25
  "cmake-js": "^8.0.0",
26
26
  "express-rate-limit": "^8.5.1",
27
- "hono": "^4.12.18",
27
+ "hono": "^4.12.23",
28
28
  "@hono/node-server": "^1.19.13",
29
29
  "tar": "^7.5.11"
30
30
  },
@@ -1,6 +1,7 @@
1
1
  import crypto from "node:crypto";
2
2
  import fs from "node:fs";
3
3
  import path from "node:path";
4
+ import { pathToFileURL } from "node:url";
4
5
  import { env, pipeline } from "@huggingface/transformers";
5
6
  import { readJsonl, asString, asNumber, asBoolean } from "./jsonl.js";
6
7
  import { CACHE_DIR, PATHS } from "./paths.js";
@@ -11,9 +12,11 @@ const EMBEDDINGS_MANIFEST_PATH = PATHS.embeddingsManifest;
11
12
  const MODEL_CACHE_DIR = PATHS.embeddingsModelCache;
12
13
  const EMBEDDINGS_DIR = path.dirname(EMBEDDINGS_PATH);
13
14
 
14
- const DEFAULT_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
15
- const DEFAULT_MAX_TEXT_CHARS = 7000;
16
- const CHUNK_BODY_PREVIEW_CHARS = 2000;
15
+ export const DEFAULT_MODEL_ID = "jinaai/jina-embeddings-v2-base-code";
16
+
17
+ export function resolveModelId(): string {
18
+ return (process.env.CORTEX_EMBED_MODEL ?? DEFAULT_MODEL_ID).trim() || DEFAULT_MODEL_ID;
19
+ }
17
20
 
18
21
  type FileEntity = {
19
22
  id: string;
@@ -134,10 +137,6 @@ function normalizeText(value: string): string {
134
137
  return value.replace(/\s+/g, " ").trim();
135
138
  }
136
139
 
137
- function clampText(value: string, maxChars: number): string {
138
- return value.slice(0, maxChars);
139
- }
140
-
141
140
  function writeJsonl(filePath: string, records: EmbeddingRecord[]): void {
142
141
  const body = records.map((record) => JSON.stringify(record)).join("\n");
143
142
  fs.writeFileSync(filePath, body ? `${body}\n` : "", "utf8");
@@ -157,7 +156,7 @@ function ensureRequiredFiles(): void {
157
156
  }
158
157
  }
159
158
 
160
- function parseFileEntities(raw: JsonObject[], maxChars: number): FileEntity[] {
159
+ export function parseFileEntities(raw: JsonObject[]): FileEntity[] {
161
160
  return raw
162
161
  .map((item) => {
163
162
  const id = asString(item.id);
@@ -170,7 +169,7 @@ function parseFileEntities(raw: JsonObject[], maxChars: number): FileEntity[] {
170
169
  const excerpt = asString(item.excerpt);
171
170
  const updatedAt = asString(item.updated_at);
172
171
  const checksum = asString(item.checksum, hashText(content));
173
- const text = clampText(`${filePath}\n${excerpt}\n${content}`, maxChars);
172
+ const text = `${filePath}\n${excerpt}\n${content}`;
174
173
 
175
174
  return {
176
175
  id,
@@ -189,7 +188,7 @@ function parseFileEntities(raw: JsonObject[], maxChars: number): FileEntity[] {
189
188
  .filter((value): value is FileEntity => value !== null);
190
189
  }
191
190
 
192
- function parseRuleEntities(raw: JsonObject[], maxChars: number): RuleEntity[] {
191
+ function parseRuleEntities(raw: JsonObject[]): RuleEntity[] {
193
192
  return raw
194
193
  .map((item) => {
195
194
  const id = asString(item.id);
@@ -200,7 +199,7 @@ function parseRuleEntities(raw: JsonObject[], maxChars: number): RuleEntity[] {
200
199
  const title = asString(item.title, id);
201
200
  const body = asString(item.body);
202
201
  const updatedAt = asString(item.updated_at, "");
203
- const text = clampText(`${title}\n${body}`, maxChars);
202
+ const text = `${title}\n${body}`;
204
203
 
205
204
  return {
206
205
  id,
@@ -219,7 +218,7 @@ function parseRuleEntities(raw: JsonObject[], maxChars: number): RuleEntity[] {
219
218
  .filter((value): value is RuleEntity => value !== null);
220
219
  }
221
220
 
222
- function parseAdrEntities(raw: JsonObject[], maxChars: number): AdrEntity[] {
221
+ function parseAdrEntities(raw: JsonObject[]): AdrEntity[] {
223
222
  return raw
224
223
  .map((item) => {
225
224
  const id = asString(item.id);
@@ -231,7 +230,7 @@ function parseAdrEntities(raw: JsonObject[], maxChars: number): AdrEntity[] {
231
230
  const body = asString(item.body);
232
231
  const adrPath = asString(item.path);
233
232
  const decisionDate = asString(item.decision_date, "");
234
- const text = clampText(`${adrPath}\n${title}\n${body}`, maxChars);
233
+ const text = `${adrPath}\n${title}\n${body}`;
235
234
 
236
235
  return {
237
236
  id,
@@ -250,7 +249,7 @@ function parseAdrEntities(raw: JsonObject[], maxChars: number): AdrEntity[] {
250
249
  .filter((value): value is AdrEntity => value !== null);
251
250
  }
252
251
 
253
- function parseModuleEntities(raw: JsonObject[], maxChars: number): ModuleEntity[] {
252
+ function parseModuleEntities(raw: JsonObject[]): ModuleEntity[] {
254
253
  return raw
255
254
  .map((item) => {
256
255
  const id = asString(item.id);
@@ -263,7 +262,7 @@ function parseModuleEntities(raw: JsonObject[], maxChars: number): ModuleEntity[
263
262
  const summary = asString(item.summary);
264
263
  const exportedSymbols = asString(item.exported_symbols);
265
264
  const updatedAt = asString(item.updated_at);
266
- const text = clampText(`${modulePath}\n${name}\n${summary}\n${exportedSymbols}`, maxChars);
265
+ const text = `${modulePath}\n${name}\n${summary}\n${exportedSymbols}`;
267
266
 
268
267
  return {
269
268
  id,
@@ -282,7 +281,7 @@ function parseModuleEntities(raw: JsonObject[], maxChars: number): ModuleEntity[
282
281
  .filter((value): value is ModuleEntity => value !== null);
283
282
  }
284
283
 
285
- function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string>, maxChars: number): ChunkEntity[] {
284
+ export function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string>): ChunkEntity[] {
286
285
  return raw
287
286
  .map((item) => {
288
287
  const id = asString(item.id);
@@ -298,7 +297,7 @@ function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string>
298
297
  const body = asString(item.body);
299
298
  const updatedAt = asString(item.updated_at);
300
299
  const checksum = asString(item.checksum, hashText(body));
301
- const text = clampText(`${filePath}\n${name}\n${sig}\n${description}\n${body.slice(0, CHUNK_BODY_PREVIEW_CHARS)}`, maxChars);
300
+ const text = `${filePath}\n${name}\n${sig}\n${description}\n${body}`;
302
301
 
303
302
  return {
304
303
  id,
@@ -317,7 +316,7 @@ function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string>
317
316
  .filter((value): value is ChunkEntity => value !== null);
318
317
  }
319
318
 
320
- function parseProjectEntities(raw: JsonObject[], maxChars: number): ProjectEntity[] {
319
+ function parseProjectEntities(raw: JsonObject[]): ProjectEntity[] {
321
320
  return raw
322
321
  .map((item) => {
323
322
  const id = asString(item.id);
@@ -332,10 +331,7 @@ function parseProjectEntities(raw: JsonObject[], maxChars: number): ProjectEntit
332
331
  const targetFramework = asString(item.target_framework);
333
332
  const summary = asString(item.summary);
334
333
  const updatedAt = asString(item.updated_at);
335
- const text = clampText(
336
- `${projectPath}\n${name}\n${kind}\n${language}\n${targetFramework}\n${summary}`,
337
- maxChars
338
- );
334
+ const text = `${projectPath}\n${name}\n${kind}\n${language}\n${targetFramework}\n${summary}`;
339
335
 
340
336
  return {
341
337
  id,
@@ -416,29 +412,39 @@ async function main(): Promise<void> {
416
412
  fs.mkdirSync(EMBEDDINGS_DIR, { recursive: true });
417
413
  fs.mkdirSync(MODEL_CACHE_DIR, { recursive: true });
418
414
 
419
- const modelId = (process.env.CORTEX_EMBED_MODEL ?? DEFAULT_MODEL_ID).trim() || DEFAULT_MODEL_ID;
420
- const maxChars = Number(process.env.CORTEX_EMBED_MAX_CHARS ?? DEFAULT_MAX_TEXT_CHARS);
421
- const maxTextChars = Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : DEFAULT_MAX_TEXT_CHARS;
415
+ const modelId = resolveModelId();
422
416
 
423
- const documents = parseFileEntities(readJsonl(path.join(CACHE_DIR, "documents.jsonl")), maxTextChars);
424
- const rules = parseRuleEntities(readJsonl(path.join(CACHE_DIR, "entities.rule.jsonl")), maxTextChars);
425
- const adrs = parseAdrEntities(readJsonl(path.join(CACHE_DIR, "entities.adr.jsonl")), maxTextChars);
426
- const modules = parseModuleEntities(readJsonl(path.join(CACHE_DIR, "entities.module.jsonl")), maxTextChars);
427
- const projects = parseProjectEntities(readJsonl(path.join(CACHE_DIR, "entities.project.jsonl")), maxTextChars);
417
+ const documents = parseFileEntities(readJsonl(path.join(CACHE_DIR, "documents.jsonl")));
418
+ const rules = parseRuleEntities(readJsonl(path.join(CACHE_DIR, "entities.rule.jsonl")));
419
+ const adrs = parseAdrEntities(readJsonl(path.join(CACHE_DIR, "entities.adr.jsonl")));
420
+ const modules = parseModuleEntities(readJsonl(path.join(CACHE_DIR, "entities.module.jsonl")));
421
+ const projects = parseProjectEntities(readJsonl(path.join(CACHE_DIR, "entities.project.jsonl")));
428
422
 
429
423
  // Build file path lookup for chunk embedding text (reuse already-parsed documents)
430
424
  const filePathById = new Map<string, string>();
431
425
  for (const doc of documents) {
432
426
  filePathById.set(doc.id, doc.path);
433
427
  }
434
- const chunks = parseChunkEntities(readJsonl(path.join(CACHE_DIR, "entities.chunk.jsonl")), filePathById, maxTextChars);
428
+ const chunks = parseChunkEntities(readJsonl(path.join(CACHE_DIR, "entities.chunk.jsonl")), filePathById);
435
429
 
436
430
  const entities: SearchEntity[] = [...documents, ...rules, ...adrs, ...modules, ...projects, ...chunks].sort((a, b) => a.id.localeCompare(b.id));
437
431
 
438
432
  const existing = parseExistingEmbeddings(readJsonl(EMBEDDINGS_PATH), modelId);
439
433
 
440
434
  env.cacheDir = MODEL_CACHE_DIR;
441
- const extractor = await pipeline("feature-extraction", modelId);
435
+ // Optional intra-op thread cap so co-located embedders (e.g. parallel CI
436
+ // jobs or eval containers) do not oversubscribe shared cores. Unset =
437
+ // library default (all cores), which is correct for a single instance.
438
+ const threadsRaw = Number(process.env.CORTEX_EMBED_THREADS);
439
+ const sessionOptions =
440
+ Number.isFinite(threadsRaw) && threadsRaw >= 1
441
+ ? { session_options: { intraOpNumThreads: Math.floor(threadsRaw), interOpNumThreads: 1 } }
442
+ : undefined;
443
+ const extractor = await pipeline(
444
+ "feature-extraction",
445
+ modelId,
446
+ sessionOptions as Parameters<typeof pipeline>[2]
447
+ );
442
448
 
443
449
  let reused = 0;
444
450
  let embedded = 0;
@@ -531,7 +537,10 @@ async function main(): Promise<void> {
531
537
  console.log(`[embed] manifest ${EMBEDDINGS_MANIFEST_PATH}`);
532
538
  }
533
539
 
534
- main().catch((error) => {
535
- process.stderr.write(`${error instanceof Error ? error.message : "Embedding error"}\n`);
536
- process.exit(1);
537
- });
540
+ const isMain = process.argv[1] ? pathToFileURL(process.argv[1]).href === import.meta.url : false;
541
+ if (isMain) {
542
+ main().catch((error) => {
543
+ process.stderr.write(`${error instanceof Error ? error.message : "Embedding error"}\n`);
544
+ process.exit(1);
545
+ });
546
+ }
@@ -98,9 +98,12 @@ export const PATHS = {
98
98
  transformsConfigRelations: path.join(CACHE_DIR, "relations.transforms_config.jsonl")
99
99
  };
100
100
 
101
+ // Graph degree mostly measures how many rules constrain an entity (docs are
102
+ // hubs, leaf code is not), so it gets low weight. Tuned together with the
103
+ // midrank-percentile graph_score in searchResults.ts.
101
104
  export const DEFAULT_RANKING: RankingWeights = {
102
- semantic: 0.4,
103
- graph: 0.25,
105
+ semantic: 0.55,
106
+ graph: 0.1,
104
107
  trust: 0.2,
105
108
  recency: 0.15
106
109
  };
@@ -33,6 +33,38 @@ export function buildSearchResults(params: {
33
33
  recencyScorer: (isoDate: string) => number;
34
34
  legacyDataAccessBooster: (entity: SearchEntity, queryTokens: string[], queryPhrase: string) => number;
35
35
  }): Record<string, unknown>[] {
36
+ // Graph score = midrank percentile of relation degree within the entity's
37
+ // own type. The previous min(1, degree/4) saturated at degree >= 4, which
38
+ // nearly every entity exceeds, making the graph weight a constant. Per-type
39
+ // percentiles discriminate by connectivity while staying type-neutral
40
+ // (every type averages ~0.5), so hub-heavy types like rules cannot drown
41
+ // out leaf code files or doc sections.
42
+ const sortedDegreesByType = new Map<string, number[]>();
43
+ for (const entity of params.candidates) {
44
+ const degree = params.degreeByEntity.get(entity.id) ?? 0;
45
+ const list = sortedDegreesByType.get(entity.entity_type);
46
+ if (list) {
47
+ list.push(degree);
48
+ } else {
49
+ sortedDegreesByType.set(entity.entity_type, [degree]);
50
+ }
51
+ }
52
+ for (const list of sortedDegreesByType.values()) {
53
+ list.sort((a, b) => a - b);
54
+ }
55
+ const midrankPercentile = (sorted: number[], value: number): number => {
56
+ let lo = 0;
57
+ let hi = sorted.length;
58
+ while (lo < hi) {
59
+ const mid = (lo + hi) >> 1;
60
+ if (sorted[mid] < value) lo = mid + 1;
61
+ else hi = mid;
62
+ }
63
+ let upper = lo;
64
+ while (upper < sorted.length && sorted[upper] === value) upper += 1;
65
+ return sorted.length > 0 ? (lo + upper) / (2 * sorted.length) : 0;
66
+ };
67
+
36
68
  const rawResults = params.candidates
37
69
  .map((entity) => {
38
70
  const lexicalSemantic = params.semanticScorer(params.queryTokens, params.queryPhrase, entity.text);
@@ -49,7 +81,8 @@ export function buildSearchResults(params: {
49
81
 
50
82
  const semantic =
51
83
  vectorSemantic > 0 ? vectorSemantic * 0.75 + lexicalSemantic * 0.25 : lexicalSemantic;
52
- const graphScore = Math.min(1, (params.degreeByEntity.get(entity.id) ?? 0) / 4);
84
+ const degree = params.degreeByEntity.get(entity.id) ?? 0;
85
+ const graphScore = midrankPercentile(sortedDegreesByType.get(entity.entity_type) ?? [], degree);
53
86
  const trustScore = Math.max(0, Math.min(1, entity.trust_level / 100));
54
87
  const dateScore = params.recencyScorer(entity.updated_at);
55
88
 
@@ -0,0 +1,72 @@
1
+ import test from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { parseFileEntities, parseChunkEntities, resolveModelId, DEFAULT_MODEL_ID } from "../dist/embed.js";
4
+
5
+ function fileRecord(content) {
6
+ return {
7
+ id: "file-1",
8
+ path: "src/big.ts",
9
+ content,
10
+ excerpt: "excerpt",
11
+ updated_at: "2026-01-01T00:00:00Z",
12
+ checksum: "fixed-checksum"
13
+ };
14
+ }
15
+
16
+ function chunkRecord(body) {
17
+ return {
18
+ id: "chunk-1",
19
+ file_id: "file-1",
20
+ name: "bigFunction",
21
+ signature: "function bigFunction()",
22
+ description: "",
23
+ body,
24
+ updated_at: "2026-01-01T00:00:00Z",
25
+ checksum: "fixed-checksum"
26
+ };
27
+ }
28
+
29
+ test("file embedding text is uncapped beyond the old 7000-char limit", () => {
30
+ const base = "x".repeat(8000);
31
+ const variant = `${"x".repeat(7500)}CHANGED${"x".repeat(493)}`;
32
+
33
+ const [a] = parseFileEntities([fileRecord(base)]);
34
+ const [b] = parseFileEntities([fileRecord(variant)]);
35
+
36
+ assert.ok(a.text.length > 7000);
37
+ assert.ok(a.text.includes(base));
38
+ assert.notEqual(a.signature, b.signature);
39
+ });
40
+
41
+ test("chunk embedding text includes the full body beyond the old 2000-char preview", () => {
42
+ const filePathById = new Map([["file-1", "src/big.ts"]]);
43
+ const base = "y".repeat(3000);
44
+ const variant = `${"y".repeat(2500)}CHANGED${"y".repeat(493)}`;
45
+
46
+ const [a] = parseChunkEntities([chunkRecord(base)], filePathById);
47
+ const [b] = parseChunkEntities([chunkRecord(variant)], filePathById);
48
+
49
+ assert.ok(a.text.includes(base));
50
+ assert.notEqual(a.signature, b.signature);
51
+ });
52
+
53
+ test("default embedding model is jina code model and env override still works", () => {
54
+ const saved = process.env.CORTEX_EMBED_MODEL;
55
+ try {
56
+ delete process.env.CORTEX_EMBED_MODEL;
57
+ assert.equal(DEFAULT_MODEL_ID, "jinaai/jina-embeddings-v2-base-code");
58
+ assert.equal(resolveModelId(), DEFAULT_MODEL_ID);
59
+
60
+ process.env.CORTEX_EMBED_MODEL = "Xenova/all-MiniLM-L6-v2";
61
+ assert.equal(resolveModelId(), "Xenova/all-MiniLM-L6-v2");
62
+
63
+ process.env.CORTEX_EMBED_MODEL = " ";
64
+ assert.equal(resolveModelId(), DEFAULT_MODEL_ID);
65
+ } finally {
66
+ if (saved === undefined) {
67
+ delete process.env.CORTEX_EMBED_MODEL;
68
+ } else {
69
+ process.env.CORTEX_EMBED_MODEL = saved;
70
+ }
71
+ }
72
+ });
@@ -0,0 +1,132 @@
1
+ import test from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { buildSearchResults } from "../dist/searchResults.js";
4
+
5
+ function makeEntity(id, entityType, overrides = {}) {
6
+ return {
7
+ id,
8
+ entity_type: entityType,
9
+ kind: entityType === "Chunk" ? "function" : "DOC",
10
+ label: id,
11
+ path: `src/${id}.ts`,
12
+ text: `text for ${id}`,
13
+ status: "active",
14
+ source_of_truth: false,
15
+ trust_level: 50,
16
+ updated_at: "2026-01-01T00:00:00Z",
17
+ snippet: "",
18
+ matched_rules: [],
19
+ ...overrides
20
+ };
21
+ }
22
+
23
+ function runSearch(candidates, degrees) {
24
+ return buildSearchResults({
25
+ candidates,
26
+ degreeByEntity: new Map(degrees),
27
+ queryTokens: ["query"],
28
+ queryPhrase: "query",
29
+ ranking: { semantic: 0.55, graph: 0.1, trust: 0.2, recency: 0.15 },
30
+ includeScores: true,
31
+ includeMatchedRules: false,
32
+ includeContent: false,
33
+ queryVector: null,
34
+ embeddingVectors: new Map(),
35
+ topK: 100,
36
+ minLexicalRelevance: 0,
37
+ minVectorRelevance: 0,
38
+ semanticScorer: () => 0.5,
39
+ vectorScorer: () => 0,
40
+ recencyScorer: () => 0,
41
+ legacyDataAccessBooster: () => 0
42
+ });
43
+ }
44
+
45
+ function graphScoreById(results) {
46
+ return new Map(results.map((result) => [result.id, result.graph_score]));
47
+ }
48
+
49
+ test("graph score gives midrank to ties: a mass of degree-1 chunks lands at 0.5", () => {
50
+ const candidates = [1, 2, 3, 4].map((n) => makeEntity(`chunk-${n}`, "Chunk"));
51
+ const degrees = candidates.map((entity) => [entity.id, 1]);
52
+
53
+ const scores = graphScoreById(runSearch(candidates, degrees));
54
+
55
+ for (const entity of candidates) {
56
+ assert.equal(scores.get(entity.id), 0.5);
57
+ }
58
+ });
59
+
60
+ test("graph score for a single-entity type is 0.5", () => {
61
+ const candidates = [makeEntity("only-file", "File")];
62
+
63
+ const scores = graphScoreById(runSearch(candidates, [["only-file", 7]]));
64
+
65
+ assert.equal(scores.get("only-file"), 0.5);
66
+ });
67
+
68
+ test("graph score is monotonic in degree within a type", () => {
69
+ const candidates = [
70
+ makeEntity("low", "Chunk"),
71
+ makeEntity("mid", "Chunk"),
72
+ makeEntity("high", "Chunk"),
73
+ makeEntity("top", "Chunk")
74
+ ];
75
+ const degrees = [
76
+ ["low", 0],
77
+ ["mid", 2],
78
+ ["high", 5],
79
+ ["top", 9]
80
+ ];
81
+
82
+ const scores = graphScoreById(runSearch(candidates, degrees));
83
+
84
+ assert.ok(scores.get("low") < scores.get("mid"));
85
+ assert.ok(scores.get("mid") < scores.get("high"));
86
+ assert.ok(scores.get("high") < scores.get("top"));
87
+ });
88
+
89
+ test("graph score percentiles are isolated per type: hub types cannot drown out leaf code", () => {
90
+ const chunks = [
91
+ makeEntity("leaf-a", "Chunk"),
92
+ makeEntity("leaf-b", "Chunk")
93
+ ];
94
+ const hubRules = [
95
+ makeEntity("rule-hub-1", "Rule", { trust_level: 95 }),
96
+ makeEntity("rule-hub-2", "Rule", { trust_level: 95 })
97
+ ];
98
+ const degrees = [
99
+ ["leaf-a", 1],
100
+ ["leaf-b", 3],
101
+ ["rule-hub-1", 200],
102
+ ["rule-hub-2", 400]
103
+ ];
104
+
105
+ const scores = graphScoreById(runSearch([...chunks, ...hubRules], degrees));
106
+
107
+ // Chunk percentiles are computed against chunks only: degree 1 is the lower
108
+ // half and degree 3 the upper half, despite rule degrees being 100x larger.
109
+ assert.equal(scores.get("leaf-a"), 0.25);
110
+ assert.equal(scores.get("leaf-b"), 0.75);
111
+ assert.equal(scores.get("rule-hub-1"), 0.25);
112
+ assert.equal(scores.get("rule-hub-2"), 0.75);
113
+ });
114
+
115
+ test("graph score never saturates to a shared constant for common degrees", () => {
116
+ // The old min(1, degree/4) mapped every degree >= 4 to 1.0.
117
+ const candidates = [
118
+ makeEntity("deg-4", "Chunk"),
119
+ makeEntity("deg-8", "Chunk"),
120
+ makeEntity("deg-40", "Chunk")
121
+ ];
122
+ const degrees = [
123
+ ["deg-4", 4],
124
+ ["deg-8", 8],
125
+ ["deg-40", 40]
126
+ ];
127
+
128
+ const scores = graphScoreById(runSearch(candidates, degrees));
129
+ const unique = new Set(scores.values());
130
+
131
+ assert.equal(unique.size, 3);
132
+ });
@@ -21,6 +21,7 @@ let parseJavaCode = null;
21
21
  let parseRubyCode = null;
22
22
  let parseBashCode = null;
23
23
  let parseVb6Code = null;
24
+ let parseMarkdownCode = null;
24
25
  let isVbNetParserAvailable = () => false;
25
26
  let isCSharpParserAvailable = () => false;
26
27
  let isCppParserAvailable = () => false;
@@ -83,6 +84,9 @@ async function loadOptionalParsers() {
83
84
  }),
84
85
  import("./parsers/vb6.mjs").then((module) => {
85
86
  parseVb6Code = module.parseCode;
87
+ }),
88
+ import("./parsers/markdown.mjs").then((module) => {
89
+ parseMarkdownCode = module.parseCode;
86
90
  })
87
91
  ];
88
92
 
@@ -164,7 +168,7 @@ const LEGACY_DOTNET_METADATA_EXTENSIONS = new Set([
164
168
  ]);
165
169
 
166
170
  const PROJECT_DEFINITION_EXTENSIONS = new Set([".sln", ".vbproj", ".csproj", ".fsproj", ".vcxproj"]);
167
- const STRUCTURED_NON_CODE_CHUNK_EXTENSIONS = new Set([".config", ".resx", ".settings"]);
171
+ const STRUCTURED_NON_CODE_CHUNK_EXTENSIONS = new Set([".config", ".resx", ".settings", ".md", ".mdx"]);
168
172
 
169
173
  const CODE_FILE_EXTENSIONS = new Set([
170
174
  ".ts",
@@ -298,6 +302,22 @@ const CHUNK_PARSERS = new Map([
298
302
  isAvailable: () => typeof parseSqlCode === "function"
299
303
  }
300
304
  ],
305
+ [
306
+ ".md",
307
+ {
308
+ language: "markdown",
309
+ parse: (...args) => parseMarkdownCode(...args),
310
+ isAvailable: () => typeof parseMarkdownCode === "function"
311
+ }
312
+ ],
313
+ [
314
+ ".mdx",
315
+ {
316
+ language: "markdown",
317
+ parse: (...args) => parseMarkdownCode(...args),
318
+ isAvailable: () => typeof parseMarkdownCode === "function"
319
+ }
320
+ ],
301
321
  [
302
322
  ".config",
303
323
  {
@@ -0,0 +1,63 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Markdown parser for Cortex.
4
+ * Chunks documents into heading-bounded sections (H1-H3). Headings inside
5
+ * fenced code blocks are ignored. Sections longer than the window threshold
6
+ * are split with overlap downstream by splitChunkIntoWindows in ingest.
7
+ */
8
+
9
+ const HEADING_PATTERN = /^(#{1,3})\s+(.+?)\s*#*\s*$/;
10
+ const FENCE_PATTERN = /^(```|~~~)/;
11
+
12
+ export function parseCode(code, filePath, language = "markdown") {
13
+ const lines = code.split(/\r?\n/);
14
+ const sections = [];
15
+ let current = null;
16
+ let inFence = false;
17
+
18
+ const pushCurrent = (endIndex) => {
19
+ if (!current) {
20
+ return;
21
+ }
22
+ const body = lines.slice(current.startIndex, endIndex).join("\n").trimEnd();
23
+ const contentLines = lines.slice(current.startIndex + (current.heading ? 1 : 0), endIndex);
24
+ const hasContent = contentLines.some((line) => line.trim().length > 0);
25
+ if (body.trim().length > 0 && hasContent) {
26
+ sections.push({
27
+ name: current.name,
28
+ kind: "section",
29
+ signature: current.heading ?? current.name,
30
+ body,
31
+ startLine: current.startIndex + 1,
32
+ endLine: Math.max(current.startIndex + 1, endIndex),
33
+ language,
34
+ calls: [],
35
+ imports: []
36
+ });
37
+ }
38
+ current = null;
39
+ };
40
+
41
+ for (let i = 0; i < lines.length; i += 1) {
42
+ const line = lines[i];
43
+ if (FENCE_PATTERN.test(line.trim())) {
44
+ inFence = !inFence;
45
+ continue;
46
+ }
47
+ if (inFence) {
48
+ continue;
49
+ }
50
+ const match = line.match(HEADING_PATTERN);
51
+ if (!match) {
52
+ if (!current) {
53
+ current = { name: "preamble", heading: null, startIndex: i };
54
+ }
55
+ continue;
56
+ }
57
+ pushCurrent(i);
58
+ current = { name: match[2].trim(), heading: line.trim(), startIndex: i };
59
+ }
60
+ pushCurrent(lines.length);
61
+
62
+ return { chunks: sections, errors: [] };
63
+ }