@danielblomma/cortex-mcp 2.0.19 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/bin/cortex.mjs +6 -3
- package/package.json +2 -2
- package/scaffold/.context/config.yaml +5 -2
- package/scaffold/mcp/package-lock.json +3 -3
- package/scaffold/mcp/src/embed.ts +32 -35
- package/scaffold/mcp/src/paths.ts +5 -2
- package/scaffold/mcp/src/searchResults.ts +34 -1
- package/scaffold/mcp/tests/embed-entities.test.mjs +72 -0
- package/scaffold/mcp/tests/search-graph-score.test.mjs +132 -0
- package/scaffold/scripts/ingest.mjs +21 -1
- package/scaffold/scripts/parsers/markdown.mjs +63 -0
package/README.md
CHANGED
|
@@ -87,6 +87,32 @@ The result is an assistant that behaves as if it already knows your codebase, be
|
|
|
87
87
|
npm i -g @danielblomma/cortex-mcp
|
|
88
88
|
```
|
|
89
89
|
|
|
90
|
+
## Upgrading
|
|
91
|
+
|
|
92
|
+
To upgrade an already-scaffolded project to a new Cortex version:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
npm i -g @danielblomma/cortex-mcp
|
|
96
|
+
cortex init --force # re-scaffolds .context/mcp + .context/scripts
|
|
97
|
+
cortex bootstrap
|
|
98
|
+
cortex update
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`cortex init --force` preserves per-project files: `.context/config.yaml`,
|
|
102
|
+
`.context/rules.yaml`, and your notes/decisions.
|
|
103
|
+
|
|
104
|
+
Version-specific notes (see [CHANGELOG.md](CHANGELOG.md) for details):
|
|
105
|
+
|
|
106
|
+
- **2.1.0**: the default embedding model changed, so the first
|
|
107
|
+
`cortex update` after upgrading triggers a full re-embed automatically
|
|
108
|
+
(~2 min per 1000 entities plus a one-time model download). The
|
|
109
|
+
`CORTEX_EMBED_MAX_CHARS` env var is removed and silently ignored.
|
|
110
|
+
Existing projects keep their old ranking weights in `config.yaml`; the
|
|
111
|
+
recommended block is now `semantic: 0.55, graph: 0.10, trust: 0.20,
|
|
112
|
+
recency: 0.15`. After re-embedding, restart the MCP server, and note
|
|
113
|
+
that the first search after a re-embed can hit a stale embeddings
|
|
114
|
+
cache — re-run the query.
|
|
115
|
+
|
|
90
116
|
## Quick Start
|
|
91
117
|
|
|
92
118
|
From the repository you want to index:
|
package/bin/cortex.mjs
CHANGED
|
@@ -191,7 +191,7 @@ function ensureScaffoldExists() {
|
|
|
191
191
|
|
|
192
192
|
// Files that should never be overwritten if they already exist in the target.
|
|
193
193
|
// These contain user-specific configuration that would be lost on re-init.
|
|
194
|
-
const PRESERVE_FILES = new Set(["config.yaml", "enterprise.yml", "enterprise.yaml", "CLAUDE.md", "AGENTS.md"]);
|
|
194
|
+
const PRESERVE_FILES = new Set(["config.yaml", "rules.yaml", "enterprise.yml", "enterprise.yaml", "CLAUDE.md", "AGENTS.md"]);
|
|
195
195
|
const DEFAULT_SOURCE_PATHS = [
|
|
196
196
|
"src",
|
|
197
197
|
"docs",
|
|
@@ -410,9 +410,12 @@ function buildInitialConfig(targetDir) {
|
|
|
410
410
|
" - RULE",
|
|
411
411
|
" - CODE",
|
|
412
412
|
" - WIKI",
|
|
413
|
+
"# graph weight is low because graph degree mostly measures how many rules",
|
|
414
|
+
"# constrain an entity (docs are hubs, leaf code is not). Tuned as a pair with",
|
|
415
|
+
"# the midrank-percentile graph_score.",
|
|
413
416
|
"ranking:",
|
|
414
|
-
" semantic: 0.
|
|
415
|
-
" graph: 0.
|
|
417
|
+
" semantic: 0.55",
|
|
418
|
+
" graph: 0.10",
|
|
416
419
|
" trust: 0.20",
|
|
417
420
|
" recency: 0.15",
|
|
418
421
|
"runtime:",
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danielblomma/cortex-mcp",
|
|
3
3
|
"mcpName": "io.github.DanielBlomma/cortex",
|
|
4
|
-
"version": "2.0
|
|
4
|
+
"version": "2.1.0",
|
|
5
5
|
"description": "Local, repo-scoped context platform for coding assistants. Semantic search, graph relationships, and architectural rule context.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"author": "Daniel Blomma",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
],
|
|
51
51
|
"scripts": {
|
|
52
52
|
"pretest": "test -d scaffold/scripts/parsers/node_modules || npm --prefix scaffold/scripts/parsers install --no-fund --no-update-notifier --silent",
|
|
53
|
-
"test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/javascript-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs",
|
|
53
|
+
"test": "node tests/context-regressions.test.mjs && node --test tests/ingest-units.test.mjs tests/javascript-parser.test.mjs tests/markdown-parser.test.mjs tests/sql-parser.test.mjs tests/config-parser.test.mjs tests/resources-parser.test.mjs tests/vbnet-parser.test.mjs tests/cpp-parser.test.mjs tests/dashboard.test.mjs tests/init-config.test.mjs tests/init-agents.test.mjs tests/multi-level.test.mjs tests/no-legacy-paths.test.mjs tests/tree-sitter-error-reporting.test.mjs tests/tree-sitter-body-cap.test.mjs tests/tree-sitter-exported.test.mjs tests/tree-sitter-robustness.test.mjs",
|
|
54
54
|
"release:sync-version": "node scripts/sync-release-version.mjs",
|
|
55
55
|
"release:check-version-sync": "node scripts/sync-release-version.mjs --check",
|
|
56
56
|
"prepublishOnly": "echo 'Ready to publish to npm'"
|
|
@@ -10,9 +10,12 @@ truth_order:
|
|
|
10
10
|
- RULE
|
|
11
11
|
- CODE
|
|
12
12
|
- WIKI
|
|
13
|
+
# graph weight is low because graph degree mostly measures how many rules
|
|
14
|
+
# constrain an entity (docs are hubs, leaf code is not). Tuned as a pair with
|
|
15
|
+
# the midrank-percentile graph_score.
|
|
13
16
|
ranking:
|
|
14
|
-
semantic: 0.
|
|
15
|
-
graph: 0.
|
|
17
|
+
semantic: 0.55
|
|
18
|
+
graph: 0.10
|
|
16
19
|
trust: 0.20
|
|
17
20
|
recency: 0.15
|
|
18
21
|
runtime:
|
|
@@ -2165,9 +2165,9 @@
|
|
|
2165
2165
|
}
|
|
2166
2166
|
},
|
|
2167
2167
|
"node_modules/qs": {
|
|
2168
|
-
"version": "6.15.
|
|
2169
|
-
"resolved": "https://registry.npmjs.org/qs/-/qs-6.15.
|
|
2170
|
-
"integrity": "sha512-
|
|
2168
|
+
"version": "6.15.2",
|
|
2169
|
+
"resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz",
|
|
2170
|
+
"integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==",
|
|
2171
2171
|
"license": "BSD-3-Clause",
|
|
2172
2172
|
"dependencies": {
|
|
2173
2173
|
"side-channel": "^1.1.0"
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { pathToFileURL } from "node:url";
|
|
4
5
|
import { env, pipeline } from "@huggingface/transformers";
|
|
5
6
|
import { readJsonl, asString, asNumber, asBoolean } from "./jsonl.js";
|
|
6
7
|
import { CACHE_DIR, PATHS } from "./paths.js";
|
|
@@ -11,9 +12,11 @@ const EMBEDDINGS_MANIFEST_PATH = PATHS.embeddingsManifest;
|
|
|
11
12
|
const MODEL_CACHE_DIR = PATHS.embeddingsModelCache;
|
|
12
13
|
const EMBEDDINGS_DIR = path.dirname(EMBEDDINGS_PATH);
|
|
13
14
|
|
|
14
|
-
const DEFAULT_MODEL_ID = "
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
export const DEFAULT_MODEL_ID = "jinaai/jina-embeddings-v2-base-code";
|
|
16
|
+
|
|
17
|
+
export function resolveModelId(): string {
|
|
18
|
+
return (process.env.CORTEX_EMBED_MODEL ?? DEFAULT_MODEL_ID).trim() || DEFAULT_MODEL_ID;
|
|
19
|
+
}
|
|
17
20
|
|
|
18
21
|
type FileEntity = {
|
|
19
22
|
id: string;
|
|
@@ -134,10 +137,6 @@ function normalizeText(value: string): string {
|
|
|
134
137
|
return value.replace(/\s+/g, " ").trim();
|
|
135
138
|
}
|
|
136
139
|
|
|
137
|
-
function clampText(value: string, maxChars: number): string {
|
|
138
|
-
return value.slice(0, maxChars);
|
|
139
|
-
}
|
|
140
|
-
|
|
141
140
|
function writeJsonl(filePath: string, records: EmbeddingRecord[]): void {
|
|
142
141
|
const body = records.map((record) => JSON.stringify(record)).join("\n");
|
|
143
142
|
fs.writeFileSync(filePath, body ? `${body}\n` : "", "utf8");
|
|
@@ -157,7 +156,7 @@ function ensureRequiredFiles(): void {
|
|
|
157
156
|
}
|
|
158
157
|
}
|
|
159
158
|
|
|
160
|
-
function parseFileEntities(raw: JsonObject[]
|
|
159
|
+
export function parseFileEntities(raw: JsonObject[]): FileEntity[] {
|
|
161
160
|
return raw
|
|
162
161
|
.map((item) => {
|
|
163
162
|
const id = asString(item.id);
|
|
@@ -170,7 +169,7 @@ function parseFileEntities(raw: JsonObject[], maxChars: number): FileEntity[] {
|
|
|
170
169
|
const excerpt = asString(item.excerpt);
|
|
171
170
|
const updatedAt = asString(item.updated_at);
|
|
172
171
|
const checksum = asString(item.checksum, hashText(content));
|
|
173
|
-
const text =
|
|
172
|
+
const text = `${filePath}\n${excerpt}\n${content}`;
|
|
174
173
|
|
|
175
174
|
return {
|
|
176
175
|
id,
|
|
@@ -189,7 +188,7 @@ function parseFileEntities(raw: JsonObject[], maxChars: number): FileEntity[] {
|
|
|
189
188
|
.filter((value): value is FileEntity => value !== null);
|
|
190
189
|
}
|
|
191
190
|
|
|
192
|
-
function parseRuleEntities(raw: JsonObject[]
|
|
191
|
+
function parseRuleEntities(raw: JsonObject[]): RuleEntity[] {
|
|
193
192
|
return raw
|
|
194
193
|
.map((item) => {
|
|
195
194
|
const id = asString(item.id);
|
|
@@ -200,7 +199,7 @@ function parseRuleEntities(raw: JsonObject[], maxChars: number): RuleEntity[] {
|
|
|
200
199
|
const title = asString(item.title, id);
|
|
201
200
|
const body = asString(item.body);
|
|
202
201
|
const updatedAt = asString(item.updated_at, "");
|
|
203
|
-
const text =
|
|
202
|
+
const text = `${title}\n${body}`;
|
|
204
203
|
|
|
205
204
|
return {
|
|
206
205
|
id,
|
|
@@ -219,7 +218,7 @@ function parseRuleEntities(raw: JsonObject[], maxChars: number): RuleEntity[] {
|
|
|
219
218
|
.filter((value): value is RuleEntity => value !== null);
|
|
220
219
|
}
|
|
221
220
|
|
|
222
|
-
function parseAdrEntities(raw: JsonObject[]
|
|
221
|
+
function parseAdrEntities(raw: JsonObject[]): AdrEntity[] {
|
|
223
222
|
return raw
|
|
224
223
|
.map((item) => {
|
|
225
224
|
const id = asString(item.id);
|
|
@@ -231,7 +230,7 @@ function parseAdrEntities(raw: JsonObject[], maxChars: number): AdrEntity[] {
|
|
|
231
230
|
const body = asString(item.body);
|
|
232
231
|
const adrPath = asString(item.path);
|
|
233
232
|
const decisionDate = asString(item.decision_date, "");
|
|
234
|
-
const text =
|
|
233
|
+
const text = `${adrPath}\n${title}\n${body}`;
|
|
235
234
|
|
|
236
235
|
return {
|
|
237
236
|
id,
|
|
@@ -250,7 +249,7 @@ function parseAdrEntities(raw: JsonObject[], maxChars: number): AdrEntity[] {
|
|
|
250
249
|
.filter((value): value is AdrEntity => value !== null);
|
|
251
250
|
}
|
|
252
251
|
|
|
253
|
-
function parseModuleEntities(raw: JsonObject[]
|
|
252
|
+
function parseModuleEntities(raw: JsonObject[]): ModuleEntity[] {
|
|
254
253
|
return raw
|
|
255
254
|
.map((item) => {
|
|
256
255
|
const id = asString(item.id);
|
|
@@ -263,7 +262,7 @@ function parseModuleEntities(raw: JsonObject[], maxChars: number): ModuleEntity[
|
|
|
263
262
|
const summary = asString(item.summary);
|
|
264
263
|
const exportedSymbols = asString(item.exported_symbols);
|
|
265
264
|
const updatedAt = asString(item.updated_at);
|
|
266
|
-
const text =
|
|
265
|
+
const text = `${modulePath}\n${name}\n${summary}\n${exportedSymbols}`;
|
|
267
266
|
|
|
268
267
|
return {
|
|
269
268
|
id,
|
|
@@ -282,7 +281,7 @@ function parseModuleEntities(raw: JsonObject[], maxChars: number): ModuleEntity[
|
|
|
282
281
|
.filter((value): value is ModuleEntity => value !== null);
|
|
283
282
|
}
|
|
284
283
|
|
|
285
|
-
function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string
|
|
284
|
+
export function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string>): ChunkEntity[] {
|
|
286
285
|
return raw
|
|
287
286
|
.map((item) => {
|
|
288
287
|
const id = asString(item.id);
|
|
@@ -298,7 +297,7 @@ function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string>
|
|
|
298
297
|
const body = asString(item.body);
|
|
299
298
|
const updatedAt = asString(item.updated_at);
|
|
300
299
|
const checksum = asString(item.checksum, hashText(body));
|
|
301
|
-
const text =
|
|
300
|
+
const text = `${filePath}\n${name}\n${sig}\n${description}\n${body}`;
|
|
302
301
|
|
|
303
302
|
return {
|
|
304
303
|
id,
|
|
@@ -317,7 +316,7 @@ function parseChunkEntities(raw: JsonObject[], filePathById: Map<string, string>
|
|
|
317
316
|
.filter((value): value is ChunkEntity => value !== null);
|
|
318
317
|
}
|
|
319
318
|
|
|
320
|
-
function parseProjectEntities(raw: JsonObject[]
|
|
319
|
+
function parseProjectEntities(raw: JsonObject[]): ProjectEntity[] {
|
|
321
320
|
return raw
|
|
322
321
|
.map((item) => {
|
|
323
322
|
const id = asString(item.id);
|
|
@@ -332,10 +331,7 @@ function parseProjectEntities(raw: JsonObject[], maxChars: number): ProjectEntit
|
|
|
332
331
|
const targetFramework = asString(item.target_framework);
|
|
333
332
|
const summary = asString(item.summary);
|
|
334
333
|
const updatedAt = asString(item.updated_at);
|
|
335
|
-
const text =
|
|
336
|
-
`${projectPath}\n${name}\n${kind}\n${language}\n${targetFramework}\n${summary}`,
|
|
337
|
-
maxChars
|
|
338
|
-
);
|
|
334
|
+
const text = `${projectPath}\n${name}\n${kind}\n${language}\n${targetFramework}\n${summary}`;
|
|
339
335
|
|
|
340
336
|
return {
|
|
341
337
|
id,
|
|
@@ -416,22 +412,20 @@ async function main(): Promise<void> {
|
|
|
416
412
|
fs.mkdirSync(EMBEDDINGS_DIR, { recursive: true });
|
|
417
413
|
fs.mkdirSync(MODEL_CACHE_DIR, { recursive: true });
|
|
418
414
|
|
|
419
|
-
const modelId = (
|
|
420
|
-
const maxChars = Number(process.env.CORTEX_EMBED_MAX_CHARS ?? DEFAULT_MAX_TEXT_CHARS);
|
|
421
|
-
const maxTextChars = Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : DEFAULT_MAX_TEXT_CHARS;
|
|
415
|
+
const modelId = resolveModelId();
|
|
422
416
|
|
|
423
|
-
const documents = parseFileEntities(readJsonl(path.join(CACHE_DIR, "documents.jsonl"))
|
|
424
|
-
const rules = parseRuleEntities(readJsonl(path.join(CACHE_DIR, "entities.rule.jsonl"))
|
|
425
|
-
const adrs = parseAdrEntities(readJsonl(path.join(CACHE_DIR, "entities.adr.jsonl"))
|
|
426
|
-
const modules = parseModuleEntities(readJsonl(path.join(CACHE_DIR, "entities.module.jsonl"))
|
|
427
|
-
const projects = parseProjectEntities(readJsonl(path.join(CACHE_DIR, "entities.project.jsonl"))
|
|
417
|
+
const documents = parseFileEntities(readJsonl(path.join(CACHE_DIR, "documents.jsonl")));
|
|
418
|
+
const rules = parseRuleEntities(readJsonl(path.join(CACHE_DIR, "entities.rule.jsonl")));
|
|
419
|
+
const adrs = parseAdrEntities(readJsonl(path.join(CACHE_DIR, "entities.adr.jsonl")));
|
|
420
|
+
const modules = parseModuleEntities(readJsonl(path.join(CACHE_DIR, "entities.module.jsonl")));
|
|
421
|
+
const projects = parseProjectEntities(readJsonl(path.join(CACHE_DIR, "entities.project.jsonl")));
|
|
428
422
|
|
|
429
423
|
// Build file path lookup for chunk embedding text (reuse already-parsed documents)
|
|
430
424
|
const filePathById = new Map<string, string>();
|
|
431
425
|
for (const doc of documents) {
|
|
432
426
|
filePathById.set(doc.id, doc.path);
|
|
433
427
|
}
|
|
434
|
-
const chunks = parseChunkEntities(readJsonl(path.join(CACHE_DIR, "entities.chunk.jsonl")), filePathById
|
|
428
|
+
const chunks = parseChunkEntities(readJsonl(path.join(CACHE_DIR, "entities.chunk.jsonl")), filePathById);
|
|
435
429
|
|
|
436
430
|
const entities: SearchEntity[] = [...documents, ...rules, ...adrs, ...modules, ...projects, ...chunks].sort((a, b) => a.id.localeCompare(b.id));
|
|
437
431
|
|
|
@@ -531,7 +525,10 @@ async function main(): Promise<void> {
|
|
|
531
525
|
console.log(`[embed] manifest ${EMBEDDINGS_MANIFEST_PATH}`);
|
|
532
526
|
}
|
|
533
527
|
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
});
|
|
528
|
+
const isMain = process.argv[1] ? pathToFileURL(process.argv[1]).href === import.meta.url : false;
|
|
529
|
+
if (isMain) {
|
|
530
|
+
main().catch((error) => {
|
|
531
|
+
process.stderr.write(`${error instanceof Error ? error.message : "Embedding error"}\n`);
|
|
532
|
+
process.exit(1);
|
|
533
|
+
});
|
|
534
|
+
}
|
|
@@ -98,9 +98,12 @@ export const PATHS = {
|
|
|
98
98
|
transformsConfigRelations: path.join(CACHE_DIR, "relations.transforms_config.jsonl")
|
|
99
99
|
};
|
|
100
100
|
|
|
101
|
+
// Graph degree mostly measures how many rules constrain an entity (docs are
|
|
102
|
+
// hubs, leaf code is not), so it gets low weight. Tuned together with the
|
|
103
|
+
// midrank-percentile graph_score in searchResults.ts.
|
|
101
104
|
export const DEFAULT_RANKING: RankingWeights = {
|
|
102
|
-
semantic: 0.
|
|
103
|
-
graph: 0.
|
|
105
|
+
semantic: 0.55,
|
|
106
|
+
graph: 0.1,
|
|
104
107
|
trust: 0.2,
|
|
105
108
|
recency: 0.15
|
|
106
109
|
};
|
|
@@ -33,6 +33,38 @@ export function buildSearchResults(params: {
|
|
|
33
33
|
recencyScorer: (isoDate: string) => number;
|
|
34
34
|
legacyDataAccessBooster: (entity: SearchEntity, queryTokens: string[], queryPhrase: string) => number;
|
|
35
35
|
}): Record<string, unknown>[] {
|
|
36
|
+
// Graph score = midrank percentile of relation degree within the entity's
|
|
37
|
+
// own type. The previous min(1, degree/4) saturated at degree >= 4, which
|
|
38
|
+
// nearly every entity exceeds, making the graph weight a constant. Per-type
|
|
39
|
+
// percentiles discriminate by connectivity while staying type-neutral
|
|
40
|
+
// (every type averages ~0.5), so hub-heavy types like rules cannot drown
|
|
41
|
+
// out leaf code files or doc sections.
|
|
42
|
+
const sortedDegreesByType = new Map<string, number[]>();
|
|
43
|
+
for (const entity of params.candidates) {
|
|
44
|
+
const degree = params.degreeByEntity.get(entity.id) ?? 0;
|
|
45
|
+
const list = sortedDegreesByType.get(entity.entity_type);
|
|
46
|
+
if (list) {
|
|
47
|
+
list.push(degree);
|
|
48
|
+
} else {
|
|
49
|
+
sortedDegreesByType.set(entity.entity_type, [degree]);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
for (const list of sortedDegreesByType.values()) {
|
|
53
|
+
list.sort((a, b) => a - b);
|
|
54
|
+
}
|
|
55
|
+
const midrankPercentile = (sorted: number[], value: number): number => {
|
|
56
|
+
let lo = 0;
|
|
57
|
+
let hi = sorted.length;
|
|
58
|
+
while (lo < hi) {
|
|
59
|
+
const mid = (lo + hi) >> 1;
|
|
60
|
+
if (sorted[mid] < value) lo = mid + 1;
|
|
61
|
+
else hi = mid;
|
|
62
|
+
}
|
|
63
|
+
let upper = lo;
|
|
64
|
+
while (upper < sorted.length && sorted[upper] === value) upper += 1;
|
|
65
|
+
return sorted.length > 0 ? (lo + upper) / (2 * sorted.length) : 0;
|
|
66
|
+
};
|
|
67
|
+
|
|
36
68
|
const rawResults = params.candidates
|
|
37
69
|
.map((entity) => {
|
|
38
70
|
const lexicalSemantic = params.semanticScorer(params.queryTokens, params.queryPhrase, entity.text);
|
|
@@ -49,7 +81,8 @@ export function buildSearchResults(params: {
|
|
|
49
81
|
|
|
50
82
|
const semantic =
|
|
51
83
|
vectorSemantic > 0 ? vectorSemantic * 0.75 + lexicalSemantic * 0.25 : lexicalSemantic;
|
|
52
|
-
const
|
|
84
|
+
const degree = params.degreeByEntity.get(entity.id) ?? 0;
|
|
85
|
+
const graphScore = midrankPercentile(sortedDegreesByType.get(entity.entity_type) ?? [], degree);
|
|
53
86
|
const trustScore = Math.max(0, Math.min(1, entity.trust_level / 100));
|
|
54
87
|
const dateScore = params.recencyScorer(entity.updated_at);
|
|
55
88
|
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { parseFileEntities, parseChunkEntities, resolveModelId, DEFAULT_MODEL_ID } from "../dist/embed.js";
|
|
4
|
+
|
|
5
|
+
function fileRecord(content) {
|
|
6
|
+
return {
|
|
7
|
+
id: "file-1",
|
|
8
|
+
path: "src/big.ts",
|
|
9
|
+
content,
|
|
10
|
+
excerpt: "excerpt",
|
|
11
|
+
updated_at: "2026-01-01T00:00:00Z",
|
|
12
|
+
checksum: "fixed-checksum"
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function chunkRecord(body) {
|
|
17
|
+
return {
|
|
18
|
+
id: "chunk-1",
|
|
19
|
+
file_id: "file-1",
|
|
20
|
+
name: "bigFunction",
|
|
21
|
+
signature: "function bigFunction()",
|
|
22
|
+
description: "",
|
|
23
|
+
body,
|
|
24
|
+
updated_at: "2026-01-01T00:00:00Z",
|
|
25
|
+
checksum: "fixed-checksum"
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
test("file embedding text is uncapped beyond the old 7000-char limit", () => {
|
|
30
|
+
const base = "x".repeat(8000);
|
|
31
|
+
const variant = `${"x".repeat(7500)}CHANGED${"x".repeat(493)}`;
|
|
32
|
+
|
|
33
|
+
const [a] = parseFileEntities([fileRecord(base)]);
|
|
34
|
+
const [b] = parseFileEntities([fileRecord(variant)]);
|
|
35
|
+
|
|
36
|
+
assert.ok(a.text.length > 7000);
|
|
37
|
+
assert.ok(a.text.includes(base));
|
|
38
|
+
assert.notEqual(a.signature, b.signature);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("chunk embedding text includes the full body beyond the old 2000-char preview", () => {
|
|
42
|
+
const filePathById = new Map([["file-1", "src/big.ts"]]);
|
|
43
|
+
const base = "y".repeat(3000);
|
|
44
|
+
const variant = `${"y".repeat(2500)}CHANGED${"y".repeat(493)}`;
|
|
45
|
+
|
|
46
|
+
const [a] = parseChunkEntities([chunkRecord(base)], filePathById);
|
|
47
|
+
const [b] = parseChunkEntities([chunkRecord(variant)], filePathById);
|
|
48
|
+
|
|
49
|
+
assert.ok(a.text.includes(base));
|
|
50
|
+
assert.notEqual(a.signature, b.signature);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("default embedding model is jina code model and env override still works", () => {
|
|
54
|
+
const saved = process.env.CORTEX_EMBED_MODEL;
|
|
55
|
+
try {
|
|
56
|
+
delete process.env.CORTEX_EMBED_MODEL;
|
|
57
|
+
assert.equal(DEFAULT_MODEL_ID, "jinaai/jina-embeddings-v2-base-code");
|
|
58
|
+
assert.equal(resolveModelId(), DEFAULT_MODEL_ID);
|
|
59
|
+
|
|
60
|
+
process.env.CORTEX_EMBED_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
61
|
+
assert.equal(resolveModelId(), "Xenova/all-MiniLM-L6-v2");
|
|
62
|
+
|
|
63
|
+
process.env.CORTEX_EMBED_MODEL = " ";
|
|
64
|
+
assert.equal(resolveModelId(), DEFAULT_MODEL_ID);
|
|
65
|
+
} finally {
|
|
66
|
+
if (saved === undefined) {
|
|
67
|
+
delete process.env.CORTEX_EMBED_MODEL;
|
|
68
|
+
} else {
|
|
69
|
+
process.env.CORTEX_EMBED_MODEL = saved;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
});
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { buildSearchResults } from "../dist/searchResults.js";
|
|
4
|
+
|
|
5
|
+
function makeEntity(id, entityType, overrides = {}) {
|
|
6
|
+
return {
|
|
7
|
+
id,
|
|
8
|
+
entity_type: entityType,
|
|
9
|
+
kind: entityType === "Chunk" ? "function" : "DOC",
|
|
10
|
+
label: id,
|
|
11
|
+
path: `src/${id}.ts`,
|
|
12
|
+
text: `text for ${id}`,
|
|
13
|
+
status: "active",
|
|
14
|
+
source_of_truth: false,
|
|
15
|
+
trust_level: 50,
|
|
16
|
+
updated_at: "2026-01-01T00:00:00Z",
|
|
17
|
+
snippet: "",
|
|
18
|
+
matched_rules: [],
|
|
19
|
+
...overrides
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function runSearch(candidates, degrees) {
|
|
24
|
+
return buildSearchResults({
|
|
25
|
+
candidates,
|
|
26
|
+
degreeByEntity: new Map(degrees),
|
|
27
|
+
queryTokens: ["query"],
|
|
28
|
+
queryPhrase: "query",
|
|
29
|
+
ranking: { semantic: 0.55, graph: 0.1, trust: 0.2, recency: 0.15 },
|
|
30
|
+
includeScores: true,
|
|
31
|
+
includeMatchedRules: false,
|
|
32
|
+
includeContent: false,
|
|
33
|
+
queryVector: null,
|
|
34
|
+
embeddingVectors: new Map(),
|
|
35
|
+
topK: 100,
|
|
36
|
+
minLexicalRelevance: 0,
|
|
37
|
+
minVectorRelevance: 0,
|
|
38
|
+
semanticScorer: () => 0.5,
|
|
39
|
+
vectorScorer: () => 0,
|
|
40
|
+
recencyScorer: () => 0,
|
|
41
|
+
legacyDataAccessBooster: () => 0
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function graphScoreById(results) {
|
|
46
|
+
return new Map(results.map((result) => [result.id, result.graph_score]));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
test("graph score gives midrank to ties: a mass of degree-1 chunks lands at 0.5", () => {
|
|
50
|
+
const candidates = [1, 2, 3, 4].map((n) => makeEntity(`chunk-${n}`, "Chunk"));
|
|
51
|
+
const degrees = candidates.map((entity) => [entity.id, 1]);
|
|
52
|
+
|
|
53
|
+
const scores = graphScoreById(runSearch(candidates, degrees));
|
|
54
|
+
|
|
55
|
+
for (const entity of candidates) {
|
|
56
|
+
assert.equal(scores.get(entity.id), 0.5);
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("graph score for a single-entity type is 0.5", () => {
|
|
61
|
+
const candidates = [makeEntity("only-file", "File")];
|
|
62
|
+
|
|
63
|
+
const scores = graphScoreById(runSearch(candidates, [["only-file", 7]]));
|
|
64
|
+
|
|
65
|
+
assert.equal(scores.get("only-file"), 0.5);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("graph score is monotonic in degree within a type", () => {
|
|
69
|
+
const candidates = [
|
|
70
|
+
makeEntity("low", "Chunk"),
|
|
71
|
+
makeEntity("mid", "Chunk"),
|
|
72
|
+
makeEntity("high", "Chunk"),
|
|
73
|
+
makeEntity("top", "Chunk")
|
|
74
|
+
];
|
|
75
|
+
const degrees = [
|
|
76
|
+
["low", 0],
|
|
77
|
+
["mid", 2],
|
|
78
|
+
["high", 5],
|
|
79
|
+
["top", 9]
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
const scores = graphScoreById(runSearch(candidates, degrees));
|
|
83
|
+
|
|
84
|
+
assert.ok(scores.get("low") < scores.get("mid"));
|
|
85
|
+
assert.ok(scores.get("mid") < scores.get("high"));
|
|
86
|
+
assert.ok(scores.get("high") < scores.get("top"));
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("graph score percentiles are isolated per type: hub types cannot drown out leaf code", () => {
|
|
90
|
+
const chunks = [
|
|
91
|
+
makeEntity("leaf-a", "Chunk"),
|
|
92
|
+
makeEntity("leaf-b", "Chunk")
|
|
93
|
+
];
|
|
94
|
+
const hubRules = [
|
|
95
|
+
makeEntity("rule-hub-1", "Rule", { trust_level: 95 }),
|
|
96
|
+
makeEntity("rule-hub-2", "Rule", { trust_level: 95 })
|
|
97
|
+
];
|
|
98
|
+
const degrees = [
|
|
99
|
+
["leaf-a", 1],
|
|
100
|
+
["leaf-b", 3],
|
|
101
|
+
["rule-hub-1", 200],
|
|
102
|
+
["rule-hub-2", 400]
|
|
103
|
+
];
|
|
104
|
+
|
|
105
|
+
const scores = graphScoreById(runSearch([...chunks, ...hubRules], degrees));
|
|
106
|
+
|
|
107
|
+
// Chunk percentiles are computed against chunks only: degree 1 is the lower
|
|
108
|
+
// half and degree 3 the upper half, despite rule degrees being 100x larger.
|
|
109
|
+
assert.equal(scores.get("leaf-a"), 0.25);
|
|
110
|
+
assert.equal(scores.get("leaf-b"), 0.75);
|
|
111
|
+
assert.equal(scores.get("rule-hub-1"), 0.25);
|
|
112
|
+
assert.equal(scores.get("rule-hub-2"), 0.75);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test("graph score never saturates to a shared constant for common degrees", () => {
|
|
116
|
+
// The old min(1, degree/4) mapped every degree >= 4 to 1.0.
|
|
117
|
+
const candidates = [
|
|
118
|
+
makeEntity("deg-4", "Chunk"),
|
|
119
|
+
makeEntity("deg-8", "Chunk"),
|
|
120
|
+
makeEntity("deg-40", "Chunk")
|
|
121
|
+
];
|
|
122
|
+
const degrees = [
|
|
123
|
+
["deg-4", 4],
|
|
124
|
+
["deg-8", 8],
|
|
125
|
+
["deg-40", 40]
|
|
126
|
+
];
|
|
127
|
+
|
|
128
|
+
const scores = graphScoreById(runSearch(candidates, degrees));
|
|
129
|
+
const unique = new Set(scores.values());
|
|
130
|
+
|
|
131
|
+
assert.equal(unique.size, 3);
|
|
132
|
+
});
|
|
@@ -21,6 +21,7 @@ let parseJavaCode = null;
|
|
|
21
21
|
let parseRubyCode = null;
|
|
22
22
|
let parseBashCode = null;
|
|
23
23
|
let parseVb6Code = null;
|
|
24
|
+
let parseMarkdownCode = null;
|
|
24
25
|
let isVbNetParserAvailable = () => false;
|
|
25
26
|
let isCSharpParserAvailable = () => false;
|
|
26
27
|
let isCppParserAvailable = () => false;
|
|
@@ -83,6 +84,9 @@ async function loadOptionalParsers() {
|
|
|
83
84
|
}),
|
|
84
85
|
import("./parsers/vb6.mjs").then((module) => {
|
|
85
86
|
parseVb6Code = module.parseCode;
|
|
87
|
+
}),
|
|
88
|
+
import("./parsers/markdown.mjs").then((module) => {
|
|
89
|
+
parseMarkdownCode = module.parseCode;
|
|
86
90
|
})
|
|
87
91
|
];
|
|
88
92
|
|
|
@@ -164,7 +168,7 @@ const LEGACY_DOTNET_METADATA_EXTENSIONS = new Set([
|
|
|
164
168
|
]);
|
|
165
169
|
|
|
166
170
|
const PROJECT_DEFINITION_EXTENSIONS = new Set([".sln", ".vbproj", ".csproj", ".fsproj", ".vcxproj"]);
|
|
167
|
-
const STRUCTURED_NON_CODE_CHUNK_EXTENSIONS = new Set([".config", ".resx", ".settings"]);
|
|
171
|
+
const STRUCTURED_NON_CODE_CHUNK_EXTENSIONS = new Set([".config", ".resx", ".settings", ".md", ".mdx"]);
|
|
168
172
|
|
|
169
173
|
const CODE_FILE_EXTENSIONS = new Set([
|
|
170
174
|
".ts",
|
|
@@ -298,6 +302,22 @@ const CHUNK_PARSERS = new Map([
|
|
|
298
302
|
isAvailable: () => typeof parseSqlCode === "function"
|
|
299
303
|
}
|
|
300
304
|
],
|
|
305
|
+
[
|
|
306
|
+
".md",
|
|
307
|
+
{
|
|
308
|
+
language: "markdown",
|
|
309
|
+
parse: (...args) => parseMarkdownCode(...args),
|
|
310
|
+
isAvailable: () => typeof parseMarkdownCode === "function"
|
|
311
|
+
}
|
|
312
|
+
],
|
|
313
|
+
[
|
|
314
|
+
".mdx",
|
|
315
|
+
{
|
|
316
|
+
language: "markdown",
|
|
317
|
+
parse: (...args) => parseMarkdownCode(...args),
|
|
318
|
+
isAvailable: () => typeof parseMarkdownCode === "function"
|
|
319
|
+
}
|
|
320
|
+
],
|
|
301
321
|
[
|
|
302
322
|
".config",
|
|
303
323
|
{
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Markdown parser for Cortex.
|
|
4
|
+
* Chunks documents into heading-bounded sections (H1-H3). Headings inside
|
|
5
|
+
* fenced code blocks are ignored. Sections longer than the window threshold
|
|
6
|
+
* are split with overlap downstream by splitChunkIntoWindows in ingest.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const HEADING_PATTERN = /^(#{1,3})\s+(.+?)\s*#*\s*$/;
|
|
10
|
+
const FENCE_PATTERN = /^(```|~~~)/;
|
|
11
|
+
|
|
12
|
+
export function parseCode(code, filePath, language = "markdown") {
|
|
13
|
+
const lines = code.split(/\r?\n/);
|
|
14
|
+
const sections = [];
|
|
15
|
+
let current = null;
|
|
16
|
+
let inFence = false;
|
|
17
|
+
|
|
18
|
+
const pushCurrent = (endIndex) => {
|
|
19
|
+
if (!current) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const body = lines.slice(current.startIndex, endIndex).join("\n").trimEnd();
|
|
23
|
+
const contentLines = lines.slice(current.startIndex + (current.heading ? 1 : 0), endIndex);
|
|
24
|
+
const hasContent = contentLines.some((line) => line.trim().length > 0);
|
|
25
|
+
if (body.trim().length > 0 && hasContent) {
|
|
26
|
+
sections.push({
|
|
27
|
+
name: current.name,
|
|
28
|
+
kind: "section",
|
|
29
|
+
signature: current.heading ?? current.name,
|
|
30
|
+
body,
|
|
31
|
+
startLine: current.startIndex + 1,
|
|
32
|
+
endLine: Math.max(current.startIndex + 1, endIndex),
|
|
33
|
+
language,
|
|
34
|
+
calls: [],
|
|
35
|
+
imports: []
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
current = null;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
42
|
+
const line = lines[i];
|
|
43
|
+
if (FENCE_PATTERN.test(line.trim())) {
|
|
44
|
+
inFence = !inFence;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
if (inFence) {
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const match = line.match(HEADING_PATTERN);
|
|
51
|
+
if (!match) {
|
|
52
|
+
if (!current) {
|
|
53
|
+
current = { name: "preamble", heading: null, startIndex: i };
|
|
54
|
+
}
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
pushCurrent(i);
|
|
58
|
+
current = { name: match[2].trim(), heading: line.trim(), startIndex: i };
|
|
59
|
+
}
|
|
60
|
+
pushCurrent(lines.length);
|
|
61
|
+
|
|
62
|
+
return { chunks: sections, errors: [] };
|
|
63
|
+
}
|