@danielblomma/cortex-mcp 2.2.5 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -0
- package/bin/cortex.mjs +31 -167
- package/package.json +2 -2
- package/scaffold/mcp/package-lock.json +24 -21
- package/scaffold/mcp/src/cli/query.ts +36 -2
- package/scaffold/mcp/src/embed.ts +211 -10
- package/scaffold/mcp/src/enterprise/index.ts +1 -1
- package/scaffold/mcp/src/enterprise/reviews/changed-files.ts +34 -0
- package/scaffold/mcp/src/enterprise/reviews/pattern-context.ts +231 -0
- package/scaffold/mcp/src/enterprise/tools/enterprise.ts +49 -36
- package/scaffold/mcp/src/paths.ts +3 -5
- package/scaffold/mcp/src/patternEvidence.ts +347 -0
- package/scaffold/mcp/src/search.ts +35 -9
- package/scaffold/mcp/src/searchCore.ts +254 -12
- package/scaffold/mcp/src/searchResults.ts +96 -9
- package/scaffold/mcp/src/types.ts +7 -0
- package/scaffold/mcp/tests/changed-files.test.mjs +41 -0
- package/scaffold/mcp/tests/embed-entities.test.mjs +109 -1
- package/scaffold/mcp/tests/enterprise-pattern-context.test.mjs +322 -0
- package/scaffold/mcp/tests/paths.test.mjs +11 -3
- package/scaffold/mcp/tests/pattern-evidence.test.mjs +321 -0
- package/scaffold/mcp/tests/query-cli.test.mjs +73 -1
- package/scaffold/mcp/tests/search-graph-score.test.mjs +167 -0
- package/scaffold/scripts/dashboard.mjs +29 -18
- package/scaffold/scripts/doctor.sh +26 -8
- package/scaffold/scripts/ingest.mjs +37 -4
- package/scaffold/scripts/status.sh +13 -6
|
@@ -20,6 +20,19 @@ const CONFIG_ENVIRONMENT_TOKENS = [
|
|
|
20
20
|
];
|
|
21
21
|
|
|
22
22
|
const QUERY_TOKEN_EXPANSIONS: Record<string, string[]> = {
|
|
23
|
+
dashboard: ["status"],
|
|
24
|
+
embed: ["embedding", "embeddings"],
|
|
25
|
+
embedding: ["embed", "embeddings"],
|
|
26
|
+
embeddings: ["embed", "embedding"],
|
|
27
|
+
git: ["githooks"],
|
|
28
|
+
hook: ["hooks", "githooks"],
|
|
29
|
+
hooks: ["hook", "githooks"],
|
|
30
|
+
import: ["imports"],
|
|
31
|
+
imports: ["import"],
|
|
32
|
+
javascript: ["js"],
|
|
33
|
+
parser: ["parsers"],
|
|
34
|
+
parsers: ["parser"],
|
|
35
|
+
status: ["dashboard"],
|
|
23
36
|
semantisk: ["semantic"],
|
|
24
37
|
sökning: ["search"],
|
|
25
38
|
sokning: ["search"],
|
|
@@ -30,15 +43,105 @@ const QUERY_TOKEN_EXPANSIONS: Record<string, string[]> = {
|
|
|
30
43
|
avvikelse: ["deviation"]
|
|
31
44
|
};
|
|
32
45
|
|
|
46
|
+
const STRUCTURAL_QUERY_STOP_TOKENS = new Set([
|
|
47
|
+
"about",
|
|
48
|
+
"after",
|
|
49
|
+
"and",
|
|
50
|
+
"are",
|
|
51
|
+
"before",
|
|
52
|
+
"between",
|
|
53
|
+
"does",
|
|
54
|
+
"from",
|
|
55
|
+
"happen",
|
|
56
|
+
"happens",
|
|
57
|
+
"has",
|
|
58
|
+
"how",
|
|
59
|
+
"into",
|
|
60
|
+
"the",
|
|
61
|
+
"their",
|
|
62
|
+
"then",
|
|
63
|
+
"through",
|
|
64
|
+
"when",
|
|
65
|
+
"where",
|
|
66
|
+
"while",
|
|
67
|
+
"with",
|
|
68
|
+
"what",
|
|
69
|
+
"which",
|
|
70
|
+
"who",
|
|
71
|
+
"why"
|
|
72
|
+
]);
|
|
73
|
+
|
|
74
|
+
const IDENTIFIER_COMPOUND_TOKENS = new Set(["csharp", "fsharp", "graphql", "javascript", "typescript", "vbnet"]);
|
|
75
|
+
const LANGUAGE_TOKEN_GROUPS = [
|
|
76
|
+
["javascript", "typescript", "js", "ts"],
|
|
77
|
+
["csharp", "cs"],
|
|
78
|
+
["vbnet", "vb"],
|
|
79
|
+
["fsharp", "fs"],
|
|
80
|
+
["java"],
|
|
81
|
+
["cpp"],
|
|
82
|
+
["python", "py"],
|
|
83
|
+
["rust", "rs"],
|
|
84
|
+
["ruby", "rb"],
|
|
85
|
+
["go"],
|
|
86
|
+
["php"],
|
|
87
|
+
["swift"],
|
|
88
|
+
["kotlin", "kt"],
|
|
89
|
+
["sql"]
|
|
90
|
+
];
|
|
91
|
+
|
|
33
92
|
export function normalizeText(value: string): string {
|
|
34
93
|
return value.normalize("NFKC").toLowerCase();
|
|
35
94
|
}
|
|
36
95
|
|
|
96
|
+
function normalizeLanguageAliases(value: string): string {
|
|
97
|
+
return value
|
|
98
|
+
.replace(/(^|[^\p{L}\p{N}])c#(?=$|[^\p{L}\p{N}])/giu, "$1 csharp ")
|
|
99
|
+
.replace(/(^|[^\p{L}\p{N}])f#(?=$|[^\p{L}\p{N}])/giu, "$1 fsharp ")
|
|
100
|
+
.replace(/(^|[^\p{L}\p{N}])vb\.net(?=$|[^\p{L}\p{N}])/giu, "$1 vbnet ");
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function splitIdentifierPart(part: string): string[] {
|
|
104
|
+
const split = part
|
|
105
|
+
.replace(/([\p{Ll}\p{N}])([\p{Lu}])/gu, "$1 $2")
|
|
106
|
+
.replace(/([\p{Lu}]+)([\p{Lu}][\p{Ll}])/gu, "$1 $2")
|
|
107
|
+
.split(/\s+/u)
|
|
108
|
+
.filter(Boolean);
|
|
109
|
+
if (split.length <= 1) {
|
|
110
|
+
return [];
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const merged: string[] = [];
|
|
114
|
+
for (let index = 0; index < split.length; index += 1) {
|
|
115
|
+
const current = normalizeText(split[index]);
|
|
116
|
+
const next = index + 1 < split.length ? normalizeText(split[index + 1]) : "";
|
|
117
|
+
const compound = `${current}${next}`;
|
|
118
|
+
if (next && IDENTIFIER_COMPOUND_TOKENS.has(compound)) {
|
|
119
|
+
merged.push(compound);
|
|
120
|
+
index += 1;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
merged.push(current);
|
|
124
|
+
}
|
|
125
|
+
return merged;
|
|
126
|
+
}
|
|
127
|
+
|
|
37
128
|
export function tokenize(value: string): string[] {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
129
|
+
const tokens: string[] = [];
|
|
130
|
+
for (const rawPart of normalizeLanguageAliases(value).normalize("NFKC").split(/[^\p{L}\p{N}]+/gu)) {
|
|
131
|
+
const part = rawPart.trim();
|
|
132
|
+
if (part.length < 2) {
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const variants = [part, ...splitIdentifierPart(part)];
|
|
137
|
+
for (const variant of variants) {
|
|
138
|
+
const token = normalizeText(variant).trim();
|
|
139
|
+
if (token.length >= 2) {
|
|
140
|
+
tokens.push(token);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return tokens;
|
|
42
145
|
}
|
|
43
146
|
|
|
44
147
|
export function expandQueryTokens(tokens: string[]): string[] {
|
|
@@ -55,18 +158,69 @@ export function expandQueryTokens(tokens: string[]): string[] {
|
|
|
55
158
|
return Array.from(expanded);
|
|
56
159
|
}
|
|
57
160
|
|
|
58
|
-
function
|
|
161
|
+
function relatedQueryTokens(token: string): string[] {
|
|
162
|
+
const related = new Set<string>(QUERY_TOKEN_EXPANSIONS[token] ?? []);
|
|
163
|
+
for (const [source, aliases] of Object.entries(QUERY_TOKEN_EXPANSIONS)) {
|
|
164
|
+
if (!aliases.includes(token)) {
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
related.add(source);
|
|
168
|
+
for (const alias of aliases) {
|
|
169
|
+
related.add(alias);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return Array.from(related);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function queryTokenGroups(queryTokens: string[]): string[][] {
|
|
176
|
+
const inputTokens = new Set(queryTokens);
|
|
177
|
+
const visited = new Set<string>();
|
|
178
|
+
const groups: string[][] = [];
|
|
179
|
+
|
|
180
|
+
for (const token of queryTokens) {
|
|
181
|
+
if (visited.has(token)) {
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const group = new Set<string>();
|
|
186
|
+
const queue = [token];
|
|
187
|
+
for (let index = 0; index < queue.length; index += 1) {
|
|
188
|
+
const current = queue[index];
|
|
189
|
+
if (group.has(current)) {
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
group.add(current);
|
|
193
|
+
|
|
194
|
+
for (const related of relatedQueryTokens(current)) {
|
|
195
|
+
group.add(related);
|
|
196
|
+
if (inputTokens.has(related) && !visited.has(related)) {
|
|
197
|
+
queue.push(related);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
for (const member of group) {
|
|
203
|
+
if (inputTokens.has(member)) {
|
|
204
|
+
visited.add(member);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
groups.push(Array.from(group));
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return groups;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function daysSince(isoDate: string, referenceTimeMs: number): number {
|
|
59
214
|
const timestamp = Date.parse(isoDate);
|
|
60
215
|
if (Number.isNaN(timestamp)) {
|
|
61
216
|
return 3650;
|
|
62
217
|
}
|
|
63
218
|
|
|
64
|
-
|
|
65
|
-
return Math.max(0, (now - timestamp) / (1000 * 60 * 60 * 24));
|
|
219
|
+
return Math.max(0, (referenceTimeMs - timestamp) / (1000 * 60 * 60 * 24));
|
|
66
220
|
}
|
|
67
221
|
|
|
68
|
-
export function recencyScore(isoDate: string): number {
|
|
69
|
-
const days = daysSince(isoDate);
|
|
222
|
+
export function recencyScore(isoDate: string, referenceTimeMs = Date.now()): number {
|
|
223
|
+
const days = daysSince(isoDate, referenceTimeMs);
|
|
70
224
|
return 1 / (1 + days / 30);
|
|
71
225
|
}
|
|
72
226
|
|
|
@@ -80,14 +234,15 @@ export function semanticScore(queryTokens: string[], queryPhrase: string, text:
|
|
|
80
234
|
return 0;
|
|
81
235
|
}
|
|
82
236
|
|
|
237
|
+
const tokenGroups = queryTokenGroups(queryTokens);
|
|
83
238
|
let matched = 0;
|
|
84
|
-
for (const
|
|
85
|
-
if (textTokenSet.has(token)) {
|
|
239
|
+
for (const group of tokenGroups) {
|
|
240
|
+
if (group.some((token) => textTokenSet.has(token))) {
|
|
86
241
|
matched += 1;
|
|
87
242
|
}
|
|
88
243
|
}
|
|
89
244
|
|
|
90
|
-
const overlap = matched /
|
|
245
|
+
const overlap = matched / tokenGroups.length;
|
|
91
246
|
if (overlap <= 0) {
|
|
92
247
|
return 0;
|
|
93
248
|
}
|
|
@@ -97,6 +252,93 @@ export function semanticScore(queryTokens: string[], queryPhrase: string, text:
|
|
|
97
252
|
return Math.min(1, overlap * 0.85 + phraseBonus);
|
|
98
253
|
}
|
|
99
254
|
|
|
255
|
+
function structuralQueryTokens(queryTokens: string[]): string[] {
|
|
256
|
+
return queryTokens.filter((token) => token.length >= 3 && !STRUCTURAL_QUERY_STOP_TOKENS.has(token));
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function tokenSet(value: string): Set<string> {
|
|
260
|
+
return new Set(tokenize(value));
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function overlapCount(queryTokens: string[], targetTokens: Set<string>): number {
|
|
264
|
+
let matched = 0;
|
|
265
|
+
for (const token of queryTokens) {
|
|
266
|
+
if (targetTokens.has(token)) {
|
|
267
|
+
matched += 1;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return matched;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function basenameWithoutExtension(pathValue: string): string {
|
|
274
|
+
const base = pathValue.split(/[\\/]/u).pop() ?? pathValue;
|
|
275
|
+
const extensionIndex = base.lastIndexOf(".");
|
|
276
|
+
return extensionIndex > 0 ? base.slice(0, extensionIndex) : base;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function languageGroupsForTokens(tokens: Set<string>): Set<number> {
|
|
280
|
+
const groups = new Set<number>();
|
|
281
|
+
for (let index = 0; index < LANGUAGE_TOKEN_GROUPS.length; index += 1) {
|
|
282
|
+
if (LANGUAGE_TOKEN_GROUPS[index].some((token) => tokens.has(token))) {
|
|
283
|
+
groups.add(index);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return groups;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function hasLanguageGroupOverlap(a: Set<number>, b: Set<number>): boolean {
|
|
290
|
+
for (const value of a) {
|
|
291
|
+
if (b.has(value)) {
|
|
292
|
+
return true;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return false;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export function structuralSearchBoost(entity: SearchEntity, queryTokens: string[], queryPhrase: string): number {
|
|
299
|
+
const structuralTokens = structuralQueryTokens(queryTokens);
|
|
300
|
+
if (structuralTokens.length === 0) {
|
|
301
|
+
return 0;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const pathTokens = tokenSet(entity.path);
|
|
305
|
+
const labelTokens = tokenSet(entity.label);
|
|
306
|
+
const pathMatches = overlapCount(structuralTokens, pathTokens);
|
|
307
|
+
const basenameMatches = overlapCount(structuralTokens, tokenSet(basenameWithoutExtension(entity.path)));
|
|
308
|
+
const labelMatches = overlapCount(structuralTokens, labelTokens);
|
|
309
|
+
const kindMatches = overlapCount(structuralTokens, tokenSet(entity.kind));
|
|
310
|
+
|
|
311
|
+
let boost = 0;
|
|
312
|
+
boost += Math.min(0.1, pathMatches * 0.035);
|
|
313
|
+
boost += Math.min(0.08, basenameMatches * 0.06);
|
|
314
|
+
boost += Math.min(0.08, labelMatches * 0.04);
|
|
315
|
+
boost += Math.min(0.03, kindMatches * 0.02);
|
|
316
|
+
|
|
317
|
+
const normalizedPath = normalizeText(entity.path);
|
|
318
|
+
const normalizedLabel = normalizeText(entity.label);
|
|
319
|
+
const compactPhrase = queryPhrase.replace(/[^\p{L}\p{N}]+/gu, "");
|
|
320
|
+
if (compactPhrase.length >= 8) {
|
|
321
|
+
const compactPath = normalizedPath.replace(/[^\p{L}\p{N}]+/gu, "");
|
|
322
|
+
const compactLabel = normalizedLabel.replace(/[^\p{L}\p{N}]+/gu, "");
|
|
323
|
+
if (compactPath.includes(compactPhrase) || compactLabel.includes(compactPhrase)) {
|
|
324
|
+
boost += 0.04;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const queryLanguageGroups = languageGroupsForTokens(new Set(queryTokens));
|
|
329
|
+
const pathLanguageGroups = languageGroupsForTokens(new Set([...pathTokens, ...labelTokens]));
|
|
330
|
+
if (
|
|
331
|
+
queryLanguageGroups.size > 0 &&
|
|
332
|
+
pathLanguageGroups.size > 0 &&
|
|
333
|
+
pathTokens.has("parsers") &&
|
|
334
|
+
!hasLanguageGroupOverlap(queryLanguageGroups, pathLanguageGroups)
|
|
335
|
+
) {
|
|
336
|
+
boost -= 0.08;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return Math.min(0.2, boost);
|
|
340
|
+
}
|
|
341
|
+
|
|
100
342
|
function queryHasAnyToken(queryTokens: string[], candidates: string[]): boolean {
|
|
101
343
|
return candidates.some((candidate) => queryTokens.includes(candidate));
|
|
102
344
|
}
|
|
@@ -28,13 +28,25 @@ function baseChunkLabel(label: string): string {
|
|
|
28
28
|
return markerIndex === -1 ? label : label.slice(0, markerIndex);
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
export function compareText(a: string, b: string): number {
|
|
32
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function rankedResultKey(ranked: RankedSearchResult): string {
|
|
36
|
+
return `${String(ranked.result.id ?? "")}\u0000${String(ranked.result.path ?? "")}`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function compareRankedResults(a: RankedSearchResult, b: RankedSearchResult): number {
|
|
40
|
+
return b.rankScore - a.rankScore || compareText(rankedResultKey(a), rankedResultKey(b));
|
|
41
|
+
}
|
|
42
|
+
|
|
31
43
|
function pruneRankedResults(bestById: Map<string, RankedSearchResult>, limit: number): void {
|
|
32
44
|
if (bestById.size <= limit) {
|
|
33
45
|
return;
|
|
34
46
|
}
|
|
35
47
|
|
|
36
48
|
const retained = [...bestById.entries()]
|
|
37
|
-
.sort(([, a], [, b]) => b
|
|
49
|
+
.sort(([, a], [, b]) => compareRankedResults(a, b))
|
|
38
50
|
.slice(0, limit);
|
|
39
51
|
bestById.clear();
|
|
40
52
|
for (const [id, result] of retained) {
|
|
@@ -42,6 +54,72 @@ function pruneRankedResults(bestById: Map<string, RankedSearchResult>, limit: nu
|
|
|
42
54
|
}
|
|
43
55
|
}
|
|
44
56
|
|
|
57
|
+
function secondarySignalScale(semantic: number): number {
|
|
58
|
+
return Math.min(1, 0.15 + Math.max(0, semantic) * 0.85);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function isTestEvidenceEntity(entity: SearchEntity): boolean {
|
|
62
|
+
return /(^|\/)(tests?|__tests__)\//u.test(entity.path) || /\.(test|spec)\.[^.]+$/u.test(entity.path);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function testEvidenceBoost(entity: SearchEntity, semantic: number, lexicalSemantic: number): number {
|
|
66
|
+
if (!isTestEvidenceEntity(entity)) {
|
|
67
|
+
return 0;
|
|
68
|
+
}
|
|
69
|
+
if (semantic >= 0.5 && lexicalSemantic >= 0.4) {
|
|
70
|
+
return 0.07;
|
|
71
|
+
}
|
|
72
|
+
if (semantic >= 0.4 && lexicalSemantic >= 0.25) {
|
|
73
|
+
return 0.04;
|
|
74
|
+
}
|
|
75
|
+
return 0;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function resultPathKey(result: SearchResult): string {
|
|
79
|
+
const path = typeof result.path === "string" ? result.path : "";
|
|
80
|
+
return path || String(result.id ?? "");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function selectDiverseResults(rankedResults: RankedSearchResult[], topK: number): RankedSearchResult[] {
|
|
84
|
+
const remaining = rankedResults
|
|
85
|
+
.map((ranked) => ({ ranked }))
|
|
86
|
+
.sort((a, b) => compareRankedResults(a.ranked, b.ranked));
|
|
87
|
+
const selected: RankedSearchResult[] = [];
|
|
88
|
+
const selectedByPath = new Map<string, number>();
|
|
89
|
+
|
|
90
|
+
while (selected.length < topK && remaining.length > 0) {
|
|
91
|
+
let bestIndex = 0;
|
|
92
|
+
let bestAdjustedScore = Number.NEGATIVE_INFINITY;
|
|
93
|
+
let bestRankScore = Number.NEGATIVE_INFINITY;
|
|
94
|
+
let bestKey = "";
|
|
95
|
+
|
|
96
|
+
for (let index = 0; index < remaining.length; index += 1) {
|
|
97
|
+
const candidate = remaining[index].ranked;
|
|
98
|
+
const pathCount = selectedByPath.get(resultPathKey(candidate.result)) ?? 0;
|
|
99
|
+
const samePathPenalty = pathCount === 0 ? 0 : Math.min(0.12, pathCount * 0.04);
|
|
100
|
+
const adjustedScore = candidate.rankScore - samePathPenalty;
|
|
101
|
+
const candidateKey = rankedResultKey(candidate);
|
|
102
|
+
if (
|
|
103
|
+
adjustedScore > bestAdjustedScore ||
|
|
104
|
+
(adjustedScore === bestAdjustedScore && candidate.rankScore > bestRankScore) ||
|
|
105
|
+
(adjustedScore === bestAdjustedScore && candidate.rankScore === bestRankScore && candidateKey < bestKey)
|
|
106
|
+
) {
|
|
107
|
+
bestIndex = index;
|
|
108
|
+
bestAdjustedScore = adjustedScore;
|
|
109
|
+
bestRankScore = candidate.rankScore;
|
|
110
|
+
bestKey = candidateKey;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const [picked] = remaining.splice(bestIndex, 1);
|
|
115
|
+
selected.push(picked.ranked);
|
|
116
|
+
const pathKey = resultPathKey(picked.ranked.result);
|
|
117
|
+
selectedByPath.set(pathKey, (selectedByPath.get(pathKey) ?? 0) + 1);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return selected;
|
|
121
|
+
}
|
|
122
|
+
|
|
45
123
|
function candidatesFrom(source: SearchCandidateSource): Iterable<SearchEntity> {
|
|
46
124
|
return typeof source === "function" ? source() : source;
|
|
47
125
|
}
|
|
@@ -63,6 +141,7 @@ export function buildSearchResultsWithStats(params: {
|
|
|
63
141
|
semanticScorer: (queryTokens: string[], queryPhrase: string, text: string) => number;
|
|
64
142
|
vectorScorer: (a: Float32Array, b: Float32Array) => number;
|
|
65
143
|
recencyScorer: (isoDate: string) => number;
|
|
144
|
+
structuralSearchBooster?: (entity: SearchEntity, queryTokens: string[], queryPhrase: string) => number;
|
|
66
145
|
legacyDataAccessBooster: (entity: SearchEntity, queryTokens: string[], queryPhrase: string) => number;
|
|
67
146
|
}): { results: Record<string, unknown>[]; totalCandidates: number } {
|
|
68
147
|
if (params.topK <= 0) {
|
|
@@ -112,7 +191,8 @@ export function buildSearchResultsWithStats(params: {
|
|
|
112
191
|
};
|
|
113
192
|
|
|
114
193
|
const resultLimit = Math.max(1, params.topK);
|
|
115
|
-
const
|
|
194
|
+
const diversityPoolLimit = Math.max(resultLimit * 3, resultLimit);
|
|
195
|
+
const pruneThreshold = Math.max(diversityPoolLimit * 2, 64);
|
|
116
196
|
const bestById = new Map<string, RankedSearchResult>();
|
|
117
197
|
|
|
118
198
|
for (const entity of candidatesFrom(params.candidates)) {
|
|
@@ -134,12 +214,17 @@ export function buildSearchResultsWithStats(params: {
|
|
|
134
214
|
const graphScore = midrankPercentile(sortedDegreesByType.get(entity.entity_type) ?? [], degree);
|
|
135
215
|
const trustScore = Math.max(0, Math.min(1, entity.trust_level / 100));
|
|
136
216
|
const dateScore = params.recencyScorer(entity.updated_at);
|
|
217
|
+
const structuralBoost = params.structuralSearchBooster?.(entity, params.queryTokens, params.queryPhrase) ?? 0;
|
|
218
|
+
const evidenceBoost = testEvidenceBoost(entity, semantic, lexicalSemantic);
|
|
219
|
+
const secondaryScale = secondarySignalScale(Math.max(semantic, structuralBoost));
|
|
137
220
|
|
|
138
221
|
let score = 0;
|
|
139
222
|
score += params.ranking.semantic * semantic;
|
|
140
|
-
score += params.ranking.graph * graphScore;
|
|
141
|
-
score += params.ranking.trust * trustScore;
|
|
142
|
-
score += params.ranking.recency * dateScore;
|
|
223
|
+
score += params.ranking.graph * graphScore * secondaryScale;
|
|
224
|
+
score += params.ranking.trust * trustScore * secondaryScale;
|
|
225
|
+
score += params.ranking.recency * dateScore * secondaryScale;
|
|
226
|
+
score += structuralBoost;
|
|
227
|
+
score += evidenceBoost;
|
|
143
228
|
score += params.legacyDataAccessBooster(entity, params.queryTokens, params.queryPhrase);
|
|
144
229
|
|
|
145
230
|
if (entity.source_of_truth) {
|
|
@@ -194,13 +279,15 @@ export function buildSearchResultsWithStats(params: {
|
|
|
194
279
|
}
|
|
195
280
|
|
|
196
281
|
if (bestById.size > pruneThreshold) {
|
|
197
|
-
pruneRankedResults(bestById,
|
|
282
|
+
pruneRankedResults(bestById, diversityPoolLimit);
|
|
198
283
|
}
|
|
199
284
|
}
|
|
200
285
|
|
|
201
|
-
const
|
|
202
|
-
.sort(
|
|
203
|
-
.slice(0,
|
|
286
|
+
const rankedResults = [...bestById.values()]
|
|
287
|
+
.sort(compareRankedResults)
|
|
288
|
+
.slice(0, diversityPoolLimit);
|
|
289
|
+
const results = selectDiverseResults(rankedResults, params.topK)
|
|
290
|
+
.sort(compareRankedResults)
|
|
204
291
|
.map((ranked) => ranked.result);
|
|
205
292
|
return { results, totalCandidates };
|
|
206
293
|
}
|
|
@@ -203,6 +203,13 @@ export type RulesParams = {
|
|
|
203
203
|
include_inactive: boolean;
|
|
204
204
|
};
|
|
205
205
|
|
|
206
|
+
export type PatternEvidenceParams = {
|
|
207
|
+
target: string;
|
|
208
|
+
query?: string;
|
|
209
|
+
top_k: number;
|
|
210
|
+
include_deprecated?: boolean;
|
|
211
|
+
};
|
|
212
|
+
|
|
206
213
|
export type ReloadParams = {
|
|
207
214
|
force: boolean;
|
|
208
215
|
};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { execFileSync } from "node:child_process";
|
|
7
|
+
import { resolveChangedReviewFiles } from "../dist/enterprise/reviews/changed-files.js";
|
|
8
|
+
|
|
9
|
+
function git(cwd, args) {
|
|
10
|
+
execFileSync("git", args, { cwd, stdio: "ignore" });
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
test("lists staged and untracked files in a repo without commits", () => {
|
|
14
|
+
const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), "cortex-changed-files-"));
|
|
15
|
+
try {
|
|
16
|
+
git(projectRoot, ["init"]);
|
|
17
|
+
fs.writeFileSync(path.join(projectRoot, ".gitignore"), "ignored.ts\n", "utf8");
|
|
18
|
+
fs.writeFileSync(path.join(projectRoot, "staged.ts"), "export const a = 1;\n", "utf8");
|
|
19
|
+
fs.writeFileSync(path.join(projectRoot, "untracked.ts"), "export const b = 2;\n", "utf8");
|
|
20
|
+
fs.writeFileSync(path.join(projectRoot, "ignored.ts"), "ignored\n", "utf8");
|
|
21
|
+
git(projectRoot, ["add", ".gitignore", "staged.ts"]);
|
|
22
|
+
|
|
23
|
+
assert.deepEqual(resolveChangedReviewFiles(projectRoot), [
|
|
24
|
+
".gitignore",
|
|
25
|
+
"staged.ts",
|
|
26
|
+
"untracked.ts",
|
|
27
|
+
]);
|
|
28
|
+
} finally {
|
|
29
|
+
fs.rmSync(projectRoot, { recursive: true, force: true });
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("returns null outside a git work tree", () => {
|
|
34
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cortex-changed-files-plain-"));
|
|
35
|
+
try {
|
|
36
|
+
fs.writeFileSync(path.join(dir, "a.ts"), "export const a = 1;\n", "utf8");
|
|
37
|
+
assert.equal(resolveChangedReviewFiles(dir), null);
|
|
38
|
+
} finally {
|
|
39
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
40
|
+
}
|
|
41
|
+
});
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
import test from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
COMPACT_FILE_TEXT_STRATEGY,
|
|
5
|
+
COMPACT_FILE_TEXT_TARGET_CHARS,
|
|
6
|
+
COMPACT_FILE_TEXT_THRESHOLD_CHARS,
|
|
7
|
+
DEFAULT_MODEL_ID,
|
|
8
|
+
parseChunkEntities,
|
|
9
|
+
parseFileEntities,
|
|
10
|
+
resolveEmbedTextProfile,
|
|
11
|
+
resolveModelId,
|
|
12
|
+
resolveSignatureProfile
|
|
13
|
+
} from "../dist/embed.js";
|
|
4
14
|
|
|
5
15
|
function fileRecord(content) {
|
|
6
16
|
return {
|
|
@@ -38,6 +48,68 @@ test("file embedding text is uncapped beyond the old 7000-char limit", () => {
|
|
|
38
48
|
assert.notEqual(a.signature, b.signature);
|
|
39
49
|
});
|
|
40
50
|
|
|
51
|
+
test("compact-files leaves small file text unchanged", () => {
|
|
52
|
+
const content = "export function smallApi() { return true; }";
|
|
53
|
+
|
|
54
|
+
const [full] = parseFileEntities([fileRecord(content)]);
|
|
55
|
+
const [compact] = parseFileEntities([fileRecord(content)], { textProfile: "compact-files" });
|
|
56
|
+
|
|
57
|
+
assert.equal(compact.text, full.text);
|
|
58
|
+
assert.equal(compact.signature, full.signature);
|
|
59
|
+
assert.equal(compact.text_profile, "compact-files");
|
|
60
|
+
assert.equal(compact.text_compacted, false);
|
|
61
|
+
assert.equal(compact.text_omitted_chars, 0);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("compact-files compacts only large file entities and preserves semantic anchors", () => {
|
|
65
|
+
const head = `HEAD_MARKER\n${"h".repeat(15000)}`;
|
|
66
|
+
const signalLines = [
|
|
67
|
+
"import { Owner } from './owner';",
|
|
68
|
+
"export interface UserRecord { owner: Owner; }",
|
|
69
|
+
"export function createUserRecord(owner: Owner) { return { owner }; }"
|
|
70
|
+
].join("\n");
|
|
71
|
+
const tail = `${"t".repeat(3000)}\nTAIL_MARKER`;
|
|
72
|
+
const content = `${head}\n${"m".repeat(COMPACT_FILE_TEXT_THRESHOLD_CHARS)}\n${signalLines}\n${tail}`;
|
|
73
|
+
|
|
74
|
+
const [full] = parseFileEntities([fileRecord(content)]);
|
|
75
|
+
const [compact] = parseFileEntities([fileRecord(content)], { textProfile: "compact-files" });
|
|
76
|
+
|
|
77
|
+
assert.equal(compact.text_profile, "compact-files");
|
|
78
|
+
assert.equal(compact.text_compacted, true);
|
|
79
|
+
assert.ok(full.text.length > COMPACT_FILE_TEXT_THRESHOLD_CHARS);
|
|
80
|
+
assert.ok(compact.text.length < full.text.length);
|
|
81
|
+
assert.ok(compact.text.length <= COMPACT_FILE_TEXT_TARGET_CHARS + 5120);
|
|
82
|
+
assert.notEqual(compact.signature, full.signature);
|
|
83
|
+
assert.match(compact.text, new RegExp(COMPACT_FILE_TEXT_STRATEGY));
|
|
84
|
+
assert.match(compact.text, /omitted_chars=/);
|
|
85
|
+
assert.ok(compact.text.includes("src/big.ts"));
|
|
86
|
+
assert.ok(compact.text.includes("excerpt"));
|
|
87
|
+
assert.ok(compact.text.includes("HEAD_MARKER"));
|
|
88
|
+
assert.ok(compact.text.includes("TAIL_MARKER"));
|
|
89
|
+
assert.ok(compact.text.includes("import { Owner } from './owner';"));
|
|
90
|
+
assert.ok(compact.text.includes("export interface UserRecord"));
|
|
91
|
+
assert.ok(compact.text.includes("export function createUserRecord"));
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("compact-files keeps collecting signal anchors after an oversized signal line", () => {
|
|
95
|
+
const hugeSignal = `export const generatedRoutes = "${"x".repeat(COMPACT_FILE_TEXT_THRESHOLD_CHARS)}";`;
|
|
96
|
+
const laterSignal = "export function conciseAnchor() { return true; }";
|
|
97
|
+
const content = [
|
|
98
|
+
`HEAD_MARKER\n${"h".repeat(15000)}`,
|
|
99
|
+
hugeSignal,
|
|
100
|
+
laterSignal,
|
|
101
|
+
`${"m".repeat(COMPACT_FILE_TEXT_THRESHOLD_CHARS)}\nTAIL_MARKER`
|
|
102
|
+
].join("\n");
|
|
103
|
+
|
|
104
|
+
const [compact] = parseFileEntities([fileRecord(content)], { textProfile: "compact-files" });
|
|
105
|
+
|
|
106
|
+
assert.equal(compact.text_compacted, true);
|
|
107
|
+
assert.ok(compact.text.includes("export const generatedRoutes"));
|
|
108
|
+
assert.match(compact.text, /signal_line_truncated_chars=/);
|
|
109
|
+
assert.ok(!compact.text.includes("x".repeat(1024)));
|
|
110
|
+
assert.ok(compact.text.includes(laterSignal));
|
|
111
|
+
});
|
|
112
|
+
|
|
41
113
|
test("chunk embedding text includes the full body beyond the old 2000-char preview", () => {
|
|
42
114
|
const filePathById = new Map([["file-1", "src/big.ts"]]);
|
|
43
115
|
const base = "y".repeat(3000);
|
|
@@ -50,6 +122,16 @@ test("chunk embedding text includes the full body beyond the old 2000-char previ
|
|
|
50
122
|
assert.notEqual(a.signature, b.signature);
|
|
51
123
|
});
|
|
52
124
|
|
|
125
|
+
test("chunk embedding text keeps full body while file compact profile is enabled", () => {
|
|
126
|
+
const filePathById = new Map([["file-1", "src/big.ts"]]);
|
|
127
|
+
const body = `${"z".repeat(COMPACT_FILE_TEXT_THRESHOLD_CHARS)}CHUNK_TAIL`;
|
|
128
|
+
|
|
129
|
+
const [chunk] = parseChunkEntities([chunkRecord(body)], filePathById);
|
|
130
|
+
|
|
131
|
+
assert.ok(chunk.text.includes(body));
|
|
132
|
+
assert.ok(chunk.text.includes("CHUNK_TAIL"));
|
|
133
|
+
});
|
|
134
|
+
|
|
53
135
|
test("default embedding model is jina code model and env override still works", () => {
|
|
54
136
|
const saved = process.env.CORTEX_EMBED_MODEL;
|
|
55
137
|
try {
|
|
@@ -70,3 +152,29 @@ test("default embedding model is jina code model and env override still works",
|
|
|
70
152
|
}
|
|
71
153
|
}
|
|
72
154
|
});
|
|
155
|
+
|
|
156
|
+
test("embedding text profile resolver accepts only known profiles", () => {
|
|
157
|
+
assert.equal(resolveEmbedTextProfile(undefined), "full");
|
|
158
|
+
assert.equal(resolveEmbedTextProfile(""), "full");
|
|
159
|
+
assert.equal(resolveEmbedTextProfile(" full "), "full");
|
|
160
|
+
assert.equal(resolveEmbedTextProfile(" compact-files "), "compact-files");
|
|
161
|
+
assert.throws(() => resolveEmbedTextProfile("compact"), /Unsupported CORTEX_EMBED_TEXT_PROFILE/);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test("signature profile preserves default cache and separates compact-files", () => {
|
|
165
|
+
assert.equal(resolveSignatureProfile(null, "full"), "");
|
|
166
|
+
assert.equal(resolveSignatureProfile(2048, "full"), "embed|max_tokens=2048");
|
|
167
|
+
|
|
168
|
+
const compact = resolveSignatureProfile(null, "compact-files");
|
|
169
|
+
assert.match(compact, /^embed\|/);
|
|
170
|
+
assert.match(compact, /text_profile=compact-files/);
|
|
171
|
+
assert.match(compact, new RegExp(COMPACT_FILE_TEXT_STRATEGY));
|
|
172
|
+
assert.match(compact, new RegExp(`threshold_chars=${COMPACT_FILE_TEXT_THRESHOLD_CHARS}`));
|
|
173
|
+
|
|
174
|
+
const cappedCompact = resolveSignatureProfile(2048, "compact-files");
|
|
175
|
+
assert.match(cappedCompact, /max_tokens=2048/);
|
|
176
|
+
assert.notEqual(cappedCompact, resolveSignatureProfile(2048, "full"));
|
|
177
|
+
assert.equal(resolveSignatureProfile(null, "compact-files", "Chunk"), "");
|
|
178
|
+
assert.equal(resolveSignatureProfile(2048, "compact-files", "Chunk"), "embed|max_tokens=2048");
|
|
179
|
+
assert.equal(resolveSignatureProfile(2048, "compact-files", "File"), cappedCompact);
|
|
180
|
+
});
|