@ngockhoale/ukit 3.5.3 → 3.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/manifests/documentation.yaml +12 -1
- package/manifests/platform.full.yaml +12 -0
- package/package.json +29 -2
- package/src/cli/commands/doctor.js +41 -0
- package/src/cli/commands/indexTools.js +10 -4
- package/src/cli/commands/memory.js +88 -6
- package/src/core/memory/episodes.js +106 -7
- package/src/core/memory/memoryCheck.js +241 -0
- package/src/core/memory/memoryCheckRun.js +39 -0
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/mutateMemory.js +37 -1
- package/src/core/memory/observations.js +192 -0
- package/src/core/memory/recallEngine.js +159 -10
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/records.js +23 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/memory/writeGate.js +137 -0
- package/src/core/runtimeConfig.js +56 -1
- package/src/decision/registry.js +17 -0
- package/src/index/buildIndex.js +312 -5
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +4 -1
- package/src/index/queryIndex.js +356 -14
- package/src/index/resolveContext.js +1 -1
- package/template_project/.claude/hooks/edit-hint.mjs +64 -17
- package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
- package/template_project/.claude/hooks/session-episode.sh +9 -4
- package/template_project/.claude/settings.json +2 -2
- package/template_project/.claude/ukit/index/lib/index-core.mjs +769 -18
- package/template_project/.claude/ukit/index/query-index.mjs +14 -3
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/index/route-task.mjs +4 -1
- package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +159 -10
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/.claude/ukit/runtime/memory-validity.mjs +28 -1
- package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
- package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
- package/template_project/ukit/storage/config.json +21 -2
- package/manifests/instructionRules.yaml +0 -442
- package/src/core/agentRuntime/completionGate.js +0 -179
- package/src/core/agentRuntime/context.js +0 -571
- package/src/core/agentRuntime/qualityComparison.js +0 -112
- package/src/core/agentRuntime/resourcePolicy.js +0 -78
- package/src/core/agentRuntime/runtimeSupport.js +0 -255
- package/src/core/codeintel/manifest.js +0 -176
- package/src/core/experiments/deliberation.js +0 -327
- package/src/core/experiments/dynamicWorkflow.js +0 -507
- package/src/core/handoffDocValidator.js +0 -154
- package/src/core/memory/deltaOverlays.js +0 -493
- package/src/core/memory/learningCandidates.js +0 -347
- package/src/core/observability/emit/sessionBoot.js +0 -407
- package/src/core/observability/evaluation/replay.js +0 -149
- package/src/core/observability/evaluation/scorecard.js +0 -465
- package/src/core/observability/schema/compatibility.js +0 -180
- package/src/core/observability/support/renderer.js +0 -130
- package/src/core/reviewPanelAggregate.js +0 -204
- package/src/core/taskBudgetValidator.js +0 -251
- package/src/core/taskProgressGuard.js +0 -227
- package/src/decision/reviewVerdict.js +0 -339
- package/src/decision/runtimeFilter.js +0 -150
- package/src/decision/runtimeScheduler.js +0 -239
- package/src/index/contextPack.js +0 -285
- package/src/index/impactContext.js +0 -360
- package/src/index/reviewPolicy.js +0 -368
- package/src/index/sensitivePath.js +0 -21
|
@@ -135,15 +135,17 @@ const EXCLUDED_DIR_NAMES = new Set([
|
|
|
135
135
|
'.venv',
|
|
136
136
|
'venv',
|
|
137
137
|
]);
|
|
138
|
-
const CODE_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.sql']);
|
|
138
|
+
const CODE_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.svelte', '.py', '.go', '.java', '.cs', '.php', '.rb', '.rs', '.sql']);
|
|
139
139
|
const STYLE_EXTENSIONS = new Set(['.css', '.scss', '.sass', '.less']);
|
|
140
|
-
const TRACKED_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.sql', '.json', '.yaml', '.yml', '.md', '.sh']);
|
|
140
|
+
const TRACKED_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.svelte', '.py', '.go', '.java', '.cs', '.php', '.rb', '.rs', '.sql', '.json', '.yaml', '.yml', '.md', '.sh']);
|
|
141
141
|
const DISCOVERED_EXTENSIONS = new Set([...TRACKED_EXTENSIONS, ...STYLE_EXTENSIONS]);
|
|
142
142
|
// v9: symbols.json gains global-registered ('global') and Vue 'method' rows,
|
|
143
143
|
// SQL 'column' rows with parent/schema, and calls.json gains per-object SQL
|
|
144
144
|
// reference records — a v8 cached artifact must not be reused for unchanged
|
|
145
145
|
// files or the new extractions silently vanish.
|
|
146
|
-
|
|
146
|
+
// v10: symbols.json gains unexported function/class ('local-function', 'local-class'),
|
|
147
|
+
// function-expression consts, class 'method' rows and TS interface/enum/type ('type') rows.
|
|
148
|
+
export const INDEX_SCHEMA_VERSION = 11;
|
|
147
149
|
export const DEFAULT_INDEX_CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
148
150
|
// Parse-phase read/extract concurrency — deliberately tighter than the
|
|
149
151
|
// discovery stat pool because parsing holds file content in memory.
|
|
@@ -211,6 +213,46 @@ const VIETNAMESE_PHRASE_ALIASES = [
|
|
|
211
213
|
{ regex: /\bthanh phan\b/gu, expansions: ['component'] },
|
|
212
214
|
{ regex: /\bkiem thu\b/gu, expansions: ['test', 'testing'] },
|
|
213
215
|
{ regex: /\bbo cuc\b/gu, expansions: ['layout'] },
|
|
216
|
+
// General programming vocabulary (diacritics folded; matches with or without accents).
|
|
217
|
+
{ regex: /\bnhay cam\b/gu, expansions: ['sensitive'] },
|
|
218
|
+
{ regex: /\bbi mat\b/gu, expansions: ['secret'] },
|
|
219
|
+
{ regex: /\bgia tri\b/gu, expansions: ['value'] },
|
|
220
|
+
{ regex: /\bvan ban\b/gu, expansions: ['text'] },
|
|
221
|
+
{ regex: /\bsao luu\b/gu, expansions: ['backup'] },
|
|
222
|
+
{ regex: /\bkhoi phuc\b/gu, expansions: ['restore', 'recover'] },
|
|
223
|
+
{ regex: /\bdinh danh\b/gu, expansions: ['identity', 'id'] },
|
|
224
|
+
{ regex: /\bdu an\b/gu, expansions: ['project'] },
|
|
225
|
+
{ regex: /\bthu muc\b/gu, expansions: ['directory', 'dir', 'folder'] },
|
|
226
|
+
{ regex: /\bphan loai\b/gu, expansions: ['classify', 'category'] },
|
|
227
|
+
{ regex: /\buoc tinh\b/gu, expansions: ['estimate'] },
|
|
228
|
+
{ regex: /\bcong chan\b/gu, expansions: ['gate', 'guard'] },
|
|
229
|
+
{ regex: /\bcap nhat\b/gu, expansions: ['update'] },
|
|
230
|
+
{ regex: /\bgo cai dat\b/gu, expansions: ['uninstall'] },
|
|
231
|
+
{ regex: /\bcai dat\b/gu, expansions: ['install', 'setup'] },
|
|
232
|
+
{ regex: /\bcau hinh\b/gu, expansions: ['config', 'configuration'] },
|
|
233
|
+
{ regex: /\bbao cao\b/gu, expansions: ['report'] },
|
|
234
|
+
{ regex: /\blich su\b/gu, expansions: ['history'] },
|
|
235
|
+
{ regex: /\bnhat ky\b/gu, expansions: ['log'] },
|
|
236
|
+
{ regex: /\bchi muc\b/gu, expansions: ['index'] },
|
|
237
|
+
{ regex: /\bxu ly\b/gu, expansions: ['handle', 'process'] },
|
|
238
|
+
{ regex: /\bphan tich\b/gu, expansions: ['parse', 'analyze'] },
|
|
239
|
+
{ regex: /\bhet han\b/gu, expansions: ['expire', 'expiry'] },
|
|
240
|
+
{ regex: /\bchup anh\b/gu, expansions: ['snapshot'] },
|
|
241
|
+
{ regex: /\bhoan tac\b/gu, expansions: ['rollback', 'revert'] },
|
|
242
|
+
{ regex: /\bnguoi dung\b/gu, expansions: ['user'] },
|
|
243
|
+
{ regex: /\bthong bao\b/gu, expansions: ['notify', 'message'] },
|
|
244
|
+
{ regex: /\bsap xep\b/gu, expansions: ['sort'] },
|
|
245
|
+
{ regex: /\bdang ky\b/gu, expansions: ['register'] },
|
|
246
|
+
{ regex: /\btham so\b/gu, expansions: ['argument', 'param', 'option'] },
|
|
247
|
+
{ regex: /\bket qua\b/gu, expansions: ['result'] },
|
|
248
|
+
{ regex: /\bmac dinh\b/gu, expansions: ['default'] },
|
|
249
|
+
{ regex: /\bdau vao\b/gu, expansions: ['input'] },
|
|
250
|
+
{ regex: /\bdau ra\b/gu, expansions: ['output'] },
|
|
251
|
+
{ regex: /\bhang doi\b/gu, expansions: ['queue'] },
|
|
252
|
+
{ regex: /\bma hoa\b/gu, expansions: ['encode', 'encrypt'] },
|
|
253
|
+
{ regex: /\bgiai ma\b/gu, expansions: ['decode', 'decrypt'] },
|
|
254
|
+
{ regex: /\bbo nho\b/gu, expansions: ['memory'] },
|
|
255
|
+
{ regex: /\bcanh bao\b/gu, expansions: ['warning', 'warn'] },
|
|
214
256
|
];
|
|
215
257
|
const VIETNAMESE_TOKEN_ALIASES = new Map([
|
|
216
258
|
['sua', ['fix']],
|
|
@@ -231,8 +273,58 @@ const VIETNAMESE_TOKEN_ALIASES = new Map([
|
|
|
231
273
|
['soat', ['review', 'audit']],
|
|
232
274
|
['trien', ['implement', 'build', 'ship']],
|
|
233
275
|
['khai', ['implement', 'build', 'ship']],
|
|
276
|
+
['kho', ['store', 'storage']],
|
|
277
|
+
['ghi', ['write', 'record']],
|
|
278
|
+
['doc', ['read']],
|
|
279
|
+
['luu', ['save', 'store', 'persist']],
|
|
280
|
+
['tim', ['find', 'search', 'resolve']],
|
|
281
|
+
['hong', ['broken', 'repair']],
|
|
282
|
+
['phut', ['minute', 'minutes']],
|
|
283
|
+
['ham', ['function']],
|
|
284
|
+
['bien', ['variable']],
|
|
285
|
+
['lop', ['class']],
|
|
286
|
+
['lenh', ['command']],
|
|
287
|
+
['chay', ['run', 'execute']],
|
|
288
|
+
['phien', ['session']],
|
|
289
|
+
['loc', ['filter']],
|
|
290
|
+
['gui', ['send', 'dispatch']],
|
|
291
|
+
['khoa', ['lock']],
|
|
292
|
+
['huy', ['cancel', 'abort']],
|
|
293
|
+
['quyet', ['decision']],
|
|
234
294
|
]);
|
|
235
295
|
|
|
296
|
+
// Query-time light stemming: adds the singular/base form of an English query word so
|
|
297
|
+
// 'files' reaches `file`, 'checks' reaches `check`, 'written' reaches `write`. Added next
|
|
298
|
+
// to the original token, never in place of it, and never applied to indexed text.
|
|
299
|
+
const IRREGULAR_STEMS = new Map([
|
|
300
|
+
['written', 'write'], ['wrote', 'write'], ['built', 'build'], ['ran', 'run'], ['saved', 'save'],
|
|
301
|
+
['found', 'find'], ['sent', 'send'], ['made', 'make'], ['read', 'read'], ['began', 'begin'],
|
|
302
|
+
]);
|
|
303
|
+
|
|
304
|
+
const STEM_MIN_QUERY_TOKENS = 5;
|
|
305
|
+
|
|
306
|
+
function stemToken(token) {
|
|
307
|
+
if (!/^[a-z]+$/.test(token) || token.length < 4) {
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
if (IRREGULAR_STEMS.has(token)) {
|
|
311
|
+
return IRREGULAR_STEMS.get(token);
|
|
312
|
+
}
|
|
313
|
+
let stem = null;
|
|
314
|
+
if (token.endsWith('ies') && token.length > 4) {
|
|
315
|
+
stem = `${token.slice(0, -3)}y`;
|
|
316
|
+
} else if (/(?:ches|shes|xes|sses)$/.test(token)) {
|
|
317
|
+
stem = token.slice(0, -2);
|
|
318
|
+
} else if (token.endsWith('s') && !/(?:ss|us|is|as|os)$/.test(token)) {
|
|
319
|
+
stem = token.slice(0, -1);
|
|
320
|
+
} else if (token.endsWith('ing') && token.length > 5) {
|
|
321
|
+
stem = token.slice(0, -3);
|
|
322
|
+
} else if (token.endsWith('ed') && token.length > 4) {
|
|
323
|
+
stem = token.endsWith('ied') ? `${token.slice(0, -3)}y` : token.slice(0, -2);
|
|
324
|
+
}
|
|
325
|
+
return stem && stem.length >= 3 && stem !== token ? stem : null;
|
|
326
|
+
}
|
|
327
|
+
|
|
236
328
|
// ── Build Index ──
|
|
237
329
|
|
|
238
330
|
// Bumped only when the snapshot shape itself changes; consumers must reject
|
|
@@ -1247,7 +1339,11 @@ function isFunctionLikeSignature(signature) {
|
|
|
1247
1339
|
// Reads symbols.json through the existing readArtifact cache (no new file reads, no new
|
|
1248
1340
|
// cache). Never throws: an absent file, a missing artifact, or a stale schema all resolve
|
|
1249
1341
|
// to [] so callers can render an outline block unconditionally.
|
|
1250
|
-
|
|
1342
|
+
const OUTLINE_WINDOW = 12; // matches OUTLINE_SYMBOLS_LIMIT in the CLI printers
|
|
1343
|
+
|
|
1344
|
+
// `focus`: symbol names the query matched (see queryMatchedSymbolNames); they are pinned into
|
|
1345
|
+
// the first OUTLINE_WINDOW rows even when unexported, so the printed outline shows the hit.
|
|
1346
|
+
export async function getFileOutline({ rootDir = process.cwd(), filePath, limit = 12, focus = null } = {}) {
|
|
1251
1347
|
if (!filePath) {
|
|
1252
1348
|
return Object.freeze([]);
|
|
1253
1349
|
}
|
|
@@ -1273,9 +1369,26 @@ export async function getFileOutline({ rootDir = process.cwd(), filePath, limit
|
|
|
1273
1369
|
seenNames.add(item.name);
|
|
1274
1370
|
deduped.push(item);
|
|
1275
1371
|
}
|
|
1372
|
+
// The first OUTLINE_WINDOW rows (what the CLI prints) hold exported/top-level symbols
|
|
1373
|
+
// first; unexported functions and class methods only take the remaining slots. Each
|
|
1374
|
+
// part stays in line order, and rows beyond the window follow in line order.
|
|
1375
|
+
const window = Number.isFinite(limit) ? limit : OUTLINE_WINDOW;
|
|
1376
|
+
let kept = deduped;
|
|
1377
|
+
if (deduped.length > window) {
|
|
1378
|
+
const pinned = new Set(focus ?? []);
|
|
1379
|
+
const isPinned = (item) => pinned.has(item.name);
|
|
1380
|
+
const ranked = [
|
|
1381
|
+
...deduped.filter(isPinned),
|
|
1382
|
+
...deduped.filter((item) => !isPinned(item) && !isLocalOutlineRow(item)),
|
|
1383
|
+
...deduped.filter((item) => !isPinned(item) && isLocalOutlineRow(item)),
|
|
1384
|
+
];
|
|
1385
|
+
const head = ranked.slice(0, window).sort((a, b) => a.line - b.line);
|
|
1386
|
+
const inHead = new Set(head);
|
|
1387
|
+
kept = [...head, ...deduped.filter((item) => !inHead.has(item))];
|
|
1388
|
+
}
|
|
1276
1389
|
|
|
1277
1390
|
return Object.freeze(
|
|
1278
|
-
|
|
1391
|
+
kept.slice(0, limit).map((item) => Object.freeze({
|
|
1279
1392
|
line: item.line,
|
|
1280
1393
|
name: item.name,
|
|
1281
1394
|
type: item.type,
|
|
@@ -1284,7 +1397,8 @@ export async function getFileOutline({ rootDir = process.cwd(), filePath, limit
|
|
|
1284
1397
|
);
|
|
1285
1398
|
}
|
|
1286
1399
|
|
|
1287
|
-
|
|
1400
|
+
// contentLane=false skips the error-string content search (context-resolution paths never pay for it).
|
|
1401
|
+
export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true } = {}) {
|
|
1288
1402
|
const normalizedQuery = String(query ?? '').trim();
|
|
1289
1403
|
if (!normalizedQuery) return [];
|
|
1290
1404
|
if (limit <= 0) return Object.freeze([]);
|
|
@@ -1296,6 +1410,7 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1296
1410
|
rootDir: absoluteRoot,
|
|
1297
1411
|
query: normalizedQuery,
|
|
1298
1412
|
limit,
|
|
1413
|
+
contentLane,
|
|
1299
1414
|
});
|
|
1300
1415
|
if (QUERY_RESULT_CACHE.has(queryCacheKey)) {
|
|
1301
1416
|
return QUERY_RESULT_CACHE.get(queryCacheKey);
|
|
@@ -1323,7 +1438,50 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1323
1438
|
}
|
|
1324
1439
|
}
|
|
1325
1440
|
|
|
1326
|
-
|
|
1441
|
+
// Content lane: a message-like query with no confident symbol/path hit also searches
|
|
1442
|
+
// file contents through the existing bounded fallback search. It only adds a bounded
|
|
1443
|
+
// bonus, so it can never displace a confident hit (those skip the lane entirely).
|
|
1444
|
+
let bestDirect = 0;
|
|
1445
|
+
let contentLaneRan = false;
|
|
1446
|
+
let contentCutShort = [];
|
|
1447
|
+
for (const entry of directScores.values()) {
|
|
1448
|
+
bestDirect = Math.max(bestDirect, entry.score);
|
|
1449
|
+
}
|
|
1450
|
+
if (contentLane && bestDirect < CONTENT_LANE_CONFIDENT_SCORE && queryDescriptor.tokens.size >= CONTENT_LANE_MIN_TOKENS && looksLikeLiteralMessage(normalizedQuery)) {
|
|
1451
|
+
contentLaneRan = true;
|
|
1452
|
+
// A literal message sits on one source line: every query token must share the matched
|
|
1453
|
+
// line, otherwise this is a scattered token hit, not the message. The scan order and the
|
|
1454
|
+
// count/byte caps are functions of the index and file contents only (never of load or of
|
|
1455
|
+
// ripgrep being installed); wall-clock is a safety cap that reports 'time-budget'.
|
|
1456
|
+
const messageTokens = [...new Set(normalizeIdentifierTokens(queryDescriptor.rawLower))];
|
|
1457
|
+
let lineHits = [];
|
|
1458
|
+
try {
|
|
1459
|
+
const ordered = rankContentLaneFiles({
|
|
1460
|
+
files, searchDescriptorsByFile, tokenWeights, queryTokens: queryDescriptor.tokens, includeLikelyTestFiles,
|
|
1461
|
+
});
|
|
1462
|
+
const scan = await scanContentLane({ absoluteRoot, ordered, tokens: messageTokens, limit: CONTENT_LANE_LIMIT });
|
|
1463
|
+
lineHits = scan.hits;
|
|
1464
|
+
contentCutShort = scan.truncated;
|
|
1465
|
+
} catch {
|
|
1466
|
+
contentCutShort = ['error'];
|
|
1467
|
+
}
|
|
1468
|
+
lineHits.forEach((row, index) => {
|
|
1469
|
+
const existing = directScores.get(row.filePath);
|
|
1470
|
+
const reason = `content_match:${row.filePath}${row.line ? `:${row.line}` : ''}`;
|
|
1471
|
+
if (existing) {
|
|
1472
|
+
existing.score += CONTENT_LANE_BONUS;
|
|
1473
|
+
existing.reasons = dedupe([...(existing.reasons ?? []), reason]);
|
|
1474
|
+
} else {
|
|
1475
|
+
directScores.set(row.filePath, {
|
|
1476
|
+
filePath: row.filePath,
|
|
1477
|
+
score: CONTENT_LANE_ONLY_SCORE - index * 0.1,
|
|
1478
|
+
reasons: [reason],
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1481
|
+
});
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
if (directScores.size === 0) return withContentLaneStatus(Object.freeze([]), contentLaneRan, contentCutShort);
|
|
1327
1485
|
|
|
1328
1486
|
const {
|
|
1329
1487
|
hotspotScores,
|
|
@@ -1361,9 +1519,11 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1361
1519
|
directScores,
|
|
1362
1520
|
scores: applyHotspotBoosts({ scores: relatedScores, hotspotScores }),
|
|
1363
1521
|
});
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1522
|
+
const ranked = [...boosted.values()].map((entry) => ({ ...entry, score: finalizeScore(entry) })).sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath));
|
|
1523
|
+
const fused = contentLane && ranked.length > 0 && ranked[0].score < FUSION_WEAK_SCORE && queryDescriptor.tokens.size >= FUSION_MIN_TOKENS
|
|
1524
|
+
? await fuseWeakQuery({ absoluteRoot, query: normalizedQuery, ranked, searchDescriptorsByFile, includeLikelyTestFiles })
|
|
1525
|
+
: null;
|
|
1526
|
+
return withContentLaneStatus(freezeQueryResults((fused ?? ranked).slice(0, limit)), contentLaneRan, contentCutShort);
|
|
1367
1527
|
})().catch((error) => {
|
|
1368
1528
|
QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
1369
1529
|
throw error;
|
|
@@ -1373,6 +1533,129 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1373
1533
|
return queryPromise;
|
|
1374
1534
|
}
|
|
1375
1535
|
|
|
1536
|
+
// Opt-in lane fusion (default OFF): a weak natural-language query (best score below
|
|
1537
|
+
// FUSION_WEAK_SCORE, >= FUSION_MIN_TOKENS tokens) is re-ranked by RRF over the scored list
|
|
1538
|
+
// and a hashed-embedding lane (the retriever.js vector lane). Enabled by
|
|
1539
|
+
// `.ukit/storage/config.json` -> index.fusion === true or UKIT_INDEX_FUSION=1. The flag is
|
|
1540
|
+
// read only after the weak gate passes, and contentLane:false callers (hooks/resolveContext)
|
|
1541
|
+
// never reach it. Never throws; async fs only.
|
|
1542
|
+
const FUSION_WEAK_SCORE = 6;
|
|
1543
|
+
const FUSION_MIN_TOKENS = 4;
|
|
1544
|
+
const FUSION_RRF_K = 5;
|
|
1545
|
+
const FUSION_VECTOR_WEIGHT = 1;
|
|
1546
|
+
const FUSION_LANE_LIMIT = 100;
|
|
1547
|
+
const FUSION_DOC_VECTORS = new WeakMap();
|
|
1548
|
+
|
|
1549
|
+
// Inlined copy of src/core/codeintel/vectorProvider.js hashVectorize/cosine (256 dims, FNV-1a
|
|
1550
|
+
// word + trigram features, L2-normalized) so the shipped index-core.mjs stays dependency-free.
|
|
1551
|
+
function fusionFnv1a(str) {
|
|
1552
|
+
let h = 0x811c9dc5;
|
|
1553
|
+
for (let i = 0; i < str.length; i += 1) {
|
|
1554
|
+
h ^= str.charCodeAt(i);
|
|
1555
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
1556
|
+
}
|
|
1557
|
+
return h >>> 0;
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
function hashVectorize(text) {
|
|
1561
|
+
const dims = 256;
|
|
1562
|
+
const vec = new Array(dims).fill(0);
|
|
1563
|
+
for (const token of String(text ?? '').split(/[^A-Za-z0-9_$]+/).map((t) => t.toLowerCase()).filter(Boolean)) {
|
|
1564
|
+
vec[fusionFnv1a(`w:${token}`) % dims] += 1;
|
|
1565
|
+
const padded = `#${token}#`;
|
|
1566
|
+
for (let i = 0; i + 3 <= padded.length; i += 1) vec[fusionFnv1a(`t:${padded.slice(i, i + 3)}`) % dims] += 0.1;
|
|
1567
|
+
}
|
|
1568
|
+
const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
|
|
1569
|
+
if (norm > 0) for (let i = 0; i < dims; i += 1) vec[i] /= norm;
|
|
1570
|
+
return vec;
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
function cosineSimilarity(a, b) {
|
|
1574
|
+
let dot = 0;
|
|
1575
|
+
for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
|
|
1576
|
+
return dot;
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
async function isFusionEnabled(absoluteRoot) {
|
|
1580
|
+
if (process.env.UKIT_INDEX_FUSION === '1') return true;
|
|
1581
|
+
if (process.env.UKIT_INDEX_FUSION === '0') return false;
|
|
1582
|
+
try {
|
|
1583
|
+
const config = JSON.parse(await fs.readFile(path.join(absoluteRoot, '.ukit', 'storage', 'config.json'), 'utf8'));
|
|
1584
|
+
return config?.index?.fusion === true;
|
|
1585
|
+
} catch {
|
|
1586
|
+
return false;
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
async function fuseWeakQuery({ absoluteRoot, query, ranked, searchDescriptorsByFile, includeLikelyTestFiles }) {
|
|
1591
|
+
try {
|
|
1592
|
+
if (!(await isFusionEnabled(absoluteRoot))) return null;
|
|
1593
|
+
let docVectors = FUSION_DOC_VECTORS.get(searchDescriptorsByFile);
|
|
1594
|
+
if (!docVectors) {
|
|
1595
|
+
docVectors = new Map();
|
|
1596
|
+
for (const [filePath, descriptor] of searchDescriptorsByFile) {
|
|
1597
|
+
docVectors.set(filePath, hashVectorize(`${filePath} ${(descriptor.symbols ?? []).map((s) => s.name).join(' ')}`));
|
|
1598
|
+
}
|
|
1599
|
+
FUSION_DOC_VECTORS.set(searchDescriptorsByFile, docVectors);
|
|
1600
|
+
}
|
|
1601
|
+
const queryVector = hashVectorize(query);
|
|
1602
|
+
const vectorLane = [];
|
|
1603
|
+
for (const [filePath, vector] of docVectors) {
|
|
1604
|
+
if (isLikelyTestFilePath(filePath) && !includeLikelyTestFiles) continue;
|
|
1605
|
+
const similarity = cosineSimilarity(queryVector, vector);
|
|
1606
|
+
if (similarity > 0) vectorLane.push([filePath, similarity]);
|
|
1607
|
+
}
|
|
1608
|
+
vectorLane.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
1609
|
+
const vectorPaths = vectorLane.slice(0, FUSION_LANE_LIMIT).map(([filePath]) => filePath);
|
|
1610
|
+
const indexPaths = ranked.slice(0, FUSION_LANE_LIMIT).map((entry) => entry.filePath);
|
|
1611
|
+
const rrf = new Map();
|
|
1612
|
+
indexPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + 1 / (FUSION_RRF_K + i + 1)));
|
|
1613
|
+
vectorPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + FUSION_VECTOR_WEIGHT / (FUSION_RRF_K + i + 1)));
|
|
1614
|
+
const byPath = new Map(ranked.map((entry) => [entry.filePath, entry]));
|
|
1615
|
+
const order = [...rrf.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([filePath]) => filePath);
|
|
1616
|
+
const floor = ranked[ranked.length - 1].score;
|
|
1617
|
+
const merged = order.map((filePath) => byPath.get(filePath)
|
|
1618
|
+
?? { filePath, score: Math.min(0.01, floor), reasons: ['fusion:vector'], tests: [] });
|
|
1619
|
+
await emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths: merged.map((e) => e.filePath) });
|
|
1620
|
+
return merged;
|
|
1621
|
+
} catch {
|
|
1622
|
+
return null;
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
// Same event shape and file as retriever.js emitRetrieverLaneTelemetry (query stored as sha256).
|
|
1627
|
+
async function emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths }) {
|
|
1628
|
+
try {
|
|
1629
|
+
const event = {
|
|
1630
|
+
ts: Date.now(),
|
|
1631
|
+
query: crypto.createHash('sha256').update(String(query ?? '')).digest('hex'),
|
|
1632
|
+
mode: 'index-query',
|
|
1633
|
+
lanes: {
|
|
1634
|
+
index: { hits: indexPaths.length, weightUsed: 1, paths: indexPaths },
|
|
1635
|
+
vector: { hits: vectorPaths.length, weightUsed: FUSION_VECTOR_WEIGHT, paths: vectorPaths },
|
|
1636
|
+
},
|
|
1637
|
+
mergedPaths: mergedPaths.slice(0, 100),
|
|
1638
|
+
merge: 'rrf',
|
|
1639
|
+
};
|
|
1640
|
+
const filePath = path.join(absoluteRoot, '.ukit', 'storage', 'cache', 'retriever-lanes.jsonl');
|
|
1641
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
1642
|
+
await withFileLock(filePath, async () => {
|
|
1643
|
+
try {
|
|
1644
|
+
const st = await fs.stat(filePath);
|
|
1645
|
+
if (st.size > 256 * 1024) {
|
|
1646
|
+
const kept = (await fs.readFile(filePath, 'utf8')).split('\n').filter(Boolean).slice(-2000);
|
|
1647
|
+
await fs.writeFile(filePath, kept.length ? `${kept.join('\n')}\n` : '', 'utf8');
|
|
1648
|
+
}
|
|
1649
|
+
} catch {
|
|
1650
|
+
// Missing file is fine; append below recreates it.
|
|
1651
|
+
}
|
|
1652
|
+
await fs.appendFile(filePath, `${JSON.stringify(event)}\n`, 'utf8');
|
|
1653
|
+
});
|
|
1654
|
+
} catch {
|
|
1655
|
+
// Telemetry is advisory.
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1376
1659
|
// ── Analog Query ──
|
|
1377
1660
|
export async function queryAnalog({ rootDir = process.cwd(), filePath, limit = 5 } = {}) {
|
|
1378
1661
|
const normalizedFilePath = String(filePath ?? '').trim();
|
|
@@ -1442,6 +1725,30 @@ export function clearIndexArtifactCache(rootDir = null) {
|
|
|
1442
1725
|
RELATED_TEST_LOOKUP_CACHE.delete(absoluteRoot);
|
|
1443
1726
|
}
|
|
1444
1727
|
|
|
1728
|
+
// Names of the symbols a query result matched, parsed from its `symbol_exact:` /
|
|
1729
|
+
// `symbol_token:` reasons; feed to getFileOutline({ focus }).
|
|
1730
|
+
export function queryMatchedSymbolNames(result) {
|
|
1731
|
+
const names = [];
|
|
1732
|
+
for (const reason of result?.reasons ?? []) {
|
|
1733
|
+
const match = /^symbol_(?:exact|compact|contains):([^:]+)$/.exec(reason) ?? /^symbol_token:([^:]+):/.exec(reason);
|
|
1734
|
+
if (match) {
|
|
1735
|
+
names.push(match[1]);
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
return names;
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
// Non-enumerable (array equality and JSON stay unchanged), like boundedSourceSearch rows.truncated:
|
|
1742
|
+
// `contentLane` is true when the content search ran; `contentLaneTruncated` lists why it was cut
|
|
1743
|
+
// short ('verify-cap', 'time-budget', 'max-files', ...) so a partial scan never reads as "no match".
|
|
1744
|
+
function withContentLaneStatus(results, ran, truncated) {
|
|
1745
|
+
// results arrive frozen, so copy before attaching the status and freeze again.
|
|
1746
|
+
const out = [...results];
|
|
1747
|
+
Object.defineProperty(out, 'contentLane', { value: ran, enumerable: false });
|
|
1748
|
+
Object.defineProperty(out, 'contentLaneTruncated', { value: Object.freeze([...truncated]), enumerable: false });
|
|
1749
|
+
return Object.freeze(out);
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1445
1752
|
function freezeQueryResults(results) {
|
|
1446
1753
|
return Object.freeze(
|
|
1447
1754
|
results.map((item) => Object.freeze({
|
|
@@ -1586,7 +1893,7 @@ export async function resolveContext({
|
|
|
1586
1893
|
}
|
|
1587
1894
|
|
|
1588
1895
|
if (queryWasUsed) {
|
|
1589
|
-
const queryResults = await queryCodeIndex({ rootDir, query: intent, limit: budget.maxFiles });
|
|
1896
|
+
const queryResults = await queryCodeIndex({ rootDir, query: intent, limit: budget.maxFiles, contentLane: false });
|
|
1590
1897
|
for (const qr of queryResults) {
|
|
1591
1898
|
addExplainedFile(result, 'primaryTargets', qr.filePath, buildQueryReason(qr));
|
|
1592
1899
|
}
|
|
@@ -2683,7 +2990,7 @@ function findScriptClose(content, startIndex) {
|
|
|
2683
2990
|
}
|
|
2684
2991
|
|
|
2685
2992
|
export function extractScriptContent(filePath, content) {
|
|
2686
|
-
if (
|
|
2993
|
+
if (!/\.(?:vue|svelte)$/.test(filePath)) {
|
|
2687
2994
|
return content;
|
|
2688
2995
|
}
|
|
2689
2996
|
|
|
@@ -2772,6 +3079,280 @@ function computeSignature(content, lineStartIndex) {
|
|
|
2772
3079
|
return `${collapsed.slice(0, 199)}…`;
|
|
2773
3080
|
}
|
|
2774
3081
|
|
|
3082
|
+
// ── Non-JS languages (3.5.5): line/indent/brace based, no parser. Rows are { name, type, index, parent? }.
|
|
3083
|
+
// Kinds follow the JS lane: exported/public top-level -> function|class, private -> local-*, members -> method,
|
|
3084
|
+
// interface/enum/struct/trait/type aliases -> type.
|
|
3085
|
+
const POLYGLOT_EXT = /\.(?:py|go|java|cs|php|rb|rs)$/i;
|
|
3086
|
+
const POLY_MEMBER_SKIP = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'new', 'else', 'throw', 'using', 'lock', 'foreach', 'fixed', 'match', 'await', 'yield', 'do', 'try', 'sizeof', 'typeof', 'nameof', 'default', 'when', 'base', 'this']);
|
|
3087
|
+
const POLY_MODS = '(?:(?:public|protected|private|internal|static|final|abstract|sealed|partial|readonly|virtual|override|async|extern|unsafe|new|synchronized|native|strictfp|default|open)\\s+)*';
|
|
3088
|
+
const POLY_JAVA_TYPE_RE = new RegExp(`^[ \\t]*(?:@\\w+(?:\\([^)]*\\))?\\s+)*${POLY_MODS}(class|interface|enum|struct|record(?:\\s+(?:class|struct))?)\\s+([A-Za-z_]\\w*)`);
|
|
3089
|
+
const POLY_JAVA_METHOD_RE = new RegExp(`^[ \\t]*(?:@\\w+(?:\\([^)]*\\))?\\s+)*(?:<[^>]+>\\s+)?(?:[\\w.<>\\[\\],?]+\\s+)*?([A-Za-z_]\\w*)[ \\t]*(?:<[^>()]*>[ \\t]*)?\\(`);
|
|
3090
|
+
const POLY_PHP_TYPE_RE = /^[ \t]*(?:(?:abstract|final|readonly)\s+)*(class|interface|trait|enum)\s+([A-Za-z_]\w*)/;
|
|
3091
|
+
const POLY_PHP_FUNC_RE = /^[ \t]*(?:(?:public|protected|private|static|abstract|final)\s+)*function[ \t]+(?:&[ \t]*)?([A-Za-z_]\w*)/;
|
|
3092
|
+
const POLY_RS_PUB = '(?:pub(?:\\([^)]*\\))?\\s+)?';
|
|
3093
|
+
const POLY_RS_FN_RE = new RegExp(`^[ \\t]*(${POLY_RS_PUB})(?:(?:default|const|async|unsafe)\\s+)*(?:extern\\s+(?:"[^"]*"\\s+)?)?fn\\s+([A-Za-z_]\\w*)`);
|
|
3094
|
+
const POLY_RS_TYPE_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}(struct|enum|union|type)\\s+([A-Za-z_]\\w*)`);
|
|
3095
|
+
const POLY_RS_TRAIT_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}(?:unsafe\\s+)?trait\\s+([A-Za-z_]\\w*)`);
|
|
3096
|
+
// Bounds against pathological input (regex stalls): files above 512 KB (the content-lane per-file cap) are not
|
|
3097
|
+
// extracted, and any line over 2000 chars is blanked in place (offsets preserved) before the per-line patterns run.
|
|
3098
|
+
const POLY_MAX_FILE_CHARS = 512 * 1024;
|
|
3099
|
+
const POLY_MAX_LINE_CHARS = 2000;
|
|
3100
|
+
// impl header -> implemented type name, without a backtracking pattern: skip a balanced <generics>, then the
|
|
3101
|
+
// type after the first `for` (trait impl) or the first identifier (inherent impl).
|
|
3102
|
+
function polyRsImplName(line) {
|
|
3103
|
+
const head = /^[ \t]*(?:unsafe[ \t]+)?impl\b/.exec(line);
|
|
3104
|
+
if (!head) return null;
|
|
3105
|
+
let rest = line.slice(head[0].length).replace(/^[ \t]+/, '');
|
|
3106
|
+
if (rest[0] === '<') {
|
|
3107
|
+
let depth = 0;
|
|
3108
|
+
let k = 0;
|
|
3109
|
+
for (; k < rest.length; k += 1) {
|
|
3110
|
+
if (rest[k] === '<') depth += 1;
|
|
3111
|
+
else if (rest[k] === '>' && (depth -= 1) === 0) break;
|
|
3112
|
+
}
|
|
3113
|
+
rest = rest.slice(k + 1);
|
|
3114
|
+
}
|
|
3115
|
+
const cut = rest.search(/[{]|[ \t]where\b/);
|
|
3116
|
+
const header = cut < 0 ? rest : rest.slice(0, cut);
|
|
3117
|
+
const forAt = header.search(/(?:^|[ \t])for[ \t]/);
|
|
3118
|
+
const target = forAt < 0 ? header : header.slice(forAt).replace(/^[ \t]*for[ \t]+/, '');
|
|
3119
|
+
return /^[ \t]*([A-Za-z_]\w*)/.exec(target)?.[1] ?? null;
|
|
3120
|
+
}
|
|
3121
|
+
const POLY_RS_MOD_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}mod\\s+([A-Za-z_]\\w*)`);
|
|
3122
|
+
|
|
3123
|
+
// Same-length copy with comments (and, per mode, string bodies) blanked, so offsets/lines stay valid.
|
|
3124
|
+
// mode 0 = comments only, 1 = + "strings", 2 = + "strings" and 'chars'.
|
|
3125
|
+
function maskCLike(content, mode) {
|
|
3126
|
+
const out = content.split('');
|
|
3127
|
+
const n = content.length;
|
|
3128
|
+
const blank = (from, to) => {
|
|
3129
|
+
for (let k = from; k < to; k += 1) {
|
|
3130
|
+
if (out[k] !== '\n') out[k] = ' ';
|
|
3131
|
+
}
|
|
3132
|
+
};
|
|
3133
|
+
let i = 0;
|
|
3134
|
+
while (i < n) {
|
|
3135
|
+
const c = content[i];
|
|
3136
|
+
const d = content[i + 1];
|
|
3137
|
+
if (c === '/' && d === '/') {
|
|
3138
|
+
let end = content.indexOf('\n', i);
|
|
3139
|
+
if (end < 0) end = n;
|
|
3140
|
+
blank(i, end);
|
|
3141
|
+
i = end;
|
|
3142
|
+
} else if (c === '/' && d === '*') {
|
|
3143
|
+
let end = content.indexOf('*/', i + 2);
|
|
3144
|
+
end = end < 0 ? n : end + 2;
|
|
3145
|
+
blank(i, end);
|
|
3146
|
+
i = end;
|
|
3147
|
+
} else if ((mode >= 1 && c === '"') || (mode >= 2 && c === "'")) {
|
|
3148
|
+
let k = i + 1;
|
|
3149
|
+
while (k < n && content[k] !== c && content[k] !== '\n') {
|
|
3150
|
+
if (content[k] === '\\') k += 1;
|
|
3151
|
+
k += 1;
|
|
3152
|
+
}
|
|
3153
|
+
blank(i + 1, Math.min(k, n));
|
|
3154
|
+
i = k + 1;
|
|
3155
|
+
} else {
|
|
3156
|
+
i += 1;
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
3159
|
+
return out.join('');
|
|
3160
|
+
}
|
|
3161
|
+
|
|
3162
|
+
function extractIndentLanguageSymbols(filePath, content) {
|
|
3163
|
+
const isPy = /\.py$/i.test(filePath);
|
|
3164
|
+
const scopeRe = isPy ? /^([ \t]*)class[ \t]+([A-Za-z_]\w*)/ : /^([ \t]*)(?:module|class)[ \t]+([A-Z][\w:]*)/;
|
|
3165
|
+
const defRe = isPy ? /^([ \t]*)(?:async[ \t]+)?def[ \t]+([A-Za-z_]\w*)/ : /^([ \t]*)def[ \t]+(?:[A-Za-z_]\w*\.)?([A-Za-z_]\w*[?!=]?)/;
|
|
3166
|
+
const rows = [];
|
|
3167
|
+
const stack = [];
|
|
3168
|
+
let docQuote = null;
|
|
3169
|
+
let offset = 0;
|
|
3170
|
+
for (const line of content.split('\n')) {
|
|
3171
|
+
const lineStart = offset;
|
|
3172
|
+
offset += line.length + 1;
|
|
3173
|
+
const startedInDoc = docQuote !== null;
|
|
3174
|
+
if (isPy) {
|
|
3175
|
+
for (const quote of ['"""', "'''"]) {
|
|
3176
|
+
const count = line.split(quote).length - 1;
|
|
3177
|
+
if (count % 2 === 1) docQuote = docQuote === quote ? null : (docQuote ?? quote);
|
|
3178
|
+
}
|
|
3179
|
+
if (startedInDoc) continue;
|
|
3180
|
+
}
|
|
3181
|
+
const scope = scopeRe.exec(line);
|
|
3182
|
+
const def = scope ? null : defRe.exec(line);
|
|
3183
|
+
const m = scope ?? def;
|
|
3184
|
+
if (!m) continue;
|
|
3185
|
+
const indent = m[1].length;
|
|
3186
|
+
while (stack.length > 0 && stack[stack.length - 1].indent >= indent) stack.pop();
|
|
3187
|
+
const top = stack[stack.length - 1];
|
|
3188
|
+
const name = m[2];
|
|
3189
|
+
const index = lineStart + indent;
|
|
3190
|
+
if (scope) {
|
|
3191
|
+
if (!top || top.kind === 'scope') {
|
|
3192
|
+
rows.push({ name, type: isPy && name.startsWith('_') ? 'local-class' : 'class', index, parent: top?.name });
|
|
3193
|
+
}
|
|
3194
|
+
stack.push({ indent, kind: 'scope', name });
|
|
3195
|
+
} else {
|
|
3196
|
+
if (!top) {
|
|
3197
|
+
rows.push({ name, type: isPy && name.startsWith('_') ? 'local-function' : 'function', index });
|
|
3198
|
+
} else if (top.kind === 'scope') {
|
|
3199
|
+
rows.push({ name, type: 'method', index, parent: top.name });
|
|
3200
|
+
}
|
|
3201
|
+
stack.push({ indent, kind: 'def', name });
|
|
3202
|
+
}
|
|
3203
|
+
}
|
|
3204
|
+
return rows;
|
|
3205
|
+
}
|
|
3206
|
+
|
|
3207
|
+
function extractGoSymbols(content) {
|
|
3208
|
+
const text = maskCLike(content, 0);
|
|
3209
|
+
const rows = [];
|
|
3210
|
+
for (const m of text.matchAll(/^type[ \t]+([A-Za-z_]\w*)/gm)) {
|
|
3211
|
+
rows.push({ name: m[1], type: 'type', index: m.index });
|
|
3212
|
+
}
|
|
3213
|
+
for (const m of text.matchAll(/^func[ \t]+([A-Za-z_]\w*)[ \t]*[[(]/gm)) {
|
|
3214
|
+
rows.push({ name: m[1], type: /^[A-Z]/.test(m[1]) ? 'function' : 'local-function', index: m.index });
|
|
3215
|
+
}
|
|
3216
|
+
for (const m of text.matchAll(/^func[ \t]*\([ \t]*(?:[A-Za-z_]\w*[ \t]+)?(?:\*[ \t]*)?([A-Za-z_]\w*)(?:\[[^\]\n]*\])?[ \t]*\)[ \t]*([A-Za-z_]\w*)/gm)) {
|
|
3217
|
+
rows.push({ name: m[2], type: 'method', index: m.index, parent: m[1] });
|
|
3218
|
+
}
|
|
3219
|
+
return rows;
|
|
3220
|
+
}
|
|
3221
|
+
|
|
3222
|
+
// Java, C#, PHP and Rust: depth-tracked on a comment/string-masked copy. A container
|
|
3223
|
+
// (class, trait, impl, mod ...) owns the lines exactly one brace level below its declaration.
|
|
3224
|
+
function extractBraceLanguageSymbols(filePath, content) {
|
|
3225
|
+
const ext = filePath.slice(filePath.lastIndexOf('.') + 1).toLowerCase();
|
|
3226
|
+
const text = maskCLike(content, ext === 'rs' ? 1 : 2);
|
|
3227
|
+
const rows = [];
|
|
3228
|
+
const stack = [];
|
|
3229
|
+
let depth = 0;
|
|
3230
|
+
let offset = 0;
|
|
3231
|
+
for (const line of text.split('\n')) {
|
|
3232
|
+
const lineStart = offset;
|
|
3233
|
+
offset += line.length + 1;
|
|
3234
|
+
while (stack.length > 0 && stack[stack.length - 1].opened && depth <= stack[stack.length - 1].depth) stack.pop();
|
|
3235
|
+
const top = stack[stack.length - 1];
|
|
3236
|
+
const atMember = top ? depth === top.depth + 1 : false;
|
|
3237
|
+
const index = lineStart + Math.max(0, line.search(/\S/));
|
|
3238
|
+
const pushContainer = (kind, name) => {
|
|
3239
|
+
if (line.includes(';') && !line.includes('{')) return;
|
|
3240
|
+
stack.push({ kind, name, depth, opened: false });
|
|
3241
|
+
};
|
|
3242
|
+
|
|
3243
|
+
if (ext === 'java' || ext === 'cs') {
|
|
3244
|
+
const typeMatch = POLY_JAVA_TYPE_RE.exec(line);
|
|
3245
|
+
if (typeMatch) {
|
|
3246
|
+
const kind = typeMatch[1].split(/\s/)[0];
|
|
3247
|
+
rows.push({ name: typeMatch[2], type: kind === 'class' ? 'class' : 'type', index, parent: top?.name });
|
|
3248
|
+
pushContainer(kind, typeMatch[2]);
|
|
3249
|
+
} else if (atMember) {
|
|
3250
|
+
const m = POLY_JAVA_METHOD_RE.exec(line);
|
|
3251
|
+
if (m && !POLY_MEMBER_SKIP.has(m[1]) && !(top.kind === 'enum' && /^[A-Z0-9_]+$/.test(m[1]))) {
|
|
3252
|
+
rows.push({ name: m[1], type: 'method', index, parent: top.name });
|
|
3253
|
+
}
|
|
3254
|
+
}
|
|
3255
|
+
} else if (ext === 'php') {
|
|
3256
|
+
const typeMatch = POLY_PHP_TYPE_RE.exec(line);
|
|
3257
|
+
const fn = typeMatch ? null : POLY_PHP_FUNC_RE.exec(line);
|
|
3258
|
+
if (typeMatch) {
|
|
3259
|
+
rows.push({ name: typeMatch[2], type: typeMatch[1] === 'class' ? 'class' : 'type', index, parent: top?.name });
|
|
3260
|
+
pushContainer(typeMatch[1], typeMatch[2]);
|
|
3261
|
+
} else if (fn && !top) {
|
|
3262
|
+
rows.push({ name: fn[1], type: 'function', index });
|
|
3263
|
+
} else if (fn && atMember) {
|
|
3264
|
+
rows.push({ name: fn[1], type: 'method', index, parent: top.name });
|
|
3265
|
+
}
|
|
3266
|
+
} else {
|
|
3267
|
+
const fn = POLY_RS_FN_RE.exec(line);
|
|
3268
|
+
const trait = fn ? null : POLY_RS_TRAIT_RE.exec(line);
|
|
3269
|
+
const impl = fn || trait ? null : polyRsImplName(line);
|
|
3270
|
+
const mod = fn || trait || impl ? null : POLY_RS_MOD_RE.exec(line);
|
|
3271
|
+
const typeMatch = fn || trait || impl || mod ? null : POLY_RS_TYPE_RE.exec(line);
|
|
3272
|
+
const atModuleLevel = top ? top.kind === 'mod' && atMember : depth === 0;
|
|
3273
|
+
if (fn) {
|
|
3274
|
+
const container = top && (top.kind === 'impl' || top.kind === 'trait');
|
|
3275
|
+
if (container && atMember) {
|
|
3276
|
+
rows.push({ name: fn[2], type: 'method', index, parent: top.name });
|
|
3277
|
+
} else if (atModuleLevel) {
|
|
3278
|
+
rows.push({ name: fn[2], type: fn[1].trim() ? 'function' : 'local-function', index });
|
|
3279
|
+
}
|
|
3280
|
+
} else if (trait) {
|
|
3281
|
+
if (atModuleLevel) rows.push({ name: trait[1], type: 'type', index });
|
|
3282
|
+
pushContainer('trait', trait[1]);
|
|
3283
|
+
} else if (impl) {
|
|
3284
|
+
pushContainer('impl', impl);
|
|
3285
|
+
} else if (mod) {
|
|
3286
|
+
pushContainer('mod', mod[1]);
|
|
3287
|
+
} else if (typeMatch && atModuleLevel) {
|
|
3288
|
+
rows.push({ name: typeMatch[2], type: 'type', index });
|
|
3289
|
+
}
|
|
3290
|
+
}
|
|
3291
|
+
|
|
3292
|
+
let maxDepth = depth;
|
|
3293
|
+
for (let k = 0; k < line.length; k += 1) {
|
|
3294
|
+
if (line[k] === '{') {
|
|
3295
|
+
depth += 1;
|
|
3296
|
+
if (depth > maxDepth) maxDepth = depth;
|
|
3297
|
+
} else if (line[k] === '}' && depth > 0) {
|
|
3298
|
+
depth -= 1;
|
|
3299
|
+
}
|
|
3300
|
+
}
|
|
3301
|
+
for (const entry of stack) {
|
|
3302
|
+
if (!entry.opened && maxDepth > entry.depth) entry.opened = true;
|
|
3303
|
+
}
|
|
3304
|
+
}
|
|
3305
|
+
return rows;
|
|
3306
|
+
}
|
|
3307
|
+
|
|
3308
|
+
function extractPolyglotSymbols(filePath, rawContent) {
|
|
3309
|
+
if (rawContent.length > POLY_MAX_FILE_CHARS) return [];
|
|
3310
|
+
const parts = rawContent.split('\n');
|
|
3311
|
+
const content = parts.some((l) => l.length > POLY_MAX_LINE_CHARS)
|
|
3312
|
+
? parts.map((l) => (l.length > POLY_MAX_LINE_CHARS ? ' '.repeat(l.length) : l)).join('\n')
|
|
3313
|
+
: rawContent;
|
|
3314
|
+
const ext = filePath.slice(filePath.lastIndexOf('.') + 1).toLowerCase();
|
|
3315
|
+
let rows;
|
|
3316
|
+
if (ext === 'py' || ext === 'rb') rows = extractIndentLanguageSymbols(filePath, content);
|
|
3317
|
+
else if (ext === 'go') rows = extractGoSymbols(content);
|
|
3318
|
+
else rows = extractBraceLanguageSymbols(filePath, content);
|
|
3319
|
+
const seen = new Set();
|
|
3320
|
+
return rows.filter((row) => {
|
|
3321
|
+
const key = `${row.type}:${row.name}:${row.parent ?? ''}`;
|
|
3322
|
+
if (seen.has(key)) return false;
|
|
3323
|
+
seen.add(key);
|
|
3324
|
+
return true;
|
|
3325
|
+
});
|
|
3326
|
+
}
|
|
3327
|
+
|
|
3328
|
+
// `const x = (` also opens plain parenthesised expressions (`const n = (a ?? 0)`), so only a
|
|
3329
|
+
// parameter list whose closing paren is followed by `=>` (optionally after a return type) is a function.
|
|
3330
|
+
function findParenArrowDeclarations(content) {
|
|
3331
|
+
const found = [];
|
|
3332
|
+
for (const match of content.matchAll(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?\(/g)) {
|
|
3333
|
+
const open = match.index + match[0].length - 1;
|
|
3334
|
+
const limit = Math.min(content.length, open + 2000);
|
|
3335
|
+
let depth = 0;
|
|
3336
|
+
let close = -1;
|
|
3337
|
+
for (let i = open; i < limit; i += 1) {
|
|
3338
|
+
if (content[i] === '(') depth += 1;
|
|
3339
|
+
else if (content[i] === ')') {
|
|
3340
|
+
depth -= 1;
|
|
3341
|
+
if (depth === 0) {
|
|
3342
|
+
close = i;
|
|
3343
|
+
break;
|
|
3344
|
+
}
|
|
3345
|
+
}
|
|
3346
|
+
}
|
|
3347
|
+
if (close >= 0 && /^\s*(?::\s*[^;\n{}=]*?(?:=>\s*[^;\n{}=]*?)?)?=>/.test(content.slice(close + 1, close + 201))) {
|
|
3348
|
+
found.push({ name: match[1], index: match.index });
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
return found;
|
|
3352
|
+
}
|
|
3353
|
+
|
|
3354
|
+
const CLASS_METHOD_SKIP = new Set(['if', 'for', 'while', 'switch', 'catch', 'function', 'return', 'with', 'constructor', 'super']);
|
|
3355
|
+
|
|
2775
3356
|
export function extractSymbols(filePath, content) {
|
|
2776
3357
|
const out = [];
|
|
2777
3358
|
const lineStarts = computeLineStarts(content);
|
|
@@ -2793,6 +3374,13 @@ export function extractSymbols(filePath, content) {
|
|
|
2793
3374
|
}
|
|
2794
3375
|
};
|
|
2795
3376
|
|
|
3377
|
+
if (POLYGLOT_EXT.test(filePath)) {
|
|
3378
|
+
for (const row of extractPolyglotSymbols(filePath, content)) {
|
|
3379
|
+
addSymbol(row.name, row.type, row.index, row.parent ? { parent: row.parent } : null);
|
|
3380
|
+
}
|
|
3381
|
+
return out;
|
|
3382
|
+
}
|
|
3383
|
+
|
|
2796
3384
|
if (filePath.endsWith('.sql')) {
|
|
2797
3385
|
// Capture the object name, skipping an optional schema qualifier (qas.fn_rpt_billing → fn_rpt_billing).
|
|
2798
3386
|
const sqlIdent = '(?:(?:["`]?[A-Za-z_][A-Za-z0-9_$]*["`]?)\\s*\\.\\s*)?["`]?([A-Za-z_][A-Za-z0-9_$]*)["`]?';
|
|
@@ -2824,8 +3412,10 @@ export function extractSymbols(filePath, content) {
|
|
|
2824
3412
|
} else {
|
|
2825
3413
|
addMatches(/export\s+(?:async\s+)?function\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'function');
|
|
2826
3414
|
addMatches(/export\s+class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'class');
|
|
2827
|
-
|
|
2828
|
-
|
|
3415
|
+
for (const decl of findParenArrowDeclarations(content)) {
|
|
3416
|
+
addSymbol(decl.name, 'callable', decl.index);
|
|
3417
|
+
}
|
|
3418
|
+
addMatches(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s+)?[A-Za-z_$][A-Za-z0-9_$]*\s*=>/g, 'callable');
|
|
2829
3419
|
addMatches(/export\s+default\s+(?:async\s+)?function\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'default-function');
|
|
2830
3420
|
addMatches(/export\s+default\s+class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'default-class');
|
|
2831
3421
|
addMatches(/export\s+default\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*(?:;|\n|$)/g, 'default-reference');
|
|
@@ -2833,6 +3423,30 @@ export function extractSymbols(filePath, content) {
|
|
|
2833
3423
|
addMatches(/defineComponent\(\s*\{[\s\S]*?name\s*:\s*['"]([^'"]+)['"]/g, 'component-name');
|
|
2834
3424
|
addMatches(/export\s+default\s*\{[\s\S]*?name\s*:\s*['"]([^'"]+)['"]/g, 'component-name');
|
|
2835
3425
|
|
|
3426
|
+
// Unexported declarations (3.5.4). Line-anchored, so `export function` / `export class`
|
|
3427
|
+
// (indexed above as exported) never double-count. Local types rank below exported ones.
|
|
3428
|
+
addMatches(/^[ \t]*(?:async\s+)?function\s*\*?\s*([A-Za-z_$][A-Za-z0-9_$]*)\s*[(<]/gm, 'local-function');
|
|
3429
|
+
addMatches(/^[ \t]*(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s+)?function\b/gm, 'callable');
|
|
3430
|
+
addMatches(/^[ \t]*(?:abstract\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'local-class');
|
|
3431
|
+
if (/\.(?:ts|tsx|mts|cts|vue|svelte)$/.test(filePath)) {
|
|
3432
|
+
addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?(?:const\s+)?(?:interface|enum)\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'type');
|
|
3433
|
+
addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?type\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*(?:<[^=\n]*>)?\s*=/gm, 'type');
|
|
3434
|
+
}
|
|
3435
|
+
// Class methods: indented `name(args) {` lines inside a balanced class body.
|
|
3436
|
+
for (const classMatch of content.matchAll(/(?:^|[^\w$.])class\s+[A-Za-z_$][A-Za-z0-9_$]*(?:\s+extends\s+[\w$.]+(?:\([^)]*\))?)?\s*\{/g)) {
|
|
3437
|
+
const bodyStart = classMatch.index + classMatch[0].length;
|
|
3438
|
+
const classBody = sliceBalancedBlock(content, bodyStart);
|
|
3439
|
+
if (!classBody) {
|
|
3440
|
+
continue;
|
|
3441
|
+
}
|
|
3442
|
+
for (const methodMatch of classBody.matchAll(/^[ \t]+(?:(?:static|async|get|set)\s+)*\*?\s*(#?[A-Za-z_$][A-Za-z0-9_$]*)\s*\([^()]*\)\s*\{/gm)) {
|
|
3443
|
+
if (CLASS_METHOD_SKIP.has(methodMatch[1])) {
|
|
3444
|
+
continue;
|
|
3445
|
+
}
|
|
3446
|
+
addSymbol(methodMatch[1], 'method', bodyStart + methodMatch.index + methodMatch[0].indexOf(methodMatch[1]));
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
|
|
2836
3450
|
// Global-registered frontend functions: reachable from anywhere without an
|
|
2837
3451
|
// import edge, so impact analysis must treat them as verified dependencies.
|
|
2838
3452
|
addMatches(/\b(?:window|globalThis|global)\s*\.\s*([A-Za-z_$][A-Za-z0-9_$]*)\s*=/g, 'global');
|
|
@@ -3405,10 +4019,22 @@ const PATH_EXACT_SCORE = 60;
|
|
|
3405
4019
|
const PATH_SUFFIX_SCORE = 48;
|
|
3406
4020
|
const PATH_BASENAME_SCORE = 40;
|
|
3407
4021
|
const SYMBOL_EXACT_SCORE = 40;
|
|
4022
|
+
const LOCAL_SYMBOL_FACTOR = 0.85; // unexported symbols (local function/class, method, unexported type) rank below an exported symbol of the same name
|
|
3408
4023
|
const SYMBOL_DECAY = 0.35; // k-th best symbol of a file adds score * SYMBOL_DECAY^k
|
|
3409
4024
|
const SYMBOL_REASON_LIMIT = 3;
|
|
3410
4025
|
const MIN_SUBSTRING_TOKEN_LENGTH = 3;
|
|
3411
4026
|
const BOOST_LIFT_RATIO = 0.2; // import/hotspot/archetype boosts may add at most this share of a direct score (+1)
|
|
4027
|
+
const CONTENT_LANE_CONFIDENT_SCORE = 30; // a best direct score at/above this (exact symbol or path hit) skips the content lane
|
|
4028
|
+
const CONTENT_LANE_MIN_TOKENS = 3; // shorter queries are names, not messages
|
|
4029
|
+
const CONTENT_LANE_LIMIT = 40; // candidates only share tokens somewhere in the file; the same-line check below keeps the real ones
|
|
4030
|
+
const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|svelte|sql|sh|py|go|java|cs|php|rb|rs)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
|
|
4031
|
+
const CONTENT_LANE_BONUS = 14; // added to a file that already has a direct score
|
|
4032
|
+
const CONTENT_LANE_ONLY_SCORE = 13; // base for a file found only by content; below CONTENT_LANE_CONFIDENT_SCORE by design
|
|
4033
|
+
const CONTENT_LANE_MAX_FILES = 1500; // files read per query, in priority order (primary bound)
|
|
4034
|
+
const CONTENT_LANE_MAX_BYTES = 24 * 1024 * 1024; // characters read per query (primary bound)
|
|
4035
|
+
const CONTENT_LANE_MAX_FILE_BYTES = 512 * 1024; // larger files are generated/data, never scanned
|
|
4036
|
+
const CONTENT_LANE_BATCH = 24;
|
|
4037
|
+
const CONTENT_LANE_SAFETY_MS = 5000; // safety cap only: hitting it reports 'time-budget', normal runs never do
|
|
3412
4038
|
const BOOST_ONLY_RATIO = 0.3; // files with no direct match stay below this share of the best direct score
|
|
3413
4039
|
|
|
3414
4040
|
// Rarity weight per token in (0,1]: a token present in nearly every file is worth
|
|
@@ -3439,10 +4065,10 @@ function buildTokenWeights(searchDescriptorsByFile) {
|
|
|
3439
4065
|
// hiding the mirror: root installed copies < template_project < everything else.
|
|
3440
4066
|
function mirrorPrior(filePath) {
|
|
3441
4067
|
if (/^\.(?:claude|codex|omp)\//.test(filePath)) {
|
|
3442
|
-
return 0.
|
|
4068
|
+
return 0.8;
|
|
3443
4069
|
}
|
|
3444
4070
|
if (filePath.startsWith('template_project/')) {
|
|
3445
|
-
return 0.
|
|
4071
|
+
return 0.85;
|
|
3446
4072
|
}
|
|
3447
4073
|
return 1;
|
|
3448
4074
|
}
|
|
@@ -3510,7 +4136,7 @@ function scoreDirectFileMatch({ queryDescriptor, rawQuery = '', filePath, search
|
|
|
3510
4136
|
queryDescriptor.compact.length >= 4
|
|
3511
4137
|
&& symbolDescriptor.descriptor.compact === queryDescriptor.compact
|
|
3512
4138
|
) {
|
|
3513
|
-
symbolHits.push({ score: SYMBOL_EXACT_SCORE, reasons: [`symbol_exact:${symbolDescriptor.name}`] });
|
|
4139
|
+
symbolHits.push({ score: SYMBOL_EXACT_SCORE * (symbolDescriptor.local ? LOCAL_SYMBOL_FACTOR : 1), reasons: [`symbol_exact:${symbolDescriptor.name}`] });
|
|
3514
4140
|
continue;
|
|
3515
4141
|
}
|
|
3516
4142
|
|
|
@@ -3550,7 +4176,7 @@ function scoreDirectFileMatch({ queryDescriptor, rawQuery = '', filePath, search
|
|
|
3550
4176
|
}
|
|
3551
4177
|
|
|
3552
4178
|
if (symbolScore > 0) {
|
|
3553
|
-
symbolHits.push({ score: symbolScore, reasons: symbolReasons });
|
|
4179
|
+
symbolHits.push({ score: symbolScore * (symbolDescriptor.local ? LOCAL_SYMBOL_FACTOR : 1), reasons: symbolReasons });
|
|
3554
4180
|
}
|
|
3555
4181
|
}
|
|
3556
4182
|
|
|
@@ -4069,11 +4695,124 @@ function buildFileSearchDescriptors(files, symbolMap) {
|
|
|
4069
4695
|
return descriptors;
|
|
4070
4696
|
}
|
|
4071
4697
|
|
|
4698
|
+
// Rows whose declaration is not exported. 'callable' keeps its pre-3.5.4 weight.
|
|
4699
|
+
// Outline-only: a 'callable' const without `export` is also unexported.
|
|
4700
|
+
function isLocalOutlineRow(item) {
|
|
4701
|
+
return isLocalSymbolRow(item) || (item?.type === 'callable' && !/^\s*export\b/.test(item?.signature ?? ''));
|
|
4702
|
+
}
|
|
4703
|
+
|
|
4704
|
+
function isLocalSymbolRow(symbol) {
|
|
4705
|
+
const type = symbol?.type;
|
|
4706
|
+
if (type === 'local-function' || type === 'local-class' || type === 'method') {
|
|
4707
|
+
return true;
|
|
4708
|
+
}
|
|
4709
|
+
return type === 'type' && !/^\s*export\b/.test(symbol?.signature ?? '');
|
|
4710
|
+
}
|
|
4711
|
+
|
|
4712
|
+
// Single identifiers, questions and non-ASCII (Vietnamese) prose are descriptions of behaviour, not strings
|
|
4713
|
+
// that appear in source, so they never pay for a content search.
|
|
4714
|
+
function looksLikeLiteralMessage(query) {
|
|
4715
|
+
return /^[\x20-\x7e]+$/.test(query)
|
|
4716
|
+
&& query.split(/\s+/).length >= CONTENT_LANE_MIN_TOKENS // identifiers and short phrases are symbol queries, never a message
|
|
4717
|
+
&& !/^(?:how|where|what|why|which|when|who|does|do|is|are|can|find|show|list)\b/i.test(query);
|
|
4718
|
+
}
|
|
4719
|
+
|
|
4720
|
+
// Stable verify order: index evidence first (rarity-weighted overlap of the query tokens with
|
|
4721
|
+
// each file's path and symbol tokens), originals before mirrors, then path order.
|
|
4722
|
+
function rankContentLaneFiles({ files, searchDescriptorsByFile, tokenWeights, queryTokens, includeLikelyTestFiles }) {
|
|
4723
|
+
const rows = [];
|
|
4724
|
+
for (const file of safeItems(files)) {
|
|
4725
|
+
const filePath = file?.filePath;
|
|
4726
|
+
if (typeof filePath !== 'string' || !CONTENT_LANE_CODE_EXT.test(filePath)) {
|
|
4727
|
+
continue;
|
|
4728
|
+
}
|
|
4729
|
+
if (isLikelyTestFilePath(filePath) && !includeLikelyTestFiles) {
|
|
4730
|
+
continue;
|
|
4731
|
+
}
|
|
4732
|
+
let overlap = 0;
|
|
4733
|
+
const descriptor = searchDescriptorsByFile.get(filePath);
|
|
4734
|
+
if (descriptor) {
|
|
4735
|
+
const seen = new Set(descriptor.path.tokens);
|
|
4736
|
+
for (const symbol of descriptor.symbols) {
|
|
4737
|
+
for (const token of symbol.descriptor.tokens) {
|
|
4738
|
+
seen.add(token);
|
|
4739
|
+
}
|
|
4740
|
+
}
|
|
4741
|
+
for (const token of queryTokens) {
|
|
4742
|
+
if (seen.has(token)) {
|
|
4743
|
+
overlap += tokenWeights?.get(token) ?? 1;
|
|
4744
|
+
}
|
|
4745
|
+
}
|
|
4746
|
+
}
|
|
4747
|
+
rows.push({ filePath, overlap, mirror: mirrorPrior(filePath) < 1 ? 1 : 0, size: Number.isFinite(Number(file.size)) && Number(file.size) >= 0 ? Number(file.size) : null });
|
|
4748
|
+
}
|
|
4749
|
+
rows.sort((a, b) => (b.overlap - a.overlap) || (a.mirror - b.mirror) || (a.filePath < b.filePath ? -1 : a.filePath > b.filePath ? 1 : 0));
|
|
4750
|
+
return rows;
|
|
4751
|
+
}
|
|
4752
|
+
|
|
4753
|
+
// Reads files in priority order (batched for speed, consumed in order) until the hit limit or a
|
|
4754
|
+
// count/byte cap. Caps are checked at batch boundaries on counts and byte totals only, so the
|
|
4755
|
+
// same tree yields the same hits regardless of machine load.
|
|
4756
|
+
async function scanContentLane({ absoluteRoot, ordered, tokens, limit }) {
|
|
4757
|
+
const hits = [];
|
|
4758
|
+
const truncated = new Set();
|
|
4759
|
+
if (tokens.length === 0) {
|
|
4760
|
+
return { hits, truncated: [] };
|
|
4761
|
+
}
|
|
4762
|
+
let scanned = 0;
|
|
4763
|
+
let bytes = 0;
|
|
4764
|
+
const deadline = Date.now() + CONTENT_LANE_SAFETY_MS;
|
|
4765
|
+
for (let start = 0; start < ordered.length && hits.length < limit; start += CONTENT_LANE_BATCH) {
|
|
4766
|
+
if (scanned >= CONTENT_LANE_MAX_FILES) { truncated.add('max-files'); break; }
|
|
4767
|
+
if (bytes >= CONTENT_LANE_MAX_BYTES) { truncated.add('max-bytes'); break; }
|
|
4768
|
+
if (Date.now() > deadline) { truncated.add('time-budget'); break; }
|
|
4769
|
+
const batch = ordered.slice(start, start + Math.min(CONTENT_LANE_BATCH, CONTENT_LANE_MAX_FILES - scanned));
|
|
4770
|
+
// Size comes from the index entry, else a stat; an oversized file is skipped before any read.
|
|
4771
|
+
const texts = await Promise.all(batch.map(async (row) => {
|
|
4772
|
+
try {
|
|
4773
|
+
const absolute = path.join(absoluteRoot, row.filePath);
|
|
4774
|
+
const size = row.size ?? (await fs.stat(absolute)).size;
|
|
4775
|
+
if (size > CONTENT_LANE_MAX_FILE_BYTES) {
|
|
4776
|
+
return { text: '', size: 0 };
|
|
4777
|
+
}
|
|
4778
|
+
const text = await fs.readFile(absolute, 'utf8');
|
|
4779
|
+
return { text, size: Buffer.byteLength(text, 'utf8') };
|
|
4780
|
+
} catch {
|
|
4781
|
+
return { text: '', size: 0 };
|
|
4782
|
+
}
|
|
4783
|
+
}));
|
|
4784
|
+
for (const [index, { text, size }] of texts.entries()) {
|
|
4785
|
+
scanned += 1;
|
|
4786
|
+
bytes += size;
|
|
4787
|
+
if (text.length === 0 || size > CONTENT_LANE_MAX_FILE_BYTES) {
|
|
4788
|
+
continue;
|
|
4789
|
+
}
|
|
4790
|
+
const lower = text.toLowerCase();
|
|
4791
|
+
if (!tokens.every((token) => lower.includes(token))) {
|
|
4792
|
+
continue;
|
|
4793
|
+
}
|
|
4794
|
+
const lines = lower.split('\n');
|
|
4795
|
+
const lineIndex = lines.findIndex((line) => tokens.every((token) => line.includes(token)));
|
|
4796
|
+
if (lineIndex >= 0) {
|
|
4797
|
+
hits.push({ filePath: batch[index].filePath, line: lineIndex + 1 });
|
|
4798
|
+
if (hits.length >= limit) {
|
|
4799
|
+
break;
|
|
4800
|
+
}
|
|
4801
|
+
}
|
|
4802
|
+
}
|
|
4803
|
+
}
|
|
4804
|
+
if (scanned >= CONTENT_LANE_MAX_FILES && scanned < ordered.length && hits.length < limit) {
|
|
4805
|
+
truncated.add('max-files');
|
|
4806
|
+
}
|
|
4807
|
+
return { hits, truncated: [...truncated] };
|
|
4808
|
+
}
|
|
4809
|
+
|
|
4072
4810
|
function createFileSearchDescriptor({ filePath, symbols }) {
|
|
4073
4811
|
return {
|
|
4074
4812
|
path: buildTextSearchDescriptor(filePath),
|
|
4075
4813
|
symbols: symbols.map((symbol) => ({
|
|
4076
4814
|
name: symbol.name,
|
|
4815
|
+
local: isLocalSymbolRow(symbol),
|
|
4077
4816
|
descriptor: buildTextSearchDescriptor(symbol.name),
|
|
4078
4817
|
})),
|
|
4079
4818
|
};
|
|
@@ -4114,6 +4853,16 @@ export function buildSearchDescriptor(value, { expandVietnameseAliases = false }
|
|
|
4114
4853
|
for (const token of collectVietnameseAliasTokens(foldedLower, tokens)) {
|
|
4115
4854
|
tokens.add(token);
|
|
4116
4855
|
}
|
|
4856
|
+
// Stems only for question-like queries: identifier-style queries (findRelatedItemIds,
|
|
4857
|
+
// collectVietnameseAliasTokens) must keep matching exact tokens.
|
|
4858
|
+
if (tokens.size >= STEM_MIN_QUERY_TOKENS) {
|
|
4859
|
+
for (const token of [...tokens]) {
|
|
4860
|
+
const stem = stemToken(token);
|
|
4861
|
+
if (stem && !GENERIC_STOPWORDS.has(stem)) {
|
|
4862
|
+
tokens.add(stem);
|
|
4863
|
+
}
|
|
4864
|
+
}
|
|
4865
|
+
}
|
|
4117
4866
|
}
|
|
4118
4867
|
|
|
4119
4868
|
return {
|
|
@@ -4485,6 +5234,7 @@ function isLikelyTestFilePath(filePath) {
|
|
|
4485
5234
|
|| lower.endsWith('.spec.jsx')
|
|
4486
5235
|
|| lower.endsWith('.spec.mjs')
|
|
4487
5236
|
|| lower.endsWith('.spec.cjs')
|
|
5237
|
+
|| /(?:_test\.(?:go|py|rb|rs)|_spec\.rb|(?:^|\/)test_[^/]+\.py|[a-z0-9]Tests?\.(?:java|cs))$/.test(filePath)
|
|
4488
5238
|
);
|
|
4489
5239
|
}
|
|
4490
5240
|
|
|
@@ -5118,6 +5868,7 @@ function isLikelyTestFile(filePath) {
|
|
|
5118
5868
|
|| lower.endsWith('.spec.jsx')
|
|
5119
5869
|
|| lower.endsWith('.spec.mjs')
|
|
5120
5870
|
|| lower.endsWith('.spec.cjs')
|
|
5871
|
+
|| /(?:_test\.(?:go|py|rb|rs)|_spec\.rb|(?:^|\/)test_[^/]+\.py|[a-z0-9]Tests?\.(?:java|cs))$/.test(filePath)
|
|
5121
5872
|
);
|
|
5122
5873
|
}
|
|
5123
5874
|
|