@ngockhoale/ukit 3.5.4 → 3.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/manifests/engineConformance.yaml +0 -13
- package/manifests/platform.full.yaml +25 -0
- package/package.json +3 -9
- package/src/cli/commands/indexTools.js +1 -0
- package/src/cli/commands/memory.js +44 -4
- package/src/core/applyPlan.js +25 -72
- package/src/core/diffPlan.js +38 -9
- package/src/core/managedTemplateMerge.js +146 -0
- package/src/core/memory/memoryCheckRun.js +13 -1
- package/src/core/memory/memoryFlags.js +5 -6
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/policy.js +5 -0
- package/src/core/memory/recallEngine.js +39 -3
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/store.js +17 -0
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/metadata.js +12 -8
- package/src/core/runInstallPipeline.js +16 -10
- package/src/core/runtimeConfig.js +59 -83
- package/src/core/uninstall.js +8 -1
- package/src/decision/registry.js +33 -16
- package/src/decision/runtimeDecide.js +5 -5
- package/src/index/buildIndex.js +287 -6
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +2 -1
- package/src/index/queryIndex.js +174 -10
- package/src/index/routeResolver.js +2 -2
- package/src/index/semanticLane.js +269 -0
- package/src/learning/lessonClass.js +110 -0
- package/src/learning/patternProposals.js +111 -7
- package/src/learning/reeditMiner.js +104 -0
- package/src/learning/selfImprove.js +9 -1
- package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
- package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
- package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
- package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
- package/template_project/.claude/ukit/index/lib/index-core.mjs +561 -14
- package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
- package/template_project/.claude/ukit/index/query-index.mjs +9 -1
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/ukit/storage/config.json +19 -34
- package/manifests/instructionRules.yaml +0 -442
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
|
@@ -4,6 +4,7 @@ import crypto from 'node:crypto';
|
|
|
4
4
|
import { execFile, spawnSync } from 'node:child_process';
|
|
5
5
|
import { withFileLock } from '../../runtime/token-utils.mjs';
|
|
6
6
|
import { withAsyncLock } from '../../runtime/async-lock.mjs';
|
|
7
|
+
import { embedConfig, looksLikeQuestion, semanticCandidates, fuseSemantic, semanticStoreSignature } from './semantic-lane.mjs';
|
|
7
8
|
// edit-graph.mjs is optional: a partial install must not lose the whole index core.
|
|
8
9
|
let editGraphModule;
|
|
9
10
|
function loadEditGraphModule() {
|
|
@@ -135,9 +136,9 @@ const EXCLUDED_DIR_NAMES = new Set([
|
|
|
135
136
|
'.venv',
|
|
136
137
|
'venv',
|
|
137
138
|
]);
|
|
138
|
-
const CODE_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.sql']);
|
|
139
|
+
const CODE_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.svelte', '.py', '.go', '.java', '.cs', '.php', '.rb', '.rs', '.sql']);
|
|
139
140
|
const STYLE_EXTENSIONS = new Set(['.css', '.scss', '.sass', '.less']);
|
|
140
|
-
const TRACKED_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.sql', '.json', '.yaml', '.yml', '.md', '.sh']);
|
|
141
|
+
const TRACKED_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.svelte', '.py', '.go', '.java', '.cs', '.php', '.rb', '.rs', '.sql', '.json', '.yaml', '.yml', '.md', '.sh']);
|
|
141
142
|
const DISCOVERED_EXTENSIONS = new Set([...TRACKED_EXTENSIONS, ...STYLE_EXTENSIONS]);
|
|
142
143
|
// v9: symbols.json gains global-registered ('global') and Vue 'method' rows,
|
|
143
144
|
// SQL 'column' rows with parent/schema, and calls.json gains per-object SQL
|
|
@@ -145,7 +146,7 @@ const DISCOVERED_EXTENSIONS = new Set([...TRACKED_EXTENSIONS, ...STYLE_EXTENSION
|
|
|
145
146
|
// files or the new extractions silently vanish.
|
|
146
147
|
// v10: symbols.json gains unexported function/class ('local-function', 'local-class'),
|
|
147
148
|
// function-expression consts, class 'method' rows and TS interface/enum/type ('type') rows.
|
|
148
|
-
export const INDEX_SCHEMA_VERSION =
|
|
149
|
+
export const INDEX_SCHEMA_VERSION = 11;
|
|
149
150
|
export const DEFAULT_INDEX_CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
150
151
|
// Parse-phase read/extract concurrency — deliberately tighter than the
|
|
151
152
|
// discovery stat pool because parsing holds file content in memory.
|
|
@@ -213,6 +214,46 @@ const VIETNAMESE_PHRASE_ALIASES = [
|
|
|
213
214
|
{ regex: /\bthanh phan\b/gu, expansions: ['component'] },
|
|
214
215
|
{ regex: /\bkiem thu\b/gu, expansions: ['test', 'testing'] },
|
|
215
216
|
{ regex: /\bbo cuc\b/gu, expansions: ['layout'] },
|
|
217
|
+
// General programming vocabulary (diacritics folded; matches with or without accents).
|
|
218
|
+
{ regex: /\bnhay cam\b/gu, expansions: ['sensitive'] },
|
|
219
|
+
{ regex: /\bbi mat\b/gu, expansions: ['secret'] },
|
|
220
|
+
{ regex: /\bgia tri\b/gu, expansions: ['value'] },
|
|
221
|
+
{ regex: /\bvan ban\b/gu, expansions: ['text'] },
|
|
222
|
+
{ regex: /\bsao luu\b/gu, expansions: ['backup'] },
|
|
223
|
+
{ regex: /\bkhoi phuc\b/gu, expansions: ['restore', 'recover'] },
|
|
224
|
+
{ regex: /\bdinh danh\b/gu, expansions: ['identity', 'id'] },
|
|
225
|
+
{ regex: /\bdu an\b/gu, expansions: ['project'] },
|
|
226
|
+
{ regex: /\bthu muc\b/gu, expansions: ['directory', 'dir', 'folder'] },
|
|
227
|
+
{ regex: /\bphan loai\b/gu, expansions: ['classify', 'category'] },
|
|
228
|
+
{ regex: /\buoc tinh\b/gu, expansions: ['estimate'] },
|
|
229
|
+
{ regex: /\bcong chan\b/gu, expansions: ['gate', 'guard'] },
|
|
230
|
+
{ regex: /\bcap nhat\b/gu, expansions: ['update'] },
|
|
231
|
+
{ regex: /\bgo cai dat\b/gu, expansions: ['uninstall'] },
|
|
232
|
+
{ regex: /\bcai dat\b/gu, expansions: ['install', 'setup'] },
|
|
233
|
+
{ regex: /\bcau hinh\b/gu, expansions: ['config', 'configuration'] },
|
|
234
|
+
{ regex: /\bbao cao\b/gu, expansions: ['report'] },
|
|
235
|
+
{ regex: /\blich su\b/gu, expansions: ['history'] },
|
|
236
|
+
{ regex: /\bnhat ky\b/gu, expansions: ['log'] },
|
|
237
|
+
{ regex: /\bchi muc\b/gu, expansions: ['index'] },
|
|
238
|
+
{ regex: /\bxu ly\b/gu, expansions: ['handle', 'process'] },
|
|
239
|
+
{ regex: /\bphan tich\b/gu, expansions: ['parse', 'analyze'] },
|
|
240
|
+
{ regex: /\bhet han\b/gu, expansions: ['expire', 'expiry'] },
|
|
241
|
+
{ regex: /\bchup anh\b/gu, expansions: ['snapshot'] },
|
|
242
|
+
{ regex: /\bhoan tac\b/gu, expansions: ['rollback', 'revert'] },
|
|
243
|
+
{ regex: /\bnguoi dung\b/gu, expansions: ['user'] },
|
|
244
|
+
{ regex: /\bthong bao\b/gu, expansions: ['notify', 'message'] },
|
|
245
|
+
{ regex: /\bsap xep\b/gu, expansions: ['sort'] },
|
|
246
|
+
{ regex: /\bdang ky\b/gu, expansions: ['register'] },
|
|
247
|
+
{ regex: /\btham so\b/gu, expansions: ['argument', 'param', 'option'] },
|
|
248
|
+
{ regex: /\bket qua\b/gu, expansions: ['result'] },
|
|
249
|
+
{ regex: /\bmac dinh\b/gu, expansions: ['default'] },
|
|
250
|
+
{ regex: /\bdau vao\b/gu, expansions: ['input'] },
|
|
251
|
+
{ regex: /\bdau ra\b/gu, expansions: ['output'] },
|
|
252
|
+
{ regex: /\bhang doi\b/gu, expansions: ['queue'] },
|
|
253
|
+
{ regex: /\bma hoa\b/gu, expansions: ['encode', 'encrypt'] },
|
|
254
|
+
{ regex: /\bgiai ma\b/gu, expansions: ['decode', 'decrypt'] },
|
|
255
|
+
{ regex: /\bbo nho\b/gu, expansions: ['memory'] },
|
|
256
|
+
{ regex: /\bcanh bao\b/gu, expansions: ['warning', 'warn'] },
|
|
216
257
|
];
|
|
217
258
|
const VIETNAMESE_TOKEN_ALIASES = new Map([
|
|
218
259
|
['sua', ['fix']],
|
|
@@ -233,8 +274,58 @@ const VIETNAMESE_TOKEN_ALIASES = new Map([
|
|
|
233
274
|
['soat', ['review', 'audit']],
|
|
234
275
|
['trien', ['implement', 'build', 'ship']],
|
|
235
276
|
['khai', ['implement', 'build', 'ship']],
|
|
277
|
+
['kho', ['store', 'storage']],
|
|
278
|
+
['ghi', ['write', 'record']],
|
|
279
|
+
['doc', ['read']],
|
|
280
|
+
['luu', ['save', 'store', 'persist']],
|
|
281
|
+
['tim', ['find', 'search', 'resolve']],
|
|
282
|
+
['hong', ['broken', 'repair']],
|
|
283
|
+
['phut', ['minute', 'minutes']],
|
|
284
|
+
['ham', ['function']],
|
|
285
|
+
['bien', ['variable']],
|
|
286
|
+
['lop', ['class']],
|
|
287
|
+
['lenh', ['command']],
|
|
288
|
+
['chay', ['run', 'execute']],
|
|
289
|
+
['phien', ['session']],
|
|
290
|
+
['loc', ['filter']],
|
|
291
|
+
['gui', ['send', 'dispatch']],
|
|
292
|
+
['khoa', ['lock']],
|
|
293
|
+
['huy', ['cancel', 'abort']],
|
|
294
|
+
['quyet', ['decision']],
|
|
236
295
|
]);
|
|
237
296
|
|
|
297
|
+
// Query-time light stemming: adds the singular/base form of an English query word so
|
|
298
|
+
// 'files' reaches `file`, 'checks' reaches `check`, 'written' reaches `write`. Added next
|
|
299
|
+
// to the original token, never in place of it, and never applied to indexed text.
|
|
300
|
+
const IRREGULAR_STEMS = new Map([
|
|
301
|
+
['written', 'write'], ['wrote', 'write'], ['built', 'build'], ['ran', 'run'], ['saved', 'save'],
|
|
302
|
+
['found', 'find'], ['sent', 'send'], ['made', 'make'], ['read', 'read'], ['began', 'begin'],
|
|
303
|
+
]);
|
|
304
|
+
|
|
305
|
+
const STEM_MIN_QUERY_TOKENS = 5;
|
|
306
|
+
|
|
307
|
+
function stemToken(token) {
|
|
308
|
+
if (!/^[a-z]+$/.test(token) || token.length < 4) {
|
|
309
|
+
return null;
|
|
310
|
+
}
|
|
311
|
+
if (IRREGULAR_STEMS.has(token)) {
|
|
312
|
+
return IRREGULAR_STEMS.get(token);
|
|
313
|
+
}
|
|
314
|
+
let stem = null;
|
|
315
|
+
if (token.endsWith('ies') && token.length > 4) {
|
|
316
|
+
stem = `${token.slice(0, -3)}y`;
|
|
317
|
+
} else if (/(?:ches|shes|xes|sses)$/.test(token)) {
|
|
318
|
+
stem = token.slice(0, -2);
|
|
319
|
+
} else if (token.endsWith('s') && !/(?:ss|us|is|as|os)$/.test(token)) {
|
|
320
|
+
stem = token.slice(0, -1);
|
|
321
|
+
} else if (token.endsWith('ing') && token.length > 5) {
|
|
322
|
+
stem = token.slice(0, -3);
|
|
323
|
+
} else if (token.endsWith('ed') && token.length > 4) {
|
|
324
|
+
stem = token.endsWith('ied') ? `${token.slice(0, -3)}y` : token.slice(0, -2);
|
|
325
|
+
}
|
|
326
|
+
return stem && stem.length >= 3 && stem !== token ? stem : null;
|
|
327
|
+
}
|
|
328
|
+
|
|
238
329
|
// ── Build Index ──
|
|
239
330
|
|
|
240
331
|
// Bumped only when the snapshot shape itself changes; consumers must reject
|
|
@@ -1308,7 +1399,7 @@ export async function getFileOutline({ rootDir = process.cwd(), filePath, limit
|
|
|
1308
1399
|
}
|
|
1309
1400
|
|
|
1310
1401
|
// contentLane=false skips the error-string content search (context-resolution paths never pay for it).
|
|
1311
|
-
export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true } = {}) {
|
|
1402
|
+
export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true, semantic = false } = {}) {
|
|
1312
1403
|
const normalizedQuery = String(query ?? '').trim();
|
|
1313
1404
|
if (!normalizedQuery) return [];
|
|
1314
1405
|
if (limit <= 0) return Object.freeze([]);
|
|
@@ -1321,11 +1412,15 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1321
1412
|
query: normalizedQuery,
|
|
1322
1413
|
limit,
|
|
1323
1414
|
contentLane,
|
|
1415
|
+
semantic: semantic && embedConfig() ? `1:${await semanticStoreSignature(absoluteRoot)}` : 0,
|
|
1324
1416
|
});
|
|
1325
1417
|
if (QUERY_RESULT_CACHE.has(queryCacheKey)) {
|
|
1326
1418
|
return QUERY_RESULT_CACHE.get(queryCacheKey);
|
|
1327
1419
|
}
|
|
1328
1420
|
const queryPromise = (async () => {
|
|
1421
|
+
// `laneStatus.fellBack`: the lane was asked for (question-shaped, lexical not decisive) and
|
|
1422
|
+
// produced nothing (timeout / HTTP error / no vectors). Such a result must not be cached.
|
|
1423
|
+
const laneStatus = { fellBack: false };
|
|
1329
1424
|
const { files, searchDescriptorsByFile, tokenWeights } = await loadQuerySearchBundle(absoluteRoot);
|
|
1330
1425
|
const preferredArchetypes = inferPreferredArchetypes(queryDescriptor.tokens);
|
|
1331
1426
|
const includeLikelyTestFiles = shouldIncludeLikelyTestFiles(queryDescriptor);
|
|
@@ -1391,7 +1486,12 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1391
1486
|
});
|
|
1392
1487
|
}
|
|
1393
1488
|
|
|
1394
|
-
if (directScores.size === 0)
|
|
1489
|
+
if (directScores.size === 0) {
|
|
1490
|
+
const onlySemantic = semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked: [], includeLikelyTestFiles, files, laneStatus }) : null;
|
|
1491
|
+
if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
1492
|
+
if (onlySemantic) return withContentLaneStatus(freezeQueryResults(onlySemantic.slice(0, limit)), contentLaneRan, contentCutShort);
|
|
1493
|
+
return withContentLaneStatus(Object.freeze([]), contentLaneRan, contentCutShort, laneStatus.fellBack);
|
|
1494
|
+
}
|
|
1395
1495
|
|
|
1396
1496
|
const {
|
|
1397
1497
|
hotspotScores,
|
|
@@ -1429,9 +1529,13 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1429
1529
|
directScores,
|
|
1430
1530
|
scores: applyHotspotBoosts({ scores: relatedScores, hotspotScores }),
|
|
1431
1531
|
});
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1532
|
+
const ranked = [...boosted.values()].map((entry) => ({ ...entry, score: finalizeScore(entry) })).sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath));
|
|
1533
|
+
const fused = contentLane && ranked.length > 0 && ranked[0].score < FUSION_WEAK_SCORE && queryDescriptor.tokens.size >= FUSION_MIN_TOKENS
|
|
1534
|
+
? await fuseWeakQuery({ absoluteRoot, query: normalizedQuery, ranked, searchDescriptorsByFile, includeLikelyTestFiles })
|
|
1535
|
+
: null;
|
|
1536
|
+
const semanticMerged = !fused && semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked, includeLikelyTestFiles, files, laneStatus }) : null;
|
|
1537
|
+
if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
1538
|
+
return withContentLaneStatus(freezeQueryResults((fused ?? semanticMerged ?? ranked).slice(0, limit)), contentLaneRan, contentCutShort, laneStatus.fellBack);
|
|
1435
1539
|
})().catch((error) => {
|
|
1436
1540
|
QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
1437
1541
|
throw error;
|
|
@@ -1441,6 +1545,154 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1441
1545
|
return queryPromise;
|
|
1442
1546
|
}
|
|
1443
1547
|
|
|
1548
|
+
// Real-embedding lane: only when the CLI passes semantic:true AND UKIT_INDEX_EMBED_URL is set.
|
|
1549
|
+
// Applies to question-shaped queries only; any problem returns null (lexical result unchanged).
|
|
1550
|
+
const SEMANTIC_CONFIDENT_MARGIN = 2;
|
|
1551
|
+
async function applySemanticLane({ absoluteRoot, query, ranked, includeLikelyTestFiles, files, laneStatus = {} }) {
|
|
1552
|
+
try {
|
|
1553
|
+
if (!embedConfig() || !looksLikeQuestion(query)) return null;
|
|
1554
|
+
// Lexical is already decisive (top-1 clear of top-2): keep it exactly, no network call.
|
|
1555
|
+
if (ranked.length > 1 && ranked[0].score - ranked[1].score >= SEMANTIC_CONFIDENT_MARGIN) return null;
|
|
1556
|
+
const known = new Set(safeItems(files).map((f) => f?.filePath));
|
|
1557
|
+
const candidates = await semanticCandidates({
|
|
1558
|
+
rootDir: absoluteRoot,
|
|
1559
|
+
query,
|
|
1560
|
+
// Vectors can outlive a deleted/renamed file: only paths in the current index qualify.
|
|
1561
|
+
accept: (filePath) => known.has(filePath) && (includeLikelyTestFiles || !isLikelyTestFilePath(filePath)),
|
|
1562
|
+
weight: mirrorPrior,
|
|
1563
|
+
});
|
|
1564
|
+
const fused = fuseSemantic({ ranked, candidates });
|
|
1565
|
+
if (!fused) laneStatus.fellBack = true;
|
|
1566
|
+
return fused;
|
|
1567
|
+
} catch {
|
|
1568
|
+
laneStatus.fellBack = true;
|
|
1569
|
+
return null;
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
// Opt-in lane fusion (default OFF): a weak natural-language query (best score below
|
|
1574
|
+
// FUSION_WEAK_SCORE, >= FUSION_MIN_TOKENS tokens) is re-ranked by RRF over the scored list
|
|
1575
|
+
// and a hashed-embedding lane (the retriever.js vector lane). Enabled by
|
|
1576
|
+
// `.ukit/storage/config.json` -> index.fusion === true or UKIT_INDEX_FUSION=1. The flag is
|
|
1577
|
+
// read only after the weak gate passes, and contentLane:false callers (hooks/resolveContext)
|
|
1578
|
+
// never reach it. Never throws; async fs only.
|
|
1579
|
+
const FUSION_WEAK_SCORE = 6;
|
|
1580
|
+
const FUSION_MIN_TOKENS = 4;
|
|
1581
|
+
const FUSION_RRF_K = 5;
|
|
1582
|
+
const FUSION_VECTOR_WEIGHT = 1;
|
|
1583
|
+
const FUSION_LANE_LIMIT = 100;
|
|
1584
|
+
const FUSION_DOC_VECTORS = new WeakMap();
|
|
1585
|
+
|
|
1586
|
+
// Inlined copy of src/core/codeintel/vectorProvider.js hashVectorize/cosine (256 dims, FNV-1a
|
|
1587
|
+
// word + trigram features, L2-normalized) so the shipped index-core.mjs stays dependency-free.
|
|
1588
|
+
function fusionFnv1a(str) {
|
|
1589
|
+
let h = 0x811c9dc5;
|
|
1590
|
+
for (let i = 0; i < str.length; i += 1) {
|
|
1591
|
+
h ^= str.charCodeAt(i);
|
|
1592
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
1593
|
+
}
|
|
1594
|
+
return h >>> 0;
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
function hashVectorize(text) {
|
|
1598
|
+
const dims = 256;
|
|
1599
|
+
const vec = new Array(dims).fill(0);
|
|
1600
|
+
for (const token of String(text ?? '').split(/[^A-Za-z0-9_$]+/).map((t) => t.toLowerCase()).filter(Boolean)) {
|
|
1601
|
+
vec[fusionFnv1a(`w:${token}`) % dims] += 1;
|
|
1602
|
+
const padded = `#${token}#`;
|
|
1603
|
+
for (let i = 0; i + 3 <= padded.length; i += 1) vec[fusionFnv1a(`t:${padded.slice(i, i + 3)}`) % dims] += 0.1;
|
|
1604
|
+
}
|
|
1605
|
+
const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
|
|
1606
|
+
if (norm > 0) for (let i = 0; i < dims; i += 1) vec[i] /= norm;
|
|
1607
|
+
return vec;
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
function cosineSimilarity(a, b) {
|
|
1611
|
+
let dot = 0;
|
|
1612
|
+
for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
|
|
1613
|
+
return dot;
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
async function isFusionEnabled(absoluteRoot) {
|
|
1617
|
+
if (process.env.UKIT_INDEX_FUSION === '1') return true;
|
|
1618
|
+
if (process.env.UKIT_INDEX_FUSION === '0') return false;
|
|
1619
|
+
try {
|
|
1620
|
+
const config = JSON.parse(await fs.readFile(path.join(absoluteRoot, '.ukit', 'storage', 'config.json'), 'utf8'));
|
|
1621
|
+
return config?.index?.fusion === true;
|
|
1622
|
+
} catch {
|
|
1623
|
+
return false;
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
async function fuseWeakQuery({ absoluteRoot, query, ranked, searchDescriptorsByFile, includeLikelyTestFiles }) {
|
|
1628
|
+
try {
|
|
1629
|
+
if (!(await isFusionEnabled(absoluteRoot))) return null;
|
|
1630
|
+
let docVectors = FUSION_DOC_VECTORS.get(searchDescriptorsByFile);
|
|
1631
|
+
if (!docVectors) {
|
|
1632
|
+
docVectors = new Map();
|
|
1633
|
+
for (const [filePath, descriptor] of searchDescriptorsByFile) {
|
|
1634
|
+
docVectors.set(filePath, hashVectorize(`${filePath} ${(descriptor.symbols ?? []).map((s) => s.name).join(' ')}`));
|
|
1635
|
+
}
|
|
1636
|
+
FUSION_DOC_VECTORS.set(searchDescriptorsByFile, docVectors);
|
|
1637
|
+
}
|
|
1638
|
+
const queryVector = hashVectorize(query);
|
|
1639
|
+
const vectorLane = [];
|
|
1640
|
+
for (const [filePath, vector] of docVectors) {
|
|
1641
|
+
if (isLikelyTestFilePath(filePath) && !includeLikelyTestFiles) continue;
|
|
1642
|
+
const similarity = cosineSimilarity(queryVector, vector);
|
|
1643
|
+
if (similarity > 0) vectorLane.push([filePath, similarity]);
|
|
1644
|
+
}
|
|
1645
|
+
vectorLane.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
1646
|
+
const vectorPaths = vectorLane.slice(0, FUSION_LANE_LIMIT).map(([filePath]) => filePath);
|
|
1647
|
+
const indexPaths = ranked.slice(0, FUSION_LANE_LIMIT).map((entry) => entry.filePath);
|
|
1648
|
+
const rrf = new Map();
|
|
1649
|
+
indexPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + 1 / (FUSION_RRF_K + i + 1)));
|
|
1650
|
+
vectorPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + FUSION_VECTOR_WEIGHT / (FUSION_RRF_K + i + 1)));
|
|
1651
|
+
const byPath = new Map(ranked.map((entry) => [entry.filePath, entry]));
|
|
1652
|
+
const order = [...rrf.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([filePath]) => filePath);
|
|
1653
|
+
const floor = ranked[ranked.length - 1].score;
|
|
1654
|
+
const merged = order.map((filePath) => byPath.get(filePath)
|
|
1655
|
+
?? { filePath, score: Math.min(0.01, floor), reasons: ['fusion:vector'], tests: [] });
|
|
1656
|
+
await emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths: merged.map((e) => e.filePath) });
|
|
1657
|
+
return merged;
|
|
1658
|
+
} catch {
|
|
1659
|
+
return null;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
// Same event shape and file as retriever.js emitRetrieverLaneTelemetry (query stored as sha256).
|
|
1664
|
+
async function emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths }) {
|
|
1665
|
+
try {
|
|
1666
|
+
const event = {
|
|
1667
|
+
ts: Date.now(),
|
|
1668
|
+
query: crypto.createHash('sha256').update(String(query ?? '')).digest('hex'),
|
|
1669
|
+
mode: 'index-query',
|
|
1670
|
+
lanes: {
|
|
1671
|
+
index: { hits: indexPaths.length, weightUsed: 1, paths: indexPaths },
|
|
1672
|
+
vector: { hits: vectorPaths.length, weightUsed: FUSION_VECTOR_WEIGHT, paths: vectorPaths },
|
|
1673
|
+
},
|
|
1674
|
+
mergedPaths: mergedPaths.slice(0, 100),
|
|
1675
|
+
merge: 'rrf',
|
|
1676
|
+
};
|
|
1677
|
+
const filePath = path.join(absoluteRoot, '.ukit', 'storage', 'cache', 'retriever-lanes.jsonl');
|
|
1678
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
1679
|
+
await withFileLock(filePath, async () => {
|
|
1680
|
+
try {
|
|
1681
|
+
const st = await fs.stat(filePath);
|
|
1682
|
+
if (st.size > 256 * 1024) {
|
|
1683
|
+
const kept = (await fs.readFile(filePath, 'utf8')).split('\n').filter(Boolean).slice(-2000);
|
|
1684
|
+
await fs.writeFile(filePath, kept.length ? `${kept.join('\n')}\n` : '', 'utf8');
|
|
1685
|
+
}
|
|
1686
|
+
} catch {
|
|
1687
|
+
// Missing file is fine; append below recreates it.
|
|
1688
|
+
}
|
|
1689
|
+
await fs.appendFile(filePath, `${JSON.stringify(event)}\n`, 'utf8');
|
|
1690
|
+
});
|
|
1691
|
+
} catch {
|
|
1692
|
+
// Telemetry is advisory.
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1444
1696
|
// ── Analog Query ──
|
|
1445
1697
|
export async function queryAnalog({ rootDir = process.cwd(), filePath, limit = 5 } = {}) {
|
|
1446
1698
|
const normalizedFilePath = String(filePath ?? '').trim();
|
|
@@ -1526,9 +1778,11 @@ export function queryMatchedSymbolNames(result) {
|
|
|
1526
1778
|
// Non-enumerable (array equality and JSON stay unchanged), like boundedSourceSearch rows.truncated:
|
|
1527
1779
|
// `contentLane` is true when the content search ran; `contentLaneTruncated` lists why it was cut
|
|
1528
1780
|
// short ('verify-cap', 'time-budget', 'max-files', ...) so a partial scan never reads as "no match".
|
|
1529
|
-
function withContentLaneStatus(results, ran, truncated) {
|
|
1781
|
+
function withContentLaneStatus(results, ran, truncated, semanticFellBack = false) {
|
|
1530
1782
|
// results arrive frozen, so copy before attaching the status and freeze again.
|
|
1531
1783
|
const out = [...results];
|
|
1784
|
+
// true when the embedding lane was asked for and returned nothing: callers must not persist this result.
|
|
1785
|
+
Object.defineProperty(out, 'semanticFellBack', { value: Boolean(semanticFellBack), enumerable: false });
|
|
1532
1786
|
Object.defineProperty(out, 'contentLane', { value: ran, enumerable: false });
|
|
1533
1787
|
Object.defineProperty(out, 'contentLaneTruncated', { value: Object.freeze([...truncated]), enumerable: false });
|
|
1534
1788
|
return Object.freeze(out);
|
|
@@ -2775,7 +3029,7 @@ function findScriptClose(content, startIndex) {
|
|
|
2775
3029
|
}
|
|
2776
3030
|
|
|
2777
3031
|
export function extractScriptContent(filePath, content) {
|
|
2778
|
-
if (
|
|
3032
|
+
if (!/\.(?:vue|svelte)$/.test(filePath)) {
|
|
2779
3033
|
return content;
|
|
2780
3034
|
}
|
|
2781
3035
|
|
|
@@ -2864,6 +3118,278 @@ function computeSignature(content, lineStartIndex) {
|
|
|
2864
3118
|
return `${collapsed.slice(0, 199)}…`;
|
|
2865
3119
|
}
|
|
2866
3120
|
|
|
3121
|
+
// ── Non-JS languages (3.5.5): line/indent/brace based, no parser. Rows are { name, type, index, parent? }.
|
|
3122
|
+
// Kinds follow the JS lane: exported/public top-level -> function|class, private -> local-*, members -> method,
|
|
3123
|
+
// interface/enum/struct/trait/type aliases -> type.
|
|
3124
|
+
const POLYGLOT_EXT = /\.(?:py|go|java|cs|php|rb|rs)$/i;
|
|
3125
|
+
const POLY_MEMBER_SKIP = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'new', 'else', 'throw', 'using', 'lock', 'foreach', 'fixed', 'match', 'await', 'yield', 'do', 'try', 'sizeof', 'typeof', 'nameof', 'default', 'when', 'base', 'this']);
|
|
3126
|
+
const POLY_MODS = '(?:(?:public|protected|private|internal|static|final|abstract|sealed|partial|readonly|virtual|override|async|extern|unsafe|new|synchronized|native|strictfp|default|open)\\s+)*';
|
|
3127
|
+
const POLY_JAVA_TYPE_RE = new RegExp(`^[ \\t]*(?:@\\w+(?:\\([^)]*\\))?\\s+)*${POLY_MODS}(class|interface|enum|struct|record(?:\\s+(?:class|struct))?)\\s+([A-Za-z_]\\w*)`);
|
|
3128
|
+
const POLY_JAVA_METHOD_RE = new RegExp(`^[ \\t]*(?:@\\w+(?:\\([^)]*\\))?\\s+)*(?:<[^>]+>\\s+)?(?:[\\w.<>\\[\\],?]+\\s+)*?([A-Za-z_]\\w*)[ \\t]*(?:<[^>()]*>[ \\t]*)?\\(`);
|
|
3129
|
+
const POLY_PHP_TYPE_RE = /^[ \t]*(?:(?:abstract|final|readonly)\s+)*(class|interface|trait|enum)\s+([A-Za-z_]\w*)/;
|
|
3130
|
+
const POLY_PHP_FUNC_RE = /^[ \t]*(?:(?:public|protected|private|static|abstract|final)\s+)*function[ \t]+(?:&[ \t]*)?([A-Za-z_]\w*)/;
|
|
3131
|
+
const POLY_RS_PUB = '(?:pub(?:\\([^)]*\\))?\\s+)?';
|
|
3132
|
+
const POLY_RS_FN_RE = new RegExp(`^[ \\t]*(${POLY_RS_PUB})(?:(?:default|const|async|unsafe)\\s+)*(?:extern\\s+(?:"[^"]*"\\s+)?)?fn\\s+([A-Za-z_]\\w*)`);
|
|
3133
|
+
const POLY_RS_TYPE_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}(struct|enum|union|type)\\s+([A-Za-z_]\\w*)`);
|
|
3134
|
+
const POLY_RS_TRAIT_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}(?:unsafe\\s+)?trait\\s+([A-Za-z_]\\w*)`);
|
|
3135
|
+
// Bounds against pathological input (regex stalls): files above 512 KB (the content-lane per-file cap) are not
|
|
3136
|
+
// extracted, and any line over 2000 chars is blanked in place (offsets preserved) before the per-line patterns run.
|
|
3137
|
+
const POLY_MAX_FILE_CHARS = 512 * 1024;
|
|
3138
|
+
const POLY_MAX_LINE_CHARS = 2000;
|
|
3139
|
+
// impl header -> implemented type name, without a backtracking pattern: skip a balanced <generics>, then the
|
|
3140
|
+
// type after the first `for` (trait impl) or the first identifier (inherent impl).
|
|
3141
|
+
function polyRsImplName(line) {
|
|
3142
|
+
const head = /^[ \t]*(?:unsafe[ \t]+)?impl\b/.exec(line);
|
|
3143
|
+
if (!head) return null;
|
|
3144
|
+
let rest = line.slice(head[0].length).replace(/^[ \t]+/, '');
|
|
3145
|
+
if (rest[0] === '<') {
|
|
3146
|
+
let depth = 0;
|
|
3147
|
+
let k = 0;
|
|
3148
|
+
for (; k < rest.length; k += 1) {
|
|
3149
|
+
if (rest[k] === '<') depth += 1;
|
|
3150
|
+
else if (rest[k] === '>' && (depth -= 1) === 0) break;
|
|
3151
|
+
}
|
|
3152
|
+
rest = rest.slice(k + 1);
|
|
3153
|
+
}
|
|
3154
|
+
const cut = rest.search(/[{]|[ \t]where\b/);
|
|
3155
|
+
const header = cut < 0 ? rest : rest.slice(0, cut);
|
|
3156
|
+
const forAt = header.search(/(?:^|[ \t])for[ \t]/);
|
|
3157
|
+
const target = forAt < 0 ? header : header.slice(forAt).replace(/^[ \t]*for[ \t]+/, '');
|
|
3158
|
+
return /^[ \t]*([A-Za-z_]\w*)/.exec(target)?.[1] ?? null;
|
|
3159
|
+
}
|
|
3160
|
+
const POLY_RS_MOD_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}mod\\s+([A-Za-z_]\\w*)`);
|
|
3161
|
+
|
|
3162
|
+
// Same-length copy with comments (and, per mode, string bodies) blanked, so offsets/lines stay valid.
|
|
3163
|
+
// mode 0 = comments only, 1 = + "strings", 2 = + "strings" and 'chars'.
|
|
3164
|
+
function maskCLike(content, mode) {
|
|
3165
|
+
const out = content.split('');
|
|
3166
|
+
const n = content.length;
|
|
3167
|
+
const blank = (from, to) => {
|
|
3168
|
+
for (let k = from; k < to; k += 1) {
|
|
3169
|
+
if (out[k] !== '\n') out[k] = ' ';
|
|
3170
|
+
}
|
|
3171
|
+
};
|
|
3172
|
+
let i = 0;
|
|
3173
|
+
while (i < n) {
|
|
3174
|
+
const c = content[i];
|
|
3175
|
+
const d = content[i + 1];
|
|
3176
|
+
if (c === '/' && d === '/') {
|
|
3177
|
+
let end = content.indexOf('\n', i);
|
|
3178
|
+
if (end < 0) end = n;
|
|
3179
|
+
blank(i, end);
|
|
3180
|
+
i = end;
|
|
3181
|
+
} else if (c === '/' && d === '*') {
|
|
3182
|
+
let end = content.indexOf('*/', i + 2);
|
|
3183
|
+
end = end < 0 ? n : end + 2;
|
|
3184
|
+
blank(i, end);
|
|
3185
|
+
i = end;
|
|
3186
|
+
} else if ((mode >= 1 && c === '"') || (mode >= 2 && c === "'")) {
|
|
3187
|
+
let k = i + 1;
|
|
3188
|
+
while (k < n && content[k] !== c && content[k] !== '\n') {
|
|
3189
|
+
if (content[k] === '\\') k += 1;
|
|
3190
|
+
k += 1;
|
|
3191
|
+
}
|
|
3192
|
+
blank(i + 1, Math.min(k, n));
|
|
3193
|
+
i = k + 1;
|
|
3194
|
+
} else {
|
|
3195
|
+
i += 1;
|
|
3196
|
+
}
|
|
3197
|
+
}
|
|
3198
|
+
return out.join('');
|
|
3199
|
+
}
|
|
3200
|
+
|
|
3201
|
+
function extractIndentLanguageSymbols(filePath, content) {
|
|
3202
|
+
const isPy = /\.py$/i.test(filePath);
|
|
3203
|
+
const scopeRe = isPy ? /^([ \t]*)class[ \t]+([A-Za-z_]\w*)/ : /^([ \t]*)(?:module|class)[ \t]+([A-Z][\w:]*)/;
|
|
3204
|
+
const defRe = isPy ? /^([ \t]*)(?:async[ \t]+)?def[ \t]+([A-Za-z_]\w*)/ : /^([ \t]*)def[ \t]+(?:[A-Za-z_]\w*\.)?([A-Za-z_]\w*[?!=]?)/;
|
|
3205
|
+
const rows = [];
|
|
3206
|
+
const stack = [];
|
|
3207
|
+
let docQuote = null;
|
|
3208
|
+
let offset = 0;
|
|
3209
|
+
for (const line of content.split('\n')) {
|
|
3210
|
+
const lineStart = offset;
|
|
3211
|
+
offset += line.length + 1;
|
|
3212
|
+
const startedInDoc = docQuote !== null;
|
|
3213
|
+
if (isPy) {
|
|
3214
|
+
for (const quote of ['"""', "'''"]) {
|
|
3215
|
+
const count = line.split(quote).length - 1;
|
|
3216
|
+
if (count % 2 === 1) docQuote = docQuote === quote ? null : (docQuote ?? quote);
|
|
3217
|
+
}
|
|
3218
|
+
if (startedInDoc) continue;
|
|
3219
|
+
}
|
|
3220
|
+
const scope = scopeRe.exec(line);
|
|
3221
|
+
const def = scope ? null : defRe.exec(line);
|
|
3222
|
+
const m = scope ?? def;
|
|
3223
|
+
if (!m) continue;
|
|
3224
|
+
const indent = m[1].length;
|
|
3225
|
+
while (stack.length > 0 && stack[stack.length - 1].indent >= indent) stack.pop();
|
|
3226
|
+
const top = stack[stack.length - 1];
|
|
3227
|
+
const name = m[2];
|
|
3228
|
+
const index = lineStart + indent;
|
|
3229
|
+
if (scope) {
|
|
3230
|
+
if (!top || top.kind === 'scope') {
|
|
3231
|
+
rows.push({ name, type: isPy && name.startsWith('_') ? 'local-class' : 'class', index, parent: top?.name });
|
|
3232
|
+
}
|
|
3233
|
+
stack.push({ indent, kind: 'scope', name });
|
|
3234
|
+
} else {
|
|
3235
|
+
if (!top) {
|
|
3236
|
+
rows.push({ name, type: isPy && name.startsWith('_') ? 'local-function' : 'function', index });
|
|
3237
|
+
} else if (top.kind === 'scope') {
|
|
3238
|
+
rows.push({ name, type: 'method', index, parent: top.name });
|
|
3239
|
+
}
|
|
3240
|
+
stack.push({ indent, kind: 'def', name });
|
|
3241
|
+
}
|
|
3242
|
+
}
|
|
3243
|
+
return rows;
|
|
3244
|
+
}
|
|
3245
|
+
|
|
3246
|
+
function extractGoSymbols(content) {
|
|
3247
|
+
const text = maskCLike(content, 0);
|
|
3248
|
+
const rows = [];
|
|
3249
|
+
for (const m of text.matchAll(/^type[ \t]+([A-Za-z_]\w*)/gm)) {
|
|
3250
|
+
rows.push({ name: m[1], type: 'type', index: m.index });
|
|
3251
|
+
}
|
|
3252
|
+
for (const m of text.matchAll(/^func[ \t]+([A-Za-z_]\w*)[ \t]*[[(]/gm)) {
|
|
3253
|
+
rows.push({ name: m[1], type: /^[A-Z]/.test(m[1]) ? 'function' : 'local-function', index: m.index });
|
|
3254
|
+
}
|
|
3255
|
+
for (const m of text.matchAll(/^func[ \t]*\([ \t]*(?:[A-Za-z_]\w*[ \t]+)?(?:\*[ \t]*)?([A-Za-z_]\w*)(?:\[[^\]\n]*\])?[ \t]*\)[ \t]*([A-Za-z_]\w*)/gm)) {
|
|
3256
|
+
rows.push({ name: m[2], type: 'method', index: m.index, parent: m[1] });
|
|
3257
|
+
}
|
|
3258
|
+
return rows;
|
|
3259
|
+
}
|
|
3260
|
+
|
|
3261
|
+
// Java, C#, PHP and Rust: depth-tracked on a comment/string-masked copy. A container
|
|
3262
|
+
// (class, trait, impl, mod ...) owns the lines exactly one brace level below its declaration.
|
|
3263
|
+
function extractBraceLanguageSymbols(filePath, content) {
|
|
3264
|
+
const ext = filePath.slice(filePath.lastIndexOf('.') + 1).toLowerCase();
|
|
3265
|
+
const text = maskCLike(content, ext === 'rs' ? 1 : 2);
|
|
3266
|
+
const rows = [];
|
|
3267
|
+
const stack = [];
|
|
3268
|
+
let depth = 0;
|
|
3269
|
+
let offset = 0;
|
|
3270
|
+
for (const line of text.split('\n')) {
|
|
3271
|
+
const lineStart = offset;
|
|
3272
|
+
offset += line.length + 1;
|
|
3273
|
+
while (stack.length > 0 && stack[stack.length - 1].opened && depth <= stack[stack.length - 1].depth) stack.pop();
|
|
3274
|
+
const top = stack[stack.length - 1];
|
|
3275
|
+
const atMember = top ? depth === top.depth + 1 : false;
|
|
3276
|
+
const index = lineStart + Math.max(0, line.search(/\S/));
|
|
3277
|
+
const pushContainer = (kind, name) => {
|
|
3278
|
+
if (line.includes(';') && !line.includes('{')) return;
|
|
3279
|
+
stack.push({ kind, name, depth, opened: false });
|
|
3280
|
+
};
|
|
3281
|
+
|
|
3282
|
+
if (ext === 'java' || ext === 'cs') {
|
|
3283
|
+
const typeMatch = POLY_JAVA_TYPE_RE.exec(line);
|
|
3284
|
+
if (typeMatch) {
|
|
3285
|
+
const kind = typeMatch[1].split(/\s/)[0];
|
|
3286
|
+
rows.push({ name: typeMatch[2], type: kind === 'class' ? 'class' : 'type', index, parent: top?.name });
|
|
3287
|
+
pushContainer(kind, typeMatch[2]);
|
|
3288
|
+
} else if (atMember) {
|
|
3289
|
+
const m = POLY_JAVA_METHOD_RE.exec(line);
|
|
3290
|
+
if (m && !POLY_MEMBER_SKIP.has(m[1]) && !(top.kind === 'enum' && /^[A-Z0-9_]+$/.test(m[1]))) {
|
|
3291
|
+
rows.push({ name: m[1], type: 'method', index, parent: top.name });
|
|
3292
|
+
}
|
|
3293
|
+
}
|
|
3294
|
+
} else if (ext === 'php') {
|
|
3295
|
+
const typeMatch = POLY_PHP_TYPE_RE.exec(line);
|
|
3296
|
+
const fn = typeMatch ? null : POLY_PHP_FUNC_RE.exec(line);
|
|
3297
|
+
if (typeMatch) {
|
|
3298
|
+
rows.push({ name: typeMatch[2], type: typeMatch[1] === 'class' ? 'class' : 'type', index, parent: top?.name });
|
|
3299
|
+
pushContainer(typeMatch[1], typeMatch[2]);
|
|
3300
|
+
} else if (fn && !top) {
|
|
3301
|
+
rows.push({ name: fn[1], type: 'function', index });
|
|
3302
|
+
} else if (fn && atMember) {
|
|
3303
|
+
rows.push({ name: fn[1], type: 'method', index, parent: top.name });
|
|
3304
|
+
}
|
|
3305
|
+
} else {
|
|
3306
|
+
const fn = POLY_RS_FN_RE.exec(line);
|
|
3307
|
+
const trait = fn ? null : POLY_RS_TRAIT_RE.exec(line);
|
|
3308
|
+
const impl = fn || trait ? null : polyRsImplName(line);
|
|
3309
|
+
const mod = fn || trait || impl ? null : POLY_RS_MOD_RE.exec(line);
|
|
3310
|
+
const typeMatch = fn || trait || impl || mod ? null : POLY_RS_TYPE_RE.exec(line);
|
|
3311
|
+
const atModuleLevel = top ? top.kind === 'mod' && atMember : depth === 0;
|
|
3312
|
+
if (fn) {
|
|
3313
|
+
const container = top && (top.kind === 'impl' || top.kind === 'trait');
|
|
3314
|
+
if (container && atMember) {
|
|
3315
|
+
rows.push({ name: fn[2], type: 'method', index, parent: top.name });
|
|
3316
|
+
} else if (atModuleLevel) {
|
|
3317
|
+
rows.push({ name: fn[2], type: fn[1].trim() ? 'function' : 'local-function', index });
|
|
3318
|
+
}
|
|
3319
|
+
} else if (trait) {
|
|
3320
|
+
if (atModuleLevel) rows.push({ name: trait[1], type: 'type', index });
|
|
3321
|
+
pushContainer('trait', trait[1]);
|
|
3322
|
+
} else if (impl) {
|
|
3323
|
+
pushContainer('impl', impl);
|
|
3324
|
+
} else if (mod) {
|
|
3325
|
+
pushContainer('mod', mod[1]);
|
|
3326
|
+
} else if (typeMatch && atModuleLevel) {
|
|
3327
|
+
rows.push({ name: typeMatch[2], type: 'type', index });
|
|
3328
|
+
}
|
|
3329
|
+
}
|
|
3330
|
+
|
|
3331
|
+
let maxDepth = depth;
|
|
3332
|
+
for (let k = 0; k < line.length; k += 1) {
|
|
3333
|
+
if (line[k] === '{') {
|
|
3334
|
+
depth += 1;
|
|
3335
|
+
if (depth > maxDepth) maxDepth = depth;
|
|
3336
|
+
} else if (line[k] === '}' && depth > 0) {
|
|
3337
|
+
depth -= 1;
|
|
3338
|
+
}
|
|
3339
|
+
}
|
|
3340
|
+
for (const entry of stack) {
|
|
3341
|
+
if (!entry.opened && maxDepth > entry.depth) entry.opened = true;
|
|
3342
|
+
}
|
|
3343
|
+
}
|
|
3344
|
+
return rows;
|
|
3345
|
+
}
|
|
3346
|
+
|
|
3347
|
+
function extractPolyglotSymbols(filePath, rawContent) {
|
|
3348
|
+
if (rawContent.length > POLY_MAX_FILE_CHARS) return [];
|
|
3349
|
+
const parts = rawContent.split('\n');
|
|
3350
|
+
const content = parts.some((l) => l.length > POLY_MAX_LINE_CHARS)
|
|
3351
|
+
? parts.map((l) => (l.length > POLY_MAX_LINE_CHARS ? ' '.repeat(l.length) : l)).join('\n')
|
|
3352
|
+
: rawContent;
|
|
3353
|
+
const ext = filePath.slice(filePath.lastIndexOf('.') + 1).toLowerCase();
|
|
3354
|
+
let rows;
|
|
3355
|
+
if (ext === 'py' || ext === 'rb') rows = extractIndentLanguageSymbols(filePath, content);
|
|
3356
|
+
else if (ext === 'go') rows = extractGoSymbols(content);
|
|
3357
|
+
else rows = extractBraceLanguageSymbols(filePath, content);
|
|
3358
|
+
const seen = new Set();
|
|
3359
|
+
return rows.filter((row) => {
|
|
3360
|
+
const key = `${row.type}:${row.name}:${row.parent ?? ''}`;
|
|
3361
|
+
if (seen.has(key)) return false;
|
|
3362
|
+
seen.add(key);
|
|
3363
|
+
return true;
|
|
3364
|
+
});
|
|
3365
|
+
}
|
|
3366
|
+
|
|
3367
|
+
// `const x = (` also opens plain parenthesised expressions (`const n = (a ?? 0)`), so only a
|
|
3368
|
+
// parameter list whose closing paren is followed by `=>` (optionally after a return type) is a function.
|
|
3369
|
+
function findParenArrowDeclarations(content) {
|
|
3370
|
+
const found = [];
|
|
3371
|
+
for (const match of content.matchAll(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?\(/g)) {
|
|
3372
|
+
const open = match.index + match[0].length - 1;
|
|
3373
|
+
const limit = Math.min(content.length, open + 2000);
|
|
3374
|
+
let depth = 0;
|
|
3375
|
+
let close = -1;
|
|
3376
|
+
for (let i = open; i < limit; i += 1) {
|
|
3377
|
+
if (content[i] === '(') depth += 1;
|
|
3378
|
+
else if (content[i] === ')') {
|
|
3379
|
+
depth -= 1;
|
|
3380
|
+
if (depth === 0) {
|
|
3381
|
+
close = i;
|
|
3382
|
+
break;
|
|
3383
|
+
}
|
|
3384
|
+
}
|
|
3385
|
+
}
|
|
3386
|
+
if (close >= 0 && /^\s*(?::\s*[^;\n{}=]*?(?:=>\s*[^;\n{}=]*?)?)?=>/.test(content.slice(close + 1, close + 201))) {
|
|
3387
|
+
found.push({ name: match[1], index: match.index });
|
|
3388
|
+
}
|
|
3389
|
+
}
|
|
3390
|
+
return found;
|
|
3391
|
+
}
|
|
3392
|
+
|
|
2867
3393
|
const CLASS_METHOD_SKIP = new Set(['if', 'for', 'while', 'switch', 'catch', 'function', 'return', 'with', 'constructor', 'super']);
|
|
2868
3394
|
|
|
2869
3395
|
export function extractSymbols(filePath, content) {
|
|
@@ -2887,6 +3413,13 @@ export function extractSymbols(filePath, content) {
|
|
|
2887
3413
|
}
|
|
2888
3414
|
};
|
|
2889
3415
|
|
|
3416
|
+
if (POLYGLOT_EXT.test(filePath)) {
|
|
3417
|
+
for (const row of extractPolyglotSymbols(filePath, content)) {
|
|
3418
|
+
addSymbol(row.name, row.type, row.index, row.parent ? { parent: row.parent } : null);
|
|
3419
|
+
}
|
|
3420
|
+
return out;
|
|
3421
|
+
}
|
|
3422
|
+
|
|
2890
3423
|
if (filePath.endsWith('.sql')) {
|
|
2891
3424
|
// Capture the object name, skipping an optional schema qualifier (qas.fn_rpt_billing → fn_rpt_billing).
|
|
2892
3425
|
const sqlIdent = '(?:(?:["`]?[A-Za-z_][A-Za-z0-9_$]*["`]?)\\s*\\.\\s*)?["`]?([A-Za-z_][A-Za-z0-9_$]*)["`]?';
|
|
@@ -2918,8 +3451,10 @@ export function extractSymbols(filePath, content) {
|
|
|
2918
3451
|
} else {
|
|
2919
3452
|
addMatches(/export\s+(?:async\s+)?function\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'function');
|
|
2920
3453
|
addMatches(/export\s+class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'class');
|
|
2921
|
-
|
|
2922
|
-
|
|
3454
|
+
for (const decl of findParenArrowDeclarations(content)) {
|
|
3455
|
+
addSymbol(decl.name, 'callable', decl.index);
|
|
3456
|
+
}
|
|
3457
|
+
addMatches(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s+)?[A-Za-z_$][A-Za-z0-9_$]*\s*=>/g, 'callable');
|
|
2923
3458
|
addMatches(/export\s+default\s+(?:async\s+)?function\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'default-function');
|
|
2924
3459
|
addMatches(/export\s+default\s+class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'default-class');
|
|
2925
3460
|
addMatches(/export\s+default\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*(?:;|\n|$)/g, 'default-reference');
|
|
@@ -2932,7 +3467,7 @@ export function extractSymbols(filePath, content) {
|
|
|
2932
3467
|
addMatches(/^[ \t]*(?:async\s+)?function\s*\*?\s*([A-Za-z_$][A-Za-z0-9_$]*)\s*[(<]/gm, 'local-function');
|
|
2933
3468
|
addMatches(/^[ \t]*(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s+)?function\b/gm, 'callable');
|
|
2934
3469
|
addMatches(/^[ \t]*(?:abstract\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'local-class');
|
|
2935
|
-
if (/\.(?:ts|tsx|mts|cts|vue)$/.test(filePath)) {
|
|
3470
|
+
if (/\.(?:ts|tsx|mts|cts|vue|svelte)$/.test(filePath)) {
|
|
2936
3471
|
addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?(?:const\s+)?(?:interface|enum)\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'type');
|
|
2937
3472
|
addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?type\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*(?:<[^=\n]*>)?\s*=/gm, 'type');
|
|
2938
3473
|
}
|
|
@@ -3531,7 +4066,7 @@ const BOOST_LIFT_RATIO = 0.2; // import/hotspot/archetype boosts may add at most
|
|
|
3531
4066
|
const CONTENT_LANE_CONFIDENT_SCORE = 30; // a best direct score at/above this (exact symbol or path hit) skips the content lane
|
|
3532
4067
|
const CONTENT_LANE_MIN_TOKENS = 3; // shorter queries are names, not messages
|
|
3533
4068
|
const CONTENT_LANE_LIMIT = 40; // candidates only share tokens somewhere in the file; the same-line check below keeps the real ones
|
|
3534
|
-
const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|sql|sh)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
|
|
4069
|
+
const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|svelte|sql|sh|py|go|java|cs|php|rb|rs)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
|
|
3535
4070
|
const CONTENT_LANE_BONUS = 14; // added to a file that already has a direct score
|
|
3536
4071
|
const CONTENT_LANE_ONLY_SCORE = 13; // base for a file found only by content; below CONTENT_LANE_CONFIDENT_SCORE by design
|
|
3537
4072
|
const CONTENT_LANE_MAX_FILES = 1500; // files read per query, in priority order (primary bound)
|
|
@@ -4357,6 +4892,16 @@ export function buildSearchDescriptor(value, { expandVietnameseAliases = false }
|
|
|
4357
4892
|
for (const token of collectVietnameseAliasTokens(foldedLower, tokens)) {
|
|
4358
4893
|
tokens.add(token);
|
|
4359
4894
|
}
|
|
4895
|
+
// Stems only for question-like queries: identifier-style queries (findRelatedItemIds,
|
|
4896
|
+
// collectVietnameseAliasTokens) must keep matching exact tokens.
|
|
4897
|
+
if (tokens.size >= STEM_MIN_QUERY_TOKENS) {
|
|
4898
|
+
for (const token of [...tokens]) {
|
|
4899
|
+
const stem = stemToken(token);
|
|
4900
|
+
if (stem && !GENERIC_STOPWORDS.has(stem)) {
|
|
4901
|
+
tokens.add(stem);
|
|
4902
|
+
}
|
|
4903
|
+
}
|
|
4904
|
+
}
|
|
4360
4905
|
}
|
|
4361
4906
|
|
|
4362
4907
|
return {
|
|
@@ -4728,6 +5273,7 @@ function isLikelyTestFilePath(filePath) {
|
|
|
4728
5273
|
|| lower.endsWith('.spec.jsx')
|
|
4729
5274
|
|| lower.endsWith('.spec.mjs')
|
|
4730
5275
|
|| lower.endsWith('.spec.cjs')
|
|
5276
|
+
|| /(?:_test\.(?:go|py|rb|rs)|_spec\.rb|(?:^|\/)test_[^/]+\.py|[a-z0-9]Tests?\.(?:java|cs))$/.test(filePath)
|
|
4731
5277
|
);
|
|
4732
5278
|
}
|
|
4733
5279
|
|
|
@@ -5361,6 +5907,7 @@ function isLikelyTestFile(filePath) {
|
|
|
5361
5907
|
|| lower.endsWith('.spec.jsx')
|
|
5362
5908
|
|| lower.endsWith('.spec.mjs')
|
|
5363
5909
|
|| lower.endsWith('.spec.cjs')
|
|
5910
|
+
|| /(?:_test\.(?:go|py|rb|rs)|_spec\.rb|(?:^|\/)test_[^/]+\.py|[a-z0-9]Tests?\.(?:java|cs))$/.test(filePath)
|
|
5364
5911
|
);
|
|
5365
5912
|
}
|
|
5366
5913
|
|