@ngockhoale/ukit 3.5.4 → 3.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -135,9 +135,9 @@ const EXCLUDED_DIR_NAMES = new Set([
135
135
  '.venv',
136
136
  'venv',
137
137
  ]);
138
- const CODE_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.sql']);
138
+ const CODE_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.svelte', '.py', '.go', '.java', '.cs', '.php', '.rb', '.rs', '.sql']);
139
139
  const STYLE_EXTENSIONS = new Set(['.css', '.scss', '.sass', '.less']);
140
- const TRACKED_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.sql', '.json', '.yaml', '.yml', '.md', '.sh']);
140
+ const TRACKED_EXTENSIONS = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.vue', '.svelte', '.py', '.go', '.java', '.cs', '.php', '.rb', '.rs', '.sql', '.json', '.yaml', '.yml', '.md', '.sh']);
141
141
  const DISCOVERED_EXTENSIONS = new Set([...TRACKED_EXTENSIONS, ...STYLE_EXTENSIONS]);
142
142
  // v9: symbols.json gains global-registered ('global') and Vue 'method' rows,
143
143
  // SQL 'column' rows with parent/schema, and calls.json gains per-object SQL
@@ -145,7 +145,7 @@ const DISCOVERED_EXTENSIONS = new Set([...TRACKED_EXTENSIONS, ...STYLE_EXTENSION
145
145
  // files or the new extractions silently vanish.
146
146
  // v10: symbols.json gains unexported function/class ('local-function', 'local-class'),
147
147
  // function-expression consts, class 'method' rows and TS interface/enum/type ('type') rows.
148
- export const INDEX_SCHEMA_VERSION = 10;
148
+ export const INDEX_SCHEMA_VERSION = 11;
149
149
  export const DEFAULT_INDEX_CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
150
150
  // Parse-phase read/extract concurrency — deliberately tighter than the
151
151
  // discovery stat pool because parsing holds file content in memory.
@@ -213,6 +213,46 @@ const VIETNAMESE_PHRASE_ALIASES = [
213
213
  { regex: /\bthanh phan\b/gu, expansions: ['component'] },
214
214
  { regex: /\bkiem thu\b/gu, expansions: ['test', 'testing'] },
215
215
  { regex: /\bbo cuc\b/gu, expansions: ['layout'] },
216
+ // General programming vocabulary (diacritics folded; matches with or without accents).
217
+ { regex: /\bnhay cam\b/gu, expansions: ['sensitive'] },
218
+ { regex: /\bbi mat\b/gu, expansions: ['secret'] },
219
+ { regex: /\bgia tri\b/gu, expansions: ['value'] },
220
+ { regex: /\bvan ban\b/gu, expansions: ['text'] },
221
+ { regex: /\bsao luu\b/gu, expansions: ['backup'] },
222
+ { regex: /\bkhoi phuc\b/gu, expansions: ['restore', 'recover'] },
223
+ { regex: /\bdinh danh\b/gu, expansions: ['identity', 'id'] },
224
+ { regex: /\bdu an\b/gu, expansions: ['project'] },
225
+ { regex: /\bthu muc\b/gu, expansions: ['directory', 'dir', 'folder'] },
226
+ { regex: /\bphan loai\b/gu, expansions: ['classify', 'category'] },
227
+ { regex: /\buoc tinh\b/gu, expansions: ['estimate'] },
228
+ { regex: /\bcong chan\b/gu, expansions: ['gate', 'guard'] },
229
+ { regex: /\bcap nhat\b/gu, expansions: ['update'] },
230
+ { regex: /\bgo cai dat\b/gu, expansions: ['uninstall'] },
231
+ { regex: /\bcai dat\b/gu, expansions: ['install', 'setup'] },
232
+ { regex: /\bcau hinh\b/gu, expansions: ['config', 'configuration'] },
233
+ { regex: /\bbao cao\b/gu, expansions: ['report'] },
234
+ { regex: /\blich su\b/gu, expansions: ['history'] },
235
+ { regex: /\bnhat ky\b/gu, expansions: ['log'] },
236
+ { regex: /\bchi muc\b/gu, expansions: ['index'] },
237
+ { regex: /\bxu ly\b/gu, expansions: ['handle', 'process'] },
238
+ { regex: /\bphan tich\b/gu, expansions: ['parse', 'analyze'] },
239
+ { regex: /\bhet han\b/gu, expansions: ['expire', 'expiry'] },
240
+ { regex: /\bchup anh\b/gu, expansions: ['snapshot'] },
241
+ { regex: /\bhoan tac\b/gu, expansions: ['rollback', 'revert'] },
242
+ { regex: /\bnguoi dung\b/gu, expansions: ['user'] },
243
+ { regex: /\bthong bao\b/gu, expansions: ['notify', 'message'] },
244
+ { regex: /\bsap xep\b/gu, expansions: ['sort'] },
245
+ { regex: /\bdang ky\b/gu, expansions: ['register'] },
246
+ { regex: /\btham so\b/gu, expansions: ['argument', 'param', 'option'] },
247
+ { regex: /\bket qua\b/gu, expansions: ['result'] },
248
+ { regex: /\bmac dinh\b/gu, expansions: ['default'] },
249
+ { regex: /\bdau vao\b/gu, expansions: ['input'] },
250
+ { regex: /\bdau ra\b/gu, expansions: ['output'] },
251
+ { regex: /\bhang doi\b/gu, expansions: ['queue'] },
252
+ { regex: /\bma hoa\b/gu, expansions: ['encode', 'encrypt'] },
253
+ { regex: /\bgiai ma\b/gu, expansions: ['decode', 'decrypt'] },
254
+ { regex: /\bbo nho\b/gu, expansions: ['memory'] },
255
+ { regex: /\bcanh bao\b/gu, expansions: ['warning', 'warn'] },
216
256
  ];
217
257
  const VIETNAMESE_TOKEN_ALIASES = new Map([
218
258
  ['sua', ['fix']],
@@ -233,8 +273,58 @@ const VIETNAMESE_TOKEN_ALIASES = new Map([
233
273
  ['soat', ['review', 'audit']],
234
274
  ['trien', ['implement', 'build', 'ship']],
235
275
  ['khai', ['implement', 'build', 'ship']],
276
+ ['kho', ['store', 'storage']],
277
+ ['ghi', ['write', 'record']],
278
+ ['doc', ['read']],
279
+ ['luu', ['save', 'store', 'persist']],
280
+ ['tim', ['find', 'search', 'resolve']],
281
+ ['hong', ['broken', 'repair']],
282
+ ['phut', ['minute', 'minutes']],
283
+ ['ham', ['function']],
284
+ ['bien', ['variable']],
285
+ ['lop', ['class']],
286
+ ['lenh', ['command']],
287
+ ['chay', ['run', 'execute']],
288
+ ['phien', ['session']],
289
+ ['loc', ['filter']],
290
+ ['gui', ['send', 'dispatch']],
291
+ ['khoa', ['lock']],
292
+ ['huy', ['cancel', 'abort']],
293
+ ['quyet', ['decision']],
236
294
  ]);
237
295
 
296
+ // Query-time light stemming: adds the singular/base form of an English query word so
297
+ // 'files' reaches `file`, 'checks' reaches `check`, 'written' reaches `write`. Added next
298
+ // to the original token, never in place of it, and never applied to indexed text.
299
+ const IRREGULAR_STEMS = new Map([
300
+ ['written', 'write'], ['wrote', 'write'], ['built', 'build'], ['ran', 'run'], ['saved', 'save'],
301
+ ['found', 'find'], ['sent', 'send'], ['made', 'make'], ['read', 'read'], ['began', 'begin'],
302
+ ]);
303
+
304
+ const STEM_MIN_QUERY_TOKENS = 5;
305
+
306
+ function stemToken(token) {
307
+ if (!/^[a-z]+$/.test(token) || token.length < 4) {
308
+ return null;
309
+ }
310
+ if (IRREGULAR_STEMS.has(token)) {
311
+ return IRREGULAR_STEMS.get(token);
312
+ }
313
+ let stem = null;
314
+ if (token.endsWith('ies') && token.length > 4) {
315
+ stem = `${token.slice(0, -3)}y`;
316
+ } else if (/(?:ches|shes|xes|sses)$/.test(token)) {
317
+ stem = token.slice(0, -2);
318
+ } else if (token.endsWith('s') && !/(?:ss|us|is|as|os)$/.test(token)) {
319
+ stem = token.slice(0, -1);
320
+ } else if (token.endsWith('ing') && token.length > 5) {
321
+ stem = token.slice(0, -3);
322
+ } else if (token.endsWith('ed') && token.length > 4) {
323
+ stem = token.endsWith('ied') ? `${token.slice(0, -3)}y` : token.slice(0, -2);
324
+ }
325
+ return stem && stem.length >= 3 && stem !== token ? stem : null;
326
+ }
327
+
238
328
  // ── Build Index ──
239
329
 
240
330
  // Bumped only when the snapshot shape itself changes; consumers must reject
@@ -1429,9 +1519,11 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
1429
1519
  directScores,
1430
1520
  scores: applyHotspotBoosts({ scores: relatedScores, hotspotScores }),
1431
1521
  });
1432
- return withContentLaneStatus(freezeQueryResults(
1433
- [...boosted.values()].map((entry) => ({ ...entry, score: finalizeScore(entry) })).sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath)).slice(0, limit),
1434
- ), contentLaneRan, contentCutShort);
1522
+ const ranked = [...boosted.values()].map((entry) => ({ ...entry, score: finalizeScore(entry) })).sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath));
1523
+ const fused = contentLane && ranked.length > 0 && ranked[0].score < FUSION_WEAK_SCORE && queryDescriptor.tokens.size >= FUSION_MIN_TOKENS
1524
+ ? await fuseWeakQuery({ absoluteRoot, query: normalizedQuery, ranked, searchDescriptorsByFile, includeLikelyTestFiles })
1525
+ : null;
1526
+ return withContentLaneStatus(freezeQueryResults((fused ?? ranked).slice(0, limit)), contentLaneRan, contentCutShort);
1435
1527
  })().catch((error) => {
1436
1528
  QUERY_RESULT_CACHE.delete(queryCacheKey);
1437
1529
  throw error;
@@ -1441,6 +1533,129 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
1441
1533
  return queryPromise;
1442
1534
  }
1443
1535
 
1536
+ // Opt-in lane fusion (default OFF): a weak natural-language query (best score below
1537
+ // FUSION_WEAK_SCORE, >= FUSION_MIN_TOKENS tokens) is re-ranked by RRF over the scored list
1538
+ // and a hashed-embedding lane (the retriever.js vector lane). Enabled by
1539
+ // `.ukit/storage/config.json` -> index.fusion === true or UKIT_INDEX_FUSION=1. The flag is
1540
+ // read only after the weak gate passes, and contentLane:false callers (hooks/resolveContext)
1541
+ // never reach it. Never throws; async fs only.
1542
+ const FUSION_WEAK_SCORE = 6;
1543
+ const FUSION_MIN_TOKENS = 4;
1544
+ const FUSION_RRF_K = 5;
1545
+ const FUSION_VECTOR_WEIGHT = 1;
1546
+ const FUSION_LANE_LIMIT = 100;
1547
+ const FUSION_DOC_VECTORS = new WeakMap();
1548
+
1549
+ // Inlined copy of src/core/codeintel/vectorProvider.js hashVectorize/cosine (256 dims, FNV-1a
1550
+ // word + trigram features, L2-normalized) so the shipped index-core.mjs stays dependency-free.
1551
+ function fusionFnv1a(str) {
1552
+ let h = 0x811c9dc5;
1553
+ for (let i = 0; i < str.length; i += 1) {
1554
+ h ^= str.charCodeAt(i);
1555
+ h = Math.imul(h, 0x01000193) >>> 0;
1556
+ }
1557
+ return h >>> 0;
1558
+ }
1559
+
1560
+ function hashVectorize(text) {
1561
+ const dims = 256;
1562
+ const vec = new Array(dims).fill(0);
1563
+ for (const token of String(text ?? '').split(/[^A-Za-z0-9_$]+/).map((t) => t.toLowerCase()).filter(Boolean)) {
1564
+ vec[fusionFnv1a(`w:${token}`) % dims] += 1;
1565
+ const padded = `#${token}#`;
1566
+ for (let i = 0; i + 3 <= padded.length; i += 1) vec[fusionFnv1a(`t:${padded.slice(i, i + 3)}`) % dims] += 0.1;
1567
+ }
1568
+ const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
1569
+ if (norm > 0) for (let i = 0; i < dims; i += 1) vec[i] /= norm;
1570
+ return vec;
1571
+ }
1572
+
1573
+ function cosineSimilarity(a, b) {
1574
+ let dot = 0;
1575
+ for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
1576
+ return dot;
1577
+ }
1578
+
1579
+ async function isFusionEnabled(absoluteRoot) {
1580
+ if (process.env.UKIT_INDEX_FUSION === '1') return true;
1581
+ if (process.env.UKIT_INDEX_FUSION === '0') return false;
1582
+ try {
1583
+ const config = JSON.parse(await fs.readFile(path.join(absoluteRoot, '.ukit', 'storage', 'config.json'), 'utf8'));
1584
+ return config?.index?.fusion === true;
1585
+ } catch {
1586
+ return false;
1587
+ }
1588
+ }
1589
+
1590
+ async function fuseWeakQuery({ absoluteRoot, query, ranked, searchDescriptorsByFile, includeLikelyTestFiles }) {
1591
+ try {
1592
+ if (!(await isFusionEnabled(absoluteRoot))) return null;
1593
+ let docVectors = FUSION_DOC_VECTORS.get(searchDescriptorsByFile);
1594
+ if (!docVectors) {
1595
+ docVectors = new Map();
1596
+ for (const [filePath, descriptor] of searchDescriptorsByFile) {
1597
+ docVectors.set(filePath, hashVectorize(`${filePath} ${(descriptor.symbols ?? []).map((s) => s.name).join(' ')}`));
1598
+ }
1599
+ FUSION_DOC_VECTORS.set(searchDescriptorsByFile, docVectors);
1600
+ }
1601
+ const queryVector = hashVectorize(query);
1602
+ const vectorLane = [];
1603
+ for (const [filePath, vector] of docVectors) {
1604
+ if (isLikelyTestFilePath(filePath) && !includeLikelyTestFiles) continue;
1605
+ const similarity = cosineSimilarity(queryVector, vector);
1606
+ if (similarity > 0) vectorLane.push([filePath, similarity]);
1607
+ }
1608
+ vectorLane.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
1609
+ const vectorPaths = vectorLane.slice(0, FUSION_LANE_LIMIT).map(([filePath]) => filePath);
1610
+ const indexPaths = ranked.slice(0, FUSION_LANE_LIMIT).map((entry) => entry.filePath);
1611
+ const rrf = new Map();
1612
+ indexPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + 1 / (FUSION_RRF_K + i + 1)));
1613
+ vectorPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + FUSION_VECTOR_WEIGHT / (FUSION_RRF_K + i + 1)));
1614
+ const byPath = new Map(ranked.map((entry) => [entry.filePath, entry]));
1615
+ const order = [...rrf.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([filePath]) => filePath);
1616
+ const floor = ranked[ranked.length - 1].score;
1617
+ const merged = order.map((filePath) => byPath.get(filePath)
1618
+ ?? { filePath, score: Math.min(0.01, floor), reasons: ['fusion:vector'], tests: [] });
1619
+ await emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths: merged.map((e) => e.filePath) });
1620
+ return merged;
1621
+ } catch {
1622
+ return null;
1623
+ }
1624
+ }
1625
+
1626
+ // Same event shape and file as retriever.js emitRetrieverLaneTelemetry (query stored as sha256).
1627
+ async function emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths }) {
1628
+ try {
1629
+ const event = {
1630
+ ts: Date.now(),
1631
+ query: crypto.createHash('sha256').update(String(query ?? '')).digest('hex'),
1632
+ mode: 'index-query',
1633
+ lanes: {
1634
+ index: { hits: indexPaths.length, weightUsed: 1, paths: indexPaths },
1635
+ vector: { hits: vectorPaths.length, weightUsed: FUSION_VECTOR_WEIGHT, paths: vectorPaths },
1636
+ },
1637
+ mergedPaths: mergedPaths.slice(0, 100),
1638
+ merge: 'rrf',
1639
+ };
1640
+ const filePath = path.join(absoluteRoot, '.ukit', 'storage', 'cache', 'retriever-lanes.jsonl');
1641
+ await fs.mkdir(path.dirname(filePath), { recursive: true });
1642
+ await withFileLock(filePath, async () => {
1643
+ try {
1644
+ const st = await fs.stat(filePath);
1645
+ if (st.size > 256 * 1024) {
1646
+ const kept = (await fs.readFile(filePath, 'utf8')).split('\n').filter(Boolean).slice(-2000);
1647
+ await fs.writeFile(filePath, kept.length ? `${kept.join('\n')}\n` : '', 'utf8');
1648
+ }
1649
+ } catch {
1650
+ // Missing file is fine; append below recreates it.
1651
+ }
1652
+ await fs.appendFile(filePath, `${JSON.stringify(event)}\n`, 'utf8');
1653
+ });
1654
+ } catch {
1655
+ // Telemetry is advisory.
1656
+ }
1657
+ }
1658
+
1444
1659
  // ── Analog Query ──
1445
1660
  export async function queryAnalog({ rootDir = process.cwd(), filePath, limit = 5 } = {}) {
1446
1661
  const normalizedFilePath = String(filePath ?? '').trim();
@@ -2775,7 +2990,7 @@ function findScriptClose(content, startIndex) {
2775
2990
  }
2776
2991
 
2777
2992
  export function extractScriptContent(filePath, content) {
2778
- if (!filePath.endsWith('.vue')) {
2993
+ if (!/\.(?:vue|svelte)$/.test(filePath)) {
2779
2994
  return content;
2780
2995
  }
2781
2996
 
@@ -2864,6 +3079,278 @@ function computeSignature(content, lineStartIndex) {
2864
3079
  return `${collapsed.slice(0, 199)}…`;
2865
3080
  }
2866
3081
 
3082
+ // ── Non-JS languages (3.5.5): line/indent/brace based, no parser. Rows are { name, type, index, parent? }.
3083
+ // Kinds follow the JS lane: exported/public top-level -> function|class, private -> local-*, members -> method,
3084
+ // interface/enum/struct/trait/type aliases -> type.
3085
+ const POLYGLOT_EXT = /\.(?:py|go|java|cs|php|rb|rs)$/i;
3086
+ const POLY_MEMBER_SKIP = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'new', 'else', 'throw', 'using', 'lock', 'foreach', 'fixed', 'match', 'await', 'yield', 'do', 'try', 'sizeof', 'typeof', 'nameof', 'default', 'when', 'base', 'this']);
3087
+ const POLY_MODS = '(?:(?:public|protected|private|internal|static|final|abstract|sealed|partial|readonly|virtual|override|async|extern|unsafe|new|synchronized|native|strictfp|default|open)\\s+)*';
3088
+ const POLY_JAVA_TYPE_RE = new RegExp(`^[ \\t]*(?:@\\w+(?:\\([^)]*\\))?\\s+)*${POLY_MODS}(class|interface|enum|struct|record(?:\\s+(?:class|struct))?)\\s+([A-Za-z_]\\w*)`);
3089
+ const POLY_JAVA_METHOD_RE = new RegExp(`^[ \\t]*(?:@\\w+(?:\\([^)]*\\))?\\s+)*(?:<[^>]+>\\s+)?(?:[\\w.<>\\[\\],?]+\\s+)*?([A-Za-z_]\\w*)[ \\t]*(?:<[^>()]*>[ \\t]*)?\\(`);
3090
+ const POLY_PHP_TYPE_RE = /^[ \t]*(?:(?:abstract|final|readonly)\s+)*(class|interface|trait|enum)\s+([A-Za-z_]\w*)/;
3091
+ const POLY_PHP_FUNC_RE = /^[ \t]*(?:(?:public|protected|private|static|abstract|final)\s+)*function[ \t]+(?:&[ \t]*)?([A-Za-z_]\w*)/;
3092
+ const POLY_RS_PUB = '(?:pub(?:\\([^)]*\\))?\\s+)?';
3093
+ const POLY_RS_FN_RE = new RegExp(`^[ \\t]*(${POLY_RS_PUB})(?:(?:default|const|async|unsafe)\\s+)*(?:extern\\s+(?:"[^"]*"\\s+)?)?fn\\s+([A-Za-z_]\\w*)`);
3094
+ const POLY_RS_TYPE_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}(struct|enum|union|type)\\s+([A-Za-z_]\\w*)`);
3095
+ const POLY_RS_TRAIT_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}(?:unsafe\\s+)?trait\\s+([A-Za-z_]\\w*)`);
3096
+ // Bounds against pathological input (regex stalls): files above 512 KB (the content-lane per-file cap) are not
3097
+ // extracted, and any line over 2000 chars is blanked in place (offsets preserved) before the per-line patterns run.
3098
+ const POLY_MAX_FILE_CHARS = 512 * 1024;
3099
+ const POLY_MAX_LINE_CHARS = 2000;
3100
+ // impl header -> implemented type name, without a backtracking pattern: skip a balanced <generics>, then the
3101
+ // type after the first `for` (trait impl) or the first identifier (inherent impl).
3102
+ function polyRsImplName(line) {
3103
+ const head = /^[ \t]*(?:unsafe[ \t]+)?impl\b/.exec(line);
3104
+ if (!head) return null;
3105
+ let rest = line.slice(head[0].length).replace(/^[ \t]+/, '');
3106
+ if (rest[0] === '<') {
3107
+ let depth = 0;
3108
+ let k = 0;
3109
+ for (; k < rest.length; k += 1) {
3110
+ if (rest[k] === '<') depth += 1;
3111
+ else if (rest[k] === '>' && (depth -= 1) === 0) break;
3112
+ }
3113
+ rest = rest.slice(k + 1);
3114
+ }
3115
+ const cut = rest.search(/[{]|[ \t]where\b/);
3116
+ const header = cut < 0 ? rest : rest.slice(0, cut);
3117
+ const forAt = header.search(/(?:^|[ \t])for[ \t]/);
3118
+ const target = forAt < 0 ? header : header.slice(forAt).replace(/^[ \t]*for[ \t]+/, '');
3119
+ return /^[ \t]*([A-Za-z_]\w*)/.exec(target)?.[1] ?? null;
3120
+ }
3121
+ const POLY_RS_MOD_RE = new RegExp(`^[ \\t]*${POLY_RS_PUB}mod\\s+([A-Za-z_]\\w*)`);
3122
+
3123
+ // Same-length copy with comments (and, per mode, string bodies) blanked, so offsets/lines stay valid.
3124
+ // mode 0 = comments only, 1 = + "strings", 2 = + "strings" and 'chars'.
3125
+ function maskCLike(content, mode) {
3126
+ const out = content.split('');
3127
+ const n = content.length;
3128
+ const blank = (from, to) => {
3129
+ for (let k = from; k < to; k += 1) {
3130
+ if (out[k] !== '\n') out[k] = ' ';
3131
+ }
3132
+ };
3133
+ let i = 0;
3134
+ while (i < n) {
3135
+ const c = content[i];
3136
+ const d = content[i + 1];
3137
+ if (c === '/' && d === '/') {
3138
+ let end = content.indexOf('\n', i);
3139
+ if (end < 0) end = n;
3140
+ blank(i, end);
3141
+ i = end;
3142
+ } else if (c === '/' && d === '*') {
3143
+ let end = content.indexOf('*/', i + 2);
3144
+ end = end < 0 ? n : end + 2;
3145
+ blank(i, end);
3146
+ i = end;
3147
+ } else if ((mode >= 1 && c === '"') || (mode >= 2 && c === "'")) {
3148
+ let k = i + 1;
3149
+ while (k < n && content[k] !== c && content[k] !== '\n') {
3150
+ if (content[k] === '\\') k += 1;
3151
+ k += 1;
3152
+ }
3153
+ blank(i + 1, Math.min(k, n));
3154
+ i = k + 1;
3155
+ } else {
3156
+ i += 1;
3157
+ }
3158
+ }
3159
+ return out.join('');
3160
+ }
3161
+
3162
+ function extractIndentLanguageSymbols(filePath, content) {
3163
+ const isPy = /\.py$/i.test(filePath);
3164
+ const scopeRe = isPy ? /^([ \t]*)class[ \t]+([A-Za-z_]\w*)/ : /^([ \t]*)(?:module|class)[ \t]+([A-Z][\w:]*)/;
3165
+ const defRe = isPy ? /^([ \t]*)(?:async[ \t]+)?def[ \t]+([A-Za-z_]\w*)/ : /^([ \t]*)def[ \t]+(?:[A-Za-z_]\w*\.)?([A-Za-z_]\w*[?!=]?)/;
3166
+ const rows = [];
3167
+ const stack = [];
3168
+ let docQuote = null;
3169
+ let offset = 0;
3170
+ for (const line of content.split('\n')) {
3171
+ const lineStart = offset;
3172
+ offset += line.length + 1;
3173
+ const startedInDoc = docQuote !== null;
3174
+ if (isPy) {
3175
+ for (const quote of ['"""', "'''"]) {
3176
+ const count = line.split(quote).length - 1;
3177
+ if (count % 2 === 1) docQuote = docQuote === quote ? null : (docQuote ?? quote);
3178
+ }
3179
+ if (startedInDoc) continue;
3180
+ }
3181
+ const scope = scopeRe.exec(line);
3182
+ const def = scope ? null : defRe.exec(line);
3183
+ const m = scope ?? def;
3184
+ if (!m) continue;
3185
+ const indent = m[1].length;
3186
+ while (stack.length > 0 && stack[stack.length - 1].indent >= indent) stack.pop();
3187
+ const top = stack[stack.length - 1];
3188
+ const name = m[2];
3189
+ const index = lineStart + indent;
3190
+ if (scope) {
3191
+ if (!top || top.kind === 'scope') {
3192
+ rows.push({ name, type: isPy && name.startsWith('_') ? 'local-class' : 'class', index, parent: top?.name });
3193
+ }
3194
+ stack.push({ indent, kind: 'scope', name });
3195
+ } else {
3196
+ if (!top) {
3197
+ rows.push({ name, type: isPy && name.startsWith('_') ? 'local-function' : 'function', index });
3198
+ } else if (top.kind === 'scope') {
3199
+ rows.push({ name, type: 'method', index, parent: top.name });
3200
+ }
3201
+ stack.push({ indent, kind: 'def', name });
3202
+ }
3203
+ }
3204
+ return rows;
3205
+ }
3206
+
3207
+ function extractGoSymbols(content) {
3208
+ const text = maskCLike(content, 0);
3209
+ const rows = [];
3210
+ for (const m of text.matchAll(/^type[ \t]+([A-Za-z_]\w*)/gm)) {
3211
+ rows.push({ name: m[1], type: 'type', index: m.index });
3212
+ }
3213
+ for (const m of text.matchAll(/^func[ \t]+([A-Za-z_]\w*)[ \t]*[[(]/gm)) {
3214
+ rows.push({ name: m[1], type: /^[A-Z]/.test(m[1]) ? 'function' : 'local-function', index: m.index });
3215
+ }
3216
+ for (const m of text.matchAll(/^func[ \t]*\([ \t]*(?:[A-Za-z_]\w*[ \t]+)?(?:\*[ \t]*)?([A-Za-z_]\w*)(?:\[[^\]\n]*\])?[ \t]*\)[ \t]*([A-Za-z_]\w*)/gm)) {
3217
+ rows.push({ name: m[2], type: 'method', index: m.index, parent: m[1] });
3218
+ }
3219
+ return rows;
3220
+ }
3221
+
3222
+ // Java, C#, PHP and Rust: depth-tracked on a comment/string-masked copy. A container
3223
+ // (class, trait, impl, mod ...) owns the lines exactly one brace level below its declaration.
3224
+ function extractBraceLanguageSymbols(filePath, content) {
3225
+ const ext = filePath.slice(filePath.lastIndexOf('.') + 1).toLowerCase();
3226
+ const text = maskCLike(content, ext === 'rs' ? 1 : 2);
3227
+ const rows = [];
3228
+ const stack = [];
3229
+ let depth = 0;
3230
+ let offset = 0;
3231
+ for (const line of text.split('\n')) {
3232
+ const lineStart = offset;
3233
+ offset += line.length + 1;
3234
+ while (stack.length > 0 && stack[stack.length - 1].opened && depth <= stack[stack.length - 1].depth) stack.pop();
3235
+ const top = stack[stack.length - 1];
3236
+ const atMember = top ? depth === top.depth + 1 : false;
3237
+ const index = lineStart + Math.max(0, line.search(/\S/));
3238
+ const pushContainer = (kind, name) => {
3239
+ if (line.includes(';') && !line.includes('{')) return;
3240
+ stack.push({ kind, name, depth, opened: false });
3241
+ };
3242
+
3243
+ if (ext === 'java' || ext === 'cs') {
3244
+ const typeMatch = POLY_JAVA_TYPE_RE.exec(line);
3245
+ if (typeMatch) {
3246
+ const kind = typeMatch[1].split(/\s/)[0];
3247
+ rows.push({ name: typeMatch[2], type: kind === 'class' ? 'class' : 'type', index, parent: top?.name });
3248
+ pushContainer(kind, typeMatch[2]);
3249
+ } else if (atMember) {
3250
+ const m = POLY_JAVA_METHOD_RE.exec(line);
3251
+ if (m && !POLY_MEMBER_SKIP.has(m[1]) && !(top.kind === 'enum' && /^[A-Z0-9_]+$/.test(m[1]))) {
3252
+ rows.push({ name: m[1], type: 'method', index, parent: top.name });
3253
+ }
3254
+ }
3255
+ } else if (ext === 'php') {
3256
+ const typeMatch = POLY_PHP_TYPE_RE.exec(line);
3257
+ const fn = typeMatch ? null : POLY_PHP_FUNC_RE.exec(line);
3258
+ if (typeMatch) {
3259
+ rows.push({ name: typeMatch[2], type: typeMatch[1] === 'class' ? 'class' : 'type', index, parent: top?.name });
3260
+ pushContainer(typeMatch[1], typeMatch[2]);
3261
+ } else if (fn && !top) {
3262
+ rows.push({ name: fn[1], type: 'function', index });
3263
+ } else if (fn && atMember) {
3264
+ rows.push({ name: fn[1], type: 'method', index, parent: top.name });
3265
+ }
3266
+ } else {
3267
+ const fn = POLY_RS_FN_RE.exec(line);
3268
+ const trait = fn ? null : POLY_RS_TRAIT_RE.exec(line);
3269
+ const impl = fn || trait ? null : polyRsImplName(line);
3270
+ const mod = fn || trait || impl ? null : POLY_RS_MOD_RE.exec(line);
3271
+ const typeMatch = fn || trait || impl || mod ? null : POLY_RS_TYPE_RE.exec(line);
3272
+ const atModuleLevel = top ? top.kind === 'mod' && atMember : depth === 0;
3273
+ if (fn) {
3274
+ const container = top && (top.kind === 'impl' || top.kind === 'trait');
3275
+ if (container && atMember) {
3276
+ rows.push({ name: fn[2], type: 'method', index, parent: top.name });
3277
+ } else if (atModuleLevel) {
3278
+ rows.push({ name: fn[2], type: fn[1].trim() ? 'function' : 'local-function', index });
3279
+ }
3280
+ } else if (trait) {
3281
+ if (atModuleLevel) rows.push({ name: trait[1], type: 'type', index });
3282
+ pushContainer('trait', trait[1]);
3283
+ } else if (impl) {
3284
+ pushContainer('impl', impl);
3285
+ } else if (mod) {
3286
+ pushContainer('mod', mod[1]);
3287
+ } else if (typeMatch && atModuleLevel) {
3288
+ rows.push({ name: typeMatch[2], type: 'type', index });
3289
+ }
3290
+ }
3291
+
3292
+ let maxDepth = depth;
3293
+ for (let k = 0; k < line.length; k += 1) {
3294
+ if (line[k] === '{') {
3295
+ depth += 1;
3296
+ if (depth > maxDepth) maxDepth = depth;
3297
+ } else if (line[k] === '}' && depth > 0) {
3298
+ depth -= 1;
3299
+ }
3300
+ }
3301
+ for (const entry of stack) {
3302
+ if (!entry.opened && maxDepth > entry.depth) entry.opened = true;
3303
+ }
3304
+ }
3305
+ return rows;
3306
+ }
3307
+
3308
+ function extractPolyglotSymbols(filePath, rawContent) {
3309
+ if (rawContent.length > POLY_MAX_FILE_CHARS) return [];
3310
+ const parts = rawContent.split('\n');
3311
+ const content = parts.some((l) => l.length > POLY_MAX_LINE_CHARS)
3312
+ ? parts.map((l) => (l.length > POLY_MAX_LINE_CHARS ? ' '.repeat(l.length) : l)).join('\n')
3313
+ : rawContent;
3314
+ const ext = filePath.slice(filePath.lastIndexOf('.') + 1).toLowerCase();
3315
+ let rows;
3316
+ if (ext === 'py' || ext === 'rb') rows = extractIndentLanguageSymbols(filePath, content);
3317
+ else if (ext === 'go') rows = extractGoSymbols(content);
3318
+ else rows = extractBraceLanguageSymbols(filePath, content);
3319
+ const seen = new Set();
3320
+ return rows.filter((row) => {
3321
+ const key = `${row.type}:${row.name}:${row.parent ?? ''}`;
3322
+ if (seen.has(key)) return false;
3323
+ seen.add(key);
3324
+ return true;
3325
+ });
3326
+ }
3327
+
3328
+ // `const x = (` also opens plain parenthesised expressions (`const n = (a ?? 0)`), so only a
3329
+ // parameter list whose closing paren is followed by `=>` (optionally after a return type) is a function.
3330
+ function findParenArrowDeclarations(content) {
3331
+ const found = [];
3332
+ for (const match of content.matchAll(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?\(/g)) {
3333
+ const open = match.index + match[0].length - 1;
3334
+ const limit = Math.min(content.length, open + 2000);
3335
+ let depth = 0;
3336
+ let close = -1;
3337
+ for (let i = open; i < limit; i += 1) {
3338
+ if (content[i] === '(') depth += 1;
3339
+ else if (content[i] === ')') {
3340
+ depth -= 1;
3341
+ if (depth === 0) {
3342
+ close = i;
3343
+ break;
3344
+ }
3345
+ }
3346
+ }
3347
+ if (close >= 0 && /^\s*(?::\s*[^;\n{}=]*?(?:=>\s*[^;\n{}=]*?)?)?=>/.test(content.slice(close + 1, close + 201))) {
3348
+ found.push({ name: match[1], index: match.index });
3349
+ }
3350
+ }
3351
+ return found;
3352
+ }
3353
+
2867
3354
  const CLASS_METHOD_SKIP = new Set(['if', 'for', 'while', 'switch', 'catch', 'function', 'return', 'with', 'constructor', 'super']);
2868
3355
 
2869
3356
  export function extractSymbols(filePath, content) {
@@ -2887,6 +3374,13 @@ export function extractSymbols(filePath, content) {
2887
3374
  }
2888
3375
  };
2889
3376
 
3377
+ if (POLYGLOT_EXT.test(filePath)) {
3378
+ for (const row of extractPolyglotSymbols(filePath, content)) {
3379
+ addSymbol(row.name, row.type, row.index, row.parent ? { parent: row.parent } : null);
3380
+ }
3381
+ return out;
3382
+ }
3383
+
2890
3384
  if (filePath.endsWith('.sql')) {
2891
3385
  // Capture the object name, skipping an optional schema qualifier (qas.fn_rpt_billing → fn_rpt_billing).
2892
3386
  const sqlIdent = '(?:(?:["`]?[A-Za-z_][A-Za-z0-9_$]*["`]?)\\s*\\.\\s*)?["`]?([A-Za-z_][A-Za-z0-9_$]*)["`]?';
@@ -2918,8 +3412,10 @@ export function extractSymbols(filePath, content) {
2918
3412
  } else {
2919
3413
  addMatches(/export\s+(?:async\s+)?function\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'function');
2920
3414
  addMatches(/export\s+class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'class');
2921
- addMatches(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?\(/g, 'callable');
2922
- addMatches(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][A-Za-z0-9_$]*)\s*=>/g, 'callable');
3415
+ for (const decl of findParenArrowDeclarations(content)) {
3416
+ addSymbol(decl.name, 'callable', decl.index);
3417
+ }
3418
+ addMatches(/(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s+)?[A-Za-z_$][A-Za-z0-9_$]*\s*=>/g, 'callable');
2923
3419
  addMatches(/export\s+default\s+(?:async\s+)?function\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'default-function');
2924
3420
  addMatches(/export\s+default\s+class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g, 'default-class');
2925
3421
  addMatches(/export\s+default\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*(?:;|\n|$)/g, 'default-reference');
@@ -2932,7 +3428,7 @@ export function extractSymbols(filePath, content) {
2932
3428
  addMatches(/^[ \t]*(?:async\s+)?function\s*\*?\s*([A-Za-z_$][A-Za-z0-9_$]*)\s*[(<]/gm, 'local-function');
2933
3429
  addMatches(/^[ \t]*(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s+)?function\b/gm, 'callable');
2934
3430
  addMatches(/^[ \t]*(?:abstract\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'local-class');
2935
- if (/\.(?:ts|tsx|mts|cts|vue)$/.test(filePath)) {
3431
+ if (/\.(?:ts|tsx|mts|cts|vue|svelte)$/.test(filePath)) {
2936
3432
  addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?(?:const\s+)?(?:interface|enum)\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'type');
2937
3433
  addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?type\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*(?:<[^=\n]*>)?\s*=/gm, 'type');
2938
3434
  }
@@ -3531,7 +4027,7 @@ const BOOST_LIFT_RATIO = 0.2; // import/hotspot/archetype boosts may add at most
3531
4027
  const CONTENT_LANE_CONFIDENT_SCORE = 30; // a best direct score at/above this (exact symbol or path hit) skips the content lane
3532
4028
  const CONTENT_LANE_MIN_TOKENS = 3; // shorter queries are names, not messages
3533
4029
  const CONTENT_LANE_LIMIT = 40; // candidates only share tokens somewhere in the file; the same-line check below keeps the real ones
3534
- const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|sql|sh)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
4030
+ const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|svelte|sql|sh|py|go|java|cs|php|rb|rs)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
3535
4031
  const CONTENT_LANE_BONUS = 14; // added to a file that already has a direct score
3536
4032
  const CONTENT_LANE_ONLY_SCORE = 13; // base for a file found only by content; below CONTENT_LANE_CONFIDENT_SCORE by design
3537
4033
  const CONTENT_LANE_MAX_FILES = 1500; // files read per query, in priority order (primary bound)
@@ -4357,6 +4853,16 @@ export function buildSearchDescriptor(value, { expandVietnameseAliases = false }
4357
4853
  for (const token of collectVietnameseAliasTokens(foldedLower, tokens)) {
4358
4854
  tokens.add(token);
4359
4855
  }
4856
+ // Stems only for question-like queries: identifier-style queries (findRelatedItemIds,
4857
+ // collectVietnameseAliasTokens) must keep matching exact tokens.
4858
+ if (tokens.size >= STEM_MIN_QUERY_TOKENS) {
4859
+ for (const token of [...tokens]) {
4860
+ const stem = stemToken(token);
4861
+ if (stem && !GENERIC_STOPWORDS.has(stem)) {
4862
+ tokens.add(stem);
4863
+ }
4864
+ }
4865
+ }
4360
4866
  }
4361
4867
 
4362
4868
  return {
@@ -4728,6 +5234,7 @@ function isLikelyTestFilePath(filePath) {
4728
5234
  || lower.endsWith('.spec.jsx')
4729
5235
  || lower.endsWith('.spec.mjs')
4730
5236
  || lower.endsWith('.spec.cjs')
5237
+ || /(?:_test\.(?:go|py|rb|rs)|_spec\.rb|(?:^|\/)test_[^/]+\.py|[a-z0-9]Tests?\.(?:java|cs))$/.test(filePath)
4731
5238
  );
4732
5239
  }
4733
5240
 
@@ -5361,6 +5868,7 @@ function isLikelyTestFile(filePath) {
5361
5868
  || lower.endsWith('.spec.jsx')
5362
5869
  || lower.endsWith('.spec.mjs')
5363
5870
  || lower.endsWith('.spec.cjs')
5871
+ || /(?:_test\.(?:go|py|rb|rs)|_spec\.rb|(?:^|\/)test_[^/]+\.py|[a-z0-9]Tests?\.(?:java|cs))$/.test(filePath)
5364
5872
  );
5365
5873
  }
5366
5874
 
@@ -74,6 +74,17 @@ if (changedFiles.length > 0 && !force) {
74
74
  }
75
75
  }
76
76
 
77
+ // 3.5.5: keep existing topic pages current. Rebuilds only pages that already exist
78
+ // and are stale (agent block and cited notes preserved); never creates pages; never
79
+ // fails the refresh. Runs here (git hook, backgrounded + deadline-armed), not in a hook path.
80
+ try {
81
+ if (process.env.UKIT_MEMORY_TOPICS !== '0') {
82
+ const { refreshTopics } = await import('../runtime/memory-topics.mjs');
83
+ const res = await refreshTopics(rootDir);
84
+ if (res.rebuilt.length > 0) console.log(`topics-rebuilt: ${res.rebuilt.length}`);
85
+ }
86
+ } catch { /* topic pages are optional */ }
87
+
77
88
  function getRootDir(argv) {
78
89
  const value = readFlagValue(argv, '--root');
79
90
  return value ? path.resolve(value) : process.cwd();