@ngockhoale/ukit 3.5.2 → 3.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -90
- package/README.md +1 -0
- package/package.json +8 -1
- package/src/cli/commands/doctor.js +14 -2
- package/src/cli/commands/memory.js +30 -2
- package/src/core/memory/memoryHit.js +2 -0
- package/src/core/memory/mutateMemory.js +9 -0
- package/src/core/memory/projectIdentity.js +9 -2
- package/src/core/memory/recallEngine.js +86 -4
- package/src/core/memory/registryPrune.js +230 -0
- package/src/core/memory/retrieval.js +12 -3
- package/src/core/runtimeConfig.js +1 -1
- package/src/index/queryIndex.js +151 -20
- package/template_project/.claude/ukit/index/edit-graph.mjs +141 -11
- package/template_project/.claude/ukit/index/lib/index-core.mjs +149 -19
- package/template_project/.claude/ukit/index/resolve-context.mjs +1 -1
- package/template_project/.claude/ukit/index/route-task.mjs +4 -1
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +44 -22
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +86 -4
- package/template_project/.claude/ukit/runtime/memory-validity.mjs +232 -0
- package/template_project/.claude/ukit/runtime/reinject-context.mjs +4 -2
- package/src/core/observability/evaluation/experimentPlan.js +0 -95
- package/src/core/observability/evaluation/perturbation.js +0 -273
- package/src/diagnostics/classifyHang.js +0 -246
- package/src/diagnostics/ompStallSnapshot.js +0 -199
- package/src/render/instructionRenderer.js +0 -290
- package/src/skill/auditSkill.js +0 -140
- package/template_project/instructions/overlays/repo.md +0 -9
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { withFileLock, writeJson, readJsonIfExists } from '../fileOps.js';
|
|
4
|
+
import { buildUserPaths } from '../userPaths.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Stale-entry pruning for the user-level project registry
|
|
8
|
+
* (`~/.ukit/storage/projects/registry.json`, owned by projectIdentity.js).
|
|
9
|
+
*
|
|
10
|
+
* An entry is stale only when EVERY root AND every git common dir provably no longer exists
|
|
11
|
+
* (a project can resolve through its git common dir after the worktree that registered it is gone):
|
|
12
|
+
* - stat of the path fails with ENOENT/ENOTDIR (any other error => keep), AND
|
|
13
|
+
* - the nearest mount-level ancestor is present and populated (an unmounted
|
|
14
|
+
* external/network volume must never make its projects look deleted), AND
|
|
15
|
+
* - the entry is not the project being registered, AND
|
|
16
|
+
* - no user-level memory record is attributed to the entry's project id.
|
|
17
|
+
* Mutation re-reads the registry under the same lock and uses the same atomic
|
|
18
|
+
* writer as registration; decisions made outside the lock are re-validated.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
export const AUTO_PRUNE_MIN_ENTRIES = 500;
|
|
22
|
+
export const AUTO_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
23
|
+
export const AUTO_PRUNE_BUDGET_MS = 1500;
|
|
24
|
+
const STAT_CONCURRENCY = 32;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Pure: nearest mount-level ancestor whose presence is required before a missing
|
|
28
|
+
* root may be called deleted. Returns null when the path has no such ancestor
|
|
29
|
+
* (plain local path: the filesystem root is always there).
|
|
30
|
+
* `emptyIsUnmounted` tells the caller an empty mount-point dir is not proof.
|
|
31
|
+
*/
|
|
32
|
+
export function mountAncestorOf(rootPath, platform = process.platform) {
|
|
33
|
+
if (typeof rootPath !== 'string' || rootPath === '') return null;
|
|
34
|
+
if (platform === 'win32') {
|
|
35
|
+
const p = path.win32;
|
|
36
|
+
const parsed = p.parse(p.normalize(rootPath));
|
|
37
|
+
if (!parsed.root) return null;
|
|
38
|
+
// Drive root (C:\) or UNC share root (\\server\share\).
|
|
39
|
+
return { path: parsed.root, emptyIsUnmounted: false };
|
|
40
|
+
}
|
|
41
|
+
const parts = path.posix.normalize(rootPath).split('/').filter(Boolean);
|
|
42
|
+
const take = (n) => ({ path: `/${parts.slice(0, n).join('/')}`, emptyIsUnmounted: true });
|
|
43
|
+
if (parts[0] === 'Volumes' && parts.length >= 2) {
|
|
44
|
+
return { path: `/${parts.slice(0, 2).join('/')}`, emptyIsUnmounted: false };
|
|
45
|
+
}
|
|
46
|
+
if (parts[0] === 'mnt' && parts.length >= 2) return take(2);
|
|
47
|
+
if (parts[0] === 'media' && parts.length >= 3) return take(3);
|
|
48
|
+
if (parts[0] === 'run' && parts[1] === 'media' && parts.length >= 4) return take(4);
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Classify one root. `io` = { stat(p), readdir(p) } (injectable for tests).
|
|
54
|
+
* @returns {Promise<'missing'|'present'|'unproven'>}
|
|
55
|
+
*/
|
|
56
|
+
export async function classifyRoot(rootPath, { platform = process.platform, io, mountCache } = {}) {
|
|
57
|
+
const fsio = io ?? { stat: (p) => fs.stat(p), readdir: (p) => fs.readdir(p) };
|
|
58
|
+
try {
|
|
59
|
+
await fsio.stat(rootPath);
|
|
60
|
+
return 'present';
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (error?.code !== 'ENOENT' && error?.code !== 'ENOTDIR') return 'unproven';
|
|
63
|
+
}
|
|
64
|
+
const mount = mountAncestorOf(rootPath, platform);
|
|
65
|
+
if (!mount) return 'missing';
|
|
66
|
+
let ok = mountCache?.get(mount.path);
|
|
67
|
+
if (ok === undefined) {
|
|
68
|
+
ok = false;
|
|
69
|
+
try {
|
|
70
|
+
await fsio.stat(mount.path);
|
|
71
|
+
ok = mount.emptyIsUnmounted ? (await fsio.readdir(mount.path)).length > 0 : true;
|
|
72
|
+
} catch {
|
|
73
|
+
ok = false;
|
|
74
|
+
}
|
|
75
|
+
mountCache?.set(mount.path, ok);
|
|
76
|
+
}
|
|
77
|
+
return ok ? 'missing' : 'unproven';
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function entrySignature(entry) {
|
|
81
|
+
return JSON.stringify([entry?.roots, entry?.gitCommonDirs ?? []]);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function registryPaths(homeDir) {
|
|
85
|
+
const user = buildUserPaths(homeDir === undefined ? {} : { homeDir });
|
|
86
|
+
const regPath = path.join(user.storageRoot, 'projects', 'registry.json');
|
|
87
|
+
return { user, regPath, stampPath: path.join(path.dirname(regPath), 'registry.prune-stamp') };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function mapLimit(items, limit, fn, shouldStop) {
|
|
91
|
+
let next = 0;
|
|
92
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
93
|
+
while (next < items.length) {
|
|
94
|
+
if (shouldStop?.()) return;
|
|
95
|
+
const i = next++;
|
|
96
|
+
await fn(items[i], i);
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
await Promise.all(workers);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function rebuildAliasIndex(registry, removedAliases) {
|
|
103
|
+
const names = new Set(removedAliases);
|
|
104
|
+
for (const name of names) {
|
|
105
|
+
const ids = Object.entries(registry.projects)
|
|
106
|
+
.filter(([, entry]) => Array.isArray(entry?.aliases) && entry.aliases.includes(name))
|
|
107
|
+
.map(([id]) => id);
|
|
108
|
+
if (ids.length === 0) delete registry.aliasIndex[name];
|
|
109
|
+
else registry.aliasIndex[name] = ids.length > 1 ? 'ambiguous' : ids;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Prune stale registry entries.
|
|
115
|
+
* @param {object} o
|
|
116
|
+
* @param {string} [o.homeDir]
|
|
117
|
+
* @param {boolean} [o.dryRun] decide + count only, never write
|
|
118
|
+
* @param {string[]} [o.keepRoots] roots that must never be pruned (current project)
|
|
119
|
+
* @param {number} [o.budgetMs] stop stat-ing after this long (partial result)
|
|
120
|
+
* @param {string} [o.platform]
|
|
121
|
+
* @param {object} [o.io] injectable stat/readdir
|
|
122
|
+
* @returns {Promise<{status:string, total:number, checked:number, stale:number, removed:number,
|
|
123
|
+
* keptWithMemory:number, unproven:number, truncated:boolean}>}
|
|
124
|
+
*/
|
|
125
|
+
export async function pruneRegistry({
|
|
126
|
+
homeDir, dryRun = false, keepRoots = [], budgetMs = Infinity, platform = process.platform, io,
|
|
127
|
+
} = {}) {
|
|
128
|
+
const { user, regPath } = registryPaths(homeDir);
|
|
129
|
+
const result = { status: 'ok', total: 0, checked: 0, stale: 0, removed: 0, keptWithMemory: 0, unproven: 0, truncated: false };
|
|
130
|
+
|
|
131
|
+
let registry;
|
|
132
|
+
try {
|
|
133
|
+
registry = await readJsonIfExists(regPath);
|
|
134
|
+
} catch {
|
|
135
|
+
return { ...result, status: 'registry-corrupt' };
|
|
136
|
+
}
|
|
137
|
+
if (!registry || typeof registry.projects !== 'object' || registry.projects === null) {
|
|
138
|
+
return { ...result, status: registry ? 'registry-corrupt' : 'no-registry' };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// Conservative: read the RAW user-memory document (not normalized) so even records this build
|
|
142
|
+
// would skip still protect their project id. Project-scoped memory lives under the project's own
|
|
143
|
+
// `.ukit/` (gone with the root) and is unreachable; the user-level store is the only data keyed by id.
|
|
144
|
+
let userDoc;
|
|
145
|
+
try {
|
|
146
|
+
userDoc = await readJsonIfExists(user.memoryV2RecordsPath);
|
|
147
|
+
} catch {
|
|
148
|
+
return { ...result, status: 'user-memory-unreadable' };
|
|
149
|
+
}
|
|
150
|
+
const memoryIds = new Set(
|
|
151
|
+
(Array.isArray(userDoc?.records) ? userDoc.records : [])
|
|
152
|
+
.map((r) => r?.project_id).filter((id) => typeof id === 'string'),
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
const keep = new Set(keepRoots);
|
|
156
|
+
const entries = Object.entries(registry.projects);
|
|
157
|
+
result.total = entries.length;
|
|
158
|
+
const started = Date.now();
|
|
159
|
+
const mountCache = new Map();
|
|
160
|
+
const staleIds = new Map(); // id -> roots signature
|
|
161
|
+
const shouldStop = () => Date.now() - started > budgetMs;
|
|
162
|
+
|
|
163
|
+
await mapLimit(entries, STAT_CONCURRENCY, async ([id, entry]) => {
|
|
164
|
+
result.checked += 1;
|
|
165
|
+
const roots = entry?.roots;
|
|
166
|
+
if (!Array.isArray(roots) || roots.length === 0 || roots.some((r) => typeof r !== 'string')) return;
|
|
167
|
+
// gitCommonDirs: absent => none; present but malformed => keep (cannot prove anything).
|
|
168
|
+
const gitDirs = entry?.gitCommonDirs ?? [];
|
|
169
|
+
if (!Array.isArray(gitDirs) || gitDirs.some((g) => typeof g !== 'string')) return;
|
|
170
|
+
if (roots.some((r) => keep.has(r))) return;
|
|
171
|
+
for (const p of [...roots, ...gitDirs]) {
|
|
172
|
+
const verdict = await classifyRoot(p, { platform, io, mountCache });
|
|
173
|
+
if (verdict !== 'missing') {
|
|
174
|
+
if (verdict === 'unproven') result.unproven += 1;
|
|
175
|
+
return; // any present/unproven root or git directory keeps the entry
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (memoryIds.has(id)) { result.keptWithMemory += 1; return; }
|
|
179
|
+
staleIds.set(id, entrySignature(entry));
|
|
180
|
+
}, shouldStop);
|
|
181
|
+
|
|
182
|
+
result.truncated = result.checked < entries.length && shouldStop();
|
|
183
|
+
result.stale = staleIds.size;
|
|
184
|
+
if (dryRun || staleIds.size === 0) return result;
|
|
185
|
+
|
|
186
|
+
const outcome = await withFileLock(regPath, async () => {
|
|
187
|
+
const fresh = await readJsonIfExists(regPath).catch(() => null);
|
|
188
|
+
if (!fresh || typeof fresh.projects !== 'object' || fresh.projects === null) return 0;
|
|
189
|
+
if (typeof fresh.aliasIndex !== 'object' || fresh.aliasIndex === null) fresh.aliasIndex = {};
|
|
190
|
+
const removedAliases = [];
|
|
191
|
+
let removed = 0;
|
|
192
|
+
for (const [id, signature] of staleIds) {
|
|
193
|
+
const current = fresh.projects[id];
|
|
194
|
+
// Re-validate: entry must still exist unchanged (a concurrent writer may have re-bound it).
|
|
195
|
+
if (!current || entrySignature(current) !== signature) continue;
|
|
196
|
+
if (Array.isArray(current.aliases)) removedAliases.push(...current.aliases);
|
|
197
|
+
delete fresh.projects[id];
|
|
198
|
+
removed += 1;
|
|
199
|
+
}
|
|
200
|
+
if (removed === 0) return 0;
|
|
201
|
+
rebuildAliasIndex(fresh, removedAliases);
|
|
202
|
+
await writeJson(regPath, fresh);
|
|
203
|
+
return removed;
|
|
204
|
+
});
|
|
205
|
+
if (outcome === undefined) return { ...result, status: 'lock-busy' };
|
|
206
|
+
return { ...result, removed: outcome };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Bounded automatic prune used during registration. Never throws; async fs only;
|
|
211
|
+
* skipped below the entry threshold or when run within the last day.
|
|
212
|
+
*/
|
|
213
|
+
export async function maybeAutoPrune({ homeDir, entryCount, currentRoot, now = Date.now() } = {}) {
|
|
214
|
+
try {
|
|
215
|
+
if (!(entryCount >= AUTO_PRUNE_MIN_ENTRIES)) return null;
|
|
216
|
+
const { stampPath } = registryPaths(homeDir);
|
|
217
|
+
try {
|
|
218
|
+
const st = await fs.stat(stampPath);
|
|
219
|
+
if (now - st.mtimeMs < AUTO_PRUNE_INTERVAL_MS) return null;
|
|
220
|
+
} catch { /* no stamp yet */ }
|
|
221
|
+
// Stamp first: concurrent/failed runs do not retry within the interval.
|
|
222
|
+
await fs.mkdir(path.dirname(stampPath), { recursive: true });
|
|
223
|
+
await fs.writeFile(stampPath, `${new Date(now).toISOString()}\n`);
|
|
224
|
+
return await pruneRegistry({
|
|
225
|
+
homeDir, keepRoots: currentRoot ? [currentRoot] : [], budgetMs: AUTO_PRUNE_BUDGET_MS,
|
|
226
|
+
});
|
|
227
|
+
} catch {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
@@ -16,6 +16,8 @@ import { loadMarkdownDocs, normalizeMemorySources, renderMarkdownDoc } from './r
|
|
|
16
16
|
import { resolveRecordFreshness } from './memoryFreshness.js';
|
|
17
17
|
import { getHeadSha } from '../codeintel/freshness.js';
|
|
18
18
|
import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
|
|
19
|
+
import { withValidity } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
|
|
20
|
+
import { STALE_MARKER } from './recallEngine.js';
|
|
19
21
|
|
|
20
22
|
function compactPhraseList(values, { limit = 1, fallback = 'n/a' } = {}) {
|
|
21
23
|
const normalized = [...new Set(
|
|
@@ -329,8 +331,11 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
|
|
|
329
331
|
if (!v2AllowsLegacyFallback(state)) {
|
|
330
332
|
// One engine (recallEngine.js BM25): eligibility first, then IDF over the
|
|
331
333
|
// eligible set + markdown bullets, floor, explicit empty result.
|
|
332
|
-
|
|
333
|
-
|
|
334
|
+
// Source validity (3.5.3): anchored items whose file changed/vanished are
|
|
335
|
+
// demoted and labelled. The CLI is the lazy-capture point for never-seen items.
|
|
336
|
+
const anchored = await withValidity(projectRoot, [...records, ...markdownDocs], { capture: true });
|
|
337
|
+
const ranked = rankHitsDetailed(anchored.slice(0, records.length), {
|
|
338
|
+
query, eligibleCtx, policy, scope, limit, now, extraDocs: anchored.slice(records.length),
|
|
334
339
|
});
|
|
335
340
|
const results = await attachFreshness(ranked.hits, records, { projectRoot, watermarks, now });
|
|
336
341
|
emitQuery({ lane: 'v2', indexed: false, candidateCount: ranked.candidateCount, results });
|
|
@@ -628,7 +633,9 @@ export async function getContextInjection(
|
|
|
628
633
|
// `path:line`; they are never records and carry no evidence/freshness.
|
|
629
634
|
const markdownRecord = (entry) => ({
|
|
630
635
|
id: entry.id, type: 'markdown', text: entry.snippet, provenance: entry.provenance, evidence: [],
|
|
636
|
+
...(entry.validity ? { validity: entry.validity } : {}),
|
|
631
637
|
});
|
|
638
|
+
const validityOf = new Map(rankedItems.filter((entry) => entry.validity).map((entry) => [entry.id, entry.validity]));
|
|
632
639
|
const markdownSelected = rankedItems.filter((entry) => entry.type === 'markdown').map(markdownRecord);
|
|
633
640
|
const selectedItems = v2Active
|
|
634
641
|
? dedupeItemsById(rankedItems
|
|
@@ -636,6 +643,7 @@ export async function getContextInjection(
|
|
|
636
643
|
? markdownRecord(entry)
|
|
637
644
|
: records.find((record) => record.id === entry.id)))
|
|
638
645
|
.filter(Boolean)
|
|
646
|
+
.map((record) => (validityOf.has(record.id) && !record.validity ? { ...record, validity: validityOf.get(record.id) } : record))
|
|
639
647
|
.filter((record) => record.type === 'markdown' || eligible(record, eligibleCtx, policy).ok))
|
|
640
648
|
.slice(0, Math.max(1, recallPlan.limit))
|
|
641
649
|
: [
|
|
@@ -697,7 +705,8 @@ export async function getContextInjection(
|
|
|
697
705
|
|
|
698
706
|
// FR-005: ' [stale]' appended only when freshness resolves stale; unknown
|
|
699
707
|
// adds no token.
|
|
700
|
-
const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '')
|
|
708
|
+
const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '')
|
|
709
|
+
+ (record.validity && record.type !== 'markdown' ? ` ${STALE_MARKER}` : '');
|
|
701
710
|
const rawBodyLines = v2Active
|
|
702
711
|
? selectedItems.map((record) => `- [${record.type}] ${buildRecordSnippet(record)}${staleTag(record)}`)
|
|
703
712
|
: selectedItems.map((item) => (item.type === 'markdown'
|
|
@@ -532,7 +532,7 @@ export function buildDefaultRuntimeConfig(overrides = {}) {
|
|
|
532
532
|
maxPatternCandidates: 50,
|
|
533
533
|
redactSecrets: true,
|
|
534
534
|
sources: ['docs/MEMORY.md'],
|
|
535
|
-
editHints: { enabled: true, mirrors: [], ignoreMirrors: [] },
|
|
535
|
+
editHints: { enabled: true, mirrors: [], ignoreMirrors: [], coChange: false },
|
|
536
536
|
},
|
|
537
537
|
validation: {
|
|
538
538
|
enabled: true,
|
package/src/index/queryIndex.js
CHANGED
|
@@ -78,7 +78,7 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
78
78
|
return QUERY_RESULT_CACHE.get(queryCacheKey);
|
|
79
79
|
}
|
|
80
80
|
const queryPromise = (async () => {
|
|
81
|
-
const { files, searchDescriptorsByFile } = await loadQuerySearchBundle(absoluteRoot);
|
|
81
|
+
const { files, searchDescriptorsByFile, tokenWeights } = await loadQuerySearchBundle(absoluteRoot);
|
|
82
82
|
const preferredArchetypes = inferPreferredArchetypes(queryDescriptor.tokens);
|
|
83
83
|
const includeLikelyTestFiles = shouldIncludeLikelyTestFiles(queryDescriptor);
|
|
84
84
|
|
|
@@ -92,8 +92,10 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
92
92
|
const searchDescriptor = searchDescriptorsByFile.get(filePath) ?? createFileSearchDescriptor({ filePath, symbols: [] });
|
|
93
93
|
const { score, reasons } = scoreDirectFileMatch({
|
|
94
94
|
queryDescriptor,
|
|
95
|
+
rawQuery: normalizedQuery,
|
|
95
96
|
filePath,
|
|
96
97
|
searchDescriptor,
|
|
98
|
+
tokenWeights,
|
|
97
99
|
});
|
|
98
100
|
|
|
99
101
|
if (score > 0) {
|
|
@@ -141,13 +143,17 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
141
143
|
importersByTarget,
|
|
142
144
|
testsMapBySource,
|
|
143
145
|
});
|
|
144
|
-
const finalScores =
|
|
145
|
-
|
|
146
|
-
|
|
146
|
+
const finalScores = capBoostLift({
|
|
147
|
+
directScores,
|
|
148
|
+
scores: applyHotspotBoosts({
|
|
149
|
+
scores: relatedScores,
|
|
150
|
+
hotspotScores,
|
|
151
|
+
}),
|
|
147
152
|
});
|
|
148
153
|
|
|
149
154
|
return freezeQueryResults(
|
|
150
155
|
[...finalScores.values()]
|
|
156
|
+
.map((entry) => ({ ...entry, score: finalizeScore(entry) }))
|
|
151
157
|
.sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath))
|
|
152
158
|
.slice(0, limit),
|
|
153
159
|
);
|
|
@@ -259,9 +265,11 @@ async function loadQuerySearchBundle(rootDir) {
|
|
|
259
265
|
const safeFiles = safeSchemaArtifact(files);
|
|
260
266
|
const safeSymbols = safeSchemaArtifact(symbols);
|
|
261
267
|
const symbolMap = groupBy(safeItems(safeSymbols), (item) => item.filePath);
|
|
268
|
+
const searchDescriptorsByFile = buildFileSearchDescriptors(safeItems(safeFiles), symbolMap);
|
|
262
269
|
return {
|
|
263
270
|
files: safeFiles,
|
|
264
|
-
searchDescriptorsByFile
|
|
271
|
+
searchDescriptorsByFile,
|
|
272
|
+
tokenWeights: buildTokenWeights(searchDescriptorsByFile),
|
|
265
273
|
};
|
|
266
274
|
})
|
|
267
275
|
.catch((error) => {
|
|
@@ -330,9 +338,91 @@ function dedupe(values) {
|
|
|
330
338
|
return [...new Set(values)];
|
|
331
339
|
}
|
|
332
340
|
|
|
333
|
-
|
|
341
|
+
// ── Ranking constants (kept identical in template index-core.mjs) ──
|
|
342
|
+
const PATH_EXACT_SCORE = 60;
|
|
343
|
+
const PATH_SUFFIX_SCORE = 48;
|
|
344
|
+
const PATH_BASENAME_SCORE = 40;
|
|
345
|
+
const SYMBOL_EXACT_SCORE = 40;
|
|
346
|
+
const SYMBOL_DECAY = 0.35; // k-th best symbol of a file adds score * SYMBOL_DECAY^k
|
|
347
|
+
const SYMBOL_REASON_LIMIT = 3;
|
|
348
|
+
const MIN_SUBSTRING_TOKEN_LENGTH = 3;
|
|
349
|
+
const BOOST_LIFT_RATIO = 0.2; // import/hotspot/archetype boosts may add at most this share of a direct score (+1)
|
|
350
|
+
const BOOST_ONLY_RATIO = 0.3; // files with no direct match stay below this share of the best direct score
|
|
351
|
+
|
|
352
|
+
// Rarity weight per token in (0,1]: a token present in nearly every file is worth
|
|
353
|
+
// ~ln2/ln(1+N), a token unique to one file is worth 1. Computed once per bundle.
|
|
354
|
+
function buildTokenWeights(searchDescriptorsByFile) {
|
|
355
|
+
const docFrequency = new Map();
|
|
356
|
+
for (const descriptor of searchDescriptorsByFile.values()) {
|
|
357
|
+
const seen = new Set(descriptor.path.tokens);
|
|
358
|
+
for (const symbol of descriptor.symbols) {
|
|
359
|
+
for (const token of symbol.descriptor.tokens) {
|
|
360
|
+
seen.add(token);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
for (const token of seen) {
|
|
364
|
+
docFrequency.set(token, (docFrequency.get(token) ?? 0) + 1);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
const total = searchDescriptorsByFile.size;
|
|
368
|
+
const denominator = Math.log(1 + total) || 1;
|
|
369
|
+
const weights = new Map();
|
|
370
|
+
for (const [token, count] of docFrequency) {
|
|
371
|
+
weights.set(token, Math.max(0.05, Math.log(1 + total / count) / denominator));
|
|
372
|
+
}
|
|
373
|
+
return weights;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Prior applied after scoring so an original beats its mirror copy on ties without
|
|
377
|
+
// hiding the mirror: root installed copies < template_project < everything else.
|
|
378
|
+
function mirrorPrior(filePath) {
|
|
379
|
+
if (/^\.(?:claude|codex|omp)\//.test(filePath)) {
|
|
380
|
+
return 0.88;
|
|
381
|
+
}
|
|
382
|
+
if (filePath.startsWith('template_project/')) {
|
|
383
|
+
return 0.94;
|
|
384
|
+
}
|
|
385
|
+
return 1;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function normalizePathQuery(rawQuery) {
|
|
389
|
+
const value = String(rawQuery ?? '').trim().toLowerCase().replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\/+/, '');
|
|
390
|
+
return value.length >= 3 && !/\s/.test(value) ? value : '';
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Exact path / path-suffix (whole segments) / basename match for single-term queries.
|
|
394
|
+
function scorePathQuery(pathQuery, filePath) {
|
|
395
|
+
if (!pathQuery) {
|
|
396
|
+
return null;
|
|
397
|
+
}
|
|
398
|
+
const lower = filePath.toLowerCase();
|
|
399
|
+
const noExt = lower.replace(/\.[a-z0-9]+$/, '');
|
|
400
|
+
if (pathQuery === lower) {
|
|
401
|
+
return { score: PATH_EXACT_SCORE, reason: `path_exact:${filePath}` };
|
|
402
|
+
}
|
|
403
|
+
if (pathQuery.includes('/')) {
|
|
404
|
+
if (noExt === pathQuery || lower.endsWith(`/${pathQuery}`) || noExt.endsWith(`/${pathQuery}`)) {
|
|
405
|
+
return { score: PATH_SUFFIX_SCORE, reason: `path_suffix:${pathQuery}` };
|
|
406
|
+
}
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
const base = lower.slice(lower.lastIndexOf('/') + 1);
|
|
410
|
+
if (pathQuery === base || pathQuery === base.replace(/\.[a-z0-9]+$/, '')) {
|
|
411
|
+
return { score: PATH_BASENAME_SCORE, reason: `path_basename:${pathQuery}` };
|
|
412
|
+
}
|
|
413
|
+
return null;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function scoreDirectFileMatch({ queryDescriptor, rawQuery = '', filePath, searchDescriptor, tokenWeights = null }) {
|
|
334
417
|
let score = 0;
|
|
335
418
|
const reasons = [];
|
|
419
|
+
const weightOf = (token) => tokenWeights?.get(token) ?? 1;
|
|
420
|
+
|
|
421
|
+
const pathHit = scorePathQuery(normalizePathQuery(rawQuery), filePath);
|
|
422
|
+
if (pathHit) {
|
|
423
|
+
score += pathHit.score;
|
|
424
|
+
reasons.push(pathHit.reason);
|
|
425
|
+
}
|
|
336
426
|
|
|
337
427
|
if (
|
|
338
428
|
queryDescriptor.compact.length >= 6
|
|
@@ -344,43 +434,45 @@ function scoreDirectFileMatch({ queryDescriptor, filePath, searchDescriptor }) {
|
|
|
344
434
|
|
|
345
435
|
for (const token of queryDescriptor.tokens) {
|
|
346
436
|
if (searchDescriptor.path.tokens.has(token)) {
|
|
347
|
-
score += 3;
|
|
437
|
+
score += 3 * weightOf(token);
|
|
348
438
|
reasons.push(`path_token:${token}`);
|
|
349
|
-
} else if (searchDescriptor.path.rawLower.includes(token)) {
|
|
350
|
-
score +=
|
|
439
|
+
} else if (token.length >= MIN_SUBSTRING_TOKEN_LENGTH && searchDescriptor.path.rawLower.includes(token)) {
|
|
440
|
+
score += weightOf(token);
|
|
351
441
|
reasons.push(`path_contains:${token}`);
|
|
352
442
|
}
|
|
353
443
|
}
|
|
354
444
|
|
|
445
|
+
const symbolHits = [];
|
|
355
446
|
for (const symbolDescriptor of searchDescriptor.symbols) {
|
|
356
447
|
if (
|
|
357
448
|
queryDescriptor.compact.length >= 4
|
|
358
449
|
&& symbolDescriptor.descriptor.compact === queryDescriptor.compact
|
|
359
450
|
) {
|
|
360
|
-
score
|
|
361
|
-
reasons.push(`symbol_exact:${symbolDescriptor.name}`);
|
|
451
|
+
symbolHits.push({ score: SYMBOL_EXACT_SCORE, reasons: [`symbol_exact:${symbolDescriptor.name}`] });
|
|
362
452
|
continue;
|
|
363
453
|
}
|
|
364
454
|
|
|
455
|
+
let symbolScore = 0;
|
|
456
|
+
const symbolReasons = [];
|
|
365
457
|
if (
|
|
366
458
|
queryDescriptor.compact.length >= 6
|
|
367
459
|
&& symbolDescriptor.descriptor.compact.includes(queryDescriptor.compact)
|
|
368
460
|
) {
|
|
369
|
-
|
|
370
|
-
|
|
461
|
+
symbolScore += 5;
|
|
462
|
+
symbolReasons.push(`symbol_compact:${symbolDescriptor.name}`);
|
|
371
463
|
}
|
|
372
464
|
|
|
373
465
|
let symbolContributed = false;
|
|
374
466
|
|
|
375
467
|
for (const token of queryDescriptor.tokens) {
|
|
376
468
|
if (symbolDescriptor.descriptor.tokens.has(token)) {
|
|
377
|
-
|
|
469
|
+
symbolScore += 3 * weightOf(token);
|
|
378
470
|
symbolContributed = true;
|
|
379
|
-
|
|
380
|
-
} else if (symbolDescriptor.descriptor.rawLower.includes(token)) {
|
|
381
|
-
|
|
471
|
+
symbolReasons.push(`symbol_token:${symbolDescriptor.name}:${token}`);
|
|
472
|
+
} else if (token.length >= MIN_SUBSTRING_TOKEN_LENGTH && symbolDescriptor.descriptor.rawLower.includes(token)) {
|
|
473
|
+
symbolScore += weightOf(token);
|
|
382
474
|
symbolContributed = true;
|
|
383
|
-
|
|
475
|
+
symbolReasons.push(`symbol_contains:${symbolDescriptor.name}`);
|
|
384
476
|
}
|
|
385
477
|
}
|
|
386
478
|
|
|
@@ -390,12 +482,28 @@ function scoreDirectFileMatch({ queryDescriptor, filePath, searchDescriptor }) {
|
|
|
390
482
|
// from the stronger tiers above.
|
|
391
483
|
if (!symbolContributed && queryDescriptor.compact.length >= 6) {
|
|
392
484
|
if (symbolFuzzyMatches(queryDescriptor.compact, symbolDescriptor.descriptor.compact)) {
|
|
393
|
-
|
|
394
|
-
|
|
485
|
+
symbolScore += 3;
|
|
486
|
+
symbolReasons.push(`symbol_fuzzy:${symbolDescriptor.name}`);
|
|
395
487
|
}
|
|
396
488
|
}
|
|
489
|
+
|
|
490
|
+
if (symbolScore > 0) {
|
|
491
|
+
symbolHits.push({ score: symbolScore, reasons: symbolReasons });
|
|
492
|
+
}
|
|
397
493
|
}
|
|
398
494
|
|
|
495
|
+
// Best symbol dominates; further symbols add with sharp diminishing returns so
|
|
496
|
+
// file size (symbol count) cannot outvote match quality.
|
|
497
|
+
symbolHits.sort((a, b) => b.score - a.score);
|
|
498
|
+
let decay = 1;
|
|
499
|
+
symbolHits.forEach((hit, index) => {
|
|
500
|
+
score += hit.score * decay;
|
|
501
|
+
decay *= SYMBOL_DECAY;
|
|
502
|
+
if (index < SYMBOL_REASON_LIMIT) {
|
|
503
|
+
reasons.push(...hit.reasons);
|
|
504
|
+
}
|
|
505
|
+
});
|
|
506
|
+
|
|
399
507
|
return {
|
|
400
508
|
score,
|
|
401
509
|
reasons: dedupe(reasons),
|
|
@@ -403,6 +511,29 @@ function scoreDirectFileMatch({ queryDescriptor, filePath, searchDescriptor }) {
|
|
|
403
511
|
};
|
|
404
512
|
}
|
|
405
513
|
|
|
514
|
+
// Import-graph, hotspot and archetype boosts are context, not evidence: they may
|
|
515
|
+
// reorder near-equal direct matches but never lift a weak match over a strong one,
|
|
516
|
+
// and files with no direct match stay below the best direct match.
|
|
517
|
+
function capBoostLift({ directScores, scores }) {
|
|
518
|
+
let maxDirect = 0;
|
|
519
|
+
for (const entry of directScores.values()) {
|
|
520
|
+
maxDirect = Math.max(maxDirect, entry.score);
|
|
521
|
+
}
|
|
522
|
+
for (const [filePath, entry] of scores) {
|
|
523
|
+
const base = directScores.get(filePath)?.score;
|
|
524
|
+
if (base === undefined) {
|
|
525
|
+
entry.score = Math.min(entry.score, BOOST_ONLY_RATIO * maxDirect);
|
|
526
|
+
} else {
|
|
527
|
+
entry.score = base + Math.min(Math.max(entry.score - base, 0), BOOST_LIFT_RATIO * base + 1);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
return scores;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function finalizeScore(entry) {
|
|
534
|
+
return Number((entry.score * mirrorPrior(entry.filePath)).toFixed(2));
|
|
535
|
+
}
|
|
536
|
+
|
|
406
537
|
function groupBy(items, keyFn) {
|
|
407
538
|
const map = new Map();
|
|
408
539
|
for (const item of items) {
|