@ngockhoale/ukit 3.5.1 → 3.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +35 -297
  2. package/README.md +1 -0
  3. package/manifests/platform.full.yaml +23 -0
  4. package/package.json +8 -1
  5. package/src/cli/commands/doctor.js +14 -2
  6. package/src/cli/commands/memory.js +30 -2
  7. package/src/core/memory/memoryHit.js +2 -0
  8. package/src/core/memory/mutateMemory.js +9 -0
  9. package/src/core/memory/projectIdentity.js +9 -2
  10. package/src/core/memory/recallEngine.js +86 -4
  11. package/src/core/memory/registryPrune.js +230 -0
  12. package/src/core/memory/retrieval.js +12 -3
  13. package/src/core/runtimeConfig.js +1 -0
  14. package/src/index/buildIndex.js +22 -0
  15. package/src/index/impactContext.js +11 -1
  16. package/src/index/queryIndex.js +151 -20
  17. package/template_project/.claude/hooks/edit-hint.mjs +119 -0
  18. package/template_project/.claude/settings.json +2 -2
  19. package/template_project/.claude/ukit/index/edit-graph.mjs +404 -0
  20. package/template_project/.claude/ukit/index/lib/index-core.mjs +185 -20
  21. package/template_project/.claude/ukit/index/post-edit-verify.mjs +8 -5
  22. package/template_project/.claude/ukit/index/resolve-context.mjs +17 -0
  23. package/template_project/.claude/ukit/index/route-task.mjs +4 -1
  24. package/template_project/.claude/ukit/runtime/hook-chain-runner.mjs +62 -5
  25. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +259 -0
  26. package/template_project/.claude/ukit/runtime/memory-recall.mjs +86 -4
  27. package/template_project/.claude/ukit/runtime/memory-validity.mjs +232 -0
  28. package/template_project/.claude/ukit/runtime/reinject-context.mjs +4 -2
  29. package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
  30. package/template_project/AGENTS.md +1 -1
  31. package/template_project/CLAUDE.md +1 -1
  32. package/template_project/instructions/core.md +1 -1
  33. package/src/core/observability/evaluation/experimentPlan.js +0 -95
  34. package/src/core/observability/evaluation/perturbation.js +0 -273
  35. package/src/diagnostics/classifyHang.js +0 -246
  36. package/src/diagnostics/ompStallSnapshot.js +0 -199
  37. package/src/render/instructionRenderer.js +0 -290
  38. package/src/skill/auditSkill.js +0 -140
  39. package/template_project/instructions/overlays/repo.md +0 -9
@@ -31,6 +31,8 @@ import { pathExists } from '../../core/fileOps.js';
31
31
  import { detectProjectContext } from '../../context/detectProjectContext.js';
32
32
  import { episodeText, readRouteState, resolveLedger, writeEpisode } from '../../core/memory/episodes.js';
33
33
  import { archiveLegacyEpisodes, purgeArchivedEpisode, restoreArchivedEpisodes } from '../../core/memory/episodeArchive.js';
34
+ import { loadMarkdownDocs, normalizeMemorySources } from '../../core/memory/recallEngine.js';
35
+ import { captureAnchors } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
34
36
  import fs from 'node:fs/promises';
35
37
  import os from 'node:os';
36
38
  import path from 'node:path';
@@ -167,6 +169,32 @@ async function runMemoryV2(projectRoot, homeDir, args) {
167
169
  return;
168
170
  }
169
171
 
172
+ if (op === 'confirm') {
173
+ // Re-anchor a note to its named files' current state (clears the
174
+ // "[source changed since noted — verify]" marker). Record id or markdown path:line.
175
+ const id = rest.join(' ').trim();
176
+ if (!id) {
177
+ throw new Error('Missing id. Usage: ukit memory v2 confirm <record-id | path:line>');
178
+ }
179
+ let doc = null;
180
+ const record = await getRecord(projectRoot, id, { homeDir });
181
+ if (record) {
182
+ doc = { id: `record:${record.id}`, type: record.type, provenance: record.provenance, text: record.text, evidence: record.evidence };
183
+ } else {
184
+ const { config } = await inspectRuntimeConfig(projectRoot, { homeDir });
185
+ const mdDocs = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config)).catch(() => []);
186
+ doc = mdDocs.find((d) => d.provenance === id || d.id === id) ?? null;
187
+ }
188
+ if (!doc) {
189
+ throw new Error(`Memory item not found: ${id}`);
190
+ }
191
+ const n = await captureAnchors(projectRoot, [doc]);
192
+ console.log(n > 0
193
+ ? `[UKit] confirmed ${id}: anchored to current source state.`
194
+ : `[UKit] ${id} names no existing file; nothing to anchor.`);
195
+ return;
196
+ }
197
+
170
198
  if (op === 'promote') {
171
199
  const id = rest.join(' ').trim();
172
200
  if (!id) {
@@ -276,7 +304,7 @@ async function runMemoryV2(projectRoot, homeDir, args) {
276
304
  console.log(`[UKit] v2 store: ${stats.total} record(s) — ${JSON.stringify(stats.byType)}`);
277
305
  return;
278
306
  }
279
- throw new Error(`Unknown memory v2 op: ${op}. Expected list|show|promote|stale|migrate.`);
307
+ throw new Error(`Unknown memory v2 op: ${op}. Expected list|show|confirm|promote|stale|migrate.`);
280
308
  }
281
309
 
282
310
  // ---- `ukit memory --user` — user-level store at ~/.ukit (FR-011) ----
@@ -1029,6 +1057,6 @@ export function printMemoryHelp() {
1029
1057
  console.log(' backup [--out <dir>] [--user] Snapshot v2 store(s) + sha256 manifest into a backup dir');
1030
1058
  console.log(' restore <dir> [--dry-run] [--user] Restore store(s) from a backup dir (plan only with --dry-run)');
1031
1059
  console.log(' doctor [--user] Diagnose v2 store state + counts + quarantine (read-only)');
1032
- console.log(' v2 <list|show|promote|stale|migrate> Operate on the project v2 record store (migrate supports --dry-run plan output)');
1060
+ console.log(' v2 <list|show|confirm|promote|stale|migrate> Operate on the project v2 record store (migrate supports --dry-run plan output)');
1033
1061
  console.log(' --user [v2] <list|add|get|update|forget|purge|stats> Operate on the user-level store at ~/.ukit/storage/memory/v2/records.json (purge requires --confirm)');
1034
1062
  }
@@ -67,6 +67,7 @@ export function buildMemoryHit(record, { freshness = null, score = 0, reasons =
67
67
  summary: snippet,
68
68
  source: record?.type ?? null,
69
69
  timestamp: createdAt,
70
+ ...(record?.validity ? { validity: record.validity } : {}),
70
71
  };
71
72
  }
72
73
 
@@ -115,6 +116,7 @@ export function buildMarkdownHit(doc, { score = 0 } = {}) {
115
116
  source: 'markdown',
116
117
  timestamp: 0,
117
118
  provenance: doc.provenance,
119
+ ...(doc.validity ? { validity: doc.validity } : {}),
118
120
  };
119
121
  }
120
122
 
@@ -28,6 +28,7 @@ import { guardWritePayload, redactWritePayload } from './writeGuard.js';
28
28
  import { classifyWrite, mergeEvidence, REVIEW_QUEUE_MAX } from './writeClassification.js';
29
29
  import { resolveMemoryStage, appendRolloutReceipt, latencyBandForMs } from './memoryFlags.js';
30
30
  import { resolveProjectIdentity } from './projectIdentity.js';
31
+ import { captureAnchors } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
31
32
 
32
33
  const JOURNAL_MAX = 256;
33
34
  const OPS = new Set(['add', 'update', 'archive', 'forget', 'purge', 'restore', 'relocate']);
@@ -351,6 +352,14 @@ export async function mutateMemory(target, envelope = {}, { classifyAdd, guard,
351
352
  latencyBand: latencyBandForMs(Date.now() - startedAt),
352
353
  });
353
354
  }
355
+ // 3.5.3: anchor a freshly written note to the files it names (sidecar only,
356
+ // outside the store lock, never fails the write).
357
+ if (result?.status === 'ok' && op === 'add' && target.kind === 'project' && result.record && result.reason !== 'review') {
358
+ await captureAnchors(target.projectRoot, [{
359
+ id: `record:${result.record.id}`, type: result.record.type, provenance: result.record.provenance,
360
+ text: result.record.text, evidence: result.record.evidence,
361
+ }]).catch(() => 0);
362
+ }
354
363
  return result;
355
364
  } catch (error) {
356
365
  if (error instanceof StoreCorruptError || error?.name === 'StoreCorruptError') {
@@ -3,6 +3,7 @@ import path from 'node:path';
3
3
  import crypto from 'node:crypto';
4
4
  import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
5
5
  import { buildUserPaths } from '../userPaths.js';
6
+ import { maybeAutoPrune } from './registryPrune.js';
6
7
 
7
8
  /**
8
9
  * Host-owned stable project identity resolver (M01-01, SPEC §5 FR-001..003).
@@ -225,7 +226,7 @@ export async function resolveProjectIdentity(projectRoot, { homeDir, mint = true
225
226
  if (fresh === null) return null; // became corrupt between read and lock
226
227
  const existing = findByRoot(fresh, realRoot)
227
228
  ?? (commonDir ? findByGitCommonDir(fresh, commonDir) : null);
228
- if (existing) return existing;
229
+ if (existing) return { projectId: existing, entryCount: 0 };
229
230
 
230
231
  const now = new Date().toISOString();
231
232
  const projectId = mintProjectId(realRoot);
@@ -238,10 +239,16 @@ export async function resolveProjectIdentity(projectRoot, { homeDir, mint = true
238
239
  };
239
240
  indexAliases(fresh, projectId, aliases);
240
241
  await writeJson(regPath, fresh);
241
- return projectId;
242
+ return { projectId, entryCount: Object.keys(fresh.projects).length };
242
243
  });
243
244
 
244
245
  if (minted === undefined || minted === null) return unverified;
246
+ if (typeof minted === 'object') {
247
+ // Amortised stale-entry cleanup (never awaited by the caller's deadline path beyond its
248
+ // own bounded budget; failure-proof). Runs after the lock is released.
249
+ await maybeAutoPrune({ homeDir, entryCount: minted.entryCount, currentRoot: realRoot });
250
+ return { projectId: minted.projectId, source: 'registry', ambiguous: false, aliases };
251
+ }
245
252
  return { projectId: minted, source: 'registry', ambiguous: false, aliases };
246
253
  }
247
254
 
@@ -161,6 +161,69 @@ export function docTokens(doc) {
161
161
  return tokenize(text.replace(EPISODE_LABEL_RE, ' '));
162
162
  }
163
163
 
164
+ // Source-validity marker shown next to an item whose anchored file changed or
165
+ // vanished (set on docs as `validity` by memory-validity.mjs).
166
+ export const STALE_MARKER = '[source changed since noted — verify]';
167
+ export const RRF_K = 60;
168
+ export const validityRank = (doc) => (doc?.validity === 'missing' ? 2 : doc?.validity === 'changed' ? 1 : 0);
169
+ const bareId = (id) => String(id ?? '').replace(/^record:/, '');
170
+
171
+ // A doc is superseded when it says so (`superseded_by`) or another candidate
172
+ // lists it in `supersedes`: the superseding doc wins even if the old text
173
+ // scores higher, so the old one never reaches the ranking.
174
+ export function dropSuperseded(docs) {
175
+ const replaced = new Set();
176
+ for (const doc of docs) {
177
+ if (!doc || !Array.isArray(doc.supersedes)) continue;
178
+ for (const id of doc.supersedes) replaced.add(bareId(id));
179
+ }
180
+ return docs.filter((doc) => doc != null && !doc.superseded_by && !replaced.has(bareId(doc.id)));
181
+ }
182
+
183
+ // Reciprocal rank fusion over ordered id lists (best first); lists are lanes.
184
+ // Score = sum weight/(k + rank). Ties break on id, never on a timestamp.
185
+ export function fuseRanked(lanes, { k = RRF_K, weights = [] } = {}) {
186
+ const scores = new Map();
187
+ (Array.isArray(lanes) ? lanes : []).forEach((lane, idx) => {
188
+ const weight = Number.isFinite(weights[idx]) ? weights[idx] : 1;
189
+ const seen = new Set();
190
+ let rank = 0;
191
+ for (const id of lane ?? []) {
192
+ if (seen.has(id)) continue;
193
+ seen.add(id);
194
+ rank += 1;
195
+ scores.set(id, (scores.get(id) ?? 0) + weight / (k + rank));
196
+ }
197
+ });
198
+ return [...scores].map(([id, score]) => ({ id, score }))
199
+ .sort((a, b) => b.score - a.score || String(a.id).localeCompare(String(b.id)));
200
+ }
201
+
202
+ // Validity as one more fusion lane (3.5.3). `changed` (an anchored source was
203
+ // edited) is always labelled but only mildly demoted: lane 1 is the incoming
204
+ // relevance order, lane 2 the same items valid-first, fused with a small weight,
205
+ // so a clearly more relevant changed note still beats a weakly relevant valid
206
+ // one. `missing` (file gone) is labelled and ranked last. Nothing is dropped.
207
+ // Measurement knob UKIT_MEMORY_VALIDITY_RANK: `hard` (every changed below every
208
+ // valid), `label` (marker only, no rank change), `mild[:<weight>]` (default).
209
+ export const VALIDITY_WEIGHT = 0.35;
210
+ const pad = (i) => String(i).padStart(6, '0');
211
+ export function demoteByValidity(items, docOf = (item) => item?.doc ?? item) {
212
+ const list = Array.isArray(items) ? [...items] : [];
213
+ if (!list.some((item) => validityRank(docOf(item)) > 0)) return list;
214
+ const [mode, raw] = String(process.env.UKIT_MEMORY_VALIDITY_RANK ?? 'mild').split(':');
215
+ if (mode === 'label') return list;
216
+ const rank = (item) => validityRank(docOf(item));
217
+ if (mode === 'hard') return list.sort((a, b) => rank(a) - rank(b));
218
+ const weight = Number.isFinite(Number(raw)) && raw !== undefined && raw !== '' ? Number(raw) : VALIDITY_WEIGHT;
219
+ const byIndex = list.map((item, i) => ({ item, i }));
220
+ const lane1 = byIndex.map((e) => pad(e.i));
221
+ const lane2 = [...byIndex].sort((a, b) => rank(a.item) - rank(b.item)).map((e) => pad(e.i));
222
+ return fuseRanked([lane1, lane2], { weights: [1, weight] })
223
+ .map((e) => list[Number(e.id)])
224
+ .sort((a, b) => Number(rank(a) === 2) - Number(rank(b) === 2));
225
+ }
226
+
164
227
  function idf(totalDocs, docFreq) {
165
228
  return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
166
229
  }
@@ -176,6 +239,8 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
176
239
  ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
177
240
  };
178
241
  if (!Array.isArray(docs) || docs.length === 0 || queryTokens.length === 0) return empty;
242
+ docs = dropSuperseded(docs);
243
+ if (docs.length === 0) return empty;
179
244
 
180
245
  const stats = [];
181
246
  const docFreq = new Map();
@@ -222,7 +287,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
222
287
  scored.sort((a, b) => b.score - a.score
223
288
  || String(a.doc.id ?? '').localeCompare(String(b.doc.id ?? '')));
224
289
  const top = scored.length > 0 ? scored[0].score : 0;
225
- const ranked = scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top).slice(0, limit);
290
+ // Temporal lane (3.5.3): validity is one more fusion lane over the docs that
291
+ // cleared the floor and cutoff (see demoteByValidity); nothing is dropped.
292
+ const ranked = demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc)
293
+ .slice(0, limit);
226
294
  return {
227
295
  ranked,
228
296
  candidateCount,
@@ -359,16 +427,30 @@ const SNIPPET_CHARS = 160;
359
427
 
360
428
  // Shared hook lane: score ALL eligible docs (no newest-N pre-filter) together
361
429
  // with the markdown source bullets, then cut. Async I/O only; never throws.
362
- export async function recallForHook({ projectRoot, query, docs = [], config = null, limit = 2 } = {}) {
430
+ // `annotate(docs) -> Promise<docs>` is an optional caller-injected async step
431
+ // (source validity, 3.5.3); this file stays free of non-node imports. A failing
432
+ // or slow-to-throw annotate never affects recall.
433
+ export async function recallForHook({
434
+ projectRoot, query, docs = [], config = null, limit = 2, annotate = null,
435
+ } = {}) {
363
436
  let markdown = [];
364
437
  try {
365
438
  markdown = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config));
366
439
  } catch {
367
440
  markdown = [];
368
441
  }
442
+ let pool = [...docs, ...markdown];
443
+ if (typeof annotate === 'function') {
444
+ try {
445
+ const annotated = await annotate(pool);
446
+ if (Array.isArray(annotated) && annotated.length === pool.length) pool = annotated;
447
+ } catch {
448
+ // keep the unannotated pool
449
+ }
450
+ }
369
451
  let result;
370
452
  try {
371
- result = rankDocs([...docs, ...markdown], query, { limit });
453
+ result = rankDocs(pool, query, { limit });
372
454
  } catch {
373
455
  result = rankDocs([], '', { limit });
374
456
  }
@@ -419,5 +501,5 @@ async function emitRecallTelemetry(projectRoot, config, result) {
419
501
  export function renderMarkdownDoc(doc, maxChars = SNIPPET_CHARS) {
420
502
  const text = String(doc?.text ?? '');
421
503
  const body = text.length > maxChars ? `${text.slice(0, maxChars - 3)}...` : text;
422
- return `[${doc?.provenance ?? 'markdown'}] ${body}`;
504
+ return `[${doc?.provenance ?? 'markdown'}] ${body}${doc?.validity ? ` ${STALE_MARKER}` : ''}`;
423
505
  }
@@ -0,0 +1,230 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { withFileLock, writeJson, readJsonIfExists } from '../fileOps.js';
4
+ import { buildUserPaths } from '../userPaths.js';
5
+
6
+ /**
7
+ * Stale-entry pruning for the user-level project registry
8
+ * (`~/.ukit/storage/projects/registry.json`, owned by projectIdentity.js).
9
+ *
10
+ * An entry is stale only when EVERY root AND every git common dir provably no longer exists
11
+ * (a project can resolve through its git common dir after the worktree that registered it is gone):
12
+ * - stat of the path fails with ENOENT/ENOTDIR (any other error => keep), AND
13
+ * - the nearest mount-level ancestor is present and populated (an unmounted
14
+ * external/network volume must never make its projects look deleted), AND
15
+ * - the entry is not the project being registered, AND
16
+ * - no user-level memory record is attributed to the entry's project id.
17
+ * Mutation re-reads the registry under the same lock and uses the same atomic
18
+ * writer as registration; decisions made outside the lock are re-validated.
19
+ */
20
+
21
+ export const AUTO_PRUNE_MIN_ENTRIES = 500;
22
+ export const AUTO_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
23
+ export const AUTO_PRUNE_BUDGET_MS = 1500;
24
+ const STAT_CONCURRENCY = 32;
25
+
26
+ /**
27
+ * Pure: nearest mount-level ancestor whose presence is required before a missing
28
+ * root may be called deleted. Returns null when the path has no such ancestor
29
+ * (plain local path: the filesystem root is always there).
30
+ * `emptyIsUnmounted` tells the caller an empty mount-point dir is not proof.
31
+ */
32
+ export function mountAncestorOf(rootPath, platform = process.platform) {
33
+ if (typeof rootPath !== 'string' || rootPath === '') return null;
34
+ if (platform === 'win32') {
35
+ const p = path.win32;
36
+ const parsed = p.parse(p.normalize(rootPath));
37
+ if (!parsed.root) return null;
38
+ // Drive root (C:\) or UNC share root (\\server\share\).
39
+ return { path: parsed.root, emptyIsUnmounted: false };
40
+ }
41
+ const parts = path.posix.normalize(rootPath).split('/').filter(Boolean);
42
+ const take = (n) => ({ path: `/${parts.slice(0, n).join('/')}`, emptyIsUnmounted: true });
43
+ if (parts[0] === 'Volumes' && parts.length >= 2) {
44
+ return { path: `/${parts.slice(0, 2).join('/')}`, emptyIsUnmounted: false };
45
+ }
46
+ if (parts[0] === 'mnt' && parts.length >= 2) return take(2);
47
+ if (parts[0] === 'media' && parts.length >= 3) return take(3);
48
+ if (parts[0] === 'run' && parts[1] === 'media' && parts.length >= 4) return take(4);
49
+ return null;
50
+ }
51
+
52
+ /**
53
+ * Classify one root. `io` = { stat(p), readdir(p) } (injectable for tests).
54
+ * @returns {Promise<'missing'|'present'|'unproven'>}
55
+ */
56
+ export async function classifyRoot(rootPath, { platform = process.platform, io, mountCache } = {}) {
57
+ const fsio = io ?? { stat: (p) => fs.stat(p), readdir: (p) => fs.readdir(p) };
58
+ try {
59
+ await fsio.stat(rootPath);
60
+ return 'present';
61
+ } catch (error) {
62
+ if (error?.code !== 'ENOENT' && error?.code !== 'ENOTDIR') return 'unproven';
63
+ }
64
+ const mount = mountAncestorOf(rootPath, platform);
65
+ if (!mount) return 'missing';
66
+ let ok = mountCache?.get(mount.path);
67
+ if (ok === undefined) {
68
+ ok = false;
69
+ try {
70
+ await fsio.stat(mount.path);
71
+ ok = mount.emptyIsUnmounted ? (await fsio.readdir(mount.path)).length > 0 : true;
72
+ } catch {
73
+ ok = false;
74
+ }
75
+ mountCache?.set(mount.path, ok);
76
+ }
77
+ return ok ? 'missing' : 'unproven';
78
+ }
79
+
80
+ function entrySignature(entry) {
81
+ return JSON.stringify([entry?.roots, entry?.gitCommonDirs ?? []]);
82
+ }
83
+
84
+ function registryPaths(homeDir) {
85
+ const user = buildUserPaths(homeDir === undefined ? {} : { homeDir });
86
+ const regPath = path.join(user.storageRoot, 'projects', 'registry.json');
87
+ return { user, regPath, stampPath: path.join(path.dirname(regPath), 'registry.prune-stamp') };
88
+ }
89
+
90
+ async function mapLimit(items, limit, fn, shouldStop) {
91
+ let next = 0;
92
+ const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
93
+ while (next < items.length) {
94
+ if (shouldStop?.()) return;
95
+ const i = next++;
96
+ await fn(items[i], i);
97
+ }
98
+ });
99
+ await Promise.all(workers);
100
+ }
101
+
102
+ function rebuildAliasIndex(registry, removedAliases) {
103
+ const names = new Set(removedAliases);
104
+ for (const name of names) {
105
+ const ids = Object.entries(registry.projects)
106
+ .filter(([, entry]) => Array.isArray(entry?.aliases) && entry.aliases.includes(name))
107
+ .map(([id]) => id);
108
+ if (ids.length === 0) delete registry.aliasIndex[name];
109
+ else registry.aliasIndex[name] = ids.length > 1 ? 'ambiguous' : ids;
110
+ }
111
+ }
112
+
113
+ /**
114
+ * Prune stale registry entries.
115
+ * @param {object} o
116
+ * @param {string} [o.homeDir]
117
+ * @param {boolean} [o.dryRun] decide + count only, never write
118
+ * @param {string[]} [o.keepRoots] roots that must never be pruned (current project)
119
+ * @param {number} [o.budgetMs] stop stat-ing after this long (partial result)
120
+ * @param {string} [o.platform]
121
+ * @param {object} [o.io] injectable stat/readdir
122
+ * @returns {Promise<{status:string, total:number, checked:number, stale:number, removed:number,
123
+ * keptWithMemory:number, unproven:number, truncated:boolean}>}
124
+ */
125
+ export async function pruneRegistry({
126
+ homeDir, dryRun = false, keepRoots = [], budgetMs = Infinity, platform = process.platform, io,
127
+ } = {}) {
128
+ const { user, regPath } = registryPaths(homeDir);
129
+ const result = { status: 'ok', total: 0, checked: 0, stale: 0, removed: 0, keptWithMemory: 0, unproven: 0, truncated: false };
130
+
131
+ let registry;
132
+ try {
133
+ registry = await readJsonIfExists(regPath);
134
+ } catch {
135
+ return { ...result, status: 'registry-corrupt' };
136
+ }
137
+ if (!registry || typeof registry.projects !== 'object' || registry.projects === null) {
138
+ return { ...result, status: registry ? 'registry-corrupt' : 'no-registry' };
139
+ }
140
+
141
+ // Conservative: read the RAW user-memory document (not normalized) so even records this build
142
+ // would skip still protect their project id. Project-scoped memory lives under the project's own
143
+ // `.ukit/` (gone with the root) and is unreachable; the user-level store is the only data keyed by id.
144
+ let userDoc;
145
+ try {
146
+ userDoc = await readJsonIfExists(user.memoryV2RecordsPath);
147
+ } catch {
148
+ return { ...result, status: 'user-memory-unreadable' };
149
+ }
150
+ const memoryIds = new Set(
151
+ (Array.isArray(userDoc?.records) ? userDoc.records : [])
152
+ .map((r) => r?.project_id).filter((id) => typeof id === 'string'),
153
+ );
154
+
155
+ const keep = new Set(keepRoots);
156
+ const entries = Object.entries(registry.projects);
157
+ result.total = entries.length;
158
+ const started = Date.now();
159
+ const mountCache = new Map();
160
+ const staleIds = new Map(); // id -> roots signature
161
+ const shouldStop = () => Date.now() - started > budgetMs;
162
+
163
+ await mapLimit(entries, STAT_CONCURRENCY, async ([id, entry]) => {
164
+ result.checked += 1;
165
+ const roots = entry?.roots;
166
+ if (!Array.isArray(roots) || roots.length === 0 || roots.some((r) => typeof r !== 'string')) return;
167
+ // gitCommonDirs: absent => none; present but malformed => keep (cannot prove anything).
168
+ const gitDirs = entry?.gitCommonDirs ?? [];
169
+ if (!Array.isArray(gitDirs) || gitDirs.some((g) => typeof g !== 'string')) return;
170
+ if (roots.some((r) => keep.has(r))) return;
171
+ for (const p of [...roots, ...gitDirs]) {
172
+ const verdict = await classifyRoot(p, { platform, io, mountCache });
173
+ if (verdict !== 'missing') {
174
+ if (verdict === 'unproven') result.unproven += 1;
175
+ return; // any present/unproven root or git directory keeps the entry
176
+ }
177
+ }
178
+ if (memoryIds.has(id)) { result.keptWithMemory += 1; return; }
179
+ staleIds.set(id, entrySignature(entry));
180
+ }, shouldStop);
181
+
182
+ result.truncated = result.checked < entries.length && shouldStop();
183
+ result.stale = staleIds.size;
184
+ if (dryRun || staleIds.size === 0) return result;
185
+
186
+ const outcome = await withFileLock(regPath, async () => {
187
+ const fresh = await readJsonIfExists(regPath).catch(() => null);
188
+ if (!fresh || typeof fresh.projects !== 'object' || fresh.projects === null) return 0;
189
+ if (typeof fresh.aliasIndex !== 'object' || fresh.aliasIndex === null) fresh.aliasIndex = {};
190
+ const removedAliases = [];
191
+ let removed = 0;
192
+ for (const [id, signature] of staleIds) {
193
+ const current = fresh.projects[id];
194
+ // Re-validate: entry must still exist unchanged (a concurrent writer may have re-bound it).
195
+ if (!current || entrySignature(current) !== signature) continue;
196
+ if (Array.isArray(current.aliases)) removedAliases.push(...current.aliases);
197
+ delete fresh.projects[id];
198
+ removed += 1;
199
+ }
200
+ if (removed === 0) return 0;
201
+ rebuildAliasIndex(fresh, removedAliases);
202
+ await writeJson(regPath, fresh);
203
+ return removed;
204
+ });
205
+ if (outcome === undefined) return { ...result, status: 'lock-busy' };
206
+ return { ...result, removed: outcome };
207
+ }
208
+
209
+ /**
210
+ * Bounded automatic prune used during registration. Never throws; async fs only;
211
+ * skipped below the entry threshold or when run within the last day.
212
+ */
213
+ export async function maybeAutoPrune({ homeDir, entryCount, currentRoot, now = Date.now() } = {}) {
214
+ try {
215
+ if (!(entryCount >= AUTO_PRUNE_MIN_ENTRIES)) return null;
216
+ const { stampPath } = registryPaths(homeDir);
217
+ try {
218
+ const st = await fs.stat(stampPath);
219
+ if (now - st.mtimeMs < AUTO_PRUNE_INTERVAL_MS) return null;
220
+ } catch { /* no stamp yet */ }
221
+ // Stamp first: concurrent/failed runs do not retry within the interval.
222
+ await fs.mkdir(path.dirname(stampPath), { recursive: true });
223
+ await fs.writeFile(stampPath, `${new Date(now).toISOString()}\n`);
224
+ return await pruneRegistry({
225
+ homeDir, keepRoots: currentRoot ? [currentRoot] : [], budgetMs: AUTO_PRUNE_BUDGET_MS,
226
+ });
227
+ } catch {
228
+ return null;
229
+ }
230
+ }
@@ -16,6 +16,8 @@ import { loadMarkdownDocs, normalizeMemorySources, renderMarkdownDoc } from './r
16
16
  import { resolveRecordFreshness } from './memoryFreshness.js';
17
17
  import { getHeadSha } from '../codeintel/freshness.js';
18
18
  import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
19
+ import { withValidity } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
20
+ import { STALE_MARKER } from './recallEngine.js';
19
21
 
20
22
  function compactPhraseList(values, { limit = 1, fallback = 'n/a' } = {}) {
21
23
  const normalized = [...new Set(
@@ -329,8 +331,11 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
329
331
  if (!v2AllowsLegacyFallback(state)) {
330
332
  // One engine (recallEngine.js BM25): eligibility first, then IDF over the
331
333
  // eligible set + markdown bullets, floor, explicit empty result.
332
- const ranked = rankHitsDetailed(records, {
333
- query, eligibleCtx, policy, scope, limit, now, extraDocs: markdownDocs,
334
+ // Source validity (3.5.3): anchored items whose file changed/vanished are
335
+ // demoted and labelled. The CLI is the lazy-capture point for never-seen items.
336
+ const anchored = await withValidity(projectRoot, [...records, ...markdownDocs], { capture: true });
337
+ const ranked = rankHitsDetailed(anchored.slice(0, records.length), {
338
+ query, eligibleCtx, policy, scope, limit, now, extraDocs: anchored.slice(records.length),
334
339
  });
335
340
  const results = await attachFreshness(ranked.hits, records, { projectRoot, watermarks, now });
336
341
  emitQuery({ lane: 'v2', indexed: false, candidateCount: ranked.candidateCount, results });
@@ -628,7 +633,9 @@ export async function getContextInjection(
628
633
  // `path:line`; they are never records and carry no evidence/freshness.
629
634
  const markdownRecord = (entry) => ({
630
635
  id: entry.id, type: 'markdown', text: entry.snippet, provenance: entry.provenance, evidence: [],
636
+ ...(entry.validity ? { validity: entry.validity } : {}),
631
637
  });
638
+ const validityOf = new Map(rankedItems.filter((entry) => entry.validity).map((entry) => [entry.id, entry.validity]));
632
639
  const markdownSelected = rankedItems.filter((entry) => entry.type === 'markdown').map(markdownRecord);
633
640
  const selectedItems = v2Active
634
641
  ? dedupeItemsById(rankedItems
@@ -636,6 +643,7 @@ export async function getContextInjection(
636
643
  ? markdownRecord(entry)
637
644
  : records.find((record) => record.id === entry.id)))
638
645
  .filter(Boolean)
646
+ .map((record) => (validityOf.has(record.id) && !record.validity ? { ...record, validity: validityOf.get(record.id) } : record))
639
647
  .filter((record) => record.type === 'markdown' || eligible(record, eligibleCtx, policy).ok))
640
648
  .slice(0, Math.max(1, recallPlan.limit))
641
649
  : [
@@ -697,7 +705,8 @@ export async function getContextInjection(
697
705
 
698
706
  // FR-005: ' [stale]' appended only when freshness resolves stale; unknown
699
707
  // adds no token.
700
- const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '');
708
+ const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '')
709
+ + (record.validity && record.type !== 'markdown' ? ` ${STALE_MARKER}` : '');
701
710
  const rawBodyLines = v2Active
702
711
  ? selectedItems.map((record) => `- [${record.type}] ${buildRecordSnippet(record)}${staleTag(record)}`)
703
712
  : selectedItems.map((item) => (item.type === 'markdown'
@@ -532,6 +532,7 @@ export function buildDefaultRuntimeConfig(overrides = {}) {
532
532
  maxPatternCandidates: 50,
533
533
  redactSecrets: true,
534
534
  sources: ['docs/MEMORY.md'],
535
+ editHints: { enabled: true, mirrors: [], ignoreMirrors: [], coChange: false },
535
536
  },
536
537
  validation: {
537
538
  enabled: true,
@@ -11,6 +11,8 @@ import { withAsyncLock } from '../../template_project/.claude/ukit/runtime/async
11
11
 
12
12
  import { getArtifactPath, getIndexDir, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION, isLikelyTestFilePath, normalizeRelative } from './paths.js';
13
13
  import { importsNeedAliasContext, loadImportAliasContextState, resolveImportSpecifier } from './importResolution.js';
14
+ import { ensureEditGraph } from '../../template_project/.claude/ukit/index/edit-graph.mjs';
15
+ import { MIRROR_PAIRS } from './impactCatalog.js';
14
16
  import { clearIndexArtifactCache } from './queryIndex.js';
15
17
  import { clearRelatedTestArtifactCache } from './relatedTests.js';
16
18
 
@@ -582,6 +584,7 @@ async function buildCodeIndexInner({ rootDir = process.cwd(), changedFiles = nul
582
584
  [INDEX_ARTIFACTS.relations]: relations.length,
583
585
  },
584
586
  });
587
+ await refreshEditGraph(absoluteRoot, sourceFingerprint.digest, fileRecords, codeFiles, imports, importAliasContext, signal);
585
588
  const preservedRuntimeCaches = canReuseFilesArtifact
586
589
  && canReuseAllParsedArtifacts
587
590
  && canReuseTestsMap
@@ -1241,6 +1244,7 @@ async function tryIncrementalIndexUpdate({ absoluteRoot, indexDir, changedPaths,
1241
1244
  clearIndexArtifactCache(absoluteRoot);
1242
1245
  clearRelatedTestArtifactCache(absoluteRoot);
1243
1246
  }
1247
+ await refreshEditGraph(absoluteRoot, refreshedFingerprint.digest, mergedFileRecords, codeFiles, imports, importAliasContext, signal);
1244
1248
 
1245
1249
  return {
1246
1250
  indexDir,
@@ -2573,6 +2577,24 @@ function areSourceFingerprintsEqual(previousFingerprint, nextFingerprint) {
2573
2577
  && previousFingerprint?.digest === nextFingerprint?.digest;
2574
2578
  }
2575
2579
 
2580
+ // Edit graph (mirror/test/import edges): rebuilt only when missing or its digest
2581
+ // differs from the index fingerprint. Never fails the index build.
2582
+ async function refreshEditGraph(rootDir, indexDigest, fileRecords, codeFiles, imports, aliasContext, signal) {
2583
+ await ensureEditGraph({
2584
+ rootDir,
2585
+ indexDigest,
2586
+ signal,
2587
+ inputs: () => {
2588
+ const resolved = buildResolvedImportMap(rootDir, imports, new Set(codeFiles.map((f) => f.filePath)), aliasContext);
2589
+ return {
2590
+ files: fileRecords,
2591
+ imports: resolved,
2592
+ declaredMirrors: MIRROR_PAIRS,
2593
+ };
2594
+ },
2595
+ });
2596
+ }
2597
+
2576
2598
  async function writeArtifact(rootDir, artifactName, payload) {
2577
2599
  const artifactPath = getArtifactPath(rootDir, artifactName);
2578
2600
  const temporaryPath = `${artifactPath}.tmp-${Date.now()}-${Math.random().toString(16).slice(2)}`;
@@ -2,6 +2,7 @@ import fs from 'node:fs/promises';
2
2
  import path from 'node:path';
3
3
 
4
4
  import { getArtifactPath, INDEX_ARTIFACTS, isLikelyTestFilePath } from './paths.js';
5
+ import { loadEditGraph, neighboursOf } from '../../template_project/.claude/ukit/index/edit-graph.mjs';
5
6
  import { classifyImpactRisk, findMirrorCounterparts } from './impactCatalog.js';
6
7
  import { inferRelatedTestsFromArtifacts, loadRelatedTestArtifacts } from './relatedTests.js';
7
8
  import { importsNeedAliasContext, loadImportAliasContext, resolveImportSpecifier } from './importResolution.js';
@@ -108,7 +109,16 @@ export async function resolveImpactContext({
108
109
  const callItems = safeItems(callsArtifact);
109
110
  const importItems = safeItems(importsArtifact);
110
111
  const risk = classifyImpactRisk(normalizedChangedFiles);
111
- const mirrorCounterparts = findMirrorCounterparts(normalizedChangedFiles).slice(0, budget.maxMirrors);
112
+ const editGraph = await loadEditGraph(absoluteRoot);
113
+ const mirrorCounterparts = [...findMirrorCounterparts(normalizedChangedFiles)];
114
+ for (const changed of normalizedChangedFiles) {
115
+ for (const mirror of neighboursOf(editGraph, changed).mirrors) {
116
+ if (!mirrorCounterparts.some((m) => m.filePath === mirror.path && m.source === changed)) {
117
+ mirrorCounterparts.push({ filePath: mirror.path, reason: `edit-graph ${mirror.kind} mirror`, source: changed });
118
+ }
119
+ }
120
+ }
121
+ mirrorCounterparts.splice(budget.maxMirrors);
112
122
 
113
123
  const changedFileSet = new Set(normalizedChangedFiles);
114
124
  const changedSymbolSet = new Set(normalizedChangedSymbols);