@ngockhoale/ukit 3.5.2 → 3.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/CHANGELOG.md +38 -90
  2. package/README.md +1 -0
  3. package/manifests/documentation.yaml +12 -1
  4. package/manifests/platform.full.yaml +12 -0
  5. package/package.json +34 -1
  6. package/src/cli/commands/doctor.js +55 -2
  7. package/src/cli/commands/indexTools.js +10 -4
  8. package/src/cli/commands/memory.js +107 -8
  9. package/src/core/memory/episodes.js +106 -7
  10. package/src/core/memory/memoryCheck.js +241 -0
  11. package/src/core/memory/memoryCheckRun.js +27 -0
  12. package/src/core/memory/memoryHit.js +2 -0
  13. package/src/core/memory/mutateMemory.js +46 -1
  14. package/src/core/memory/observations.js +192 -0
  15. package/src/core/memory/projectIdentity.js +9 -2
  16. package/src/core/memory/recallEngine.js +204 -9
  17. package/src/core/memory/records.js +23 -0
  18. package/src/core/memory/registryPrune.js +230 -0
  19. package/src/core/memory/retrieval.js +12 -3
  20. package/src/core/memory/writeGate.js +137 -0
  21. package/src/core/runtimeConfig.js +12 -2
  22. package/src/index/buildIndex.js +26 -0
  23. package/src/index/paths.js +3 -1
  24. package/src/index/queryIndex.js +372 -26
  25. package/src/index/resolveContext.js +1 -1
  26. package/template_project/.claude/hooks/edit-hint.mjs +64 -17
  27. package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
  28. package/template_project/.claude/hooks/session-episode.sh +9 -4
  29. package/template_project/.claude/settings.json +2 -2
  30. package/template_project/.claude/ukit/index/edit-graph.mjs +141 -11
  31. package/template_project/.claude/ukit/index/lib/index-core.mjs +400 -27
  32. package/template_project/.claude/ukit/index/query-index.mjs +14 -3
  33. package/template_project/.claude/ukit/index/resolve-context.mjs +1 -1
  34. package/template_project/.claude/ukit/index/route-task.mjs +8 -2
  35. package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
  36. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +44 -22
  37. package/template_project/.claude/ukit/runtime/memory-recall.mjs +204 -9
  38. package/template_project/.claude/ukit/runtime/memory-validity.mjs +259 -0
  39. package/template_project/.claude/ukit/runtime/reinject-context.mjs +4 -2
  40. package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
  41. package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
  42. package/template_project/ukit/storage/config.json +4 -2
  43. package/src/core/agentRuntime/completionGate.js +0 -179
  44. package/src/core/agentRuntime/context.js +0 -571
  45. package/src/core/agentRuntime/qualityComparison.js +0 -112
  46. package/src/core/agentRuntime/resourcePolicy.js +0 -78
  47. package/src/core/agentRuntime/runtimeSupport.js +0 -255
  48. package/src/core/codeintel/manifest.js +0 -176
  49. package/src/core/experiments/deliberation.js +0 -327
  50. package/src/core/experiments/dynamicWorkflow.js +0 -507
  51. package/src/core/handoffDocValidator.js +0 -154
  52. package/src/core/memory/deltaOverlays.js +0 -493
  53. package/src/core/memory/learningCandidates.js +0 -347
  54. package/src/core/observability/emit/sessionBoot.js +0 -407
  55. package/src/core/observability/evaluation/experimentPlan.js +0 -95
  56. package/src/core/observability/evaluation/perturbation.js +0 -273
  57. package/src/core/observability/evaluation/replay.js +0 -149
  58. package/src/core/observability/evaluation/scorecard.js +0 -465
  59. package/src/core/observability/schema/compatibility.js +0 -180
  60. package/src/core/observability/support/renderer.js +0 -130
  61. package/src/core/reviewPanelAggregate.js +0 -204
  62. package/src/core/taskBudgetValidator.js +0 -251
  63. package/src/core/taskProgressGuard.js +0 -227
  64. package/src/decision/reviewVerdict.js +0 -339
  65. package/src/decision/runtimeFilter.js +0 -150
  66. package/src/decision/runtimeScheduler.js +0 -239
  67. package/src/diagnostics/classifyHang.js +0 -246
  68. package/src/diagnostics/ompStallSnapshot.js +0 -199
  69. package/src/index/contextPack.js +0 -285
  70. package/src/index/impactContext.js +0 -360
  71. package/src/index/reviewPolicy.js +0 -368
  72. package/src/index/sensitivePath.js +0 -21
  73. package/src/render/instructionRenderer.js +0 -290
  74. package/src/skill/auditSkill.js +0 -140
  75. package/template_project/instructions/overlays/repo.md +0 -9
@@ -23,7 +23,12 @@ import fsp from 'node:fs/promises';
23
23
  import path from 'node:path';
24
24
 
25
25
  export const RECALL_ENGINE_VERSION = 'bm25-1';
26
- export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md']);
26
+ // Capped core + every detail file of docs/memory/ (3.5.4). An entry ending in
27
+ // `/*.md` names one directory level, expanded by expandMemorySources; a missing
28
+ // file or directory contributes nothing.
29
+ export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md', 'docs/memory/*.md']);
30
+ const DIR_GLOB_RE = /^[^*?[\]{}]+\/\*\.md$/;
31
+ const MAX_DIR_FILES = 32;
27
32
 
28
33
  const K1 = 1.2;
29
34
  const B = 0.75;
@@ -37,6 +42,8 @@ const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
37
42
  // ... and the IDF mass of the matched tokens reaches this fraction of the
38
43
  // largest IDF the corpus can produce (a token present in exactly one doc).
39
44
  const MIN_IDF_MASS_RATIO = 0.5;
45
+ // Episodes only: matched IDF mass must reach this fraction of the whole query's.
46
+ const EPISODE_MIN_COVERAGE = 0.5;
40
47
  // Results scoring below this fraction of the best hit are noise tail.
41
48
  const RELATIVE_CUTOFF = 0.35;
42
49
  // Query-dependent type prior: multiplies the lexical score, so it can only
@@ -46,6 +53,37 @@ const RELATIVE_CUTOFF = 0.35;
46
53
  const TYPE_PRIOR = Object.freeze({
47
54
  project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
48
55
  });
56
+ // Core/detail twins (3.5.4): a capped-core one-liner ends in "Detail: <file>" and
57
+ // its verbatim note sits in that file. Both score on the same words, so one fact
58
+ // would spend two result slots. collapseTwins keeps one of each pair that is
59
+ // present in the same ranked list: `keep` 'detail' (default: the full note) or 'core'.
60
+ const DETAIL_POINTER_RE = /\bDetail:\s+(\S+?\.md)\b/;
61
+ const TWIN_MIN_OVERLAP = 0.5;
62
+ export function collapseTwins(entries, keep = 'detail') {
63
+ if (keep === 'off' || !Array.isArray(entries) || entries.length < 2) return entries;
64
+ const drop = new Set();
65
+ for (const core of entries) {
66
+ const doc = core.doc;
67
+ if (doc?.type !== 'markdown' || drop.has(core)) continue;
68
+ const m = DETAIL_POINTER_RE.exec(doc.text ?? '');
69
+ if (!m) continue;
70
+ const coreTokens = new Set(tokenize(doc.text));
71
+ if (coreTokens.size === 0) continue;
72
+ let best = null;
73
+ let bestOverlap = 0;
74
+ for (const other of entries) {
75
+ if (other === core || drop.has(other) || other.doc?.type !== 'markdown') continue;
76
+ if (!String(other.doc.provenance ?? '').startsWith(`${m[1]}:`)) continue;
77
+ const otherTokens = new Set(tokenize(other.doc.text));
78
+ let shared = 0;
79
+ for (const t of coreTokens) if (otherTokens.has(t)) shared += 1;
80
+ const overlap = shared / coreTokens.size;
81
+ if (overlap > bestOverlap) { bestOverlap = overlap; best = other; }
82
+ }
83
+ if (best && bestOverlap >= TWIN_MIN_OVERLAP) drop.add(keep === 'core' ? best : core);
84
+ }
85
+ return drop.size === 0 ? entries : entries.filter((e) => !drop.has(e));
86
+ }
49
87
  const MAX_SOURCES = 8;
50
88
  const MAX_SOURCE_BYTES = 512 * 1024;
51
89
  const MIN_BULLET_CHARS = 12;
@@ -161,6 +199,69 @@ export function docTokens(doc) {
161
199
  return tokenize(text.replace(EPISODE_LABEL_RE, ' '));
162
200
  }
163
201
 
202
+ // Source-validity marker shown next to an item whose anchored file changed or
203
+ // vanished (set on docs as `validity` by memory-validity.mjs).
204
+ export const STALE_MARKER = '[source changed since noted — verify]';
205
+ export const RRF_K = 60;
206
+ export const validityRank = (doc) => (doc?.validity === 'missing' ? 2 : doc?.validity === 'changed' ? 1 : 0);
207
+ const bareId = (id) => String(id ?? '').replace(/^record:/, '');
208
+
209
+ // A doc is superseded when it says so (`superseded_by`) or another candidate
210
+ // lists it in `supersedes`: the superseding doc wins even if the old text
211
+ // scores higher, so the old one never reaches the ranking.
212
+ export function dropSuperseded(docs) {
213
+ const replaced = new Set();
214
+ for (const doc of docs) {
215
+ if (!doc || !Array.isArray(doc.supersedes)) continue;
216
+ for (const id of doc.supersedes) replaced.add(bareId(id));
217
+ }
218
+ return docs.filter((doc) => doc != null && !doc.superseded_by && !replaced.has(bareId(doc.id)));
219
+ }
220
+
221
+ // Reciprocal rank fusion over ordered id lists (best first); lists are lanes.
222
+ // Score = sum weight/(k + rank). Ties break on id, never on a timestamp.
223
+ export function fuseRanked(lanes, { k = RRF_K, weights = [] } = {}) {
224
+ const scores = new Map();
225
+ (Array.isArray(lanes) ? lanes : []).forEach((lane, idx) => {
226
+ const weight = Number.isFinite(weights[idx]) ? weights[idx] : 1;
227
+ const seen = new Set();
228
+ let rank = 0;
229
+ for (const id of lane ?? []) {
230
+ if (seen.has(id)) continue;
231
+ seen.add(id);
232
+ rank += 1;
233
+ scores.set(id, (scores.get(id) ?? 0) + weight / (k + rank));
234
+ }
235
+ });
236
+ return [...scores].map(([id, score]) => ({ id, score }))
237
+ .sort((a, b) => b.score - a.score || String(a.id).localeCompare(String(b.id)));
238
+ }
239
+
240
+ // Validity as one more fusion lane (3.5.3). `changed` (an anchored source was
241
+ // edited) is always labelled but only mildly demoted: lane 1 is the incoming
242
+ // relevance order, lane 2 the same items valid-first, fused with a small weight,
243
+ // so a clearly more relevant changed note still beats a weakly relevant valid
244
+ // one. `missing` (file gone) is labelled and ranked last. Nothing is dropped.
245
+ // Measurement knob UKIT_MEMORY_VALIDITY_RANK: `hard` (every changed below every
246
+ // valid), `label` (marker only, no rank change), `mild[:<weight>]` (default).
247
+ export const VALIDITY_WEIGHT = 0.35;
248
+ const pad = (i) => String(i).padStart(6, '0');
249
+ export function demoteByValidity(items, docOf = (item) => item?.doc ?? item) {
250
+ const list = Array.isArray(items) ? [...items] : [];
251
+ if (!list.some((item) => validityRank(docOf(item)) > 0)) return list;
252
+ const [mode, raw] = String(process.env.UKIT_MEMORY_VALIDITY_RANK ?? 'mild').split(':');
253
+ if (mode === 'label') return list;
254
+ const rank = (item) => validityRank(docOf(item));
255
+ if (mode === 'hard') return list.sort((a, b) => rank(a) - rank(b));
256
+ const weight = Number.isFinite(Number(raw)) && raw !== undefined && raw !== '' ? Number(raw) : VALIDITY_WEIGHT;
257
+ const byIndex = list.map((item, i) => ({ item, i }));
258
+ const lane1 = byIndex.map((e) => pad(e.i));
259
+ const lane2 = [...byIndex].sort((a, b) => rank(a.item) - rank(b.item)).map((e) => pad(e.i));
260
+ return fuseRanked([lane1, lane2], { weights: [1, weight] })
261
+ .map((e) => list[Number(e.id)])
262
+ .sort((a, b) => Number(rank(a) === 2) - Number(rank(b) === 2));
263
+ }
264
+
164
265
  function idf(totalDocs, docFreq) {
165
266
  return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
166
267
  }
@@ -170,12 +271,14 @@ function idf(totalDocs, docFreq) {
170
271
  // abstained }
171
272
  // `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
172
273
  // `query` is a string or a pre-tokenized array. Never throws.
173
- export function rankDocs(docs, query, { limit = 10 } = {}) {
274
+ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
174
275
  const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
175
276
  const empty = {
176
277
  ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
177
278
  };
178
279
  if (!Array.isArray(docs) || docs.length === 0 || queryTokens.length === 0) return empty;
280
+ docs = dropSuperseded(docs);
281
+ if (docs.length === 0) return empty;
179
282
 
180
283
  const stats = [];
181
284
  const docFreq = new Map();
@@ -196,6 +299,8 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
196
299
  // IDF cannot discriminate in a tiny corpus, so the multi-token requirement
197
300
  // only applies once there are enough documents for it to mean something.
198
301
  const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
302
+ let queryMass = 0;
303
+ for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
199
304
 
200
305
  const scored = [];
201
306
  let candidateCount = 0;
@@ -215,6 +320,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
215
320
  candidateCount += 1;
216
321
  if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
217
322
  if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
323
+ // An episode is a long raw prompt: it holds many common words, so two of them
324
+ // clear the absolute floor above while the query's real subject is absent.
325
+ // It must also cover a fraction of the query's own IDF mass.
326
+ if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
218
327
  scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
219
328
  }
220
329
  // Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
@@ -222,7 +331,12 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
222
331
  scored.sort((a, b) => b.score - a.score
223
332
  || String(a.doc.id ?? '').localeCompare(String(b.doc.id ?? '')));
224
333
  const top = scored.length > 0 ? scored[0].score : 0;
225
- const ranked = scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top).slice(0, limit);
334
+ // Temporal lane (3.5.3): validity is one more fusion lane over the docs that
335
+ // cleared the floor and cutoff (see demoteByValidity); nothing is dropped.
336
+ const ranked = collapseTwins(
337
+ demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc),
338
+ twins,
339
+ ).slice(0, limit);
226
340
  return {
227
341
  ranked,
228
342
  candidateCount,
@@ -244,6 +358,7 @@ export function normalizeMemorySources(config) {
244
358
  const rel = entry.trim().replace(/\\/g, '/');
245
359
  if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
246
360
  if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
361
+ if (rel.includes('*') && !DIR_GLOB_RE.test(rel)) continue;
247
362
  seen.add(rel);
248
363
  out.push(rel);
249
364
  if (out.length >= MAX_SOURCES) break;
@@ -251,6 +366,44 @@ export function normalizeMemorySources(config) {
251
366
  return out;
252
367
  }
253
368
 
369
+ // matchesMemorySource(sources, rel) -> boolean. Sync, no I/O: is `rel` one of the
370
+ // (possibly `dir/*.md`) entries? Lets a hook skip the directory listing for edits
371
+ // that are nowhere near a memory source.
372
+ export function matchesMemorySource(sources, rel) {
373
+ for (const entry of sources) {
374
+ if (entry === rel) return true;
375
+ if (entry.endsWith('/*.md') && DIR_GLOB_RE.test(entry)) {
376
+ const dir = entry.slice(0, -4);
377
+ if (rel.startsWith(dir) && !rel.slice(dir.length).includes('/') && /\.md$/i.test(rel)) return true;
378
+ }
379
+ }
380
+ return false;
381
+ }
382
+
383
+ // expandMemorySources(projectRoot, sources) -> concrete rel .md paths. `dir/*.md`
384
+ // lists that one directory level (async, sorted, at most MAX_DIR_FILES, dot-files
385
+ // skipped); a missing directory yields nothing. Explicit entries keep their order.
386
+ export async function expandMemorySources(projectRoot, sources) {
387
+ const out = [];
388
+ const seen = new Set();
389
+ const push = (rel) => { if (!seen.has(rel)) { seen.add(rel); out.push(rel); } };
390
+ for (const entry of sources) {
391
+ if (!entry.includes('*')) { push(entry); continue; }
392
+ if (!DIR_GLOB_RE.test(entry)) continue;
393
+ const dir = entry.slice(0, -4);
394
+ let names = [];
395
+ try {
396
+ const items = await fsp.readdir(path.resolve(projectRoot, dir), { withFileTypes: true });
397
+ names = items.filter((d) => (d.isFile() || d.isSymbolicLink()) && /\.md$/i.test(d.name) && !d.name.startsWith('.')).map((d) => d.name);
398
+ } catch {
399
+ names = [];
400
+ }
401
+ names.sort();
402
+ for (const name of names.slice(0, MAX_DIR_FILES)) push(`${dir}${name}`);
403
+ }
404
+ return out;
405
+ }
406
+
254
407
  function cleanMarkdown(text) {
255
408
  return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
256
409
  }
@@ -260,6 +413,22 @@ function cleanMarkdown(text) {
260
413
  // indexing it too would return the same rule twice (record + bullet).
261
414
  const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
262
415
  const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
416
+ const MD_ID_TAG_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
417
+ const MD_SUPERSEDES_TAG_RE = /\[supersedes:\s*([a-z0-9][a-z0-9._,\s-]*)\]/gi;
418
+
419
+ // Resolve slug tags to doc ids across every loaded source: the replacing doc
420
+ // gets `supersedes: [md:<rel>:<line>]` so dropSuperseded() removes the old one.
421
+ // Returns copies only for docs that carry tags; cached docs are never mutated.
422
+ function resolveMarkdownSupersession(docs) {
423
+ const bySlug = new Map();
424
+ for (const doc of docs) if (doc.slug && !bySlug.has(doc.slug)) bySlug.set(doc.slug, doc.id);
425
+ if (bySlug.size === 0) return docs;
426
+ return docs.map((doc) => {
427
+ if (!doc.supersedesSlugs) return doc;
428
+ const ids = doc.supersedesSlugs.map((s) => bySlug.get(s)).filter((id) => id && id !== doc.id);
429
+ return ids.length > 0 ? { ...doc, supersedes: ids } : doc;
430
+ });
431
+ }
263
432
 
264
433
  // Top-level bullets ("- " / "* ") with their indented continuation lines.
265
434
  export function parseMarkdownBullets(content, relPath) {
@@ -271,9 +440,21 @@ export function parseMarkdownBullets(content, relPath) {
271
440
  let current = null;
272
441
  const flush = () => {
273
442
  if (!current) return;
274
- const text = cleanMarkdown(current.parts.join(' '));
443
+ // Supersession tags (3.5.4): `[id: slug]` names a bullet, `[supersedes: a, b]`
444
+ // says it replaces the bullets so named. Tags are stripped from the text.
445
+ let slug = null;
446
+ const replacesSlugs = [];
447
+ const raw = current.parts.join(' ')
448
+ .replace(MD_ID_TAG_RE, (_m, s) => { slug ??= s.toLowerCase(); return ' '; })
449
+ .replace(MD_SUPERSEDES_TAG_RE, (_m, list) => {
450
+ for (const s of list.split(/[\s,]+/)) if (s) replacesSlugs.push(s.toLowerCase());
451
+ return ' ';
452
+ });
453
+ const text = cleanMarkdown(raw);
275
454
  if (text.length >= MIN_BULLET_CHARS) {
276
455
  docs.push({
456
+ ...(slug ? { slug } : {}),
457
+ ...(replacesSlugs.length > 0 ? { supersedesSlugs: replacesSlugs } : {}),
277
458
  id: `md:${relPath}:${current.line}`,
278
459
  type: 'markdown',
279
460
  source: 'markdown',
@@ -330,7 +511,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
330
511
  const root = path.resolve(projectRoot);
331
512
  const docs = [];
332
513
  let realRoot = null;
333
- for (const rel of sources) {
514
+ for (const rel of await expandMemorySources(root, sources)) {
334
515
  const abs = path.resolve(root, rel);
335
516
  if (abs !== root && !abs.startsWith(root + path.sep)) continue;
336
517
  try {
@@ -352,23 +533,37 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
352
533
  markdownCache.delete(abs);
353
534
  }
354
535
  }
355
- return docs;
536
+ return resolveMarkdownSupersession(docs);
356
537
  }
357
538
 
358
539
  const SNIPPET_CHARS = 160;
359
540
 
360
541
  // Shared hook lane: score ALL eligible docs (no newest-N pre-filter) together
361
542
  // with the markdown source bullets, then cut. Async I/O only; never throws.
362
- export async function recallForHook({ projectRoot, query, docs = [], config = null, limit = 2 } = {}) {
543
+ // `annotate(docs) -> Promise<docs>` is an optional caller-injected async step
544
+ // (source validity, 3.5.3); this file stays free of non-node imports. A failing
545
+ // or slow-to-throw annotate never affects recall.
546
+ export async function recallForHook({
547
+ projectRoot, query, docs = [], config = null, limit = 2, annotate = null,
548
+ } = {}) {
363
549
  let markdown = [];
364
550
  try {
365
551
  markdown = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config));
366
552
  } catch {
367
553
  markdown = [];
368
554
  }
555
+ let pool = [...docs, ...markdown];
556
+ if (typeof annotate === 'function') {
557
+ try {
558
+ const annotated = await annotate(pool);
559
+ if (Array.isArray(annotated) && annotated.length === pool.length) pool = annotated;
560
+ } catch {
561
+ // keep the unannotated pool
562
+ }
563
+ }
369
564
  let result;
370
565
  try {
371
- result = rankDocs([...docs, ...markdown], query, { limit });
566
+ result = rankDocs(pool, query, { limit });
372
567
  } catch {
373
568
  result = rankDocs([], '', { limit });
374
569
  }
@@ -419,5 +614,5 @@ async function emitRecallTelemetry(projectRoot, config, result) {
419
614
  export function renderMarkdownDoc(doc, maxChars = SNIPPET_CHARS) {
420
615
  const text = String(doc?.text ?? '');
421
616
  const body = text.length > maxChars ? `${text.slice(0, maxChars - 3)}...` : text;
422
- return `[${doc?.provenance ?? 'markdown'}] ${body}`;
617
+ return `[${doc?.provenance ?? 'markdown'}] ${body}${doc?.validity ? ` ${STALE_MARKER}` : ''}`;
423
618
  }
@@ -8,6 +8,7 @@ export const RECORD_TYPES = Object.freeze([
8
8
  'derived_fact',
9
9
  'episode',
10
10
  'procedure',
11
+ 'observation',
11
12
  ]);
12
13
 
13
14
  export const RECORD_SCOPES = Object.freeze([
@@ -66,6 +67,27 @@ function newId() {
66
67
  return `mem_${crypto.randomBytes(6).toString('hex')}`;
67
68
  }
68
69
 
70
+ // 3.5.4: an observation is a consolidated note backed by >= 2 distinct episodes.
71
+ // Structural check only (no store lookup); existence is validateObservation's job.
72
+ export const MIN_OBSERVATION_EVIDENCE = 2;
73
+ export function observationEvidenceErrors(record) {
74
+ const ev = record?.meta?.evidence;
75
+ if (!ev || typeof ev !== 'object' || Array.isArray(ev) || !Array.isArray(ev.episodes)) {
76
+ return ['observation requires meta.evidence = { episodes: [ids], count }'];
77
+ }
78
+ const errors = [];
79
+ const ids = ev.episodes;
80
+ if (ids.some((id) => typeof id !== 'string' || !RECORD_ID_RE.test(id))) {
81
+ errors.push('observation evidence episodes must be record ids');
82
+ }
83
+ const distinct = new Set(ids).size;
84
+ if (distinct < MIN_OBSERVATION_EVIDENCE) {
85
+ errors.push(`observation needs at least ${MIN_OBSERVATION_EVIDENCE} distinct evidence episodes`);
86
+ }
87
+ if (ev.count !== distinct) errors.push('observation evidence count must equal the distinct episode count');
88
+ return errors;
89
+ }
90
+
69
91
  /**
70
92
  * validateRecord(record) → { ok: boolean, errors: string[] }
71
93
  */
@@ -147,6 +169,7 @@ export function validateRecord(record) {
147
169
  }
148
170
  }
149
171
  }
172
+ if (record.type === 'observation') errors.push(...observationEvidenceErrors(record));
150
173
  if (record.valid_from != null
151
174
  && (typeof record.valid_from !== 'number' || !Number.isFinite(record.valid_from))) {
152
175
  errors.push('valid_from must be epoch ms | null');
@@ -0,0 +1,230 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { withFileLock, writeJson, readJsonIfExists } from '../fileOps.js';
4
+ import { buildUserPaths } from '../userPaths.js';
5
+
6
+ /**
7
+ * Stale-entry pruning for the user-level project registry
8
+ * (`~/.ukit/storage/projects/registry.json`, owned by projectIdentity.js).
9
+ *
10
+ * An entry is stale only when EVERY root AND every git common dir provably no longer exists
11
+ * (a project can resolve through its git common dir after the worktree that registered it is gone):
12
+ * - stat of the path fails with ENOENT/ENOTDIR (any other error => keep), AND
13
+ * - the nearest mount-level ancestor is present and populated (an unmounted
14
+ * external/network volume must never make its projects look deleted), AND
15
+ * - the entry is not the project being registered, AND
16
+ * - no user-level memory record is attributed to the entry's project id.
17
+ * Mutation re-reads the registry under the same lock and uses the same atomic
18
+ * writer as registration; decisions made outside the lock are re-validated.
19
+ */
20
+
21
+ export const AUTO_PRUNE_MIN_ENTRIES = 500;
22
+ export const AUTO_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
23
+ export const AUTO_PRUNE_BUDGET_MS = 1500;
24
+ const STAT_CONCURRENCY = 32;
25
+
26
+ /**
27
+ * Pure: nearest mount-level ancestor whose presence is required before a missing
28
+ * root may be called deleted. Returns null when the path has no such ancestor
29
+ * (plain local path: the filesystem root is always there).
30
+ * `emptyIsUnmounted` tells the caller an empty mount-point dir is not proof.
31
+ */
32
+ export function mountAncestorOf(rootPath, platform = process.platform) {
33
+ if (typeof rootPath !== 'string' || rootPath === '') return null;
34
+ if (platform === 'win32') {
35
+ const p = path.win32;
36
+ const parsed = p.parse(p.normalize(rootPath));
37
+ if (!parsed.root) return null;
38
+ // Drive root (C:\) or UNC share root (\\server\share\).
39
+ return { path: parsed.root, emptyIsUnmounted: false };
40
+ }
41
+ const parts = path.posix.normalize(rootPath).split('/').filter(Boolean);
42
+ const take = (n) => ({ path: `/${parts.slice(0, n).join('/')}`, emptyIsUnmounted: true });
43
+ if (parts[0] === 'Volumes' && parts.length >= 2) {
44
+ return { path: `/${parts.slice(0, 2).join('/')}`, emptyIsUnmounted: false };
45
+ }
46
+ if (parts[0] === 'mnt' && parts.length >= 2) return take(2);
47
+ if (parts[0] === 'media' && parts.length >= 3) return take(3);
48
+ if (parts[0] === 'run' && parts[1] === 'media' && parts.length >= 4) return take(4);
49
+ return null;
50
+ }
51
+
52
+ /**
53
+ * Classify one root. `io` = { stat(p), readdir(p) } (injectable for tests).
54
+ * @returns {Promise<'missing'|'present'|'unproven'>}
55
+ */
56
+ export async function classifyRoot(rootPath, { platform = process.platform, io, mountCache } = {}) {
57
+ const fsio = io ?? { stat: (p) => fs.stat(p), readdir: (p) => fs.readdir(p) };
58
+ try {
59
+ await fsio.stat(rootPath);
60
+ return 'present';
61
+ } catch (error) {
62
+ if (error?.code !== 'ENOENT' && error?.code !== 'ENOTDIR') return 'unproven';
63
+ }
64
+ const mount = mountAncestorOf(rootPath, platform);
65
+ if (!mount) return 'missing';
66
+ let ok = mountCache?.get(mount.path);
67
+ if (ok === undefined) {
68
+ ok = false;
69
+ try {
70
+ await fsio.stat(mount.path);
71
+ ok = mount.emptyIsUnmounted ? (await fsio.readdir(mount.path)).length > 0 : true;
72
+ } catch {
73
+ ok = false;
74
+ }
75
+ mountCache?.set(mount.path, ok);
76
+ }
77
+ return ok ? 'missing' : 'unproven';
78
+ }
79
+
80
+ function entrySignature(entry) {
81
+ return JSON.stringify([entry?.roots, entry?.gitCommonDirs ?? []]);
82
+ }
83
+
84
+ function registryPaths(homeDir) {
85
+ const user = buildUserPaths(homeDir === undefined ? {} : { homeDir });
86
+ const regPath = path.join(user.storageRoot, 'projects', 'registry.json');
87
+ return { user, regPath, stampPath: path.join(path.dirname(regPath), 'registry.prune-stamp') };
88
+ }
89
+
90
+ async function mapLimit(items, limit, fn, shouldStop) {
91
+ let next = 0;
92
+ const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
93
+ while (next < items.length) {
94
+ if (shouldStop?.()) return;
95
+ const i = next++;
96
+ await fn(items[i], i);
97
+ }
98
+ });
99
+ await Promise.all(workers);
100
+ }
101
+
102
+ function rebuildAliasIndex(registry, removedAliases) {
103
+ const names = new Set(removedAliases);
104
+ for (const name of names) {
105
+ const ids = Object.entries(registry.projects)
106
+ .filter(([, entry]) => Array.isArray(entry?.aliases) && entry.aliases.includes(name))
107
+ .map(([id]) => id);
108
+ if (ids.length === 0) delete registry.aliasIndex[name];
109
+ else registry.aliasIndex[name] = ids.length > 1 ? 'ambiguous' : ids;
110
+ }
111
+ }
112
+
113
+ /**
114
+ * Prune stale registry entries.
115
+ * @param {object} o
116
+ * @param {string} [o.homeDir]
117
+ * @param {boolean} [o.dryRun] decide + count only, never write
118
+ * @param {string[]} [o.keepRoots] roots that must never be pruned (current project)
119
+ * @param {number} [o.budgetMs] stop stat-ing after this long (partial result)
120
+ * @param {string} [o.platform]
121
+ * @param {object} [o.io] injectable stat/readdir
122
+ * @returns {Promise<{status:string, total:number, checked:number, stale:number, removed:number,
123
+ * keptWithMemory:number, unproven:number, truncated:boolean}>}
124
+ */
125
+ export async function pruneRegistry({
126
+ homeDir, dryRun = false, keepRoots = [], budgetMs = Infinity, platform = process.platform, io,
127
+ } = {}) {
128
+ const { user, regPath } = registryPaths(homeDir);
129
+ const result = { status: 'ok', total: 0, checked: 0, stale: 0, removed: 0, keptWithMemory: 0, unproven: 0, truncated: false };
130
+
131
+ let registry;
132
+ try {
133
+ registry = await readJsonIfExists(regPath);
134
+ } catch {
135
+ return { ...result, status: 'registry-corrupt' };
136
+ }
137
+ if (!registry || typeof registry.projects !== 'object' || registry.projects === null) {
138
+ return { ...result, status: registry ? 'registry-corrupt' : 'no-registry' };
139
+ }
140
+
141
+ // Conservative: read the RAW user-memory document (not normalized) so even records this build
142
+ // would skip still protect their project id. Project-scoped memory lives under the project's own
143
+ // `.ukit/` (gone with the root) and is unreachable; the user-level store is the only data keyed by id.
144
+ let userDoc;
145
+ try {
146
+ userDoc = await readJsonIfExists(user.memoryV2RecordsPath);
147
+ } catch {
148
+ return { ...result, status: 'user-memory-unreadable' };
149
+ }
150
+ const memoryIds = new Set(
151
+ (Array.isArray(userDoc?.records) ? userDoc.records : [])
152
+ .map((r) => r?.project_id).filter((id) => typeof id === 'string'),
153
+ );
154
+
155
+ const keep = new Set(keepRoots);
156
+ const entries = Object.entries(registry.projects);
157
+ result.total = entries.length;
158
+ const started = Date.now();
159
+ const mountCache = new Map();
160
+ const staleIds = new Map(); // id -> roots signature
161
+ const shouldStop = () => Date.now() - started > budgetMs;
162
+
163
+ await mapLimit(entries, STAT_CONCURRENCY, async ([id, entry]) => {
164
+ result.checked += 1;
165
+ const roots = entry?.roots;
166
+ if (!Array.isArray(roots) || roots.length === 0 || roots.some((r) => typeof r !== 'string')) return;
167
+ // gitCommonDirs: absent => none; present but malformed => keep (cannot prove anything).
168
+ const gitDirs = entry?.gitCommonDirs ?? [];
169
+ if (!Array.isArray(gitDirs) || gitDirs.some((g) => typeof g !== 'string')) return;
170
+ if (roots.some((r) => keep.has(r))) return;
171
+ for (const p of [...roots, ...gitDirs]) {
172
+ const verdict = await classifyRoot(p, { platform, io, mountCache });
173
+ if (verdict !== 'missing') {
174
+ if (verdict === 'unproven') result.unproven += 1;
175
+ return; // any present/unproven root or git directory keeps the entry
176
+ }
177
+ }
178
+ if (memoryIds.has(id)) { result.keptWithMemory += 1; return; }
179
+ staleIds.set(id, entrySignature(entry));
180
+ }, shouldStop);
181
+
182
+ result.truncated = result.checked < entries.length && shouldStop();
183
+ result.stale = staleIds.size;
184
+ if (dryRun || staleIds.size === 0) return result;
185
+
186
+ const outcome = await withFileLock(regPath, async () => {
187
+ const fresh = await readJsonIfExists(regPath).catch(() => null);
188
+ if (!fresh || typeof fresh.projects !== 'object' || fresh.projects === null) return 0;
189
+ if (typeof fresh.aliasIndex !== 'object' || fresh.aliasIndex === null) fresh.aliasIndex = {};
190
+ const removedAliases = [];
191
+ let removed = 0;
192
+ for (const [id, signature] of staleIds) {
193
+ const current = fresh.projects[id];
194
+ // Re-validate: entry must still exist unchanged (a concurrent writer may have re-bound it).
195
+ if (!current || entrySignature(current) !== signature) continue;
196
+ if (Array.isArray(current.aliases)) removedAliases.push(...current.aliases);
197
+ delete fresh.projects[id];
198
+ removed += 1;
199
+ }
200
+ if (removed === 0) return 0;
201
+ rebuildAliasIndex(fresh, removedAliases);
202
+ await writeJson(regPath, fresh);
203
+ return removed;
204
+ });
205
+ if (outcome === undefined) return { ...result, status: 'lock-busy' };
206
+ return { ...result, removed: outcome };
207
+ }
208
+
209
+ /**
210
+ * Bounded automatic prune used during registration. Never throws; async fs only;
211
+ * skipped below the entry threshold or when run within the last day.
212
+ */
213
+ export async function maybeAutoPrune({ homeDir, entryCount, currentRoot, now = Date.now() } = {}) {
214
+ try {
215
+ if (!(entryCount >= AUTO_PRUNE_MIN_ENTRIES)) return null;
216
+ const { stampPath } = registryPaths(homeDir);
217
+ try {
218
+ const st = await fs.stat(stampPath);
219
+ if (now - st.mtimeMs < AUTO_PRUNE_INTERVAL_MS) return null;
220
+ } catch { /* no stamp yet */ }
221
+ // Stamp first: concurrent/failed runs do not retry within the interval.
222
+ await fs.mkdir(path.dirname(stampPath), { recursive: true });
223
+ await fs.writeFile(stampPath, `${new Date(now).toISOString()}\n`);
224
+ return await pruneRegistry({
225
+ homeDir, keepRoots: currentRoot ? [currentRoot] : [], budgetMs: AUTO_PRUNE_BUDGET_MS,
226
+ });
227
+ } catch {
228
+ return null;
229
+ }
230
+ }
@@ -16,6 +16,8 @@ import { loadMarkdownDocs, normalizeMemorySources, renderMarkdownDoc } from './r
16
16
  import { resolveRecordFreshness } from './memoryFreshness.js';
17
17
  import { getHeadSha } from '../codeintel/freshness.js';
18
18
  import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
19
+ import { withValidity } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
20
+ import { STALE_MARKER } from './recallEngine.js';
19
21
 
20
22
  function compactPhraseList(values, { limit = 1, fallback = 'n/a' } = {}) {
21
23
  const normalized = [...new Set(
@@ -329,8 +331,11 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
329
331
  if (!v2AllowsLegacyFallback(state)) {
330
332
  // One engine (recallEngine.js BM25): eligibility first, then IDF over the
331
333
  // eligible set + markdown bullets, floor, explicit empty result.
332
- const ranked = rankHitsDetailed(records, {
333
- query, eligibleCtx, policy, scope, limit, now, extraDocs: markdownDocs,
334
+ // Source validity (3.5.3): anchored items whose file changed/vanished are
335
+ // demoted and labelled. The CLI is the lazy-capture point for never-seen items.
336
+ const anchored = await withValidity(projectRoot, [...records, ...markdownDocs], { capture: true });
337
+ const ranked = rankHitsDetailed(anchored.slice(0, records.length), {
338
+ query, eligibleCtx, policy, scope, limit, now, extraDocs: anchored.slice(records.length),
334
339
  });
335
340
  const results = await attachFreshness(ranked.hits, records, { projectRoot, watermarks, now });
336
341
  emitQuery({ lane: 'v2', indexed: false, candidateCount: ranked.candidateCount, results });
@@ -628,7 +633,9 @@ export async function getContextInjection(
628
633
  // `path:line`; they are never records and carry no evidence/freshness.
629
634
  const markdownRecord = (entry) => ({
630
635
  id: entry.id, type: 'markdown', text: entry.snippet, provenance: entry.provenance, evidence: [],
636
+ ...(entry.validity ? { validity: entry.validity } : {}),
631
637
  });
638
+ const validityOf = new Map(rankedItems.filter((entry) => entry.validity).map((entry) => [entry.id, entry.validity]));
632
639
  const markdownSelected = rankedItems.filter((entry) => entry.type === 'markdown').map(markdownRecord);
633
640
  const selectedItems = v2Active
634
641
  ? dedupeItemsById(rankedItems
@@ -636,6 +643,7 @@ export async function getContextInjection(
636
643
  ? markdownRecord(entry)
637
644
  : records.find((record) => record.id === entry.id)))
638
645
  .filter(Boolean)
646
+ .map((record) => (validityOf.has(record.id) && !record.validity ? { ...record, validity: validityOf.get(record.id) } : record))
639
647
  .filter((record) => record.type === 'markdown' || eligible(record, eligibleCtx, policy).ok))
640
648
  .slice(0, Math.max(1, recallPlan.limit))
641
649
  : [
@@ -697,7 +705,8 @@ export async function getContextInjection(
697
705
 
698
706
  // FR-005: ' [stale]' appended only when freshness resolves stale; unknown
699
707
  // adds no token.
700
- const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '');
708
+ const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '')
709
+ + (record.validity && record.type !== 'markdown' ? ` ${STALE_MARKER}` : '');
701
710
  const rawBodyLines = v2Active
702
711
  ? selectedItems.map((record) => `- [${record.type}] ${buildRecordSnippet(record)}${staleTag(record)}`)
703
712
  : selectedItems.map((item) => (item.type === 'markdown'