@ngockhoale/ukit 3.5.3 → 3.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/manifests/documentation.yaml +12 -1
  3. package/manifests/platform.full.yaml +12 -0
  4. package/package.json +27 -1
  5. package/src/cli/commands/doctor.js +41 -0
  6. package/src/cli/commands/indexTools.js +10 -4
  7. package/src/cli/commands/memory.js +77 -6
  8. package/src/core/memory/episodes.js +106 -7
  9. package/src/core/memory/memoryCheck.js +241 -0
  10. package/src/core/memory/memoryCheckRun.js +27 -0
  11. package/src/core/memory/mutateMemory.js +37 -1
  12. package/src/core/memory/observations.js +192 -0
  13. package/src/core/memory/recallEngine.js +120 -7
  14. package/src/core/memory/records.js +23 -0
  15. package/src/core/memory/writeGate.js +137 -0
  16. package/src/core/runtimeConfig.js +11 -1
  17. package/src/index/buildIndex.js +26 -0
  18. package/src/index/paths.js +3 -1
  19. package/src/index/queryIndex.js +225 -10
  20. package/src/index/resolveContext.js +1 -1
  21. package/template_project/.claude/hooks/edit-hint.mjs +64 -17
  22. package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
  23. package/template_project/.claude/hooks/session-episode.sh +9 -4
  24. package/template_project/.claude/settings.json +2 -2
  25. package/template_project/.claude/ukit/index/lib/index-core.mjs +255 -12
  26. package/template_project/.claude/ukit/index/query-index.mjs +14 -3
  27. package/template_project/.claude/ukit/index/route-task.mjs +4 -1
  28. package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
  29. package/template_project/.claude/ukit/runtime/memory-recall.mjs +120 -7
  30. package/template_project/.claude/ukit/runtime/memory-validity.mjs +28 -1
  31. package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
  32. package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
  33. package/template_project/ukit/storage/config.json +4 -2
  34. package/src/core/agentRuntime/completionGate.js +0 -179
  35. package/src/core/agentRuntime/context.js +0 -571
  36. package/src/core/agentRuntime/qualityComparison.js +0 -112
  37. package/src/core/agentRuntime/resourcePolicy.js +0 -78
  38. package/src/core/agentRuntime/runtimeSupport.js +0 -255
  39. package/src/core/codeintel/manifest.js +0 -176
  40. package/src/core/experiments/deliberation.js +0 -327
  41. package/src/core/experiments/dynamicWorkflow.js +0 -507
  42. package/src/core/handoffDocValidator.js +0 -154
  43. package/src/core/memory/deltaOverlays.js +0 -493
  44. package/src/core/memory/learningCandidates.js +0 -347
  45. package/src/core/observability/emit/sessionBoot.js +0 -407
  46. package/src/core/observability/evaluation/replay.js +0 -149
  47. package/src/core/observability/evaluation/scorecard.js +0 -465
  48. package/src/core/observability/schema/compatibility.js +0 -180
  49. package/src/core/observability/support/renderer.js +0 -130
  50. package/src/core/reviewPanelAggregate.js +0 -204
  51. package/src/core/taskBudgetValidator.js +0 -251
  52. package/src/core/taskProgressGuard.js +0 -227
  53. package/src/decision/reviewVerdict.js +0 -339
  54. package/src/decision/runtimeFilter.js +0 -150
  55. package/src/decision/runtimeScheduler.js +0 -239
  56. package/src/index/contextPack.js +0 -285
  57. package/src/index/impactContext.js +0 -360
  58. package/src/index/reviewPolicy.js +0 -368
  59. package/src/index/sensitivePath.js +0 -21
@@ -0,0 +1,241 @@
1
+ // memoryCheck.js — memory rules + deterministic checker for memory source files
2
+ // (3.5.4). ONE file, node: builtins only, shipped byte-identical as
3
+ // template_project/.claude/ukit/runtime/memory-check.mjs (the PostToolUse advisory
4
+ // and Codex-less hosts use the mirror; `ukit memory check` and the record write
5
+ // gate use this copy). The recall engine and the secret scanner are injected, so
6
+ // there is exactly one tokenizer and one scanner. Async fs only; never throws.
7
+ //
8
+ // violation (exit 1) warning (exit 0)
9
+ // core-over-lines / core-over-bytes no-evidence (bullet names no existing
10
+ // line-too-long (core) / bullet-too-long file path or commit)
11
+ // secret-detected / near-duplicate
12
+ // unknown-supersedes / duplicate-id
13
+ //
14
+ // no-evidence is only a warning for markdown: pre-3.5.4 notes without a path are
15
+ // not rejected retroactively. Record writes (writeGate.js) enforce it.
16
+ import fsp from 'node:fs/promises';
17
+ import path from 'node:path';
18
+
19
+ // Caps. Measured on the live corpus before the split: longest hand-written
20
+ // bullet 1315 chars, longest record text 114, episodes sliced to 900.
21
+ export const RECORD_TEXT_MAX = 1200;
22
+ export const CORE_MAX_LINES = 150;
23
+ export const CORE_MAX_BYTES = 12000; // "12 KB" core cap, decimal so 12,000 is a hard number
24
+ export const CORE_LINE_MAX = 400; // one line per item in the core
25
+ export const DETAIL_BULLET_MAX = 1500; // verbatim detail bullets in docs/memory/*
26
+ export const CORE_FILE = 'docs/MEMORY.md';
27
+
28
+ // Lexical near-duplicate: Jaccard over the BM25 tokens. Measured (3.5.4) on
29
+ // docs/MEMORY.md (68 notes) + live records: the highest similarity between two
30
+ // distinct hand-written notes is 0.250 (markdown/markdown), 0.080 (markdown/
31
+ // record); 0.80 flags only near-restatements. Generated "Verification pattern
32
+ // ... failed N x" records reach 0.917 with each other, so generated writers are
33
+ // exempt in writeGate.js (GENERATED_CREATORS) and never rejected.
34
+ export const NEAR_DUP_JACCARD = 0.8;
35
+ export const NEAR_DUP_MIN_TOKENS = 6;
36
+
37
+ // Machine-generated writers: templated text, expected to resemble each other.
38
+ export const GENERATED_CREATORS = Object.freeze(new Set([
39
+ 'episode-hook', 'propose', 'learning-observe', 'learning-candidate',
40
+ 'delta-overlay', 'migration', 'system', 'observe',
41
+ ]));
42
+
43
+ export function jaccard(aSet, bSet) {
44
+ if (aSet.size === 0 || bSet.size === 0) return 0;
45
+ let inter = 0;
46
+ for (const t of aSet) if (bSet.has(t)) inter += 1;
47
+ return inter / (aSet.size + bSet.size - inter);
48
+ }
49
+
50
+ /**
51
+ * findNearDuplicate(text, candidates, tokenize, threshold) -> { id, similarity } | null
52
+ * candidates: [{ id, text }]. Notes under NEAR_DUP_MIN_TOKENS tokens are skipped
53
+ * (a short note shares most of its tokens by chance).
54
+ */
55
+ export function findNearDuplicate(text, candidates, tokenize, threshold = NEAR_DUP_JACCARD) {
56
+ const a = new Set(tokenize(String(text ?? '')));
57
+ if (a.size < NEAR_DUP_MIN_TOKENS) return null;
58
+ let best = null;
59
+ for (const c of candidates) {
60
+ const b = new Set(tokenize(String(c?.text ?? '')));
61
+ if (b.size < NEAR_DUP_MIN_TOKENS) continue;
62
+ const similarity = jaccard(a, b);
63
+ if (similarity >= threshold && (!best || similarity > best.similarity)) best = { id: c.id, similarity };
64
+ }
65
+ return best;
66
+ }
67
+
68
+ // Markdown supersession convention (honoured by recallEngine.parseMarkdownBullets):
69
+ // [id: slug] names a bullet (slug = [a-z0-9][a-z0-9._-]*)
70
+ // [supersedes: slug] the bullet replaces the bullet(s) so named
71
+ const MD_ID_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
72
+ // dir/file paths, or a bare file name with a known extension
73
+ const PATH_WORD_RE = /(?<![\w./@-])(?:[\w@.-]+\/)+[\w@.-]+(?::\d+)?|(?<![\w./@-])[\w-]+\.(?:m?js|cjs|json|md|sh|ya?ml|ts|py)\b/g;
74
+ const SHA_RE = /\b[0-9a-f]{7,40}\b/;
75
+
76
+ async function exists(root, rel) {
77
+ const clean = rel.replace(/[:#].*$/, '').replace(/^\.\//, '');
78
+ if (!clean || clean.startsWith('/') || clean.split('/').includes('..')) return false;
79
+ try {
80
+ await fsp.stat(path.join(root, clean));
81
+ return true;
82
+ } catch {
83
+ return false;
84
+ }
85
+ }
86
+
87
+ // A note is "about code" when it names a code token (camelCase, snake_case, call(),
88
+ // --flag, dotted config key). Owner preferences, process rules and release history
89
+ // name none, cannot point at a file, and never warn. The heading skip covers prose
90
+ // sections whose notes happen to mention a flag.
91
+ const CODE_TOKEN_RE = /\b[a-z]+[A-Z]\w*\b|\b[a-z0-9]+_[a-z0-9_]+\b|\b\w+\(\)|(?<![\w-])--[a-z][\w-]+|\b[a-z]+\.[a-z]+\.[a-z]\w*\b|\b[\w-]+\.(?:m?js|cjs|json|sh|ya?ml|ts)\b/;
92
+ const PROSE_HEADING_RE = /protocol|release history|preference|process|owner/i;
93
+ export function isCodeNote(doc) {
94
+ return !PROSE_HEADING_RE.test(doc?.heading ?? '') && CODE_TOKEN_RE.test(doc?.text ?? '');
95
+ }
96
+
97
+ /** A bullet is evidenced when it names an existing project path (backticked) or a commit. */
98
+ export async function bulletNamesExistingFile(root, text) {
99
+ // Parsed bullet text has its backticks stripped, so match path-shaped words.
100
+ let tried = 0;
101
+ for (const m of text.matchAll(PATH_WORD_RE)) {
102
+ if (tried >= 12) break;
103
+ tried += 1;
104
+ if (await exists(root, m[0])) return true;
105
+ }
106
+ return SHA_RE.test(text) && /\b(commit|sha)\b/i.test(text);
107
+ }
108
+
109
+ const DETAIL_RE = /Detail:\s+(\S+\.md)/g;
110
+
111
+ /**
112
+ * missingDetailPointers(root, text) -> [{ line, rel, why }]: core lines whose
113
+ * `Detail: <file>.md` pointer escapes the project root or names a missing file.
114
+ */
115
+ export async function missingDetailPointers(root, text) {
116
+ const out = [];
117
+ const base = path.resolve(root);
118
+ const lines = String(text).split('\n');
119
+ for (let i = 0; i < lines.length; i += 1) {
120
+ for (const m of lines[i].replace(/`/g, '').matchAll(DETAIL_RE)) {
121
+ const abs = path.resolve(base, m[1]);
122
+ if (abs !== base && !abs.startsWith(base + path.sep)) { out.push({ line: i + 1, rel: m[1], why: 'escapes the project root' }); continue; }
123
+ try { await fsp.stat(abs); } catch { out.push({ line: i + 1, rel: m[1], why: 'does not exist' }); }
124
+ }
125
+ }
126
+ return out;
127
+ }
128
+
129
+ /**
130
+ * checkMemorySources(projectRoot, { sources, engine, scan, files }) ->
131
+ * { ok, violations: [{file,line,code,message}], warnings, stats }
132
+ * sources: relative .md paths (normalizeMemorySources). engine: { tokenize,
133
+ * parseMarkdownBullets }. scan: (text) -> { labels: string[] } (optional).
134
+ * files: limit the report to these paths (every source is still read, so
135
+ * cross-file duplicates and supersession slugs resolve).
136
+ */
137
+ export async function checkMemorySources(projectRoot, { sources, engine, scan = null, files = null } = {}) {
138
+ const root = path.resolve(projectRoot);
139
+ const violations = [];
140
+ const warnings = [];
141
+ const stats = { files: 0, bullets: 0, coreLines: 0, coreBytes: 0 };
142
+ const all = [];
143
+ const raw = new Map();
144
+ for (const rel of sources ?? []) {
145
+ let text;
146
+ try { text = await fsp.readFile(path.join(root, rel), 'utf8'); } catch { continue; }
147
+ stats.files += 1;
148
+ raw.set(rel, text);
149
+ for (const doc of engine.parseMarkdownBullets(text, rel)) all.push({ ...doc, rel });
150
+ }
151
+ const focus = (rel) => !files || files.includes(rel);
152
+ const add = (list, rel, line, code, message) => { if (focus(rel)) list.push({ file: rel, line, code, message }); };
153
+
154
+ const core = raw.get(CORE_FILE);
155
+ if (core !== undefined) {
156
+ const lines = core.split('\n');
157
+ const lineCount = core.endsWith('\n') ? lines.length - 1 : lines.length;
158
+ const bytes = Buffer.byteLength(core, 'utf8');
159
+ stats.coreLines = lineCount;
160
+ stats.coreBytes = bytes;
161
+ if (lineCount > CORE_MAX_LINES) {
162
+ add(violations, CORE_FILE, lineCount, 'core-over-lines',
163
+ `core has ${lineCount} lines (max ${CORE_MAX_LINES}); merge items or move detail to docs/memory/`);
164
+ }
165
+ if (bytes > CORE_MAX_BYTES) {
166
+ add(violations, CORE_FILE, 1, 'core-over-bytes',
167
+ `core is ${bytes} bytes (max ${CORE_MAX_BYTES}); merge items or move detail to docs/memory/`);
168
+ }
169
+ for (const p of await missingDetailPointers(root, core)) {
170
+ add(violations, CORE_FILE, p.line, 'missing-detail', `Detail pointer ${p.rel} ${p.why}; point at an existing file under docs/memory/`);
171
+ }
172
+ lines.forEach((line, i) => {
173
+ if (/^[-*]\s/.test(line) && line.length > CORE_LINE_MAX) {
174
+ add(violations, CORE_FILE, i + 1, 'line-too-long',
175
+ `core item is ${line.length} chars (max ${CORE_LINE_MAX}); keep one line, move detail to docs/memory/`);
176
+ }
177
+ });
178
+ }
179
+ for (const [rel, text] of raw) {
180
+ if (rel === CORE_FILE) continue;
181
+ text.split('\n').forEach((line, i) => {
182
+ if (/^[-*]\s/.test(line) && line.length > DETAIL_BULLET_MAX) {
183
+ add(violations, rel, i + 1, 'bullet-too-long', `bullet is ${line.length} chars (max ${DETAIL_BULLET_MAX}); split it`);
184
+ }
185
+ });
186
+ }
187
+
188
+ stats.bullets = all.length;
189
+ const slugs = new Map();
190
+ for (const [rel, text] of raw) {
191
+ text.split('\n').forEach((line, i) => {
192
+ for (const m of line.matchAll(MD_ID_RE)) {
193
+ const slug = m[1].toLowerCase();
194
+ if (slugs.has(slug)) add(violations, rel, i + 1, 'duplicate-id', `id "${slug}" already used at ${slugs.get(slug)}`);
195
+ else slugs.set(slug, `${rel}:${i + 1}`);
196
+ }
197
+ });
198
+ }
199
+ for (const doc of all) {
200
+ for (const slug of doc.supersedesSlugs ?? []) {
201
+ if (!slugs.has(slug)) add(violations, doc.rel, doc.line, 'unknown-supersedes', `supersedes "${slug}" but no bullet has [id: ${slug}]`);
202
+ }
203
+ if (scan && focus(doc.rel)) {
204
+ let labels = [];
205
+ try { labels = scan(doc.text)?.labels ?? []; } catch { labels = []; }
206
+ if (labels.length > 0) add(violations, doc.rel, doc.line, 'secret-detected', `secret-like value (${labels.join(', ')}); remove it`);
207
+ }
208
+ if (focus(doc.rel) && isCodeNote(doc) && !(await bulletNamesExistingFile(root, doc.text))) {
209
+ add(warnings, doc.rel, doc.line, 'no-evidence', 'names no existing file path or commit; add the path this note is about');
210
+ }
211
+ }
212
+ // A superseded bullet may legitimately resemble its replacement.
213
+ const replaced = new Set();
214
+ for (const doc of all) for (const s of doc.supersedesSlugs ?? []) { const t = slugs.get(s); if (t) replaced.add(t); }
215
+ const live = all.filter((d) => !replaced.has(`${d.rel}:${d.line}`));
216
+ const seen = new Set();
217
+ const tokenCache = new Map();
218
+ const memoTokenize = (t) => {
219
+ let v = tokenCache.get(t);
220
+ if (!v) { v = engine.tokenize(t); tokenCache.set(t, v); }
221
+ return v;
222
+ };
223
+ for (let i = 0; i < live.length; i += 1) {
224
+ if (!focus(live[i].rel)) continue;
225
+ const dup = findNearDuplicate(live[i].text, live.slice(0, i).map((d) => ({ id: `${d.rel}:${d.line}`, text: d.text })), memoTokenize);
226
+ if (dup && !seen.has(dup.id)) {
227
+ seen.add(dup.id);
228
+ add(violations, live[i].rel, live[i].line, 'near-duplicate',
229
+ `${Math.round(dup.similarity * 100)}% lexical overlap with ${dup.id}; merge them or tag one [supersedes: <slug>]`);
230
+ }
231
+ }
232
+ return { ok: violations.length === 0, violations, warnings, stats };
233
+ }
234
+
235
+ /** One advisory line for the first violation, '' when clean. */
236
+ export function formatCheckLine(res) {
237
+ const v = res?.violations?.[0];
238
+ if (!v) return '';
239
+ const more = res.violations.length > 1 ? ` (+${res.violations.length - 1} more)` : '';
240
+ return `[UKit memory] ${v.file}:${v.line} ${v.code}: ${v.message}${more}. Run \`ukit memory check\`.`;
241
+ }
@@ -23,7 +23,12 @@ import fsp from 'node:fs/promises';
23
23
  import path from 'node:path';
24
24
 
25
25
  export const RECALL_ENGINE_VERSION = 'bm25-1';
26
- export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md']);
26
+ // Capped core + every detail file of docs/memory/ (3.5.4). An entry ending in
27
+ // `/*.md` names one directory level, expanded by expandMemorySources; a missing
28
+ // file or directory contributes nothing.
29
+ export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md', 'docs/memory/*.md']);
30
+ const DIR_GLOB_RE = /^[^*?[\]{}]+\/\*\.md$/;
31
+ const MAX_DIR_FILES = 32;
27
32
 
28
33
  const K1 = 1.2;
29
34
  const B = 0.75;
@@ -37,6 +42,8 @@ const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
37
42
  // ... and the IDF mass of the matched tokens reaches this fraction of the
38
43
  // largest IDF the corpus can produce (a token present in exactly one doc).
39
44
  const MIN_IDF_MASS_RATIO = 0.5;
45
+ // Episodes only: matched IDF mass must reach this fraction of the whole query's.
46
+ const EPISODE_MIN_COVERAGE = 0.5;
40
47
  // Results scoring below this fraction of the best hit are noise tail.
41
48
  const RELATIVE_CUTOFF = 0.35;
42
49
  // Query-dependent type prior: multiplies the lexical score, so it can only
@@ -46,6 +53,37 @@ const RELATIVE_CUTOFF = 0.35;
46
53
  const TYPE_PRIOR = Object.freeze({
47
54
  project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
48
55
  });
56
+ // Core/detail twins (3.5.4): a capped-core one-liner ends in "Detail: <file>" and
57
+ // its verbatim note sits in that file. Both score on the same words, so one fact
58
+ // would spend two result slots. collapseTwins keeps one of each pair that is
59
+ // present in the same ranked list: `keep` 'detail' (default: the full note) or 'core'.
60
+ const DETAIL_POINTER_RE = /\bDetail:\s+(\S+?\.md)\b/;
61
+ const TWIN_MIN_OVERLAP = 0.5;
62
+ export function collapseTwins(entries, keep = 'detail') {
63
+ if (keep === 'off' || !Array.isArray(entries) || entries.length < 2) return entries;
64
+ const drop = new Set();
65
+ for (const core of entries) {
66
+ const doc = core.doc;
67
+ if (doc?.type !== 'markdown' || drop.has(core)) continue;
68
+ const m = DETAIL_POINTER_RE.exec(doc.text ?? '');
69
+ if (!m) continue;
70
+ const coreTokens = new Set(tokenize(doc.text));
71
+ if (coreTokens.size === 0) continue;
72
+ let best = null;
73
+ let bestOverlap = 0;
74
+ for (const other of entries) {
75
+ if (other === core || drop.has(other) || other.doc?.type !== 'markdown') continue;
76
+ if (!String(other.doc.provenance ?? '').startsWith(`${m[1]}:`)) continue;
77
+ const otherTokens = new Set(tokenize(other.doc.text));
78
+ let shared = 0;
79
+ for (const t of coreTokens) if (otherTokens.has(t)) shared += 1;
80
+ const overlap = shared / coreTokens.size;
81
+ if (overlap > bestOverlap) { bestOverlap = overlap; best = other; }
82
+ }
83
+ if (best && bestOverlap >= TWIN_MIN_OVERLAP) drop.add(keep === 'core' ? best : core);
84
+ }
85
+ return drop.size === 0 ? entries : entries.filter((e) => !drop.has(e));
86
+ }
49
87
  const MAX_SOURCES = 8;
50
88
  const MAX_SOURCE_BYTES = 512 * 1024;
51
89
  const MIN_BULLET_CHARS = 12;
@@ -233,7 +271,7 @@ function idf(totalDocs, docFreq) {
233
271
  // abstained }
234
272
  // `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
235
273
  // `query` is a string or a pre-tokenized array. Never throws.
236
- export function rankDocs(docs, query, { limit = 10 } = {}) {
274
+ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
237
275
  const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
238
276
  const empty = {
239
277
  ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
@@ -261,6 +299,8 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
261
299
  // IDF cannot discriminate in a tiny corpus, so the multi-token requirement
262
300
  // only applies once there are enough documents for it to mean something.
263
301
  const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
302
+ let queryMass = 0;
303
+ for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
264
304
 
265
305
  const scored = [];
266
306
  let candidateCount = 0;
@@ -280,6 +320,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
280
320
  candidateCount += 1;
281
321
  if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
282
322
  if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
323
+ // An episode is a long raw prompt: it holds many common words, so two of them
324
+ // clear the absolute floor above while the query's real subject is absent.
325
+ // It must also cover a fraction of the query's own IDF mass.
326
+ if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
283
327
  scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
284
328
  }
285
329
  // Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
@@ -289,8 +333,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
289
333
  const top = scored.length > 0 ? scored[0].score : 0;
290
334
  // Temporal lane (3.5.3): validity is one more fusion lane over the docs that
291
335
  // cleared the floor and cutoff (see demoteByValidity); nothing is dropped.
292
- const ranked = demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc)
293
- .slice(0, limit);
336
+ const ranked = collapseTwins(
337
+ demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc),
338
+ twins,
339
+ ).slice(0, limit);
294
340
  return {
295
341
  ranked,
296
342
  candidateCount,
@@ -312,6 +358,7 @@ export function normalizeMemorySources(config) {
312
358
  const rel = entry.trim().replace(/\\/g, '/');
313
359
  if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
314
360
  if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
361
+ if (rel.includes('*') && !DIR_GLOB_RE.test(rel)) continue;
315
362
  seen.add(rel);
316
363
  out.push(rel);
317
364
  if (out.length >= MAX_SOURCES) break;
@@ -319,6 +366,44 @@ export function normalizeMemorySources(config) {
319
366
  return out;
320
367
  }
321
368
 
369
+ // matchesMemorySource(sources, rel) -> boolean. Sync, no I/O: is `rel` one of the
370
+ // (possibly `dir/*.md`) entries? Lets a hook skip the directory listing for edits
371
+ // that are nowhere near a memory source.
372
+ export function matchesMemorySource(sources, rel) {
373
+ for (const entry of sources) {
374
+ if (entry === rel) return true;
375
+ if (entry.endsWith('/*.md') && DIR_GLOB_RE.test(entry)) {
376
+ const dir = entry.slice(0, -4);
377
+ if (rel.startsWith(dir) && !rel.slice(dir.length).includes('/') && /\.md$/i.test(rel)) return true;
378
+ }
379
+ }
380
+ return false;
381
+ }
382
+
383
+ // expandMemorySources(projectRoot, sources) -> concrete rel .md paths. `dir/*.md`
384
+ // lists that one directory level (async, sorted, at most MAX_DIR_FILES, dot-files
385
+ // skipped); a missing directory yields nothing. Explicit entries keep their order.
386
+ export async function expandMemorySources(projectRoot, sources) {
387
+ const out = [];
388
+ const seen = new Set();
389
+ const push = (rel) => { if (!seen.has(rel)) { seen.add(rel); out.push(rel); } };
390
+ for (const entry of sources) {
391
+ if (!entry.includes('*')) { push(entry); continue; }
392
+ if (!DIR_GLOB_RE.test(entry)) continue;
393
+ const dir = entry.slice(0, -4);
394
+ let names = [];
395
+ try {
396
+ const items = await fsp.readdir(path.resolve(projectRoot, dir), { withFileTypes: true });
397
+ names = items.filter((d) => (d.isFile() || d.isSymbolicLink()) && /\.md$/i.test(d.name) && !d.name.startsWith('.')).map((d) => d.name);
398
+ } catch {
399
+ names = [];
400
+ }
401
+ names.sort();
402
+ for (const name of names.slice(0, MAX_DIR_FILES)) push(`${dir}${name}`);
403
+ }
404
+ return out;
405
+ }
406
+
322
407
  function cleanMarkdown(text) {
323
408
  return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
324
409
  }
@@ -328,6 +413,22 @@ function cleanMarkdown(text) {
328
413
  // indexing it too would return the same rule twice (record + bullet).
329
414
  const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
330
415
  const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
416
+ const MD_ID_TAG_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
417
+ const MD_SUPERSEDES_TAG_RE = /\[supersedes:\s*([a-z0-9][a-z0-9._,\s-]*)\]/gi;
418
+
419
+ // Resolve slug tags to doc ids across every loaded source: the replacing doc
420
+ // gets `supersedes: [md:<rel>:<line>]` so dropSuperseded() removes the old one.
421
+ // Returns copies only for docs that carry tags; cached docs are never mutated.
422
+ function resolveMarkdownSupersession(docs) {
423
+ const bySlug = new Map();
424
+ for (const doc of docs) if (doc.slug && !bySlug.has(doc.slug)) bySlug.set(doc.slug, doc.id);
425
+ if (bySlug.size === 0) return docs;
426
+ return docs.map((doc) => {
427
+ if (!doc.supersedesSlugs) return doc;
428
+ const ids = doc.supersedesSlugs.map((s) => bySlug.get(s)).filter((id) => id && id !== doc.id);
429
+ return ids.length > 0 ? { ...doc, supersedes: ids } : doc;
430
+ });
431
+ }
331
432
 
332
433
  // Top-level bullets ("- " / "* ") with their indented continuation lines.
333
434
  export function parseMarkdownBullets(content, relPath) {
@@ -339,9 +440,21 @@ export function parseMarkdownBullets(content, relPath) {
339
440
  let current = null;
340
441
  const flush = () => {
341
442
  if (!current) return;
342
- const text = cleanMarkdown(current.parts.join(' '));
443
+ // Supersession tags (3.5.4): `[id: slug]` names a bullet, `[supersedes: a, b]`
444
+ // says it replaces the bullets so named. Tags are stripped from the text.
445
+ let slug = null;
446
+ const replacesSlugs = [];
447
+ const raw = current.parts.join(' ')
448
+ .replace(MD_ID_TAG_RE, (_m, s) => { slug ??= s.toLowerCase(); return ' '; })
449
+ .replace(MD_SUPERSEDES_TAG_RE, (_m, list) => {
450
+ for (const s of list.split(/[\s,]+/)) if (s) replacesSlugs.push(s.toLowerCase());
451
+ return ' ';
452
+ });
453
+ const text = cleanMarkdown(raw);
343
454
  if (text.length >= MIN_BULLET_CHARS) {
344
455
  docs.push({
456
+ ...(slug ? { slug } : {}),
457
+ ...(replacesSlugs.length > 0 ? { supersedesSlugs: replacesSlugs } : {}),
345
458
  id: `md:${relPath}:${current.line}`,
346
459
  type: 'markdown',
347
460
  source: 'markdown',
@@ -398,7 +511,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
398
511
  const root = path.resolve(projectRoot);
399
512
  const docs = [];
400
513
  let realRoot = null;
401
- for (const rel of sources) {
514
+ for (const rel of await expandMemorySources(root, sources)) {
402
515
  const abs = path.resolve(root, rel);
403
516
  if (abs !== root && !abs.startsWith(root + path.sep)) continue;
404
517
  try {
@@ -420,7 +533,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
420
533
  markdownCache.delete(abs);
421
534
  }
422
535
  }
423
- return docs;
536
+ return resolveMarkdownSupersession(docs);
424
537
  }
425
538
 
426
539
  const SNIPPET_CHARS = 160;
@@ -173,6 +173,33 @@ async function mapLimited(items, limit, fn) {
173
173
  return out;
174
174
  }
175
175
 
176
+ // Observations (3.5.4) are backed by episodes (meta.evidence.episodes), not a file:
177
+ // labelled with the live evidence count; with every evidence episode gone they are
178
+ // marked validity 'missing' (STALE_MARKER, ranked last) - the same mechanism.
179
+ const OBS_MAX_BYTES = 8 * 1024 * 1024;
180
+ const readJsonSmall = async (file) => {
181
+ try {
182
+ const st = await fsp.stat(file);
183
+ return st.isFile() && st.size <= OBS_MAX_BYTES ? JSON.parse(await fsp.readFile(file, 'utf8')) : null;
184
+ } catch { return null; }
185
+ };
186
+ async function annotateObservations(root, docs) {
187
+ if (!docs.some((d) => d?.type === 'observation')) return docs;
188
+ const dir = path.join(root, '.ukit', 'storage', 'memory', 'v2');
189
+ const [store, archive] = await Promise.all([readJsonSmall(path.join(dir, 'records.json')), readJsonSmall(path.join(dir, 'episodes-archive.json'))]);
190
+ const all = [...(Array.isArray(store?.records) ? store.records : []), ...(Array.isArray(archive?.records) ? archive.records : [])];
191
+ const known = new Set(all.filter((r) => r?.type === 'episode').map((r) => r.id));
192
+ const byId = new Map(all.map((r) => [r?.id, r]));
193
+ return docs.map((doc) => {
194
+ if (doc?.type !== 'observation') return doc;
195
+ const rec = byId.get(String(doc.id ?? '').replace(/^record:/, ''));
196
+ const ids = rec?.meta?.evidence?.episodes ?? doc.meta?.evidence?.episodes;
197
+ const live = Array.isArray(ids) ? ids.filter((id) => known.has(id)).length : 0;
198
+ const label = ` (observation, seen in ${live} episode${live === 1 ? '' : 's'})`;
199
+ return { ...doc, text: `${doc.text ?? ''}${label}`, ...(live === 0 ? { validity: 'missing' } : {}) };
200
+ });
201
+ }
202
+
176
203
  // Adds `validity: 'changed'|'missing'` to docs whose anchored source moved on;
177
204
  // every other doc is returned untouched (same object). `capture` also anchors
178
205
  // docs that have no anchor yet (CLI/write time only, never inside a hook).
@@ -200,7 +227,7 @@ export async function withValidity(root, docs, { capture = false } = {}) {
200
227
  return doc;
201
228
  });
202
229
  if (dirty) await saveAnchors(root, store);
203
- return out;
230
+ return await annotateObservations(root, out);
204
231
  } catch {
205
232
  return list;
206
233
  }