@ngockhoale/ukit 3.5.3 → 3.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/manifests/documentation.yaml +12 -1
  3. package/manifests/platform.full.yaml +12 -0
  4. package/package.json +29 -2
  5. package/src/cli/commands/doctor.js +41 -0
  6. package/src/cli/commands/indexTools.js +10 -4
  7. package/src/cli/commands/memory.js +88 -6
  8. package/src/core/memory/episodes.js +106 -7
  9. package/src/core/memory/memoryCheck.js +241 -0
  10. package/src/core/memory/memoryCheckRun.js +39 -0
  11. package/src/core/memory/memoryHit.js +31 -15
  12. package/src/core/memory/mutateMemory.js +37 -1
  13. package/src/core/memory/observations.js +192 -0
  14. package/src/core/memory/recallEngine.js +159 -10
  15. package/src/core/memory/recallLanes.js +182 -0
  16. package/src/core/memory/records.js +23 -0
  17. package/src/core/memory/retrieval.js +13 -1
  18. package/src/core/memory/topicCommand.js +75 -0
  19. package/src/core/memory/topicPages.js +545 -0
  20. package/src/core/memory/writeGate.js +137 -0
  21. package/src/core/runtimeConfig.js +56 -1
  22. package/src/decision/registry.js +17 -0
  23. package/src/index/buildIndex.js +312 -5
  24. package/src/index/languageTools.js +100 -0
  25. package/src/index/paths.js +4 -1
  26. package/src/index/queryIndex.js +356 -14
  27. package/src/index/resolveContext.js +1 -1
  28. package/template_project/.claude/hooks/edit-hint.mjs +64 -17
  29. package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
  30. package/template_project/.claude/hooks/session-episode.sh +9 -4
  31. package/template_project/.claude/settings.json +2 -2
  32. package/template_project/.claude/ukit/index/lib/index-core.mjs +769 -18
  33. package/template_project/.claude/ukit/index/query-index.mjs +14 -3
  34. package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
  35. package/template_project/.claude/ukit/index/route-task.mjs +4 -1
  36. package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
  37. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
  38. package/template_project/.claude/ukit/runtime/memory-recall.mjs +159 -10
  39. package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
  40. package/template_project/.claude/ukit/runtime/memory-validity.mjs +28 -1
  41. package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
  42. package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
  43. package/template_project/ukit/storage/config.json +21 -2
  44. package/manifests/instructionRules.yaml +0 -442
  45. package/src/core/agentRuntime/completionGate.js +0 -179
  46. package/src/core/agentRuntime/context.js +0 -571
  47. package/src/core/agentRuntime/qualityComparison.js +0 -112
  48. package/src/core/agentRuntime/resourcePolicy.js +0 -78
  49. package/src/core/agentRuntime/runtimeSupport.js +0 -255
  50. package/src/core/codeintel/manifest.js +0 -176
  51. package/src/core/experiments/deliberation.js +0 -327
  52. package/src/core/experiments/dynamicWorkflow.js +0 -507
  53. package/src/core/handoffDocValidator.js +0 -154
  54. package/src/core/memory/deltaOverlays.js +0 -493
  55. package/src/core/memory/learningCandidates.js +0 -347
  56. package/src/core/observability/emit/sessionBoot.js +0 -407
  57. package/src/core/observability/evaluation/replay.js +0 -149
  58. package/src/core/observability/evaluation/scorecard.js +0 -465
  59. package/src/core/observability/schema/compatibility.js +0 -180
  60. package/src/core/observability/support/renderer.js +0 -130
  61. package/src/core/reviewPanelAggregate.js +0 -204
  62. package/src/core/taskBudgetValidator.js +0 -251
  63. package/src/core/taskProgressGuard.js +0 -227
  64. package/src/decision/reviewVerdict.js +0 -339
  65. package/src/decision/runtimeFilter.js +0 -150
  66. package/src/decision/runtimeScheduler.js +0 -239
  67. package/src/index/contextPack.js +0 -285
  68. package/src/index/impactContext.js +0 -360
  69. package/src/index/reviewPolicy.js +0 -368
  70. package/src/index/sensitivePath.js +0 -21
@@ -0,0 +1,545 @@
1
+ // Topic pages (3.5.5, decision D3'): per-module "mental model" pages under
2
+ // docs/memory/topics/<slug>.md. A page is a DETERMINISTIC skeleton built from the
3
+ // index (symbols), the edit graph (mirrors, tests, importers) and the memory
4
+ // notes that name paths inside the module - no LLM in the generator. One marked
5
+ // section is left for the running agent to fill in; a rebuild keeps it.
6
+ //
7
+ // Rules (D3'): never loaded unless asked for (recall by query, the edit hint
8
+ // pointer, `ukit memory topic show`); every page carries a fingerprint of the
9
+ // directory it describes and its evidence; a page whose directory changed is
10
+ // labelled stale, never served as current.
11
+ //
12
+ // This file imports node builtins only and is mirrored byte-identically as
13
+ // template_project/.claude/ukit/runtime/memory-topics.mjs (hook lanes load it).
14
+ // All I/O is async; every reader swallows errors and returns an empty value.
15
+
16
+ import crypto from 'node:crypto';
17
+ import fs from 'node:fs';
18
+ import fsp from 'node:fs/promises';
19
+ import path from 'node:path';
20
+
21
+ export const TOPIC_DIR = 'docs/memory/topics';
22
+ export const TOPIC_PAGE_MAX_BYTES = 9000;
23
+ export const TOPIC_LINE_MAX = 400;
24
+ export const TOPIC_MAX_PAGES = 40;
25
+ const MAX_FILES = 400;
26
+ const MAX_FILE_BYTES = 512 * 1024;
27
+ const MAX_INDEX_BYTES = 40 * 1024 * 1024;
28
+ const SHOW_FILES = 14;
29
+ const SHOW_NOTES = 8;
30
+ const SKIP_DIRS = new Set(['node_modules', '.git', '.cache', '.ukit', 'dist', 'build', 'coverage']);
31
+ const SOURCE_RE = /\.(?:[cm]?[jt]sx?|vue|py|go|java|cs|php|rb|rs|sh|md|json|ya?ml)$/i;
32
+ const AGENT_START = '<!-- ukit-topic:agent-start -->';
33
+ const AGENT_END = '<!-- ukit-topic:agent-end -->';
34
+ const AGENT_PLACEHOLDER = '_Not filled in. The agent working in this module may replace this line with what it learned; keep it short and cite files._';
35
+ const HEADER_RE = /^<!--\s*ukit-topic\s+dir=(\S+)\s+fingerprint=([0-9a-f]{40}|none)\s+files=(\d+)\s*-->\s*$/m;
36
+ export const TOPIC_STALE_MARKER = '[module changed since this page was built - verify]';
37
+
38
+ const norm = (p) => String(p ?? '').trim().replace(/\\/g, '/').replace(/^(\.\/)+/, '').replace(/\/+$/, '');
39
+ const flat = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
40
+ const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
41
+
42
+ export function validTopicDir(dir) {
43
+ const d = norm(dir);
44
+ return d !== '' && !path.isAbsolute(d) && !d.split('/').includes('..') && /^[A-Za-z0-9._/@-]+$/.test(d);
45
+ }
46
+
47
+ export function topicSlug(dir) {
48
+ return norm(dir).replace(/[^A-Za-z0-9]+/g, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'root';
49
+ }
50
+
51
+ export const topicRel = (dir) => `${TOPIC_DIR}/${topicSlug(dir)}.md`;
52
+
53
+ async function readJson(file, maxBytes = MAX_INDEX_BYTES) {
54
+ try {
55
+ const st = await fsp.stat(file);
56
+ if (!st.isFile() || st.size > maxBytes) return null;
57
+ return JSON.parse(await fsp.readFile(file, 'utf8'));
58
+ } catch {
59
+ return null;
60
+ }
61
+ }
62
+
63
+ // ---- path safety ----------------------------------------------------------
64
+ // A page or module must never lead outside the repo: symlinks are refused for the
65
+ // topics directory chain, the page file and the module root; reads open with
66
+ // O_NOFOLLOW and require a regular file; writes go to a temp file + rename.
67
+
68
+ const NOFOLLOW = fs.constants.O_NOFOLLOW ?? 0;
69
+ const TOPIC_CHAIN = ['docs', 'docs/memory', TOPIC_DIR];
70
+
71
+ async function topicsDirOk(root) {
72
+ try {
73
+ for (const p of TOPIC_CHAIN) {
74
+ const st = await fsp.lstat(path.resolve(root, p));
75
+ if (st.isSymbolicLink() || !st.isDirectory()) return false;
76
+ }
77
+ return (await fsp.realpath(path.resolve(root, TOPIC_DIR))) === path.join(await fsp.realpath(path.resolve(root)), TOPIC_DIR);
78
+ } catch {
79
+ return false;
80
+ }
81
+ }
82
+
83
+ // Reads at most `maxBytes` (default whole file up to the page cap x4) of a regular,
84
+ // non-symlink file; null otherwise. `head` reads only the first N bytes.
85
+ async function readRegular(abs, { head = 0 } = {}) {
86
+ let fh;
87
+ try {
88
+ fh = await fsp.open(abs, fs.constants.O_RDONLY | NOFOLLOW);
89
+ const st = await fh.stat();
90
+ if (!st.isFile()) return null;
91
+ if (head > 0) {
92
+ const buf = Buffer.alloc(head);
93
+ const { bytesRead } = await fh.read(buf, 0, head, 0);
94
+ return buf.toString('utf8', 0, bytesRead);
95
+ }
96
+ if (st.size > TOPIC_PAGE_MAX_BYTES * 4) return null;
97
+ return await fh.readFile('utf8');
98
+ } catch {
99
+ return null;
100
+ } finally {
101
+ await fh?.close().catch(() => {});
102
+ }
103
+ }
104
+
105
+ async function readTopicFile(root, rel, opts) {
106
+ const abs = path.resolve(root, rel);
107
+ if (!abs.startsWith(path.resolve(root, TOPIC_DIR) + path.sep)) return null;
108
+ if (!(await topicsDirOk(root))) return null;
109
+ return readRegular(abs, opts);
110
+ }
111
+
112
+ // A module root must be a real directory inside the repo (no symlink in its path).
113
+ async function moduleRootOk(root, dir) {
114
+ try {
115
+ const abs = path.resolve(root, norm(dir));
116
+ const st = await fsp.lstat(abs);
117
+ if (st.isSymbolicLink() || !st.isDirectory()) return false;
118
+ return (await fsp.realpath(abs)) === path.join(await fsp.realpath(path.resolve(root)), ...norm(dir).split('/'));
119
+ } catch {
120
+ return false;
121
+ }
122
+ }
123
+
124
+ // ---- fingerprint ----------------------------------------------------------
125
+
126
+ const hashCache = new Map(); // abs -> {size, mtimeMs, sha}
127
+
128
+ async function fileSha(abs) {
129
+ const st = await fsp.stat(abs);
130
+ if (!st.isFile()) return null;
131
+ const cached = hashCache.get(abs);
132
+ if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) return cached.sha;
133
+ const sha = st.size > MAX_FILE_BYTES
134
+ ? `size:${st.size}`
135
+ : crypto.createHash('sha1').update(await fsp.readFile(abs)).digest('hex');
136
+ hashCache.set(abs, { size: st.size, mtimeMs: st.mtimeMs, sha });
137
+ return sha;
138
+ }
139
+
140
+ // Files of a module: every file under `dir` (recursive, dot-dirs and vendored dirs
141
+ // skipped), sorted, at most MAX_FILES. truncated=true when more existed.
142
+ export async function listModuleFiles(root, dir) {
143
+ const base = path.resolve(root, norm(dir));
144
+ const out = [];
145
+ let truncated = false;
146
+ if (!(await moduleRootOk(root, dir))) return { files: out, truncated };
147
+ const walk = async (rel) => {
148
+ let items;
149
+ try { items = await fsp.readdir(path.join(base, rel), { withFileTypes: true }); } catch { return; }
150
+ items.sort((a, b) => cmp(a.name, b.name));
151
+ for (const item of items) {
152
+ if (item.name.startsWith('.') || SKIP_DIRS.has(item.name)) continue;
153
+ const next = rel ? `${rel}/${item.name}` : item.name;
154
+ if (item.isDirectory()) { await walk(next); continue; }
155
+ if (!item.isFile() || !SOURCE_RE.test(item.name)) continue;
156
+ if (out.length >= MAX_FILES) { truncated = true; return; }
157
+ out.push(`${norm(dir)}/${next}`);
158
+ }
159
+ };
160
+ await walk('');
161
+ return { files: out.sort(cmp), truncated };
162
+ }
163
+
164
+ // fingerprintDir -> { fingerprint, files, count, truncated, exists }. Content
165
+ // based (sha1 per file), so a fresh clone or a touched mtime never reads as a change.
166
+ export async function fingerprintDir(root, dir) {
167
+ try {
168
+ const st = await fsp.lstat(path.resolve(root, norm(dir)));
169
+ if (st.isSymbolicLink()) return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false, refused: true };
170
+ if (!st.isDirectory()) return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false };
171
+ } catch {
172
+ return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false };
173
+ }
174
+ if (!(await moduleRootOk(root, dir))) return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false, refused: true };
175
+ const { files, truncated } = await listModuleFiles(root, dir);
176
+ const shas = [];
177
+ for (let i = 0; i < files.length; i += 16) {
178
+ const slice = files.slice(i, i + 16);
179
+ shas.push(...await Promise.all(slice.map((f) => fileSha(path.resolve(root, f)).catch(() => null))));
180
+ }
181
+ const h = crypto.createHash('sha1');
182
+ files.forEach((f, i) => h.update(`${f}\0${shas[i] ?? 'unreadable'}\n`));
183
+ return { fingerprint: h.digest('hex'), files, count: files.length, truncated, exists: true };
184
+ }
185
+
186
+ // ---- page parse / read ----------------------------------------------------
187
+
188
+ export function parseTopicHeader(text) {
189
+ const m = HEADER_RE.exec(String(text ?? ''));
190
+ return m ? { dir: m[1], fingerprint: m[2], files: Number(m[3]) } : null;
191
+ }
192
+
193
+ export function agentSection(text) {
194
+ const t = String(text ?? '');
195
+ const a = t.indexOf(AGENT_START);
196
+ const b = t.indexOf(AGENT_END);
197
+ if (a < 0 || b < a) return '';
198
+ const body = t.slice(a + AGENT_START.length, b).trim();
199
+ return body === AGENT_PLACEHOLDER ? '' : body;
200
+ }
201
+
202
+ // readTopicPage(root, rel) -> { rel, dir, fingerprint, current, stale, missing, text, agentText } | null
203
+ export async function readTopicPage(root, rel) {
204
+ try {
205
+ const text = await readTopicFile(root, rel);
206
+ if (text === null) return null;
207
+ const header = parseTopicHeader(text);
208
+ if (!header || !validTopicDir(header.dir)) return null;
209
+ const now = await fingerprintDir(root, header.dir);
210
+ return {
211
+ rel: norm(rel), dir: header.dir, fingerprint: header.fingerprint, currentFingerprint: now.fingerprint,
212
+ missing: !now.exists, stale: !now.exists || now.fingerprint !== header.fingerprint,
213
+ text, agentText: agentSection(text),
214
+ };
215
+ } catch {
216
+ return null;
217
+ }
218
+ }
219
+
220
+ export async function listTopicRels(root) {
221
+ try {
222
+ if (!(await topicsDirOk(root))) return [];
223
+ const items = await fsp.readdir(path.resolve(root, TOPIC_DIR), { withFileTypes: true });
224
+ return items.filter((d) => d.isFile() && /\.md$/i.test(d.name) && !d.name.startsWith('.'))
225
+ .map((d) => `${TOPIC_DIR}/${d.name}`).sort(cmp).slice(0, TOPIC_MAX_PAGES);
226
+ } catch {
227
+ return [];
228
+ }
229
+ }
230
+
231
+ export async function listTopics(root) {
232
+ const out = [];
233
+ for (const rel of await listTopicRels(root)) {
234
+ const page = await readTopicPage(root, rel);
235
+ if (page) out.push({ rel, dir: page.dir, stale: page.stale, missing: page.missing, bytes: Buffer.byteLength(page.text), filled: page.agentText !== '' });
236
+ }
237
+ return out;
238
+ }
239
+
240
+ // ---- build ----------------------------------------------------------------
241
+
242
+ async function headComment(root, file) {
243
+ try {
244
+ const fh = await fsp.open(path.resolve(root, file), 'r');
245
+ try {
246
+ const buf = Buffer.alloc(2048);
247
+ const { bytesRead } = await fh.read(buf, 0, 2048, 0);
248
+ const lines = buf.toString('utf8', 0, bytesRead).split(/\r?\n/).slice(0, 14);
249
+ const picked = [];
250
+ for (const line of lines) {
251
+ if (/^#!/.test(line)) continue;
252
+ const m = /^\s*(?:\/\/|#|\*|\/\*+)\s?(.*)$/.exec(line);
253
+ if (!m) { if (picked.length || /\S/.test(line)) break; continue; }
254
+ const body = m[1].replace(/\*\/\s*$/, '').trim();
255
+ if (body) picked.push(body);
256
+ if (picked.join(' ').length > 170) break;
257
+ }
258
+ const joined = flat(picked.join(' '));
259
+ return joined.length > 190 ? `${joined.slice(0, 187)}...` : joined;
260
+ } finally { await fh.close(); }
261
+ } catch {
262
+ return '';
263
+ }
264
+ }
265
+
266
+ function exportsByFile(symbols, files) {
267
+ const want = new Set(files);
268
+ const map = new Map();
269
+ for (const item of Array.isArray(symbols?.items) ? symbols.items : []) {
270
+ if (!want.has(item?.filePath) || !/^\s*export\b/.test(String(item.signature ?? ''))) continue;
271
+ if (!map.has(item.filePath)) map.set(item.filePath, []);
272
+ map.get(item.filePath).push(item.name);
273
+ }
274
+ return map;
275
+ }
276
+
277
+ // buildTopicPage({ root, dir, notes, existingText, now }) -> { rel, text, fingerprint, count, truncated, bytes }
278
+ // `notes` = [{ ref, text, type? }] supplied by the caller (markdown bullets and
279
+ // active records); only notes that name a path inside `dir` are used.
280
+ export async function buildTopicPage({ root, dir, notes = [], existingText = '', now = Date.now() } = {}) {
281
+ const d = norm(dir);
282
+ if (!validTopicDir(d)) throw new Error(`Invalid topic directory: ${dir}`);
283
+ const fp = await fingerprintDir(root, d);
284
+ if (fp.refused) throw new Error(`Topic directory is a symlink or leaves the repo: ${d}`);
285
+ if (!fp.exists) throw new Error(`Topic directory not found: ${d}`);
286
+ if (fp.truncated) throw new Error(`Topic directory too large (>${MAX_FILES} files): pick a subdirectory of ${d}`);
287
+ const graph = await readJson(path.resolve(root, '.cache/index/edit-graph.json'));
288
+ const symbols = await readJson(path.resolve(root, '.cache/index/symbols.json'));
289
+ const entries = graph?.files && typeof graph.files === 'object' ? graph.files : {};
290
+ const exportsOf = exportsByFile(symbols, fp.files);
291
+ const inside = (p) => p === d || p.startsWith(`${d}/`);
292
+ const info = fp.files.map((f) => {
293
+ const e = entries[f] ?? {};
294
+ const importers = Array.isArray(e.i) ? e.i.map(norm) : [];
295
+ return {
296
+ file: f,
297
+ exports: [...new Set(exportsOf.get(f) ?? [])].sort(cmp),
298
+ importers,
299
+ outside: importers.filter((p) => !inside(p)),
300
+ mirrors: (Array.isArray(e.m) ? e.m : []).map((x) => norm(x?.p ?? x)).filter(Boolean),
301
+ tests: (Array.isArray(e.t) ? e.t : []).map(norm).filter(Boolean),
302
+ };
303
+ });
304
+ const code = info.filter((x) => /\.(?:[cm]?[jt]sx?|vue|py|go|java|cs|php|rb|rs|sh)$/i.test(x.file) && !x.tests.includes(x.file));
305
+ const ranked = [...code].sort((a, b) => b.importers.length - a.importers.length || b.exports.length - a.exports.length || cmp(a.file, b.file));
306
+ const shown = ranked.slice(0, SHOW_FILES);
307
+ const bullet = (s) => `- ${flat(s).slice(0, TOPIC_LINE_MAX - 2)}`;
308
+
309
+ const lines = [];
310
+ lines.push(`# Topic: ${d}`);
311
+ lines.push(`<!-- ukit-topic dir=${d} fingerprint=${fp.fingerprint} files=${fp.count} -->`);
312
+ lines.push('');
313
+ lines.push(`Generated skeleton for \`${d}\` (${fp.count} files, built ${new Date(now).toISOString().slice(0, 10)}). Facts below come from the index, the edit graph and cited notes; rebuild with \`ukit memory topic build ${d}\`. If the module changed since the build the page is stale: verify before relying on it.`);
314
+ lines.push('');
315
+ lines.push('## What it is for');
316
+ for (const x of shown.slice(0, 6)) {
317
+ const c = await headComment(root, x.file);
318
+ if (c) lines.push(bullet(`\`${x.file}\`: ${c}`));
319
+ }
320
+ if (lines[lines.length - 1] === '## What it is for') lines.push('- No header comments found in the key files.');
321
+ lines.push('');
322
+ lines.push('## Key files');
323
+ for (const x of shown) {
324
+ const ex = x.exports.slice(0, 6).join(', ');
325
+ lines.push(bullet(`\`${x.file}\`${ex ? ` exports ${ex}` : ''}; imported by ${x.importers.length >= 5 ? '5+' : x.importers.length} file(s)`));
326
+ }
327
+ if (code.length > shown.length) lines.push(`- ... ${code.length - shown.length} more file(s) under \`${d}\`.`);
328
+ lines.push('');
329
+ lines.push('## Entry points (imported from outside the module)');
330
+ const entry = ranked.filter((x) => x.outside.length > 0).slice(0, 8);
331
+ if (entry.length === 0) lines.push('- None recorded in the edit graph.');
332
+ for (const x of entry) lines.push(bullet(`\`${x.file}\` used by ${x.outside.slice(0, 3).map((p) => `\`${p}\``).join(', ')}${x.outside.length > 3 ? ` (+${x.outside.length - 3})` : ''}`));
333
+ lines.push('');
334
+ lines.push('## Mirrors and tests');
335
+ const mirrors = [...new Set(info.flatMap((x) => x.mirrors.map((m) => `${x.file} -> ${m}`)))].sort(cmp).slice(0, 8);
336
+ const tests = [...new Set(info.flatMap((x) => x.tests))].sort(cmp);
337
+ if (mirrors.length === 0 && tests.length === 0) lines.push('- No mirror or test edges recorded in the edit graph.');
338
+ for (const m of mirrors) lines.push(bullet(`mirror: \`${m.split(' -> ')[0]}\` -> \`${m.split(' -> ')[1]}\``));
339
+ if (tests.length) lines.push(bullet(`tests (${tests.length}): ${tests.slice(0, 8).map((t) => `\`${t}\``).join(', ')}${tests.length > 8 ? ', ...' : ''}`));
340
+ lines.push('');
341
+ lines.push('## Notes naming this module');
342
+ const named = [];
343
+ const seen = new Set();
344
+ for (const n of Array.isArray(notes) ? notes : []) {
345
+ const text = flat(n?.text);
346
+ if (!text || !text.includes(`${d}/`) || seen.has(n.ref)) continue;
347
+ seen.add(n.ref);
348
+ named.push({ ref: flat(n.ref), text, type: n.type });
349
+ }
350
+ if (named.length === 0) {
351
+ // No note source supplied (index-refresh rebuild): carry the page's own cited notes.
352
+ for (const l of sectionLines(existingText, 'Notes naming this module')) {
353
+ const m = /^\[([^\]]+)\] (.*)$/.exec(l);
354
+ if (m && !seen.has(m[1])) { seen.add(m[1]); named.push({ ref: m[1], text: m[2] }); }
355
+ }
356
+ }
357
+ named.sort((a, b) => cmp(a.ref, b.ref));
358
+ if (named.length === 0) lines.push('- No memory note names a path in this module.');
359
+ for (const n of named.slice(0, SHOW_NOTES)) lines.push(bullet(`[${n.ref}] ${n.text.length > 230 ? `${n.text.slice(0, 227)}...` : n.text}`));
360
+ lines.push('');
361
+ lines.push('## Evidence');
362
+ lines.push(bullet(`built from: .cache/index/edit-graph.json, .cache/index/symbols.json, ${fp.count} file(s) under \`${d}\``));
363
+ if (named.length) lines.push(bullet(`notes: ${named.slice(0, SHOW_NOTES).map((n) => n.ref).join(', ')}`));
364
+ lines.push('');
365
+ lines.push('## Agent notes (fill in; kept across rebuilds)');
366
+ lines.push(AGENT_START);
367
+ lines.push(agentSection(existingText) || AGENT_PLACEHOLDER);
368
+ lines.push(AGENT_END);
369
+ lines.push('');
370
+ // The generator guarantees its own cap: drop the last bullet of the least valuable
371
+ // section (notes, then purpose lines, then key files, then entry points) until it fits.
372
+ const size = () => Buffer.byteLength(lines.join('\n'));
373
+ for (const title of ['Notes naming this module', 'What it is for', 'Key files', 'Entry points (imported from outside the module)']) {
374
+ while (size() > TOPIC_PAGE_MAX_BYTES) {
375
+ const at = lines.indexOf(`## ${title}`);
376
+ let end = at + 1;
377
+ while (end < lines.length && !lines[end].startsWith('## ') && lines[end] !== '') end += 1;
378
+ if (at < 0 || end - at <= 2) break;
379
+ lines.splice(end - 1, 1);
380
+ }
381
+ }
382
+ const text = lines.join('\n');
383
+ if (Buffer.byteLength(text) > TOPIC_PAGE_MAX_BYTES) {
384
+ throw new Error(`Topic page for ${d} would exceed ${TOPIC_PAGE_MAX_BYTES} bytes; pick a smaller module`);
385
+ }
386
+ return { rel: topicRel(d), text, fingerprint: fp.fingerprint, count: fp.count, truncated: false, bytes: Buffer.byteLength(text) };
387
+ }
388
+
389
+ export async function writeTopicPage(root, dir, notes = []) {
390
+ const rel = topicRel(dir);
391
+ const abs = path.resolve(root, rel);
392
+ // Refuse symlinks anywhere in docs/memory/topics (and the page itself) before any write.
393
+ for (const p of TOPIC_CHAIN) {
394
+ const st = await fsp.lstat(path.resolve(root, p)).catch(() => null);
395
+ if (st && (st.isSymbolicLink() || !st.isDirectory())) throw new Error(`Refusing to write: ${p} is a symlink or not a directory`);
396
+ }
397
+ const pst = await fsp.lstat(abs).catch(() => null);
398
+ if (pst && (pst.isSymbolicLink() || !pst.isFile())) throw new Error(`Refusing to write: ${rel} is a symlink or not a regular file`);
399
+ const existing = pst ? (await readRegular(abs)) ?? '' : '';
400
+ const built = await buildTopicPage({ root, dir, notes, existingText: existing });
401
+ await fsp.mkdir(path.dirname(abs), { recursive: true });
402
+ if (!(await topicsDirOk(root))) throw new Error(`Refusing to write: ${TOPIC_DIR} does not resolve inside the project`);
403
+ const tmp = path.join(path.dirname(abs), `.${topicSlug(dir)}.${process.pid}.${crypto.randomBytes(4).toString('hex')}.tmp`);
404
+ try {
405
+ await fsp.writeFile(tmp, built.text, { encoding: 'utf8', flag: 'wx' });
406
+ await fsp.rename(tmp, abs);
407
+ } catch (error) {
408
+ await fsp.unlink(tmp).catch(() => {});
409
+ throw error;
410
+ }
411
+ return built;
412
+ }
413
+
414
+ // refreshTopics(root) -> { rebuilt: [rel], failed: [rel] }: rebuild only pages that
415
+ // already exist and are stale (never creates a page); the agent block and the cited
416
+ // notes are carried over. Called from the index refresh, never from a hook.
417
+ export async function refreshTopics(root) {
418
+ const rebuilt = [];
419
+ const failed = [];
420
+ for (const rel of await listTopicRels(root)) {
421
+ try {
422
+ const page = await readTopicPage(root, rel);
423
+ if (!page || page.missing || !page.stale) continue;
424
+ await writeTopicPage(root, page.dir, []);
425
+ rebuilt.push(rel);
426
+ } catch {
427
+ failed.push(rel);
428
+ }
429
+ }
430
+ return { rebuilt, failed };
431
+ }
432
+
433
+ // ---- recall documents -----------------------------------------------------
434
+
435
+ const sectionLines = (text, title) => {
436
+ const lines = String(text).split(/\r?\n/);
437
+ const at = lines.findIndex((l) => l.trim() === `## ${title}`);
438
+ if (at < 0) return [];
439
+ const out = [];
440
+ for (const l of lines.slice(at + 1)) { if (/^##\s/.test(l)) break; if (/^- /.test(l)) out.push(l.slice(2)); }
441
+ return out;
442
+ };
443
+
444
+ // One recall document per page: title, purpose, key files, entry points, tests.
445
+ // type 'markdown' + kind 'topic' so it flows through every existing lane; a stale
446
+ // page carries validity 'changed' (or 'missing'), which the renderers label and
447
+ // the engine demotes. Never throws.
448
+ export async function loadTopicDocs(root, { maxPages = TOPIC_MAX_PAGES } = {}) {
449
+ const docs = [];
450
+ try {
451
+ for (const rel of (await listTopicRels(root)).slice(0, maxPages)) {
452
+ const page = await readTopicPage(root, rel);
453
+ if (!page) continue;
454
+ const strip = (s) => s.replace(/`/g, '');
455
+ // Rendered recall lines cut at ~160 chars, so the cheap-to-read part comes
456
+ // first: the module, then its key files by name relative to the module.
457
+ const rel2 = (f) => (f.startsWith(`${page.dir}/`) ? f.slice(page.dir.length + 1) : f);
458
+ const keyNames = sectionLines(page.text, 'Key files').slice(0, 10)
459
+ .map((l) => rel2(strip(l).split(' exports ')[0].split(';')[0].trim())).filter((f) => f && !f.startsWith('...'));
460
+ const parts = [
461
+ `Module ${page.dir} key files: ${keyNames.join(', ')}.`,
462
+ // Identifier-level text only: prose header comments added generic words
463
+ // ("mode", "toggle") that let a long page clear the floor on off-topic queries.
464
+ ...sectionLines(page.text, 'Key files').slice(0, 10)
465
+ .map((l) => / exports ([^;]+)/.exec(strip(l))?.[1]).filter(Boolean)
466
+ // The doc tokenizer does not split camelCase (queries are split), so spell the words out.
467
+ .map((e) => `exports ${e.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2').toLowerCase()}.`),
468
+ ...(page.agentText ? [`Agent notes: ${flat(page.agentText)}`] : []),
469
+ ];
470
+ docs.push({
471
+ id: `topic:${topicSlug(page.dir)}`, type: 'markdown', source: 'markdown', kind: 'topic',
472
+ text: flat(parts.join(' ')).slice(0, 1400), provenance: `${rel}:1`, line: 1, heading: `Topic: ${page.dir}`,
473
+ created_at: 0, ...(page.stale ? { validity: page.missing ? 'missing' : 'changed' } : {}),
474
+ });
475
+ }
476
+ } catch {
477
+ return docs;
478
+ }
479
+ return docs;
480
+ }
481
+
482
+ // Config gate: UKIT_MEMORY_TOPICS=1|0 wins over memory.topics.recall (default on:
483
+ // gold topic set 0/12 -> 9/12 with headline unchanged; inert while no page exists).
484
+ export function topicsRecallEnabled(config) {
485
+ const env = process.env.UKIT_MEMORY_TOPICS;
486
+ if (env === '1') return true;
487
+ if (env === '0') return false;
488
+ return config?.memory?.topics?.recall !== false;
489
+ }
490
+
491
+ // findTopicFor(root, relFile, { checkStale }) -> { rel, dir, stale } | null: the page
492
+ // of the deepest ancestor directory of `relFile` that has one. Probes one path per
493
+ // ancestor (the slug is derived from the directory) and reads one 600-byte header;
494
+ // with checkStale false (hooks) it never hashes the module and `stale` is null.
495
+ export async function findTopicFor(root, relFile, { checkStale = true } = {}) {
496
+ try {
497
+ const parts = norm(relFile).split('/');
498
+ if (!(await topicsDirOk(root))) return null;
499
+ for (let n = parts.length - 1; n >= 1; n -= 1) {
500
+ const dir = parts.slice(0, n).join('/');
501
+ if (!validTopicDir(dir)) continue;
502
+ const rel = topicRel(dir);
503
+ const head = (await readRegular(path.resolve(root, rel), { head: 600 })) ?? '';
504
+ const header = parseTopicHeader(head.split('\n').slice(0, 3).join('\n'));
505
+ if (!header || header.dir !== dir) continue;
506
+ if (!checkStale) return { rel, dir, stale: null };
507
+ const now = await fingerprintDir(root, dir);
508
+ return { rel, dir, stale: !now.exists || now.fingerprint !== header.fingerprint };
509
+ }
510
+ return null;
511
+ } catch {
512
+ return null;
513
+ }
514
+ }
515
+
516
+ // ---- check ----------------------------------------------------------------
517
+
518
+ // checkTopics(root, { scan }) -> { violations, warnings, pages }. Oversize,
519
+ // over-long lines, secrets (injected `scan(text) -> {labels}`, like memoryCheck) and
520
+ // an unreadable header are violations; a stale or missing-dir fingerprint is a warning.
521
+ export async function checkTopics(root, { scan = null } = {}) {
522
+ const violations = [];
523
+ const warnings = [];
524
+ const rels = await listTopicRels(root);
525
+ for (const rel of rels) {
526
+ const text = await readTopicFile(root, rel);
527
+ if (text === null) continue;
528
+ const bytes = Buffer.byteLength(text);
529
+ if (!parseTopicHeader(text)) { violations.push({ file: rel, line: 1, code: 'topic-bad-header', message: 'missing or malformed ukit-topic header' }); continue; }
530
+ if (bytes > TOPIC_PAGE_MAX_BYTES) violations.push({ file: rel, line: 1, code: 'topic-over-bytes', message: `${bytes} bytes exceeds ${TOPIC_PAGE_MAX_BYTES}` });
531
+ text.split(/\r?\n/).forEach((l, i) => {
532
+ if (l.length > TOPIC_LINE_MAX && !HEADER_RE.test(l)) violations.push({ file: rel, line: i + 1, code: 'topic-line-too-long', message: `${l.length} chars exceeds ${TOPIC_LINE_MAX}` });
533
+ });
534
+ if (typeof scan === 'function') {
535
+ try {
536
+ const res = scan(text);
537
+ if ((res?.labels?.length ?? 0) > 0) violations.push({ file: rel, line: 1, code: 'secret-detected', message: 'a secret-like value is present' });
538
+ } catch { /* scanner failure is not a violation */ }
539
+ }
540
+ const page = await readTopicPage(root, rel);
541
+ if (page?.missing) warnings.push({ file: rel, line: 1, code: 'topic-dir-missing', message: `${page.dir} no longer exists` });
542
+ else if (page?.stale) warnings.push({ file: rel, line: 1, code: 'topic-stale', message: `${page.dir} changed since the page was built; rebuild with ukit memory topic build ${page.dir}` });
543
+ }
544
+ return { violations, warnings, pages: rels.length };
545
+ }
@@ -0,0 +1,137 @@
1
+ // writeGate.js — deterministic admission gate for record writes (3.5.4).
2
+ // No LLM. Called by mutateMemory on `add` / `update` after the schema build and
3
+ // the secret guard; returns a typed rejection with a one-line reason.
4
+ //
5
+ // code reason prefix rule
6
+ // too-large too-large: record text over RECORD_TEXT_MAX chars
7
+ // no-evidence no-evidence: no source evidence (see hasSourceEvidence)
8
+ // near-duplicate near-duplicate: lexical near-duplicate of a live record
9
+ //
10
+ // (schema -> `invalid-record`, secret -> `secret-detected` stay where they were
11
+ // in mutateMemory; the codes above are the gate's own.) Stored items are never
12
+ // rejected retroactively: the gate only looks at the incoming write.
13
+ //
14
+ // Source evidence (project lane), any one of:
15
+ // - evidence[] entry kind 'file' whose locator is an existing path inside the project
16
+ // - evidence[] entry kind 'ledger' whose locator is an existing episode id (store or archive)
17
+ // - evidence[] entry kind 'command' whose locator names a commit: `commit:<sha>` or a bare 7-40 hex sha
18
+ // - an observation with meta.evidence.episodes that validateObservationEvidence accepts
19
+ // kinds 'manual' and 'import' are claims, not evidence, and do not count.
20
+ // Machine-generated writers (GENERATED_CREATORS) skip evidence and duplicate
21
+ // checks (their text is templated from a ledger); size still applies.
22
+ import fsp from 'node:fs/promises';
23
+ import path from 'node:path';
24
+
25
+ import { GENERATED_CREATORS, RECORD_TEXT_MAX, findNearDuplicate } from './memoryCheck.js';
26
+ import { tokenize } from './recallEngine.js';
27
+ import { guardWritePayload } from './writeGuard.js';
28
+ import { loadEpisodeIdsFor, validateObservationEvidence } from './observations.js';
29
+
30
+ const COMMIT_RE = /^(?:commit:)?[0-9a-f]{7,40}$/i;
31
+
32
+ async function fileExists(projectRoot, locator) {
33
+ if (typeof projectRoot !== 'string' || typeof locator !== 'string' || !locator) return false;
34
+ const root = path.resolve(projectRoot);
35
+ const abs = path.resolve(root, locator);
36
+ if (abs !== root && !abs.startsWith(root + path.sep)) return false;
37
+ try {
38
+ await fsp.stat(abs);
39
+ return true;
40
+ } catch {
41
+ return false;
42
+ }
43
+ }
44
+
45
+ /** hasSourceEvidence(record, {projectRoot}) -> Promise<{ok, detail?}> */
46
+ export async function hasSourceEvidence(record, { projectRoot } = {}) {
47
+ if (record?.type === 'observation') {
48
+ const res = await validateObservationEvidence(record, { projectRoot });
49
+ return res.ok ? { ok: true } : { ok: false, detail: res.reason };
50
+ }
51
+ let episodeIds = null;
52
+ for (const ev of Array.isArray(record?.evidence) ? record.evidence : []) {
53
+ if (!ev || typeof ev.locator !== 'string') continue;
54
+ if (ev.kind === 'file' && await fileExists(projectRoot, ev.locator)) return { ok: true };
55
+ if (ev.kind === 'command' && COMMIT_RE.test(ev.locator)) return { ok: true };
56
+ if (ev.kind === 'ledger') {
57
+ episodeIds ??= await loadEpisodeIdsFor(projectRoot).catch(() => new Set());
58
+ if (episodeIds.has(ev.locator)) return { ok: true };
59
+ }
60
+ }
61
+ return { ok: false, detail: 'add evidence: a file path that exists, a commit sha, or an episode id' };
62
+ }
63
+
64
+ /**
65
+ * checkProposalText({ text, existing, config, projectId }) -> { ok: true } | { ok, code, reason, labels? }
66
+ * Text-only gate for `ukit memory propose` (a human approves candidates, so no
67
+ * evidence rule): size cap, secret scan, near-duplicate of `existing` ([{id,text}],
68
+ * pending + approved candidates). Honours memory.writeGate 'off'.
69
+ */
70
+ export function checkProposalText({ text, existing = [], config = null, projectId = null } = {}) {
71
+ const mode = config?.memory?.writeGate;
72
+ if (mode === false || mode === 'off') return { ok: true };
73
+ const body = String(text ?? '');
74
+ if (body.length > RECORD_TEXT_MAX) {
75
+ return {
76
+ ok: false, code: 'too-large',
77
+ reason: `too-large: text is ${body.length} chars (max ${RECORD_TEXT_MAX}); move the detail to docs/memory/ and keep one line here`,
78
+ };
79
+ }
80
+ const guarded = guardWritePayload({ text: body }, { config, projectId });
81
+ if (guarded?.ok === false) {
82
+ return { ok: false, code: 'secret-detected', reason: guarded.reason ?? 'secret-detected', labels: guarded.labels };
83
+ }
84
+ const dup = findNearDuplicate(body, existing, tokenize);
85
+ if (dup) {
86
+ return {
87
+ ok: false, code: 'near-duplicate',
88
+ reason: `near-duplicate: ${Math.round(dup.similarity * 100)}% lexical overlap with ${dup.id}; approve or reject that candidate instead`,
89
+ };
90
+ }
91
+ return { ok: true };
92
+ }
93
+
94
+ const isLive = (r) => r && r.status !== 'archived' && !r.superseded_by;
95
+
96
+ /**
97
+ * checkRecordWrite({ record, records, projectRoot, op, replaces }) ->
98
+ * { ok: true } | { ok: false, code, reason }
99
+ * record: the built (validated) record or, for update, the would-be merged record.
100
+ * records: current store contents. replaces: ids this write supersedes or updates
101
+ * (excluded from the duplicate comparison). op 'update' only enforces the rules
102
+ * the patch can break (size, duplicate of changed text, evidence it supplies).
103
+ */
104
+ export async function checkRecordWrite({
105
+ record, records = [], projectRoot = null, op = 'add', replaces = [], patch = null, strict = false,
106
+ } = {}) {
107
+ const text = String(record?.text ?? '');
108
+ const touchesText = op === 'add' || (patch && 'text' in patch);
109
+ if (touchesText && text.length > RECORD_TEXT_MAX) {
110
+ return {
111
+ ok: false, code: 'too-large',
112
+ reason: `too-large: text is ${text.length} chars (max ${RECORD_TEXT_MAX}); move the detail to docs/memory/ and keep one line here`,
113
+ };
114
+ }
115
+ // Evidence and near-duplicate rules are for agent-authored notes (strict).
116
+ // Legacy callers that write bare records (tests, tools, restore flows) pass
117
+ // the size cap only; generated writers are exempt even when strict.
118
+ if (!strict || GENERATED_CREATORS.has(record?.created_by)) return { ok: true };
119
+
120
+ const touchesEvidence = op === 'add' || (patch && ('evidence' in patch || 'meta' in patch || patch.type === 'observation'));
121
+ if (touchesEvidence && projectRoot) {
122
+ const res = await hasSourceEvidence(record, { projectRoot });
123
+ if (!res.ok) return { ok: false, code: 'no-evidence', reason: `no-evidence: ${res.detail}` };
124
+ }
125
+
126
+ if (touchesText) {
127
+ const skip = new Set([record.id, ...replaces, ...(Array.isArray(record.supersedes) ? record.supersedes : [])]);
128
+ const dup = findNearDuplicate(text, records.filter((r) => isLive(r) && !skip.has(r.id)), tokenize);
129
+ if (dup) {
130
+ return {
131
+ ok: false, code: 'near-duplicate',
132
+ reason: `near-duplicate: ${Math.round(dup.similarity * 100)}% lexical overlap with ${dup.id}; update it, or pass supersedes: ["${dup.id}"] to replace it`,
133
+ };
134
+ }
135
+ }
136
+ return { ok: true };
137
+ }