@ngockhoale/ukit 3.5.4 → 3.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/package.json +3 -2
- package/src/cli/commands/memory.js +11 -0
- package/src/core/memory/memoryCheckRun.js +13 -1
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/recallEngine.js +39 -3
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/runtimeConfig.js +45 -0
- package/src/decision/registry.js +17 -0
- package/src/index/buildIndex.js +287 -6
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +2 -1
- package/src/index/queryIndex.js +134 -7
- package/template_project/.claude/ukit/index/lib/index-core.mjs +519 -11
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/ukit/storage/config.json +19 -2
- package/manifests/instructionRules.yaml +0 -442
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
demoteByValidity, dropSuperseded, fuseRanked, loadMarkdownDocs, normalizeMemorySources, rankDocs, STALE_MARKER,
|
|
7
7
|
} from './memory-recall.mjs';
|
|
8
8
|
import { withValidity } from './memory-validity.mjs';
|
|
9
|
+
import { findTopicFor } from './memory-topics.mjs';
|
|
9
10
|
import { eligible, resolveProjectIdentityMirror } from './memory-policy.mjs';
|
|
10
11
|
import { resolveMemoryStage } from './memory-flags.mjs';
|
|
11
12
|
|
|
@@ -113,10 +114,12 @@ export function recallForTarget({ target, neighbours, docs, query = '', limit =
|
|
|
113
114
|
const flat = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
|
|
114
115
|
const KIND_LABEL = { twin: 'identical at last index', declared: 'declared', drifted: 'differs now' };
|
|
115
116
|
|
|
116
|
-
export function renderEditHint({ target, neighbours, memory = [], maxChars = 600 } = {}) {
|
|
117
|
+
export function renderEditHint({ target, neighbours, memory = [], maxChars = 600, topic = null } = {}) {
|
|
117
118
|
try {
|
|
118
119
|
const n = neighbours ?? {};
|
|
119
120
|
const lines = [];
|
|
121
|
+
// 3.5.5: pointer to the module's topic page (path only; the page is read on request).
|
|
122
|
+
if (topic?.rel) lines.push(`module page: ${flat(topic.rel)} (read it for the module overview; \`ukit memory topic list\` shows if it is stale)`);
|
|
120
123
|
const mirrors = (Array.isArray(n.mirrors) ? n.mirrors : []).map((m) => (typeof m === 'string' ? { path: m, kind: 'declared' } : m))
|
|
121
124
|
.filter((m) => m?.path).sort((a, b) => String(a.path).localeCompare(String(b.path))).slice(0, 2);
|
|
122
125
|
for (const m of mirrors) lines.push(`mirror (${KIND_LABEL[m.kind] ?? 'declared'}): ${flat(m.path)}`);
|
|
@@ -250,7 +253,11 @@ export async function editHintFor({
|
|
|
250
253
|
const drop = new Set(Array.isArray(exclude?.memoryIds) ? exclude.memoryIds : []);
|
|
251
254
|
const docs = [...records, ...markdown].filter((d) => !drop.has(d.id));
|
|
252
255
|
const { ranked } = recallForTarget({ target: rel, neighbours, docs, query });
|
|
253
|
-
|
|
256
|
+
// Topic pointer: one header read per ancestor dir, async, never hashes the module;
|
|
257
|
+
// skipped when the memory budget already missed so the graph part stays fast.
|
|
258
|
+
const topic = memoryMissed || cfg?.memory?.topics?.hint === false || signal?.aborted
|
|
259
|
+
? null : await findTopicFor(projectRoot, rel, { checkStale: false });
|
|
260
|
+
const text = renderEditHint({ target: rel, neighbours, memory: ranked, topic });
|
|
254
261
|
if (!text) return { text: '', neighbours, memoryIds: [], memoryMissed };
|
|
255
262
|
return { text, neighbours, memoryIds: ranked.slice(0, 2).map((e) => e.doc.id), memoryMissed };
|
|
256
263
|
} catch {
|
|
@@ -155,9 +155,17 @@ function stem(token) {
|
|
|
155
155
|
|
|
156
156
|
// Tokens of `text`: whole tokens plus their separator-split parts
|
|
157
157
|
// (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
|
|
158
|
-
|
|
158
|
+
// A camelCase identifier also yields its words (`deriveDeterministicBucket`
|
|
159
|
+
// -> derive, deterministic, bucket); the whole identifier stays a token too.
|
|
160
|
+
const CAMEL_RE = /\b[a-z][a-z0-9]*(?:[A-Z][a-z0-9]+)+\b/g;
|
|
161
|
+
const splitCamel = (word) => `${word} ${word.replace(/([a-z0-9])([A-Z])/g, '$1 $2')}`;
|
|
162
|
+
|
|
163
|
+
// Query side only (`{ camel: true }`): in a long note the split words of its many
|
|
164
|
+
// identifiers (`postEditReviewPolicy`) would add common-word matches.
|
|
165
|
+
export function tokenize(text, { camel = false } = {}) {
|
|
159
166
|
const out = [];
|
|
160
|
-
|
|
167
|
+
const source = camel ? String(text ?? '').replace(CAMEL_RE, splitCamel) : text;
|
|
168
|
+
for (const raw of normalize(source).split(/\s+/)) {
|
|
161
169
|
if (!raw) continue;
|
|
162
170
|
const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
|
|
163
171
|
const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
|
|
@@ -169,6 +177,21 @@ export function tokenize(text) {
|
|
|
169
177
|
return out;
|
|
170
178
|
}
|
|
171
179
|
|
|
180
|
+
// File names in a query (`name.ext`): a query that names a file is only
|
|
181
|
+
// answered by a note that names it too and covers the rest of the question.
|
|
182
|
+
const QUERY_FILE_RE = /[\w-]+\.(?:test\.)?(?:m?js|cjs|tsx?|jsx|sh|md|json|py|ya?ml)\b/gi;
|
|
183
|
+
const FILE_EXT_TOKENS = new Set(['js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'sh', 'md', 'json', 'py', 'yml', 'yaml', 'test']);
|
|
184
|
+
const FILE_REST_COVERAGE = 0.3;
|
|
185
|
+
const FILE_REST_MIN_TOKENS = 2;
|
|
186
|
+
function queryFileSpec(query) {
|
|
187
|
+
if (typeof query !== 'string') return null;
|
|
188
|
+
const files = query.match(QUERY_FILE_RE);
|
|
189
|
+
if (!files) return null;
|
|
190
|
+
const fileTokens = new Set(files.flatMap((f) => tokenize(f.toLowerCase())).filter((t) => !FILE_EXT_TOKENS.has(t)));
|
|
191
|
+
const restTokens = [...new Set(tokenize(query.replace(QUERY_FILE_RE, ' '), { camel: true }))];
|
|
192
|
+
return fileTokens.size > 0 ? { fileTokens, restTokens } : null;
|
|
193
|
+
}
|
|
194
|
+
|
|
172
195
|
function episodeBody(text) {
|
|
173
196
|
return String(text ?? '')
|
|
174
197
|
.replace(/^\s*Session\s+\S+?:\s*/i, '')
|
|
@@ -272,7 +295,7 @@ function idf(totalDocs, docFreq) {
|
|
|
272
295
|
// `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
|
|
273
296
|
// `query` is a string or a pre-tokenized array. Never throws.
|
|
274
297
|
export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
275
|
-
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
|
|
298
|
+
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query, { camel: true }))];
|
|
276
299
|
const empty = {
|
|
277
300
|
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
278
301
|
};
|
|
@@ -302,6 +325,10 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
|
302
325
|
let queryMass = 0;
|
|
303
326
|
for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
|
|
304
327
|
|
|
328
|
+
const fileSpec = total >= SMALL_CORPUS_DOCS ? queryFileSpec(query) : null;
|
|
329
|
+
let restMass = 0;
|
|
330
|
+
if (fileSpec) for (const token of fileSpec.restTokens) restMass += idf(total, docFreq.get(token) ?? 0);
|
|
331
|
+
|
|
305
332
|
const scored = [];
|
|
306
333
|
let candidateCount = 0;
|
|
307
334
|
for (const { doc, tf, length } of stats) {
|
|
@@ -324,6 +351,15 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
|
324
351
|
// clear the absolute floor above while the query's real subject is absent.
|
|
325
352
|
// It must also cover a fraction of the query's own IDF mass.
|
|
326
353
|
if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
|
|
354
|
+
if (fileSpec) {
|
|
355
|
+
let named = false;
|
|
356
|
+
for (const token of fileSpec.fileTokens) if (tf.has(token)) { named = true; break; }
|
|
357
|
+
let restHit = 0;
|
|
358
|
+
for (const token of fileSpec.restTokens) if (tf.has(token)) restHit += idf(total, docFreq.get(token));
|
|
359
|
+
// One leftover word ("what does X check") is too little to judge coverage by.
|
|
360
|
+
const judgeRest = fileSpec.restTokens.length >= FILE_REST_MIN_TOKENS && restMass > 0;
|
|
361
|
+
if (!named || (judgeRest && restHit < FILE_REST_COVERAGE * restMass)) continue;
|
|
362
|
+
}
|
|
327
363
|
scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
|
|
328
364
|
}
|
|
329
365
|
// Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
|
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
// Topic pages (3.5.5, decision D3'): per-module "mental model" pages under
|
|
2
|
+
// docs/memory/topics/<slug>.md. A page is a DETERMINISTIC skeleton built from the
|
|
3
|
+
// index (symbols), the edit graph (mirrors, tests, importers) and the memory
|
|
4
|
+
// notes that name paths inside the module - no LLM in the generator. One marked
|
|
5
|
+
// section is left for the running agent to fill in; a rebuild keeps it.
|
|
6
|
+
//
|
|
7
|
+
// Rules (D3'): never loaded unless asked for (recall by query, the edit hint
|
|
8
|
+
// pointer, `ukit memory topic show`); every page carries a fingerprint of the
|
|
9
|
+
// directory it describes and its evidence; a page whose directory changed is
|
|
10
|
+
// labelled stale, never served as current.
|
|
11
|
+
//
|
|
12
|
+
// This file imports node builtins only and is mirrored byte-identically as
|
|
13
|
+
// template_project/.claude/ukit/runtime/memory-topics.mjs (hook lanes load it).
|
|
14
|
+
// All I/O is async; every reader swallows errors and returns an empty value.
|
|
15
|
+
|
|
16
|
+
import crypto from 'node:crypto';
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import fsp from 'node:fs/promises';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
|
|
21
|
+
export const TOPIC_DIR = 'docs/memory/topics';
|
|
22
|
+
export const TOPIC_PAGE_MAX_BYTES = 9000;
|
|
23
|
+
export const TOPIC_LINE_MAX = 400;
|
|
24
|
+
export const TOPIC_MAX_PAGES = 40;
|
|
25
|
+
const MAX_FILES = 400;
|
|
26
|
+
const MAX_FILE_BYTES = 512 * 1024;
|
|
27
|
+
const MAX_INDEX_BYTES = 40 * 1024 * 1024;
|
|
28
|
+
const SHOW_FILES = 14;
|
|
29
|
+
const SHOW_NOTES = 8;
|
|
30
|
+
const SKIP_DIRS = new Set(['node_modules', '.git', '.cache', '.ukit', 'dist', 'build', 'coverage']);
|
|
31
|
+
const SOURCE_RE = /\.(?:[cm]?[jt]sx?|vue|py|go|java|cs|php|rb|rs|sh|md|json|ya?ml)$/i;
|
|
32
|
+
const AGENT_START = '<!-- ukit-topic:agent-start -->';
|
|
33
|
+
const AGENT_END = '<!-- ukit-topic:agent-end -->';
|
|
34
|
+
const AGENT_PLACEHOLDER = '_Not filled in. The agent working in this module may replace this line with what it learned; keep it short and cite files._';
|
|
35
|
+
const HEADER_RE = /^<!--\s*ukit-topic\s+dir=(\S+)\s+fingerprint=([0-9a-f]{40}|none)\s+files=(\d+)\s*-->\s*$/m;
|
|
36
|
+
export const TOPIC_STALE_MARKER = '[module changed since this page was built - verify]';
|
|
37
|
+
|
|
38
|
+
const norm = (p) => String(p ?? '').trim().replace(/\\/g, '/').replace(/^(\.\/)+/, '').replace(/\/+$/, '');
|
|
39
|
+
const flat = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
|
|
40
|
+
const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
41
|
+
|
|
42
|
+
export function validTopicDir(dir) {
|
|
43
|
+
const d = norm(dir);
|
|
44
|
+
return d !== '' && !path.isAbsolute(d) && !d.split('/').includes('..') && /^[A-Za-z0-9._/@-]+$/.test(d);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function topicSlug(dir) {
|
|
48
|
+
return norm(dir).replace(/[^A-Za-z0-9]+/g, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'root';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export const topicRel = (dir) => `${TOPIC_DIR}/${topicSlug(dir)}.md`;
|
|
52
|
+
|
|
53
|
+
async function readJson(file, maxBytes = MAX_INDEX_BYTES) {
|
|
54
|
+
try {
|
|
55
|
+
const st = await fsp.stat(file);
|
|
56
|
+
if (!st.isFile() || st.size > maxBytes) return null;
|
|
57
|
+
return JSON.parse(await fsp.readFile(file, 'utf8'));
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ---- path safety ----------------------------------------------------------
|
|
64
|
+
// A page or module must never lead outside the repo: symlinks are refused for the
|
|
65
|
+
// topics directory chain, the page file and the module root; reads open with
|
|
66
|
+
// O_NOFOLLOW and require a regular file; writes go to a temp file + rename.
|
|
67
|
+
|
|
68
|
+
const NOFOLLOW = fs.constants.O_NOFOLLOW ?? 0;
|
|
69
|
+
const TOPIC_CHAIN = ['docs', 'docs/memory', TOPIC_DIR];
|
|
70
|
+
|
|
71
|
+
async function topicsDirOk(root) {
|
|
72
|
+
try {
|
|
73
|
+
for (const p of TOPIC_CHAIN) {
|
|
74
|
+
const st = await fsp.lstat(path.resolve(root, p));
|
|
75
|
+
if (st.isSymbolicLink() || !st.isDirectory()) return false;
|
|
76
|
+
}
|
|
77
|
+
return (await fsp.realpath(path.resolve(root, TOPIC_DIR))) === path.join(await fsp.realpath(path.resolve(root)), TOPIC_DIR);
|
|
78
|
+
} catch {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Reads at most `maxBytes` (default whole file up to the page cap x4) of a regular,
|
|
84
|
+
// non-symlink file; null otherwise. `head` reads only the first N bytes.
|
|
85
|
+
async function readRegular(abs, { head = 0 } = {}) {
|
|
86
|
+
let fh;
|
|
87
|
+
try {
|
|
88
|
+
fh = await fsp.open(abs, fs.constants.O_RDONLY | NOFOLLOW);
|
|
89
|
+
const st = await fh.stat();
|
|
90
|
+
if (!st.isFile()) return null;
|
|
91
|
+
if (head > 0) {
|
|
92
|
+
const buf = Buffer.alloc(head);
|
|
93
|
+
const { bytesRead } = await fh.read(buf, 0, head, 0);
|
|
94
|
+
return buf.toString('utf8', 0, bytesRead);
|
|
95
|
+
}
|
|
96
|
+
if (st.size > TOPIC_PAGE_MAX_BYTES * 4) return null;
|
|
97
|
+
return await fh.readFile('utf8');
|
|
98
|
+
} catch {
|
|
99
|
+
return null;
|
|
100
|
+
} finally {
|
|
101
|
+
await fh?.close().catch(() => {});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function readTopicFile(root, rel, opts) {
|
|
106
|
+
const abs = path.resolve(root, rel);
|
|
107
|
+
if (!abs.startsWith(path.resolve(root, TOPIC_DIR) + path.sep)) return null;
|
|
108
|
+
if (!(await topicsDirOk(root))) return null;
|
|
109
|
+
return readRegular(abs, opts);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// A module root must be a real directory inside the repo (no symlink in its path).
|
|
113
|
+
async function moduleRootOk(root, dir) {
|
|
114
|
+
try {
|
|
115
|
+
const abs = path.resolve(root, norm(dir));
|
|
116
|
+
const st = await fsp.lstat(abs);
|
|
117
|
+
if (st.isSymbolicLink() || !st.isDirectory()) return false;
|
|
118
|
+
return (await fsp.realpath(abs)) === path.join(await fsp.realpath(path.resolve(root)), ...norm(dir).split('/'));
|
|
119
|
+
} catch {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ---- fingerprint ----------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
const hashCache = new Map(); // abs -> {size, mtimeMs, sha}
|
|
127
|
+
|
|
128
|
+
async function fileSha(abs) {
|
|
129
|
+
const st = await fsp.stat(abs);
|
|
130
|
+
if (!st.isFile()) return null;
|
|
131
|
+
const cached = hashCache.get(abs);
|
|
132
|
+
if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) return cached.sha;
|
|
133
|
+
const sha = st.size > MAX_FILE_BYTES
|
|
134
|
+
? `size:${st.size}`
|
|
135
|
+
: crypto.createHash('sha1').update(await fsp.readFile(abs)).digest('hex');
|
|
136
|
+
hashCache.set(abs, { size: st.size, mtimeMs: st.mtimeMs, sha });
|
|
137
|
+
return sha;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Files of a module: every file under `dir` (recursive, dot-dirs and vendored dirs
|
|
141
|
+
// skipped), sorted, at most MAX_FILES. truncated=true when more existed.
|
|
142
|
+
export async function listModuleFiles(root, dir) {
|
|
143
|
+
const base = path.resolve(root, norm(dir));
|
|
144
|
+
const out = [];
|
|
145
|
+
let truncated = false;
|
|
146
|
+
if (!(await moduleRootOk(root, dir))) return { files: out, truncated };
|
|
147
|
+
const walk = async (rel) => {
|
|
148
|
+
let items;
|
|
149
|
+
try { items = await fsp.readdir(path.join(base, rel), { withFileTypes: true }); } catch { return; }
|
|
150
|
+
items.sort((a, b) => cmp(a.name, b.name));
|
|
151
|
+
for (const item of items) {
|
|
152
|
+
if (item.name.startsWith('.') || SKIP_DIRS.has(item.name)) continue;
|
|
153
|
+
const next = rel ? `${rel}/${item.name}` : item.name;
|
|
154
|
+
if (item.isDirectory()) { await walk(next); continue; }
|
|
155
|
+
if (!item.isFile() || !SOURCE_RE.test(item.name)) continue;
|
|
156
|
+
if (out.length >= MAX_FILES) { truncated = true; return; }
|
|
157
|
+
out.push(`${norm(dir)}/${next}`);
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
await walk('');
|
|
161
|
+
return { files: out.sort(cmp), truncated };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// fingerprintDir -> { fingerprint, files, count, truncated, exists }. Content
|
|
165
|
+
// based (sha1 per file), so a fresh clone or a touched mtime never reads as a change.
|
|
166
|
+
export async function fingerprintDir(root, dir) {
|
|
167
|
+
try {
|
|
168
|
+
const st = await fsp.lstat(path.resolve(root, norm(dir)));
|
|
169
|
+
if (st.isSymbolicLink()) return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false, refused: true };
|
|
170
|
+
if (!st.isDirectory()) return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false };
|
|
171
|
+
} catch {
|
|
172
|
+
return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false };
|
|
173
|
+
}
|
|
174
|
+
if (!(await moduleRootOk(root, dir))) return { fingerprint: 'none', files: [], count: 0, truncated: false, exists: false, refused: true };
|
|
175
|
+
const { files, truncated } = await listModuleFiles(root, dir);
|
|
176
|
+
const shas = [];
|
|
177
|
+
for (let i = 0; i < files.length; i += 16) {
|
|
178
|
+
const slice = files.slice(i, i + 16);
|
|
179
|
+
shas.push(...await Promise.all(slice.map((f) => fileSha(path.resolve(root, f)).catch(() => null))));
|
|
180
|
+
}
|
|
181
|
+
const h = crypto.createHash('sha1');
|
|
182
|
+
files.forEach((f, i) => h.update(`${f}\0${shas[i] ?? 'unreadable'}\n`));
|
|
183
|
+
return { fingerprint: h.digest('hex'), files, count: files.length, truncated, exists: true };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ---- page parse / read ----------------------------------------------------
|
|
187
|
+
|
|
188
|
+
export function parseTopicHeader(text) {
|
|
189
|
+
const m = HEADER_RE.exec(String(text ?? ''));
|
|
190
|
+
return m ? { dir: m[1], fingerprint: m[2], files: Number(m[3]) } : null;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export function agentSection(text) {
|
|
194
|
+
const t = String(text ?? '');
|
|
195
|
+
const a = t.indexOf(AGENT_START);
|
|
196
|
+
const b = t.indexOf(AGENT_END);
|
|
197
|
+
if (a < 0 || b < a) return '';
|
|
198
|
+
const body = t.slice(a + AGENT_START.length, b).trim();
|
|
199
|
+
return body === AGENT_PLACEHOLDER ? '' : body;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// readTopicPage(root, rel) -> { rel, dir, fingerprint, current, stale, missing, text, agentText } | null
|
|
203
|
+
export async function readTopicPage(root, rel) {
|
|
204
|
+
try {
|
|
205
|
+
const text = await readTopicFile(root, rel);
|
|
206
|
+
if (text === null) return null;
|
|
207
|
+
const header = parseTopicHeader(text);
|
|
208
|
+
if (!header || !validTopicDir(header.dir)) return null;
|
|
209
|
+
const now = await fingerprintDir(root, header.dir);
|
|
210
|
+
return {
|
|
211
|
+
rel: norm(rel), dir: header.dir, fingerprint: header.fingerprint, currentFingerprint: now.fingerprint,
|
|
212
|
+
missing: !now.exists, stale: !now.exists || now.fingerprint !== header.fingerprint,
|
|
213
|
+
text, agentText: agentSection(text),
|
|
214
|
+
};
|
|
215
|
+
} catch {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
export async function listTopicRels(root) {
|
|
221
|
+
try {
|
|
222
|
+
if (!(await topicsDirOk(root))) return [];
|
|
223
|
+
const items = await fsp.readdir(path.resolve(root, TOPIC_DIR), { withFileTypes: true });
|
|
224
|
+
return items.filter((d) => d.isFile() && /\.md$/i.test(d.name) && !d.name.startsWith('.'))
|
|
225
|
+
.map((d) => `${TOPIC_DIR}/${d.name}`).sort(cmp).slice(0, TOPIC_MAX_PAGES);
|
|
226
|
+
} catch {
|
|
227
|
+
return [];
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export async function listTopics(root) {
|
|
232
|
+
const out = [];
|
|
233
|
+
for (const rel of await listTopicRels(root)) {
|
|
234
|
+
const page = await readTopicPage(root, rel);
|
|
235
|
+
if (page) out.push({ rel, dir: page.dir, stale: page.stale, missing: page.missing, bytes: Buffer.byteLength(page.text), filled: page.agentText !== '' });
|
|
236
|
+
}
|
|
237
|
+
return out;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ---- build ----------------------------------------------------------------
|
|
241
|
+
|
|
242
|
+
async function headComment(root, file) {
|
|
243
|
+
try {
|
|
244
|
+
const fh = await fsp.open(path.resolve(root, file), 'r');
|
|
245
|
+
try {
|
|
246
|
+
const buf = Buffer.alloc(2048);
|
|
247
|
+
const { bytesRead } = await fh.read(buf, 0, 2048, 0);
|
|
248
|
+
const lines = buf.toString('utf8', 0, bytesRead).split(/\r?\n/).slice(0, 14);
|
|
249
|
+
const picked = [];
|
|
250
|
+
for (const line of lines) {
|
|
251
|
+
if (/^#!/.test(line)) continue;
|
|
252
|
+
const m = /^\s*(?:\/\/|#|\*|\/\*+)\s?(.*)$/.exec(line);
|
|
253
|
+
if (!m) { if (picked.length || /\S/.test(line)) break; continue; }
|
|
254
|
+
const body = m[1].replace(/\*\/\s*$/, '').trim();
|
|
255
|
+
if (body) picked.push(body);
|
|
256
|
+
if (picked.join(' ').length > 170) break;
|
|
257
|
+
}
|
|
258
|
+
const joined = flat(picked.join(' '));
|
|
259
|
+
return joined.length > 190 ? `${joined.slice(0, 187)}...` : joined;
|
|
260
|
+
} finally { await fh.close(); }
|
|
261
|
+
} catch {
|
|
262
|
+
return '';
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function exportsByFile(symbols, files) {
|
|
267
|
+
const want = new Set(files);
|
|
268
|
+
const map = new Map();
|
|
269
|
+
for (const item of Array.isArray(symbols?.items) ? symbols.items : []) {
|
|
270
|
+
if (!want.has(item?.filePath) || !/^\s*export\b/.test(String(item.signature ?? ''))) continue;
|
|
271
|
+
if (!map.has(item.filePath)) map.set(item.filePath, []);
|
|
272
|
+
map.get(item.filePath).push(item.name);
|
|
273
|
+
}
|
|
274
|
+
return map;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// buildTopicPage({ root, dir, notes, existingText, now }) -> { rel, text, fingerprint, count, truncated, bytes }
|
|
278
|
+
// `notes` = [{ ref, text, type? }] supplied by the caller (markdown bullets and
|
|
279
|
+
// active records); only notes that name a path inside `dir` are used.
|
|
280
|
+
export async function buildTopicPage({ root, dir, notes = [], existingText = '', now = Date.now() } = {}) {
|
|
281
|
+
const d = norm(dir);
|
|
282
|
+
if (!validTopicDir(d)) throw new Error(`Invalid topic directory: ${dir}`);
|
|
283
|
+
const fp = await fingerprintDir(root, d);
|
|
284
|
+
if (fp.refused) throw new Error(`Topic directory is a symlink or leaves the repo: ${d}`);
|
|
285
|
+
if (!fp.exists) throw new Error(`Topic directory not found: ${d}`);
|
|
286
|
+
if (fp.truncated) throw new Error(`Topic directory too large (>${MAX_FILES} files): pick a subdirectory of ${d}`);
|
|
287
|
+
const graph = await readJson(path.resolve(root, '.cache/index/edit-graph.json'));
|
|
288
|
+
const symbols = await readJson(path.resolve(root, '.cache/index/symbols.json'));
|
|
289
|
+
const entries = graph?.files && typeof graph.files === 'object' ? graph.files : {};
|
|
290
|
+
const exportsOf = exportsByFile(symbols, fp.files);
|
|
291
|
+
const inside = (p) => p === d || p.startsWith(`${d}/`);
|
|
292
|
+
const info = fp.files.map((f) => {
|
|
293
|
+
const e = entries[f] ?? {};
|
|
294
|
+
const importers = Array.isArray(e.i) ? e.i.map(norm) : [];
|
|
295
|
+
return {
|
|
296
|
+
file: f,
|
|
297
|
+
exports: [...new Set(exportsOf.get(f) ?? [])].sort(cmp),
|
|
298
|
+
importers,
|
|
299
|
+
outside: importers.filter((p) => !inside(p)),
|
|
300
|
+
mirrors: (Array.isArray(e.m) ? e.m : []).map((x) => norm(x?.p ?? x)).filter(Boolean),
|
|
301
|
+
tests: (Array.isArray(e.t) ? e.t : []).map(norm).filter(Boolean),
|
|
302
|
+
};
|
|
303
|
+
});
|
|
304
|
+
const code = info.filter((x) => /\.(?:[cm]?[jt]sx?|vue|py|go|java|cs|php|rb|rs|sh)$/i.test(x.file) && !x.tests.includes(x.file));
|
|
305
|
+
const ranked = [...code].sort((a, b) => b.importers.length - a.importers.length || b.exports.length - a.exports.length || cmp(a.file, b.file));
|
|
306
|
+
const shown = ranked.slice(0, SHOW_FILES);
|
|
307
|
+
const bullet = (s) => `- ${flat(s).slice(0, TOPIC_LINE_MAX - 2)}`;
|
|
308
|
+
|
|
309
|
+
const lines = [];
|
|
310
|
+
lines.push(`# Topic: ${d}`);
|
|
311
|
+
lines.push(`<!-- ukit-topic dir=${d} fingerprint=${fp.fingerprint} files=${fp.count} -->`);
|
|
312
|
+
lines.push('');
|
|
313
|
+
lines.push(`Generated skeleton for \`${d}\` (${fp.count} files, built ${new Date(now).toISOString().slice(0, 10)}). Facts below come from the index, the edit graph and cited notes; rebuild with \`ukit memory topic build ${d}\`. If the module changed since the build the page is stale: verify before relying on it.`);
|
|
314
|
+
lines.push('');
|
|
315
|
+
lines.push('## What it is for');
|
|
316
|
+
for (const x of shown.slice(0, 6)) {
|
|
317
|
+
const c = await headComment(root, x.file);
|
|
318
|
+
if (c) lines.push(bullet(`\`${x.file}\`: ${c}`));
|
|
319
|
+
}
|
|
320
|
+
if (lines[lines.length - 1] === '## What it is for') lines.push('- No header comments found in the key files.');
|
|
321
|
+
lines.push('');
|
|
322
|
+
lines.push('## Key files');
|
|
323
|
+
for (const x of shown) {
|
|
324
|
+
const ex = x.exports.slice(0, 6).join(', ');
|
|
325
|
+
lines.push(bullet(`\`${x.file}\`${ex ? ` exports ${ex}` : ''}; imported by ${x.importers.length >= 5 ? '5+' : x.importers.length} file(s)`));
|
|
326
|
+
}
|
|
327
|
+
if (code.length > shown.length) lines.push(`- ... ${code.length - shown.length} more file(s) under \`${d}\`.`);
|
|
328
|
+
lines.push('');
|
|
329
|
+
lines.push('## Entry points (imported from outside the module)');
|
|
330
|
+
const entry = ranked.filter((x) => x.outside.length > 0).slice(0, 8);
|
|
331
|
+
if (entry.length === 0) lines.push('- None recorded in the edit graph.');
|
|
332
|
+
for (const x of entry) lines.push(bullet(`\`${x.file}\` used by ${x.outside.slice(0, 3).map((p) => `\`${p}\``).join(', ')}${x.outside.length > 3 ? ` (+${x.outside.length - 3})` : ''}`));
|
|
333
|
+
lines.push('');
|
|
334
|
+
lines.push('## Mirrors and tests');
|
|
335
|
+
const mirrors = [...new Set(info.flatMap((x) => x.mirrors.map((m) => `${x.file} -> ${m}`)))].sort(cmp).slice(0, 8);
|
|
336
|
+
const tests = [...new Set(info.flatMap((x) => x.tests))].sort(cmp);
|
|
337
|
+
if (mirrors.length === 0 && tests.length === 0) lines.push('- No mirror or test edges recorded in the edit graph.');
|
|
338
|
+
for (const m of mirrors) lines.push(bullet(`mirror: \`${m.split(' -> ')[0]}\` -> \`${m.split(' -> ')[1]}\``));
|
|
339
|
+
if (tests.length) lines.push(bullet(`tests (${tests.length}): ${tests.slice(0, 8).map((t) => `\`${t}\``).join(', ')}${tests.length > 8 ? ', ...' : ''}`));
|
|
340
|
+
lines.push('');
|
|
341
|
+
lines.push('## Notes naming this module');
|
|
342
|
+
const named = [];
|
|
343
|
+
const seen = new Set();
|
|
344
|
+
for (const n of Array.isArray(notes) ? notes : []) {
|
|
345
|
+
const text = flat(n?.text);
|
|
346
|
+
if (!text || !text.includes(`${d}/`) || seen.has(n.ref)) continue;
|
|
347
|
+
seen.add(n.ref);
|
|
348
|
+
named.push({ ref: flat(n.ref), text, type: n.type });
|
|
349
|
+
}
|
|
350
|
+
if (named.length === 0) {
|
|
351
|
+
// No note source supplied (index-refresh rebuild): carry the page's own cited notes.
|
|
352
|
+
for (const l of sectionLines(existingText, 'Notes naming this module')) {
|
|
353
|
+
const m = /^\[([^\]]+)\] (.*)$/.exec(l);
|
|
354
|
+
if (m && !seen.has(m[1])) { seen.add(m[1]); named.push({ ref: m[1], text: m[2] }); }
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
named.sort((a, b) => cmp(a.ref, b.ref));
|
|
358
|
+
if (named.length === 0) lines.push('- No memory note names a path in this module.');
|
|
359
|
+
for (const n of named.slice(0, SHOW_NOTES)) lines.push(bullet(`[${n.ref}] ${n.text.length > 230 ? `${n.text.slice(0, 227)}...` : n.text}`));
|
|
360
|
+
lines.push('');
|
|
361
|
+
lines.push('## Evidence');
|
|
362
|
+
lines.push(bullet(`built from: .cache/index/edit-graph.json, .cache/index/symbols.json, ${fp.count} file(s) under \`${d}\``));
|
|
363
|
+
if (named.length) lines.push(bullet(`notes: ${named.slice(0, SHOW_NOTES).map((n) => n.ref).join(', ')}`));
|
|
364
|
+
lines.push('');
|
|
365
|
+
lines.push('## Agent notes (fill in; kept across rebuilds)');
|
|
366
|
+
lines.push(AGENT_START);
|
|
367
|
+
lines.push(agentSection(existingText) || AGENT_PLACEHOLDER);
|
|
368
|
+
lines.push(AGENT_END);
|
|
369
|
+
lines.push('');
|
|
370
|
+
// The generator guarantees its own cap: drop the last bullet of the least valuable
|
|
371
|
+
// section (notes, then purpose lines, then key files, then entry points) until it fits.
|
|
372
|
+
const size = () => Buffer.byteLength(lines.join('\n'));
|
|
373
|
+
for (const title of ['Notes naming this module', 'What it is for', 'Key files', 'Entry points (imported from outside the module)']) {
|
|
374
|
+
while (size() > TOPIC_PAGE_MAX_BYTES) {
|
|
375
|
+
const at = lines.indexOf(`## ${title}`);
|
|
376
|
+
let end = at + 1;
|
|
377
|
+
while (end < lines.length && !lines[end].startsWith('## ') && lines[end] !== '') end += 1;
|
|
378
|
+
if (at < 0 || end - at <= 2) break;
|
|
379
|
+
lines.splice(end - 1, 1);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
const text = lines.join('\n');
|
|
383
|
+
if (Buffer.byteLength(text) > TOPIC_PAGE_MAX_BYTES) {
|
|
384
|
+
throw new Error(`Topic page for ${d} would exceed ${TOPIC_PAGE_MAX_BYTES} bytes; pick a smaller module`);
|
|
385
|
+
}
|
|
386
|
+
return { rel: topicRel(d), text, fingerprint: fp.fingerprint, count: fp.count, truncated: false, bytes: Buffer.byteLength(text) };
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
export async function writeTopicPage(root, dir, notes = []) {
|
|
390
|
+
const rel = topicRel(dir);
|
|
391
|
+
const abs = path.resolve(root, rel);
|
|
392
|
+
// Refuse symlinks anywhere in docs/memory/topics (and the page itself) before any write.
|
|
393
|
+
for (const p of TOPIC_CHAIN) {
|
|
394
|
+
const st = await fsp.lstat(path.resolve(root, p)).catch(() => null);
|
|
395
|
+
if (st && (st.isSymbolicLink() || !st.isDirectory())) throw new Error(`Refusing to write: ${p} is a symlink or not a directory`);
|
|
396
|
+
}
|
|
397
|
+
const pst = await fsp.lstat(abs).catch(() => null);
|
|
398
|
+
if (pst && (pst.isSymbolicLink() || !pst.isFile())) throw new Error(`Refusing to write: ${rel} is a symlink or not a regular file`);
|
|
399
|
+
const existing = pst ? (await readRegular(abs)) ?? '' : '';
|
|
400
|
+
const built = await buildTopicPage({ root, dir, notes, existingText: existing });
|
|
401
|
+
await fsp.mkdir(path.dirname(abs), { recursive: true });
|
|
402
|
+
if (!(await topicsDirOk(root))) throw new Error(`Refusing to write: ${TOPIC_DIR} does not resolve inside the project`);
|
|
403
|
+
const tmp = path.join(path.dirname(abs), `.${topicSlug(dir)}.${process.pid}.${crypto.randomBytes(4).toString('hex')}.tmp`);
|
|
404
|
+
try {
|
|
405
|
+
await fsp.writeFile(tmp, built.text, { encoding: 'utf8', flag: 'wx' });
|
|
406
|
+
await fsp.rename(tmp, abs);
|
|
407
|
+
} catch (error) {
|
|
408
|
+
await fsp.unlink(tmp).catch(() => {});
|
|
409
|
+
throw error;
|
|
410
|
+
}
|
|
411
|
+
return built;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// refreshTopics(root) -> { rebuilt: [rel], failed: [rel] }: rebuild only pages that
|
|
415
|
+
// already exist and are stale (never creates a page); the agent block and the cited
|
|
416
|
+
// notes are carried over. Called from the index refresh, never from a hook.
|
|
417
|
+
export async function refreshTopics(root) {
|
|
418
|
+
const rebuilt = [];
|
|
419
|
+
const failed = [];
|
|
420
|
+
for (const rel of await listTopicRels(root)) {
|
|
421
|
+
try {
|
|
422
|
+
const page = await readTopicPage(root, rel);
|
|
423
|
+
if (!page || page.missing || !page.stale) continue;
|
|
424
|
+
await writeTopicPage(root, page.dir, []);
|
|
425
|
+
rebuilt.push(rel);
|
|
426
|
+
} catch {
|
|
427
|
+
failed.push(rel);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return { rebuilt, failed };
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// ---- recall documents -----------------------------------------------------
|
|
434
|
+
|
|
435
|
+
const sectionLines = (text, title) => {
|
|
436
|
+
const lines = String(text).split(/\r?\n/);
|
|
437
|
+
const at = lines.findIndex((l) => l.trim() === `## ${title}`);
|
|
438
|
+
if (at < 0) return [];
|
|
439
|
+
const out = [];
|
|
440
|
+
for (const l of lines.slice(at + 1)) { if (/^##\s/.test(l)) break; if (/^- /.test(l)) out.push(l.slice(2)); }
|
|
441
|
+
return out;
|
|
442
|
+
};
|
|
443
|
+
|
|
444
|
+
// One recall document per page: title, purpose, key files, entry points, tests.
|
|
445
|
+
// type 'markdown' + kind 'topic' so it flows through every existing lane; a stale
|
|
446
|
+
// page carries validity 'changed' (or 'missing'), which the renderers label and
|
|
447
|
+
// the engine demotes. Never throws.
|
|
448
|
+
export async function loadTopicDocs(root, { maxPages = TOPIC_MAX_PAGES } = {}) {
|
|
449
|
+
const docs = [];
|
|
450
|
+
try {
|
|
451
|
+
for (const rel of (await listTopicRels(root)).slice(0, maxPages)) {
|
|
452
|
+
const page = await readTopicPage(root, rel);
|
|
453
|
+
if (!page) continue;
|
|
454
|
+
const strip = (s) => s.replace(/`/g, '');
|
|
455
|
+
// Rendered recall lines cut at ~160 chars, so the cheap-to-read part comes
|
|
456
|
+
// first: the module, then its key files by name relative to the module.
|
|
457
|
+
const rel2 = (f) => (f.startsWith(`${page.dir}/`) ? f.slice(page.dir.length + 1) : f);
|
|
458
|
+
const keyNames = sectionLines(page.text, 'Key files').slice(0, 10)
|
|
459
|
+
.map((l) => rel2(strip(l).split(' exports ')[0].split(';')[0].trim())).filter((f) => f && !f.startsWith('...'));
|
|
460
|
+
const parts = [
|
|
461
|
+
`Module ${page.dir} key files: ${keyNames.join(', ')}.`,
|
|
462
|
+
// Identifier-level text only: prose header comments added generic words
|
|
463
|
+
// ("mode", "toggle") that let a long page clear the floor on off-topic queries.
|
|
464
|
+
...sectionLines(page.text, 'Key files').slice(0, 10)
|
|
465
|
+
.map((l) => / exports ([^;]+)/.exec(strip(l))?.[1]).filter(Boolean)
|
|
466
|
+
// The doc tokenizer does not split camelCase (queries are split), so spell the words out.
|
|
467
|
+
.map((e) => `exports ${e.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2').toLowerCase()}.`),
|
|
468
|
+
...(page.agentText ? [`Agent notes: ${flat(page.agentText)}`] : []),
|
|
469
|
+
];
|
|
470
|
+
docs.push({
|
|
471
|
+
id: `topic:${topicSlug(page.dir)}`, type: 'markdown', source: 'markdown', kind: 'topic',
|
|
472
|
+
text: flat(parts.join(' ')).slice(0, 1400), provenance: `${rel}:1`, line: 1, heading: `Topic: ${page.dir}`,
|
|
473
|
+
created_at: 0, ...(page.stale ? { validity: page.missing ? 'missing' : 'changed' } : {}),
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
} catch {
|
|
477
|
+
return docs;
|
|
478
|
+
}
|
|
479
|
+
return docs;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// Config gate: UKIT_MEMORY_TOPICS=1|0 wins over memory.topics.recall (default on:
|
|
483
|
+
// gold topic set 0/12 -> 9/12 with headline unchanged; inert while no page exists).
|
|
484
|
+
export function topicsRecallEnabled(config) {
|
|
485
|
+
const env = process.env.UKIT_MEMORY_TOPICS;
|
|
486
|
+
if (env === '1') return true;
|
|
487
|
+
if (env === '0') return false;
|
|
488
|
+
return config?.memory?.topics?.recall !== false;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// findTopicFor(root, relFile, { checkStale }) -> { rel, dir, stale } | null: the page
|
|
492
|
+
// of the deepest ancestor directory of `relFile` that has one. Probes one path per
|
|
493
|
+
// ancestor (the slug is derived from the directory) and reads one 600-byte header;
|
|
494
|
+
// with checkStale false (hooks) it never hashes the module and `stale` is null.
|
|
495
|
+
export async function findTopicFor(root, relFile, { checkStale = true } = {}) {
|
|
496
|
+
try {
|
|
497
|
+
const parts = norm(relFile).split('/');
|
|
498
|
+
if (!(await topicsDirOk(root))) return null;
|
|
499
|
+
for (let n = parts.length - 1; n >= 1; n -= 1) {
|
|
500
|
+
const dir = parts.slice(0, n).join('/');
|
|
501
|
+
if (!validTopicDir(dir)) continue;
|
|
502
|
+
const rel = topicRel(dir);
|
|
503
|
+
const head = (await readRegular(path.resolve(root, rel), { head: 600 })) ?? '';
|
|
504
|
+
const header = parseTopicHeader(head.split('\n').slice(0, 3).join('\n'));
|
|
505
|
+
if (!header || header.dir !== dir) continue;
|
|
506
|
+
if (!checkStale) return { rel, dir, stale: null };
|
|
507
|
+
const now = await fingerprintDir(root, dir);
|
|
508
|
+
return { rel, dir, stale: !now.exists || now.fingerprint !== header.fingerprint };
|
|
509
|
+
}
|
|
510
|
+
return null;
|
|
511
|
+
} catch {
|
|
512
|
+
return null;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// ---- check ----------------------------------------------------------------
|
|
517
|
+
|
|
518
|
+
// checkTopics(root, { scan }) -> { violations, warnings, pages }. Oversize,
|
|
519
|
+
// over-long lines, secrets (injected `scan(text) -> {labels}`, like memoryCheck) and
|
|
520
|
+
// an unreadable header are violations; a stale or missing-dir fingerprint is a warning.
|
|
521
|
+
export async function checkTopics(root, { scan = null } = {}) {
|
|
522
|
+
const violations = [];
|
|
523
|
+
const warnings = [];
|
|
524
|
+
const rels = await listTopicRels(root);
|
|
525
|
+
for (const rel of rels) {
|
|
526
|
+
const text = await readTopicFile(root, rel);
|
|
527
|
+
if (text === null) continue;
|
|
528
|
+
const bytes = Buffer.byteLength(text);
|
|
529
|
+
if (!parseTopicHeader(text)) { violations.push({ file: rel, line: 1, code: 'topic-bad-header', message: 'missing or malformed ukit-topic header' }); continue; }
|
|
530
|
+
if (bytes > TOPIC_PAGE_MAX_BYTES) violations.push({ file: rel, line: 1, code: 'topic-over-bytes', message: `${bytes} bytes exceeds ${TOPIC_PAGE_MAX_BYTES}` });
|
|
531
|
+
text.split(/\r?\n/).forEach((l, i) => {
|
|
532
|
+
if (l.length > TOPIC_LINE_MAX && !HEADER_RE.test(l)) violations.push({ file: rel, line: i + 1, code: 'topic-line-too-long', message: `${l.length} chars exceeds ${TOPIC_LINE_MAX}` });
|
|
533
|
+
});
|
|
534
|
+
if (typeof scan === 'function') {
|
|
535
|
+
try {
|
|
536
|
+
const res = scan(text);
|
|
537
|
+
if ((res?.labels?.length ?? 0) > 0) violations.push({ file: rel, line: 1, code: 'secret-detected', message: 'a secret-like value is present' });
|
|
538
|
+
} catch { /* scanner failure is not a violation */ }
|
|
539
|
+
}
|
|
540
|
+
const page = await readTopicPage(root, rel);
|
|
541
|
+
if (page?.missing) warnings.push({ file: rel, line: 1, code: 'topic-dir-missing', message: `${page.dir} no longer exists` });
|
|
542
|
+
else if (page?.stale) warnings.push({ file: rel, line: 1, code: 'topic-stale', message: `${page.dir} changed since the page was built; rebuild with ukit memory topic build ${page.dir}` });
|
|
543
|
+
}
|
|
544
|
+
return { violations, warnings, pages: rels.length };
|
|
545
|
+
}
|