@ngockhoale/ukit 3.5.3 → 3.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/manifests/documentation.yaml +12 -1
- package/manifests/platform.full.yaml +12 -0
- package/package.json +29 -2
- package/src/cli/commands/doctor.js +41 -0
- package/src/cli/commands/indexTools.js +10 -4
- package/src/cli/commands/memory.js +88 -6
- package/src/core/memory/episodes.js +106 -7
- package/src/core/memory/memoryCheck.js +241 -0
- package/src/core/memory/memoryCheckRun.js +39 -0
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/mutateMemory.js +37 -1
- package/src/core/memory/observations.js +192 -0
- package/src/core/memory/recallEngine.js +159 -10
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/records.js +23 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/memory/writeGate.js +137 -0
- package/src/core/runtimeConfig.js +56 -1
- package/src/decision/registry.js +17 -0
- package/src/index/buildIndex.js +312 -5
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +4 -1
- package/src/index/queryIndex.js +356 -14
- package/src/index/resolveContext.js +1 -1
- package/template_project/.claude/hooks/edit-hint.mjs +64 -17
- package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
- package/template_project/.claude/hooks/session-episode.sh +9 -4
- package/template_project/.claude/settings.json +2 -2
- package/template_project/.claude/ukit/index/lib/index-core.mjs +769 -18
- package/template_project/.claude/ukit/index/query-index.mjs +14 -3
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/index/route-task.mjs +4 -1
- package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +159 -10
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/.claude/ukit/runtime/memory-validity.mjs +28 -1
- package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
- package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
- package/template_project/ukit/storage/config.json +21 -2
- package/manifests/instructionRules.yaml +0 -442
- package/src/core/agentRuntime/completionGate.js +0 -179
- package/src/core/agentRuntime/context.js +0 -571
- package/src/core/agentRuntime/qualityComparison.js +0 -112
- package/src/core/agentRuntime/resourcePolicy.js +0 -78
- package/src/core/agentRuntime/runtimeSupport.js +0 -255
- package/src/core/codeintel/manifest.js +0 -176
- package/src/core/experiments/deliberation.js +0 -327
- package/src/core/experiments/dynamicWorkflow.js +0 -507
- package/src/core/handoffDocValidator.js +0 -154
- package/src/core/memory/deltaOverlays.js +0 -493
- package/src/core/memory/learningCandidates.js +0 -347
- package/src/core/observability/emit/sessionBoot.js +0 -407
- package/src/core/observability/evaluation/replay.js +0 -149
- package/src/core/observability/evaluation/scorecard.js +0 -465
- package/src/core/observability/schema/compatibility.js +0 -180
- package/src/core/observability/support/renderer.js +0 -130
- package/src/core/reviewPanelAggregate.js +0 -204
- package/src/core/taskBudgetValidator.js +0 -251
- package/src/core/taskProgressGuard.js +0 -227
- package/src/decision/reviewVerdict.js +0 -339
- package/src/decision/runtimeFilter.js +0 -150
- package/src/decision/runtimeScheduler.js +0 -239
- package/src/index/contextPack.js +0 -285
- package/src/index/impactContext.js +0 -360
- package/src/index/reviewPolicy.js +0 -368
- package/src/index/sensitivePath.js +0 -21
|
@@ -53,12 +53,15 @@ if (!query && !symbolQuery) {
|
|
|
53
53
|
query,
|
|
54
54
|
limit: effectiveLimit,
|
|
55
55
|
});
|
|
56
|
+
// The status rides on the array as a non-enumerable prop, so the cache stores it explicitly.
|
|
57
|
+
const contentLaneTruncated = cached ? (cached.contentLaneTruncated ?? []) : [...(results.contentLaneTruncated ?? [])];
|
|
56
58
|
|
|
57
59
|
if (!cached) {
|
|
58
60
|
await writeRecentCacheEntry(cachePath, {
|
|
59
61
|
requestKey,
|
|
60
62
|
updatedAt: Date.now(),
|
|
61
63
|
result: results,
|
|
64
|
+
contentLaneTruncated,
|
|
62
65
|
}, {
|
|
63
66
|
maxEntries: DEFAULT_RECENT_CACHE_MAX_ENTRIES,
|
|
64
67
|
});
|
|
@@ -71,6 +74,7 @@ if (!query && !symbolQuery) {
|
|
|
71
74
|
|
|
72
75
|
if (existingResults.length === 0) {
|
|
73
76
|
console.log('[ukit:index:query] no matches');
|
|
77
|
+
printContentLaneNotice(contentLaneTruncated);
|
|
74
78
|
} else {
|
|
75
79
|
console.log(`[ukit:index:query] top ${existingResults.length} matches for: ${query}`);
|
|
76
80
|
for (const [idx, item] of existingResults.entries()) {
|
|
@@ -82,9 +86,10 @@ if (!query && !symbolQuery) {
|
|
|
82
86
|
console.log(` reasons: ${item.reasons.join(', ')}`);
|
|
83
87
|
}
|
|
84
88
|
if (showOutline && idx < OUTLINE_TOP_FILES_LIMIT) {
|
|
85
|
-
await printOutlineForFile(rootDir, item.filePath);
|
|
89
|
+
await printOutlineForFile(rootDir, item.filePath, indexCore.queryMatchedSymbolNames?.(item) ?? null);
|
|
86
90
|
}
|
|
87
91
|
}
|
|
92
|
+
printContentLaneNotice(contentLaneTruncated);
|
|
88
93
|
}
|
|
89
94
|
}
|
|
90
95
|
|
|
@@ -98,6 +103,12 @@ catch (error) {
|
|
|
98
103
|
}
|
|
99
104
|
}
|
|
100
105
|
|
|
106
|
+
function printContentLaneNotice(truncated) {
|
|
107
|
+
if (Array.isArray(truncated) && truncated.length > 0) {
|
|
108
|
+
console.log(`content-lane: truncated (${truncated.join(',')}) — results may be incomplete`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
101
112
|
async function runSymbolMode(symbolQuery) {
|
|
102
113
|
try {
|
|
103
114
|
await ensureFreshIndex({ rootDir, logPrefix: 'index:query' });
|
|
@@ -177,11 +188,11 @@ async function fileExists(absolutePath) {
|
|
|
177
188
|
}
|
|
178
189
|
}
|
|
179
190
|
|
|
180
|
-
async function printOutlineForFile(root, filePath) {
|
|
191
|
+
async function printOutlineForFile(root, filePath, focus = null) {
|
|
181
192
|
if (typeof getFileOutline !== 'function') {
|
|
182
193
|
return;
|
|
183
194
|
}
|
|
184
|
-
const outline = await getFileOutline({ rootDir: root, filePath, limit: Infinity });
|
|
195
|
+
const outline = await getFileOutline({ rootDir: root, filePath, limit: Infinity, focus });
|
|
185
196
|
if (outline.length === 0) {
|
|
186
197
|
return;
|
|
187
198
|
}
|
|
@@ -74,6 +74,17 @@ if (changedFiles.length > 0 && !force) {
|
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
// 3.5.5: keep existing topic pages current. Rebuilds only pages that already exist
|
|
78
|
+
// and are stale (agent block and cited notes preserved); never creates pages; never
|
|
79
|
+
// fails the refresh. Runs here (git hook, backgrounded + deadline-armed), not in a hook path.
|
|
80
|
+
try {
|
|
81
|
+
if (process.env.UKIT_MEMORY_TOPICS !== '0') {
|
|
82
|
+
const { refreshTopics } = await import('../runtime/memory-topics.mjs');
|
|
83
|
+
const res = await refreshTopics(rootDir);
|
|
84
|
+
if (res.rebuilt.length > 0) console.log(`topics-rebuilt: ${res.rebuilt.length}`);
|
|
85
|
+
}
|
|
86
|
+
} catch { /* topic pages are optional */ }
|
|
87
|
+
|
|
77
88
|
function getRootDir(argv) {
|
|
78
89
|
const value = readFlagValue(argv, '--root');
|
|
79
90
|
return value ? path.resolve(value) : process.cwd();
|
|
@@ -1737,8 +1737,11 @@ async function main() {
|
|
|
1737
1737
|
// with UKIT_SELF_IMPROVE=0 (tests/CI) or learning.selfImprove.enabled:false.
|
|
1738
1738
|
if (process.env.UKIT_SELF_IMPROVE !== '0') {
|
|
1739
1739
|
try {
|
|
1740
|
-
const { triggerSelfImprove } = await import('../runtime/self-improve-trigger.mjs');
|
|
1740
|
+
const { triggerSelfImprove, triggerEpisodeStart } = await import('../runtime/self-improve-trigger.mjs');
|
|
1741
1741
|
triggerSelfImprove({ projectRoot: rootDir });
|
|
1742
|
+
// Codex has no SessionStart/SessionEnd: the routed prompt is its only lifecycle point.
|
|
1743
|
+
// Same 10 min marker as the other engines, so later prompts pay one async read.
|
|
1744
|
+
await triggerEpisodeStart({ projectRoot: rootDir });
|
|
1742
1745
|
} catch {
|
|
1743
1746
|
// background work only — a missing/broken trigger never affects routing
|
|
1744
1747
|
}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
// memoryCheck.js — memory rules + deterministic checker for memory source files
|
|
2
|
+
// (3.5.4). ONE file, node: builtins only, shipped byte-identical as
|
|
3
|
+
// template_project/.claude/ukit/runtime/memory-check.mjs (the PostToolUse advisory
|
|
4
|
+
// and Codex-less hosts use the mirror; `ukit memory check` and the record write
|
|
5
|
+
// gate use this copy). The recall engine and the secret scanner are injected, so
|
|
6
|
+
// there is exactly one tokenizer and one scanner. Async fs only; never throws.
|
|
7
|
+
//
|
|
8
|
+
// violation (exit 1) warning (exit 0)
|
|
9
|
+
// core-over-lines / core-over-bytes no-evidence (bullet names no existing
|
|
10
|
+
// line-too-long (core) / bullet-too-long file path or commit)
|
|
11
|
+
// secret-detected / near-duplicate
|
|
12
|
+
// unknown-supersedes / duplicate-id
|
|
13
|
+
//
|
|
14
|
+
// no-evidence is only a warning for markdown: pre-3.5.4 notes without a path are
|
|
15
|
+
// not rejected retroactively. Record writes (writeGate.js) enforce it.
|
|
16
|
+
import fsp from 'node:fs/promises';
|
|
17
|
+
import path from 'node:path';
|
|
18
|
+
|
|
19
|
+
// Caps. Measured on the live corpus before the split: longest hand-written
|
|
20
|
+
// bullet 1315 chars, longest record text 114, episodes sliced to 900.
|
|
21
|
+
export const RECORD_TEXT_MAX = 1200;
|
|
22
|
+
export const CORE_MAX_LINES = 150;
|
|
23
|
+
export const CORE_MAX_BYTES = 12000; // "12 KB" core cap, decimal so 12,000 is a hard number
|
|
24
|
+
export const CORE_LINE_MAX = 400; // one line per item in the core
|
|
25
|
+
export const DETAIL_BULLET_MAX = 1500; // verbatim detail bullets in docs/memory/*
|
|
26
|
+
export const CORE_FILE = 'docs/MEMORY.md';
|
|
27
|
+
|
|
28
|
+
// Lexical near-duplicate: Jaccard over the BM25 tokens. Measured (3.5.4) on
|
|
29
|
+
// docs/MEMORY.md (68 notes) + live records: the highest similarity between two
|
|
30
|
+
// distinct hand-written notes is 0.250 (markdown/markdown), 0.080 (markdown/
|
|
31
|
+
// record); 0.80 flags only near-restatements. Generated "Verification pattern
|
|
32
|
+
// ... failed N x" records reach 0.917 with each other, so generated writers are
|
|
33
|
+
// exempt in writeGate.js (GENERATED_CREATORS) and never rejected.
|
|
34
|
+
export const NEAR_DUP_JACCARD = 0.8;
|
|
35
|
+
export const NEAR_DUP_MIN_TOKENS = 6;
|
|
36
|
+
|
|
37
|
+
// Machine-generated writers: templated text, expected to resemble each other.
|
|
38
|
+
export const GENERATED_CREATORS = Object.freeze(new Set([
|
|
39
|
+
'episode-hook', 'propose', 'learning-observe', 'learning-candidate',
|
|
40
|
+
'delta-overlay', 'migration', 'system', 'observe',
|
|
41
|
+
]));
|
|
42
|
+
|
|
43
|
+
export function jaccard(aSet, bSet) {
|
|
44
|
+
if (aSet.size === 0 || bSet.size === 0) return 0;
|
|
45
|
+
let inter = 0;
|
|
46
|
+
for (const t of aSet) if (bSet.has(t)) inter += 1;
|
|
47
|
+
return inter / (aSet.size + bSet.size - inter);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* findNearDuplicate(text, candidates, tokenize, threshold) -> { id, similarity } | null
|
|
52
|
+
* candidates: [{ id, text }]. Notes under NEAR_DUP_MIN_TOKENS tokens are skipped
|
|
53
|
+
* (a short note shares most of its tokens by chance).
|
|
54
|
+
*/
|
|
55
|
+
export function findNearDuplicate(text, candidates, tokenize, threshold = NEAR_DUP_JACCARD) {
|
|
56
|
+
const a = new Set(tokenize(String(text ?? '')));
|
|
57
|
+
if (a.size < NEAR_DUP_MIN_TOKENS) return null;
|
|
58
|
+
let best = null;
|
|
59
|
+
for (const c of candidates) {
|
|
60
|
+
const b = new Set(tokenize(String(c?.text ?? '')));
|
|
61
|
+
if (b.size < NEAR_DUP_MIN_TOKENS) continue;
|
|
62
|
+
const similarity = jaccard(a, b);
|
|
63
|
+
if (similarity >= threshold && (!best || similarity > best.similarity)) best = { id: c.id, similarity };
|
|
64
|
+
}
|
|
65
|
+
return best;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Markdown supersession convention (honoured by recallEngine.parseMarkdownBullets):
|
|
69
|
+
// [id: slug] names a bullet (slug = [a-z0-9][a-z0-9._-]*)
|
|
70
|
+
// [supersedes: slug] the bullet replaces the bullet(s) so named
|
|
71
|
+
const MD_ID_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
|
|
72
|
+
// dir/file paths, or a bare file name with a known extension
|
|
73
|
+
const PATH_WORD_RE = /(?<![\w./@-])(?:[\w@.-]+\/)+[\w@.-]+(?::\d+)?|(?<![\w./@-])[\w-]+\.(?:m?js|cjs|json|md|sh|ya?ml|ts|py)\b/g;
|
|
74
|
+
const SHA_RE = /\b[0-9a-f]{7,40}\b/;
|
|
75
|
+
|
|
76
|
+
async function exists(root, rel) {
|
|
77
|
+
const clean = rel.replace(/[:#].*$/, '').replace(/^\.\//, '');
|
|
78
|
+
if (!clean || clean.startsWith('/') || clean.split('/').includes('..')) return false;
|
|
79
|
+
try {
|
|
80
|
+
await fsp.stat(path.join(root, clean));
|
|
81
|
+
return true;
|
|
82
|
+
} catch {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// A note is "about code" when it names a code token (camelCase, snake_case, call(),
|
|
88
|
+
// --flag, dotted config key). Owner preferences, process rules and release history
|
|
89
|
+
// name none, cannot point at a file, and never warn. The heading skip covers prose
|
|
90
|
+
// sections whose notes happen to mention a flag.
|
|
91
|
+
const CODE_TOKEN_RE = /\b[a-z]+[A-Z]\w*\b|\b[a-z0-9]+_[a-z0-9_]+\b|\b\w+\(\)|(?<![\w-])--[a-z][\w-]+|\b[a-z]+\.[a-z]+\.[a-z]\w*\b|\b[\w-]+\.(?:m?js|cjs|json|sh|ya?ml|ts)\b/;
|
|
92
|
+
const PROSE_HEADING_RE = /protocol|release history|preference|process|owner/i;
|
|
93
|
+
export function isCodeNote(doc) {
|
|
94
|
+
return !PROSE_HEADING_RE.test(doc?.heading ?? '') && CODE_TOKEN_RE.test(doc?.text ?? '');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** A bullet is evidenced when it names an existing project path (backticked) or a commit. */
|
|
98
|
+
export async function bulletNamesExistingFile(root, text) {
|
|
99
|
+
// Parsed bullet text has its backticks stripped, so match path-shaped words.
|
|
100
|
+
let tried = 0;
|
|
101
|
+
for (const m of text.matchAll(PATH_WORD_RE)) {
|
|
102
|
+
if (tried >= 12) break;
|
|
103
|
+
tried += 1;
|
|
104
|
+
if (await exists(root, m[0])) return true;
|
|
105
|
+
}
|
|
106
|
+
return SHA_RE.test(text) && /\b(commit|sha)\b/i.test(text);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const DETAIL_RE = /Detail:\s+(\S+\.md)/g;
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* missingDetailPointers(root, text) -> [{ line, rel, why }]: core lines whose
|
|
113
|
+
* `Detail: <file>.md` pointer escapes the project root or names a missing file.
|
|
114
|
+
*/
|
|
115
|
+
export async function missingDetailPointers(root, text) {
|
|
116
|
+
const out = [];
|
|
117
|
+
const base = path.resolve(root);
|
|
118
|
+
const lines = String(text).split('\n');
|
|
119
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
120
|
+
for (const m of lines[i].replace(/`/g, '').matchAll(DETAIL_RE)) {
|
|
121
|
+
const abs = path.resolve(base, m[1]);
|
|
122
|
+
if (abs !== base && !abs.startsWith(base + path.sep)) { out.push({ line: i + 1, rel: m[1], why: 'escapes the project root' }); continue; }
|
|
123
|
+
try { await fsp.stat(abs); } catch { out.push({ line: i + 1, rel: m[1], why: 'does not exist' }); }
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* checkMemorySources(projectRoot, { sources, engine, scan, files }) ->
|
|
131
|
+
* { ok, violations: [{file,line,code,message}], warnings, stats }
|
|
132
|
+
* sources: relative .md paths (normalizeMemorySources). engine: { tokenize,
|
|
133
|
+
* parseMarkdownBullets }. scan: (text) -> { labels: string[] } (optional).
|
|
134
|
+
* files: limit the report to these paths (every source is still read, so
|
|
135
|
+
* cross-file duplicates and supersession slugs resolve).
|
|
136
|
+
*/
|
|
137
|
+
export async function checkMemorySources(projectRoot, { sources, engine, scan = null, files = null } = {}) {
|
|
138
|
+
const root = path.resolve(projectRoot);
|
|
139
|
+
const violations = [];
|
|
140
|
+
const warnings = [];
|
|
141
|
+
const stats = { files: 0, bullets: 0, coreLines: 0, coreBytes: 0 };
|
|
142
|
+
const all = [];
|
|
143
|
+
const raw = new Map();
|
|
144
|
+
for (const rel of sources ?? []) {
|
|
145
|
+
let text;
|
|
146
|
+
try { text = await fsp.readFile(path.join(root, rel), 'utf8'); } catch { continue; }
|
|
147
|
+
stats.files += 1;
|
|
148
|
+
raw.set(rel, text);
|
|
149
|
+
for (const doc of engine.parseMarkdownBullets(text, rel)) all.push({ ...doc, rel });
|
|
150
|
+
}
|
|
151
|
+
const focus = (rel) => !files || files.includes(rel);
|
|
152
|
+
const add = (list, rel, line, code, message) => { if (focus(rel)) list.push({ file: rel, line, code, message }); };
|
|
153
|
+
|
|
154
|
+
const core = raw.get(CORE_FILE);
|
|
155
|
+
if (core !== undefined) {
|
|
156
|
+
const lines = core.split('\n');
|
|
157
|
+
const lineCount = core.endsWith('\n') ? lines.length - 1 : lines.length;
|
|
158
|
+
const bytes = Buffer.byteLength(core, 'utf8');
|
|
159
|
+
stats.coreLines = lineCount;
|
|
160
|
+
stats.coreBytes = bytes;
|
|
161
|
+
if (lineCount > CORE_MAX_LINES) {
|
|
162
|
+
add(violations, CORE_FILE, lineCount, 'core-over-lines',
|
|
163
|
+
`core has ${lineCount} lines (max ${CORE_MAX_LINES}); merge items or move detail to docs/memory/`);
|
|
164
|
+
}
|
|
165
|
+
if (bytes > CORE_MAX_BYTES) {
|
|
166
|
+
add(violations, CORE_FILE, 1, 'core-over-bytes',
|
|
167
|
+
`core is ${bytes} bytes (max ${CORE_MAX_BYTES}); merge items or move detail to docs/memory/`);
|
|
168
|
+
}
|
|
169
|
+
for (const p of await missingDetailPointers(root, core)) {
|
|
170
|
+
add(violations, CORE_FILE, p.line, 'missing-detail', `Detail pointer ${p.rel} ${p.why}; point at an existing file under docs/memory/`);
|
|
171
|
+
}
|
|
172
|
+
lines.forEach((line, i) => {
|
|
173
|
+
if (/^[-*]\s/.test(line) && line.length > CORE_LINE_MAX) {
|
|
174
|
+
add(violations, CORE_FILE, i + 1, 'line-too-long',
|
|
175
|
+
`core item is ${line.length} chars (max ${CORE_LINE_MAX}); keep one line, move detail to docs/memory/`);
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
for (const [rel, text] of raw) {
|
|
180
|
+
if (rel === CORE_FILE) continue;
|
|
181
|
+
text.split('\n').forEach((line, i) => {
|
|
182
|
+
if (/^[-*]\s/.test(line) && line.length > DETAIL_BULLET_MAX) {
|
|
183
|
+
add(violations, rel, i + 1, 'bullet-too-long', `bullet is ${line.length} chars (max ${DETAIL_BULLET_MAX}); split it`);
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
stats.bullets = all.length;
|
|
189
|
+
const slugs = new Map();
|
|
190
|
+
for (const [rel, text] of raw) {
|
|
191
|
+
text.split('\n').forEach((line, i) => {
|
|
192
|
+
for (const m of line.matchAll(MD_ID_RE)) {
|
|
193
|
+
const slug = m[1].toLowerCase();
|
|
194
|
+
if (slugs.has(slug)) add(violations, rel, i + 1, 'duplicate-id', `id "${slug}" already used at ${slugs.get(slug)}`);
|
|
195
|
+
else slugs.set(slug, `${rel}:${i + 1}`);
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
for (const doc of all) {
|
|
200
|
+
for (const slug of doc.supersedesSlugs ?? []) {
|
|
201
|
+
if (!slugs.has(slug)) add(violations, doc.rel, doc.line, 'unknown-supersedes', `supersedes "${slug}" but no bullet has [id: ${slug}]`);
|
|
202
|
+
}
|
|
203
|
+
if (scan && focus(doc.rel)) {
|
|
204
|
+
let labels = [];
|
|
205
|
+
try { labels = scan(doc.text)?.labels ?? []; } catch { labels = []; }
|
|
206
|
+
if (labels.length > 0) add(violations, doc.rel, doc.line, 'secret-detected', `secret-like value (${labels.join(', ')}); remove it`);
|
|
207
|
+
}
|
|
208
|
+
if (focus(doc.rel) && isCodeNote(doc) && !(await bulletNamesExistingFile(root, doc.text))) {
|
|
209
|
+
add(warnings, doc.rel, doc.line, 'no-evidence', 'names no existing file path or commit; add the path this note is about');
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// A superseded bullet may legitimately resemble its replacement.
|
|
213
|
+
const replaced = new Set();
|
|
214
|
+
for (const doc of all) for (const s of doc.supersedesSlugs ?? []) { const t = slugs.get(s); if (t) replaced.add(t); }
|
|
215
|
+
const live = all.filter((d) => !replaced.has(`${d.rel}:${d.line}`));
|
|
216
|
+
const seen = new Set();
|
|
217
|
+
const tokenCache = new Map();
|
|
218
|
+
const memoTokenize = (t) => {
|
|
219
|
+
let v = tokenCache.get(t);
|
|
220
|
+
if (!v) { v = engine.tokenize(t); tokenCache.set(t, v); }
|
|
221
|
+
return v;
|
|
222
|
+
};
|
|
223
|
+
for (let i = 0; i < live.length; i += 1) {
|
|
224
|
+
if (!focus(live[i].rel)) continue;
|
|
225
|
+
const dup = findNearDuplicate(live[i].text, live.slice(0, i).map((d) => ({ id: `${d.rel}:${d.line}`, text: d.text })), memoTokenize);
|
|
226
|
+
if (dup && !seen.has(dup.id)) {
|
|
227
|
+
seen.add(dup.id);
|
|
228
|
+
add(violations, live[i].rel, live[i].line, 'near-duplicate',
|
|
229
|
+
`${Math.round(dup.similarity * 100)}% lexical overlap with ${dup.id}; merge them or tag one [supersedes: <slug>]`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return { ok: violations.length === 0, violations, warnings, stats };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** One advisory line for the first violation, '' when clean. */
|
|
236
|
+
export function formatCheckLine(res) {
|
|
237
|
+
const v = res?.violations?.[0];
|
|
238
|
+
if (!v) return '';
|
|
239
|
+
const more = res.violations.length > 1 ? ` (+${res.violations.length - 1} more)` : '';
|
|
240
|
+
return `[UKit memory] ${v.file}:${v.line} ${v.code}: ${v.message}${more}. Run \`ukit memory check\`.`;
|
|
241
|
+
}
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
demoteByValidity, dropSuperseded, fuseRanked, loadMarkdownDocs, normalizeMemorySources, rankDocs, STALE_MARKER,
|
|
7
7
|
} from './memory-recall.mjs';
|
|
8
8
|
import { withValidity } from './memory-validity.mjs';
|
|
9
|
+
import { findTopicFor } from './memory-topics.mjs';
|
|
9
10
|
import { eligible, resolveProjectIdentityMirror } from './memory-policy.mjs';
|
|
10
11
|
import { resolveMemoryStage } from './memory-flags.mjs';
|
|
11
12
|
|
|
@@ -113,10 +114,12 @@ export function recallForTarget({ target, neighbours, docs, query = '', limit =
|
|
|
113
114
|
const flat = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
|
|
114
115
|
const KIND_LABEL = { twin: 'identical at last index', declared: 'declared', drifted: 'differs now' };
|
|
115
116
|
|
|
116
|
-
export function renderEditHint({ target, neighbours, memory = [], maxChars = 600 } = {}) {
|
|
117
|
+
export function renderEditHint({ target, neighbours, memory = [], maxChars = 600, topic = null } = {}) {
|
|
117
118
|
try {
|
|
118
119
|
const n = neighbours ?? {};
|
|
119
120
|
const lines = [];
|
|
121
|
+
// 3.5.5: pointer to the module's topic page (path only; the page is read on request).
|
|
122
|
+
if (topic?.rel) lines.push(`module page: ${flat(topic.rel)} (read it for the module overview; \`ukit memory topic list\` shows if it is stale)`);
|
|
120
123
|
const mirrors = (Array.isArray(n.mirrors) ? n.mirrors : []).map((m) => (typeof m === 'string' ? { path: m, kind: 'declared' } : m))
|
|
121
124
|
.filter((m) => m?.path).sort((a, b) => String(a.path).localeCompare(String(b.path))).slice(0, 2);
|
|
122
125
|
for (const m of mirrors) lines.push(`mirror (${KIND_LABEL[m.kind] ?? 'declared'}): ${flat(m.path)}`);
|
|
@@ -250,7 +253,11 @@ export async function editHintFor({
|
|
|
250
253
|
const drop = new Set(Array.isArray(exclude?.memoryIds) ? exclude.memoryIds : []);
|
|
251
254
|
const docs = [...records, ...markdown].filter((d) => !drop.has(d.id));
|
|
252
255
|
const { ranked } = recallForTarget({ target: rel, neighbours, docs, query });
|
|
253
|
-
|
|
256
|
+
// Topic pointer: one header read per ancestor dir, async, never hashes the module;
|
|
257
|
+
// skipped when the memory budget already missed so the graph part stays fast.
|
|
258
|
+
const topic = memoryMissed || cfg?.memory?.topics?.hint === false || signal?.aborted
|
|
259
|
+
? null : await findTopicFor(projectRoot, rel, { checkStale: false });
|
|
260
|
+
const text = renderEditHint({ target: rel, neighbours, memory: ranked, topic });
|
|
254
261
|
if (!text) return { text: '', neighbours, memoryIds: [], memoryMissed };
|
|
255
262
|
return { text, neighbours, memoryIds: ranked.slice(0, 2).map((e) => e.doc.id), memoryMissed };
|
|
256
263
|
} catch {
|
|
@@ -23,7 +23,12 @@ import fsp from 'node:fs/promises';
|
|
|
23
23
|
import path from 'node:path';
|
|
24
24
|
|
|
25
25
|
export const RECALL_ENGINE_VERSION = 'bm25-1';
|
|
26
|
-
|
|
26
|
+
// Capped core + every detail file of docs/memory/ (3.5.4). An entry ending in
|
|
27
|
+
// `/*.md` names one directory level, expanded by expandMemorySources; a missing
|
|
28
|
+
// file or directory contributes nothing.
|
|
29
|
+
export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md', 'docs/memory/*.md']);
|
|
30
|
+
const DIR_GLOB_RE = /^[^*?[\]{}]+\/\*\.md$/;
|
|
31
|
+
const MAX_DIR_FILES = 32;
|
|
27
32
|
|
|
28
33
|
const K1 = 1.2;
|
|
29
34
|
const B = 0.75;
|
|
@@ -37,6 +42,8 @@ const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
|
|
|
37
42
|
// ... and the IDF mass of the matched tokens reaches this fraction of the
|
|
38
43
|
// largest IDF the corpus can produce (a token present in exactly one doc).
|
|
39
44
|
const MIN_IDF_MASS_RATIO = 0.5;
|
|
45
|
+
// Episodes only: matched IDF mass must reach this fraction of the whole query's.
|
|
46
|
+
const EPISODE_MIN_COVERAGE = 0.5;
|
|
40
47
|
// Results scoring below this fraction of the best hit are noise tail.
|
|
41
48
|
const RELATIVE_CUTOFF = 0.35;
|
|
42
49
|
// Query-dependent type prior: multiplies the lexical score, so it can only
|
|
@@ -46,6 +53,37 @@ const RELATIVE_CUTOFF = 0.35;
|
|
|
46
53
|
const TYPE_PRIOR = Object.freeze({
|
|
47
54
|
project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
|
|
48
55
|
});
|
|
56
|
+
// Core/detail twins (3.5.4): a capped-core one-liner ends in "Detail: <file>" and
|
|
57
|
+
// its verbatim note sits in that file. Both score on the same words, so one fact
|
|
58
|
+
// would spend two result slots. collapseTwins keeps one of each pair that is
|
|
59
|
+
// present in the same ranked list: `keep` 'detail' (default: the full note) or 'core'.
|
|
60
|
+
const DETAIL_POINTER_RE = /\bDetail:\s+(\S+?\.md)\b/;
|
|
61
|
+
const TWIN_MIN_OVERLAP = 0.5;
|
|
62
|
+
export function collapseTwins(entries, keep = 'detail') {
|
|
63
|
+
if (keep === 'off' || !Array.isArray(entries) || entries.length < 2) return entries;
|
|
64
|
+
const drop = new Set();
|
|
65
|
+
for (const core of entries) {
|
|
66
|
+
const doc = core.doc;
|
|
67
|
+
if (doc?.type !== 'markdown' || drop.has(core)) continue;
|
|
68
|
+
const m = DETAIL_POINTER_RE.exec(doc.text ?? '');
|
|
69
|
+
if (!m) continue;
|
|
70
|
+
const coreTokens = new Set(tokenize(doc.text));
|
|
71
|
+
if (coreTokens.size === 0) continue;
|
|
72
|
+
let best = null;
|
|
73
|
+
let bestOverlap = 0;
|
|
74
|
+
for (const other of entries) {
|
|
75
|
+
if (other === core || drop.has(other) || other.doc?.type !== 'markdown') continue;
|
|
76
|
+
if (!String(other.doc.provenance ?? '').startsWith(`${m[1]}:`)) continue;
|
|
77
|
+
const otherTokens = new Set(tokenize(other.doc.text));
|
|
78
|
+
let shared = 0;
|
|
79
|
+
for (const t of coreTokens) if (otherTokens.has(t)) shared += 1;
|
|
80
|
+
const overlap = shared / coreTokens.size;
|
|
81
|
+
if (overlap > bestOverlap) { bestOverlap = overlap; best = other; }
|
|
82
|
+
}
|
|
83
|
+
if (best && bestOverlap >= TWIN_MIN_OVERLAP) drop.add(keep === 'core' ? best : core);
|
|
84
|
+
}
|
|
85
|
+
return drop.size === 0 ? entries : entries.filter((e) => !drop.has(e));
|
|
86
|
+
}
|
|
49
87
|
const MAX_SOURCES = 8;
|
|
50
88
|
const MAX_SOURCE_BYTES = 512 * 1024;
|
|
51
89
|
const MIN_BULLET_CHARS = 12;
|
|
@@ -117,9 +155,17 @@ function stem(token) {
|
|
|
117
155
|
|
|
118
156
|
// Tokens of `text`: whole tokens plus their separator-split parts
|
|
119
157
|
// (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
|
|
120
|
-
|
|
158
|
+
// A camelCase identifier also yields its words (`deriveDeterministicBucket`
|
|
159
|
+
// -> derive, deterministic, bucket); the whole identifier stays a token too.
|
|
160
|
+
const CAMEL_RE = /\b[a-z][a-z0-9]*(?:[A-Z][a-z0-9]+)+\b/g;
|
|
161
|
+
const splitCamel = (word) => `${word} ${word.replace(/([a-z0-9])([A-Z])/g, '$1 $2')}`;
|
|
162
|
+
|
|
163
|
+
// Query side only (`{ camel: true }`): in a long note the split words of its many
|
|
164
|
+
// identifiers (`postEditReviewPolicy`) would add common-word matches.
|
|
165
|
+
export function tokenize(text, { camel = false } = {}) {
|
|
121
166
|
const out = [];
|
|
122
|
-
|
|
167
|
+
const source = camel ? String(text ?? '').replace(CAMEL_RE, splitCamel) : text;
|
|
168
|
+
for (const raw of normalize(source).split(/\s+/)) {
|
|
123
169
|
if (!raw) continue;
|
|
124
170
|
const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
|
|
125
171
|
const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
|
|
@@ -131,6 +177,21 @@ export function tokenize(text) {
|
|
|
131
177
|
return out;
|
|
132
178
|
}
|
|
133
179
|
|
|
180
|
+
// File names in a query (`name.ext`): a query that names a file is only
|
|
181
|
+
// answered by a note that names it too and covers the rest of the question.
|
|
182
|
+
const QUERY_FILE_RE = /[\w-]+\.(?:test\.)?(?:m?js|cjs|tsx?|jsx|sh|md|json|py|ya?ml)\b/gi;
|
|
183
|
+
const FILE_EXT_TOKENS = new Set(['js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'sh', 'md', 'json', 'py', 'yml', 'yaml', 'test']);
|
|
184
|
+
const FILE_REST_COVERAGE = 0.3;
|
|
185
|
+
const FILE_REST_MIN_TOKENS = 2;
|
|
186
|
+
function queryFileSpec(query) {
|
|
187
|
+
if (typeof query !== 'string') return null;
|
|
188
|
+
const files = query.match(QUERY_FILE_RE);
|
|
189
|
+
if (!files) return null;
|
|
190
|
+
const fileTokens = new Set(files.flatMap((f) => tokenize(f.toLowerCase())).filter((t) => !FILE_EXT_TOKENS.has(t)));
|
|
191
|
+
const restTokens = [...new Set(tokenize(query.replace(QUERY_FILE_RE, ' '), { camel: true }))];
|
|
192
|
+
return fileTokens.size > 0 ? { fileTokens, restTokens } : null;
|
|
193
|
+
}
|
|
194
|
+
|
|
134
195
|
function episodeBody(text) {
|
|
135
196
|
return String(text ?? '')
|
|
136
197
|
.replace(/^\s*Session\s+\S+?:\s*/i, '')
|
|
@@ -233,8 +294,8 @@ function idf(totalDocs, docFreq) {
|
|
|
233
294
|
// abstained }
|
|
234
295
|
// `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
|
|
235
296
|
// `query` is a string or a pre-tokenized array. Never throws.
|
|
236
|
-
export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
237
|
-
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
|
|
297
|
+
export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
298
|
+
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query, { camel: true }))];
|
|
238
299
|
const empty = {
|
|
239
300
|
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
240
301
|
};
|
|
@@ -261,6 +322,12 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
261
322
|
// IDF cannot discriminate in a tiny corpus, so the multi-token requirement
|
|
262
323
|
// only applies once there are enough documents for it to mean something.
|
|
263
324
|
const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
|
|
325
|
+
let queryMass = 0;
|
|
326
|
+
for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
|
|
327
|
+
|
|
328
|
+
const fileSpec = total >= SMALL_CORPUS_DOCS ? queryFileSpec(query) : null;
|
|
329
|
+
let restMass = 0;
|
|
330
|
+
if (fileSpec) for (const token of fileSpec.restTokens) restMass += idf(total, docFreq.get(token) ?? 0);
|
|
264
331
|
|
|
265
332
|
const scored = [];
|
|
266
333
|
let candidateCount = 0;
|
|
@@ -280,6 +347,19 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
280
347
|
candidateCount += 1;
|
|
281
348
|
if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
|
|
282
349
|
if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
|
|
350
|
+
// An episode is a long raw prompt: it holds many common words, so two of them
|
|
351
|
+
// clear the absolute floor above while the query's real subject is absent.
|
|
352
|
+
// It must also cover a fraction of the query's own IDF mass.
|
|
353
|
+
if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
|
|
354
|
+
if (fileSpec) {
|
|
355
|
+
let named = false;
|
|
356
|
+
for (const token of fileSpec.fileTokens) if (tf.has(token)) { named = true; break; }
|
|
357
|
+
let restHit = 0;
|
|
358
|
+
for (const token of fileSpec.restTokens) if (tf.has(token)) restHit += idf(total, docFreq.get(token));
|
|
359
|
+
// One leftover word ("what does X check") is too little to judge coverage by.
|
|
360
|
+
const judgeRest = fileSpec.restTokens.length >= FILE_REST_MIN_TOKENS && restMass > 0;
|
|
361
|
+
if (!named || (judgeRest && restHit < FILE_REST_COVERAGE * restMass)) continue;
|
|
362
|
+
}
|
|
283
363
|
scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
|
|
284
364
|
}
|
|
285
365
|
// Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
|
|
@@ -289,8 +369,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
289
369
|
const top = scored.length > 0 ? scored[0].score : 0;
|
|
290
370
|
// Temporal lane (3.5.3): validity is one more fusion lane over the docs that
|
|
291
371
|
// cleared the floor and cutoff (see demoteByValidity); nothing is dropped.
|
|
292
|
-
const ranked =
|
|
293
|
-
.
|
|
372
|
+
const ranked = collapseTwins(
|
|
373
|
+
demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc),
|
|
374
|
+
twins,
|
|
375
|
+
).slice(0, limit);
|
|
294
376
|
return {
|
|
295
377
|
ranked,
|
|
296
378
|
candidateCount,
|
|
@@ -312,6 +394,7 @@ export function normalizeMemorySources(config) {
|
|
|
312
394
|
const rel = entry.trim().replace(/\\/g, '/');
|
|
313
395
|
if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
|
|
314
396
|
if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
|
|
397
|
+
if (rel.includes('*') && !DIR_GLOB_RE.test(rel)) continue;
|
|
315
398
|
seen.add(rel);
|
|
316
399
|
out.push(rel);
|
|
317
400
|
if (out.length >= MAX_SOURCES) break;
|
|
@@ -319,6 +402,44 @@ export function normalizeMemorySources(config) {
|
|
|
319
402
|
return out;
|
|
320
403
|
}
|
|
321
404
|
|
|
405
|
+
// matchesMemorySource(sources, rel) -> boolean. Sync, no I/O: is `rel` one of the
|
|
406
|
+
// (possibly `dir/*.md`) entries? Lets a hook skip the directory listing for edits
|
|
407
|
+
// that are nowhere near a memory source.
|
|
408
|
+
export function matchesMemorySource(sources, rel) {
|
|
409
|
+
for (const entry of sources) {
|
|
410
|
+
if (entry === rel) return true;
|
|
411
|
+
if (entry.endsWith('/*.md') && DIR_GLOB_RE.test(entry)) {
|
|
412
|
+
const dir = entry.slice(0, -4);
|
|
413
|
+
if (rel.startsWith(dir) && !rel.slice(dir.length).includes('/') && /\.md$/i.test(rel)) return true;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
return false;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// expandMemorySources(projectRoot, sources) -> concrete rel .md paths. `dir/*.md`
|
|
420
|
+
// lists that one directory level (async, sorted, at most MAX_DIR_FILES, dot-files
|
|
421
|
+
// skipped); a missing directory yields nothing. Explicit entries keep their order.
|
|
422
|
+
export async function expandMemorySources(projectRoot, sources) {
|
|
423
|
+
const out = [];
|
|
424
|
+
const seen = new Set();
|
|
425
|
+
const push = (rel) => { if (!seen.has(rel)) { seen.add(rel); out.push(rel); } };
|
|
426
|
+
for (const entry of sources) {
|
|
427
|
+
if (!entry.includes('*')) { push(entry); continue; }
|
|
428
|
+
if (!DIR_GLOB_RE.test(entry)) continue;
|
|
429
|
+
const dir = entry.slice(0, -4);
|
|
430
|
+
let names = [];
|
|
431
|
+
try {
|
|
432
|
+
const items = await fsp.readdir(path.resolve(projectRoot, dir), { withFileTypes: true });
|
|
433
|
+
names = items.filter((d) => (d.isFile() || d.isSymbolicLink()) && /\.md$/i.test(d.name) && !d.name.startsWith('.')).map((d) => d.name);
|
|
434
|
+
} catch {
|
|
435
|
+
names = [];
|
|
436
|
+
}
|
|
437
|
+
names.sort();
|
|
438
|
+
for (const name of names.slice(0, MAX_DIR_FILES)) push(`${dir}${name}`);
|
|
439
|
+
}
|
|
440
|
+
return out;
|
|
441
|
+
}
|
|
442
|
+
|
|
322
443
|
function cleanMarkdown(text) {
|
|
323
444
|
return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
|
|
324
445
|
}
|
|
@@ -328,6 +449,22 @@ function cleanMarkdown(text) {
|
|
|
328
449
|
// indexing it too would return the same rule twice (record + bullet).
|
|
329
450
|
const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
|
|
330
451
|
const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
|
|
452
|
+
const MD_ID_TAG_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
|
|
453
|
+
const MD_SUPERSEDES_TAG_RE = /\[supersedes:\s*([a-z0-9][a-z0-9._,\s-]*)\]/gi;
|
|
454
|
+
|
|
455
|
+
// Resolve slug tags to doc ids across every loaded source: the replacing doc
|
|
456
|
+
// gets `supersedes: [md:<rel>:<line>]` so dropSuperseded() removes the old one.
|
|
457
|
+
// Returns copies only for docs that carry tags; cached docs are never mutated.
|
|
458
|
+
function resolveMarkdownSupersession(docs) {
|
|
459
|
+
const bySlug = new Map();
|
|
460
|
+
for (const doc of docs) if (doc.slug && !bySlug.has(doc.slug)) bySlug.set(doc.slug, doc.id);
|
|
461
|
+
if (bySlug.size === 0) return docs;
|
|
462
|
+
return docs.map((doc) => {
|
|
463
|
+
if (!doc.supersedesSlugs) return doc;
|
|
464
|
+
const ids = doc.supersedesSlugs.map((s) => bySlug.get(s)).filter((id) => id && id !== doc.id);
|
|
465
|
+
return ids.length > 0 ? { ...doc, supersedes: ids } : doc;
|
|
466
|
+
});
|
|
467
|
+
}
|
|
331
468
|
|
|
332
469
|
// Top-level bullets ("- " / "* ") with their indented continuation lines.
|
|
333
470
|
export function parseMarkdownBullets(content, relPath) {
|
|
@@ -339,9 +476,21 @@ export function parseMarkdownBullets(content, relPath) {
|
|
|
339
476
|
let current = null;
|
|
340
477
|
const flush = () => {
|
|
341
478
|
if (!current) return;
|
|
342
|
-
|
|
479
|
+
// Supersession tags (3.5.4): `[id: slug]` names a bullet, `[supersedes: a, b]`
|
|
480
|
+
// says it replaces the bullets so named. Tags are stripped from the text.
|
|
481
|
+
let slug = null;
|
|
482
|
+
const replacesSlugs = [];
|
|
483
|
+
const raw = current.parts.join(' ')
|
|
484
|
+
.replace(MD_ID_TAG_RE, (_m, s) => { slug ??= s.toLowerCase(); return ' '; })
|
|
485
|
+
.replace(MD_SUPERSEDES_TAG_RE, (_m, list) => {
|
|
486
|
+
for (const s of list.split(/[\s,]+/)) if (s) replacesSlugs.push(s.toLowerCase());
|
|
487
|
+
return ' ';
|
|
488
|
+
});
|
|
489
|
+
const text = cleanMarkdown(raw);
|
|
343
490
|
if (text.length >= MIN_BULLET_CHARS) {
|
|
344
491
|
docs.push({
|
|
492
|
+
...(slug ? { slug } : {}),
|
|
493
|
+
...(replacesSlugs.length > 0 ? { supersedesSlugs: replacesSlugs } : {}),
|
|
345
494
|
id: `md:${relPath}:${current.line}`,
|
|
346
495
|
type: 'markdown',
|
|
347
496
|
source: 'markdown',
|
|
@@ -398,7 +547,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
|
|
|
398
547
|
const root = path.resolve(projectRoot);
|
|
399
548
|
const docs = [];
|
|
400
549
|
let realRoot = null;
|
|
401
|
-
for (const rel of sources) {
|
|
550
|
+
for (const rel of await expandMemorySources(root, sources)) {
|
|
402
551
|
const abs = path.resolve(root, rel);
|
|
403
552
|
if (abs !== root && !abs.startsWith(root + path.sep)) continue;
|
|
404
553
|
try {
|
|
@@ -420,7 +569,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
|
|
|
420
569
|
markdownCache.delete(abs);
|
|
421
570
|
}
|
|
422
571
|
}
|
|
423
|
-
return docs;
|
|
572
|
+
return resolveMarkdownSupersession(docs);
|
|
424
573
|
}
|
|
425
574
|
|
|
426
575
|
const SNIPPET_CHARS = 160;
|