@ngockhoale/ukit 3.4.85 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -857
- package/bin/ukit +9 -0
- package/manifests/documentation.yaml +12 -0
- package/package.json +1 -1
- package/src/cli/commands/install.js +12 -0
- package/src/cli/commands/memory.js +30 -3
- package/src/core/memory/episodeArchive.js +147 -0
- package/src/core/memory/episodes.js +199 -37
- package/src/core/memory/memoryHit.js +59 -40
- package/src/core/memory/mutateMemory.js +23 -3
- package/src/core/memory/recallEngine.js +423 -0
- package/src/core/memory/retrieval.js +55 -92
- package/src/core/observability/emit/lifecycle.js +30 -0
- package/src/core/runtimeConfig.js +6 -0
- package/template_project/.claude/hooks/skill-router.sh +82 -51
- package/template_project/.claude/ukit/index/route-task.mjs +29 -48
- package/template_project/.claude/ukit/runtime/execution-ledger.mjs +45 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +423 -0
- package/template_project/.claude/ukit/runtime/reinject-context.mjs +29 -45
|
@@ -1890,6 +1890,7 @@ function carriedEvidenceLedger(fresh, current) {
|
|
|
1890
1890
|
...fresh,
|
|
1891
1891
|
taskStartedAt: Number.isFinite(current.taskStartedAt) ? current.taskStartedAt : (fresh.taskStartedAt ?? null),
|
|
1892
1892
|
parentSessionId: current.parentSessionId || fresh.parentSessionId || null,
|
|
1893
|
+
taskPrompt: current.taskPrompt || fresh.taskPrompt || null,
|
|
1893
1894
|
operationBank: current.operationBank || {},
|
|
1894
1895
|
receiptSequence: current.receiptSequence || 0,
|
|
1895
1896
|
sourceSucceeded: fresh.sourceSucceeded || current.sourceSucceeded === true,
|
|
@@ -1989,9 +1990,51 @@ function evidencePromptKey(routeState) {
|
|
|
1989
1990
|
}
|
|
1990
1991
|
export { evidencePromptKey };
|
|
1991
1992
|
|
|
1993
|
+
// Episode spool (3.5.1): when a task's ledger is replaced by a different task, the outgoing
|
|
1994
|
+
// task's compact episode inputs are appended to `<ledger>.episodes` (bounded, async, advisory).
|
|
1995
|
+
// The store write stays in the CLI/SessionEnd/backfill consumer (src/core/memory/episodes.js).
|
|
1996
|
+
const EPISODE_SPOOL_MAX = 32;
|
|
1997
|
+
function boundedPrompt(text) {
|
|
1998
|
+
return String(text || '').replace(/\s+/g, ' ').trim().slice(0, 240) || null;
|
|
1999
|
+
}
|
|
2000
|
+
async function spoolOutgoingEpisode(target, current, next) {
|
|
2001
|
+
try {
|
|
2002
|
+
if (!current?.taskInstanceId || current.taskInstanceId === next?.taskInstanceId
|
|
2003
|
+
|| next?.adoptedFrom === current.taskInstanceId) return;
|
|
2004
|
+
const receipts = Array.isArray(current.receipts) ? current.receipts : [];
|
|
2005
|
+
const writes = receipts.filter((r) => r?.kind === 'write' && r.success === true && typeof r.file === 'string');
|
|
2006
|
+
if (writes.length === 0 && current.writeSucceeded !== true) return;
|
|
2007
|
+
const lastVerification = receipts.filter((r) => r?.kind === 'verification').slice(-1);
|
|
2008
|
+
const compact = [...writes.slice(-12), ...lastVerification].map((r) => ({
|
|
2009
|
+
kind: r.kind,
|
|
2010
|
+
success: r.success === true,
|
|
2011
|
+
...(typeof r.file === 'string' ? { file: r.file.slice(0, 300) } : {}),
|
|
2012
|
+
...(typeof r.command === 'string' ? { command: r.command.slice(0, 300) } : {}),
|
|
2013
|
+
}));
|
|
2014
|
+
const spoolPath = `${target}.episodes`;
|
|
2015
|
+
const list = await readJson(spoolPath, []);
|
|
2016
|
+
const kept = (Array.isArray(list) ? list : []).filter((e) => e?.taskInstanceId !== current.taskInstanceId);
|
|
2017
|
+
kept.push({
|
|
2018
|
+
sessionKey: current.sessionKey,
|
|
2019
|
+
sessionId: current.sessionId || null,
|
|
2020
|
+
taskInstanceId: current.taskInstanceId,
|
|
2021
|
+
taskPrompt: boundedPrompt(current.taskPrompt),
|
|
2022
|
+
startedAt: current.taskStartedAt ?? null,
|
|
2023
|
+
endedAt: Date.now(),
|
|
2024
|
+
writeSucceeded: true,
|
|
2025
|
+
verificationAttempted: current.verificationAttempted === true || lastVerification.length > 0,
|
|
2026
|
+
verificationSucceeded: current.verificationSucceeded === true,
|
|
2027
|
+
blocker: current.blocker ? String(current.blocker.reason || current.blocker.code || current.blocker).slice(0, 80) : null,
|
|
2028
|
+
receipts: compact,
|
|
2029
|
+
});
|
|
2030
|
+
await writeJsonAtomic(spoolPath, kept.slice(-EPISODE_SPOOL_MAX));
|
|
2031
|
+
} catch { /* advisory: a spool failure never changes the ledger event */ }
|
|
2032
|
+
}
|
|
2033
|
+
|
|
1992
2034
|
function freshLedger(payload, routeState, harness) {
|
|
1993
2035
|
return {
|
|
1994
2036
|
version: LEDGER_VERSION,
|
|
2037
|
+
taskPrompt: boundedPrompt(authoritativePromptText(routeState)),
|
|
1995
2038
|
sessionKey: sessionIdentity(payload),
|
|
1996
2039
|
sessionId: payload.session_id || payload.sessionId || null,
|
|
1997
2040
|
transcriptPath: payload.transcript_path || payload.transcriptPath || null,
|
|
@@ -3262,6 +3305,7 @@ function reconcilePredecessorLedger(state, ledger, cwd) {
|
|
|
3262
3305
|
taskInstanceId: state.taskInstanceId,
|
|
3263
3306
|
requestKey: state.requestKey || ledger.requestKey || null,
|
|
3264
3307
|
promptKey: evidencePromptKey(state),
|
|
3308
|
+
taskPrompt: boundedPrompt(authoritativePromptText(state)),
|
|
3265
3309
|
routeFingerprint: state.fingerprint || ledger.routeFingerprint || null,
|
|
3266
3310
|
adoptedFrom: ledger.taskInstanceId,
|
|
3267
3311
|
operationBank: {},
|
|
@@ -3448,6 +3492,7 @@ export async function recordLedgerEvent(event, {
|
|
|
3448
3492
|
lockedRouteState = state || null;
|
|
3449
3493
|
const current = await readExecutionLedger(projectRoot, payload);
|
|
3450
3494
|
let ledger = liveBaseLedger(event, current, payload, state, harness, fallbackLedger, projectRoot);
|
|
3495
|
+
await spoolOutgoingEpisode(target, current, ledger);
|
|
3451
3496
|
// Reconcile pending journaled events first — inside this caller's acquired lock and
|
|
3452
3497
|
// before its own evidence, so a timed-out writer's receipt is never stranded.
|
|
3453
3498
|
const drained = await drainJournal(target, { ledger, payload, signal, deadlineMs, projectRoot });
|
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
// Memory recall engine (release 3.5.1) - the ONE ranking implementation used by
|
|
2
|
+
// the CLI (src/core/memory/retrieval.js, memoryHit.js) and, through the
|
|
3
|
+
// byte-identical shipped mirror template_project/.claude/ukit/runtime/
|
|
4
|
+
// memory-recall.mjs, by the route-task / reinject-context / skill-router hook
|
|
5
|
+
// lanes. Parity is enforced by tests/consistency/memoryRecallParity.test.js:
|
|
6
|
+
// this file and the mirror must stay byte-identical, so it imports only node:
|
|
7
|
+
// builtins and is fully async on the I/O path (hook deadline rule, MEMORY.md
|
|
8
|
+
// 2026-09-19).
|
|
9
|
+
//
|
|
10
|
+
// Ranking: BM25 with IDF computed over the eligible document set the caller
|
|
11
|
+
// passes in, length normalisation, a multiplicative type prior, no wall-clock
|
|
12
|
+
// term. A hit must clear a
|
|
13
|
+
// floor (matched-token count + matched IDF mass relative to the corpus size);
|
|
14
|
+
// when nothing clears it the result is explicitly empty (abstention).
|
|
15
|
+
// Tokens that come from the pre-3.5.1 episode text template never count.
|
|
16
|
+
//
|
|
17
|
+
// Markdown source: bullets of configured .md files (default docs/MEMORY.md)
|
|
18
|
+
// become read-only documents with provenance `path:line`. They are parsed on
|
|
19
|
+
// demand, cached by file size + mtime, and never copied into records.json.
|
|
20
|
+
|
|
21
|
+
import crypto from 'node:crypto';
|
|
22
|
+
import fsp from 'node:fs/promises';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
|
|
25
|
+
export const RECALL_ENGINE_VERSION = 'bm25-1';
|
|
26
|
+
export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md']);
|
|
27
|
+
|
|
28
|
+
const K1 = 1.2;
|
|
29
|
+
const B = 0.75;
|
|
30
|
+
// A document is a hit only when it matches at least this many distinct query
|
|
31
|
+
// tokens (or all of them when the query has fewer) ...
|
|
32
|
+
const MIN_MATCHED_TOKENS = 2;
|
|
33
|
+
const SMALL_CORPUS_DOCS = 8;
|
|
34
|
+
// One matched token is enough only for curated records; markdown bullets and
|
|
35
|
+
// episodes (the noisy sources) must match two distinct tokens.
|
|
36
|
+
const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
|
|
37
|
+
// ... and the IDF mass of the matched tokens reaches this fraction of the
|
|
38
|
+
// largest IDF the corpus can produce (a token present in exactly one doc).
|
|
39
|
+
const MIN_IDF_MASS_RATIO = 0.5;
|
|
40
|
+
// Results scoring below this fraction of the best hit are noise tail.
|
|
41
|
+
const RELATIVE_CUTOFF = 0.35;
|
|
42
|
+
// Query-dependent type prior: multiplies the lexical score, so it can only
|
|
43
|
+
// reorder documents that already match (it never surfaces a non-matching one,
|
|
44
|
+
// unlike the additive type/evidence/recency bonuses removed in 3.5.1). A
|
|
45
|
+
// curated rule or decision outranks an episode with the same lexical match.
|
|
46
|
+
const TYPE_PRIOR = Object.freeze({
|
|
47
|
+
project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
|
|
48
|
+
});
|
|
49
|
+
const MAX_SOURCES = 8;
|
|
50
|
+
const MAX_SOURCE_BYTES = 512 * 1024;
|
|
51
|
+
const MIN_BULLET_CHARS = 12;
|
|
52
|
+
|
|
53
|
+
const STOPWORDS = new Set([
|
|
54
|
+
'the', 'a', 'an', 'and', 'or', 'to', 'for', 'of', 'with', 'in', 'on', 'is', 'are',
|
|
55
|
+
'this', 'that', 'it', 'as', 'by', 'be', 'use', 'using', 'implement', 'fix', 'task',
|
|
56
|
+
'what', 'how', 'why', 'when', 'where', 'which', 'who', 'whom', 'does', 'do', 'did',
|
|
57
|
+
'should', 'would', 'could', 'can', 'may', 'might', 'will', 'shall', 'from', 'at',
|
|
58
|
+
'into', 'than', 'then', 'was', 'were', 'been', 'has', 'have', 'had', 'not', 'no',
|
|
59
|
+
'but', 'if', 'so', 'about', 'after', 'before', 'there', 'their', 'its', 'our',
|
|
60
|
+
'we', 'you', 'your', 'any', 'all', 'each', 'per', 'via', 'up', 'out', 'over',
|
|
61
|
+
'only', 'also', 'just', 'too', 'very', 'us', 'me', 'my', 'he', 'she', 'they',
|
|
62
|
+
'them', 'these', 'those', 'am', 'being', 'both', 'more', 'most', 'some', 'such',
|
|
63
|
+
'cần', 'và', 'là', 'cho', 'một', 'những', 'dùng',
|
|
64
|
+
]);
|
|
65
|
+
|
|
66
|
+
// Pre-3.5.1 episode text template ("Session <id>: writeOk, verify=ok, N receipts,
|
|
67
|
+
// last command <cmd>") - template words must never produce a match.
|
|
68
|
+
const EPISODE_TEMPLATE_TOKENS = new Set([
|
|
69
|
+
'session', 'writeok', 'writefail', 'verify', 'receipts', 'receipt', 'last', 'command',
|
|
70
|
+
'ok', 'fail', 'run', 'n/a',
|
|
71
|
+
]);
|
|
72
|
+
const UUID_RE = /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/g;
|
|
73
|
+
|
|
74
|
+
export function normalize(text) {
|
|
75
|
+
return String(text ?? '')
|
|
76
|
+
.toLowerCase()
|
|
77
|
+
.replace(UUID_RE, ' ')
|
|
78
|
+
.replace(/[^\p{L}\p{N}\s./:_-]/gu, ' ')
|
|
79
|
+
.replace(/\s+/g, ' ')
|
|
80
|
+
.trim();
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const STEM_SUFFIXES = ['ations', 'ation', 'ments', 'ment', 'ings', 'ing', 'ied', 'ed', 'ers', 'er', 'es', 's', 'ly'];
|
|
84
|
+
|
|
85
|
+
// Word forms must meet on one stem: `rule`/`rules`, `file`/`files`,
|
|
86
|
+
// `update`/`updated`, `call`/`called`, `add`/`adding`. So every alphabetic token
|
|
87
|
+
// gets the same final normalisation (one of a doubled final consonant, then a
|
|
88
|
+
// trailing `e`), whether or not a suffix was stripped first.
|
|
89
|
+
function finishStem(base) {
|
|
90
|
+
let out = base;
|
|
91
|
+
if (out.length >= 3 && /(ll|tt|pp|dd|gg|nn|mm|rr)$/.test(out)) out = out.slice(0, -1);
|
|
92
|
+
if (out.length >= 4 && out.endsWith('e')) out = out.slice(0, -1);
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Forms the suffix rules get wrong: `plane` must not meet `plan`, `written` must
|
|
97
|
+
// meet `write`/`writer`, and `status` must meet `statuses`.
|
|
98
|
+
const STEM_EXCEPTIONS = new Map([
|
|
99
|
+
['plane', 'plane'], ['planes', 'plane'], ['written', 'writ'], ['status', 'status'],
|
|
100
|
+
]);
|
|
101
|
+
|
|
102
|
+
function stem(token) {
|
|
103
|
+
if (token.length < 3 || !/^[a-z]+$/.test(token)) return token;
|
|
104
|
+
const fixed = STEM_EXCEPTIONS.get(token);
|
|
105
|
+
if (fixed) return fixed;
|
|
106
|
+
if (/ie[sd]$/.test(token) && token.length > 5) return `${token.slice(0, -3)}y`;
|
|
107
|
+
if (token.length >= 4) {
|
|
108
|
+
for (const suffix of STEM_SUFFIXES) {
|
|
109
|
+
if (token.endsWith(suffix) && token.length - suffix.length >= 3) {
|
|
110
|
+
if (suffix === 's' && token.endsWith('ss')) break;
|
|
111
|
+
return finishStem(token.slice(0, -suffix.length));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return finishStem(token);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Tokens of `text`: whole tokens plus their separator-split parts
|
|
119
|
+
// (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
|
|
120
|
+
export function tokenize(text) {
|
|
121
|
+
const out = [];
|
|
122
|
+
for (const raw of normalize(text).split(/\s+/)) {
|
|
123
|
+
if (!raw) continue;
|
|
124
|
+
const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
|
|
125
|
+
const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
|
|
126
|
+
for (const candidate of candidates) {
|
|
127
|
+
if (candidate.length < 2 || STOPWORDS.has(candidate)) continue;
|
|
128
|
+
out.push(stem(candidate));
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return out;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function episodeBody(text) {
|
|
135
|
+
return String(text ?? '')
|
|
136
|
+
.replace(/^\s*Session\s+\S+?:\s*/i, '')
|
|
137
|
+
.replace(/\b(writeOk|writeFail)\b/g, ' ')
|
|
138
|
+
.replace(/\bverify=(ok|fail|not-run)\b/g, ' ')
|
|
139
|
+
.replace(/\b\d+\s+receipts?\b/g, ' ')
|
|
140
|
+
.replace(/\blast command\b/gi, ' ');
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function isTemplateText(doc) {
|
|
144
|
+
return doc?.type === 'episode' || doc?.provenance === 'exec-ledger';
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Pre-3.5.1 content-free episode shape. Only this shape gets the template-token
|
|
148
|
+
// filter: a 3.5.1 episode ("Task: … . Wrote: … . Verify: …") carries the user's
|
|
149
|
+
// own words, and dropping `session`/`command`/`run` from it would hide real tasks.
|
|
150
|
+
const LEGACY_EPISODE_RE = /^\s*Session\s+\S+?:\s*(?:writeOk|writeFail)\b/;
|
|
151
|
+
// Field labels of the 3.5.1 episode text; present in every episode, never content.
|
|
152
|
+
const EPISODE_LABEL_RE = /(?:^|\.\s)(?:Task|Wrote|Verify|Unfinished):/g;
|
|
153
|
+
|
|
154
|
+
// Distinct match tokens of a document.
|
|
155
|
+
export function docTokens(doc) {
|
|
156
|
+
if (!isTemplateText(doc)) return tokenize(doc?.text);
|
|
157
|
+
const text = String(doc.text ?? '');
|
|
158
|
+
if (LEGACY_EPISODE_RE.test(text)) {
|
|
159
|
+
return tokenize(episodeBody(text)).filter((token) => !EPISODE_TEMPLATE_TOKENS.has(token));
|
|
160
|
+
}
|
|
161
|
+
return tokenize(text.replace(EPISODE_LABEL_RE, ' '));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function idf(totalDocs, docFreq) {
|
|
165
|
+
return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// rankDocs(docs, query, {limit?}) -> {
|
|
169
|
+
// ranked:[{doc, score, matched}], candidateCount, queryTokenCount, topScore,
|
|
170
|
+
// abstained }
|
|
171
|
+
// `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
|
|
172
|
+
// `query` is a string or a pre-tokenized array. Never throws.
|
|
173
|
+
export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
174
|
+
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
|
|
175
|
+
const empty = {
|
|
176
|
+
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
177
|
+
};
|
|
178
|
+
if (!Array.isArray(docs) || docs.length === 0 || queryTokens.length === 0) return empty;
|
|
179
|
+
|
|
180
|
+
const stats = [];
|
|
181
|
+
const docFreq = new Map();
|
|
182
|
+
let totalLength = 0;
|
|
183
|
+
for (const doc of docs) {
|
|
184
|
+
if (doc == null) continue;
|
|
185
|
+
const tokens = docTokens(doc);
|
|
186
|
+
const tf = new Map();
|
|
187
|
+
for (const token of tokens) tf.set(token, (tf.get(token) ?? 0) + 1);
|
|
188
|
+
for (const token of tf.keys()) docFreq.set(token, (docFreq.get(token) ?? 0) + 1);
|
|
189
|
+
totalLength += tokens.length;
|
|
190
|
+
stats.push({ doc, tf, length: tokens.length });
|
|
191
|
+
}
|
|
192
|
+
const total = stats.length;
|
|
193
|
+
if (total === 0) return empty;
|
|
194
|
+
const avgLength = totalLength / total || 1;
|
|
195
|
+
const idfMax = idf(total, 1);
|
|
196
|
+
// IDF cannot discriminate in a tiny corpus, so the multi-token requirement
|
|
197
|
+
// only applies once there are enough documents for it to mean something.
|
|
198
|
+
const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
|
|
199
|
+
|
|
200
|
+
const scored = [];
|
|
201
|
+
let candidateCount = 0;
|
|
202
|
+
for (const { doc, tf, length } of stats) {
|
|
203
|
+
let score = 0;
|
|
204
|
+
let idfMass = 0;
|
|
205
|
+
let matched = 0;
|
|
206
|
+
for (const token of queryTokens) {
|
|
207
|
+
const freq = tf.get(token);
|
|
208
|
+
if (!freq) continue;
|
|
209
|
+
const weight = idf(total, docFreq.get(token));
|
|
210
|
+
matched += 1;
|
|
211
|
+
idfMass += weight;
|
|
212
|
+
score += weight * ((freq * (K1 + 1)) / (freq + K1 * (1 - B + B * (length / avgLength))));
|
|
213
|
+
}
|
|
214
|
+
if (matched === 0) continue;
|
|
215
|
+
candidateCount += 1;
|
|
216
|
+
if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
|
|
217
|
+
if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
|
|
218
|
+
scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
|
|
219
|
+
}
|
|
220
|
+
// Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
|
|
221
|
+
// entry must not reorder the injected block and invalidate the prompt cache.
|
|
222
|
+
scored.sort((a, b) => b.score - a.score
|
|
223
|
+
|| String(a.doc.id ?? '').localeCompare(String(b.doc.id ?? '')));
|
|
224
|
+
const top = scored.length > 0 ? scored[0].score : 0;
|
|
225
|
+
const ranked = scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top).slice(0, limit);
|
|
226
|
+
return {
|
|
227
|
+
ranked,
|
|
228
|
+
candidateCount,
|
|
229
|
+
queryTokenCount: queryTokens.length,
|
|
230
|
+
topScore: top,
|
|
231
|
+
abstained: ranked.length === 0,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// ---- Markdown source ------------------------------------------------------
|
|
236
|
+
|
|
237
|
+
export function normalizeMemorySources(config) {
|
|
238
|
+
const raw = config?.memory?.sources;
|
|
239
|
+
const list = Array.isArray(raw) ? raw : DEFAULT_MEMORY_SOURCES;
|
|
240
|
+
const seen = new Set();
|
|
241
|
+
const out = [];
|
|
242
|
+
for (const entry of list) {
|
|
243
|
+
if (typeof entry !== 'string') continue;
|
|
244
|
+
const rel = entry.trim().replace(/\\/g, '/');
|
|
245
|
+
if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
|
|
246
|
+
if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
|
|
247
|
+
seen.add(rel);
|
|
248
|
+
out.push(rel);
|
|
249
|
+
if (out.length >= MAX_SOURCES) break;
|
|
250
|
+
}
|
|
251
|
+
return out;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function cleanMarkdown(text) {
|
|
255
|
+
return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// `ukit memory promote` renders active project_rule/procedure records into this
|
|
259
|
+
// block. Those records are already recall documents, so the block is skipped:
|
|
260
|
+
// indexing it too would return the same rule twice (record + bullet).
|
|
261
|
+
const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
|
|
262
|
+
const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
|
|
263
|
+
|
|
264
|
+
// Top-level bullets ("- " / "* ") with their indented continuation lines.
|
|
265
|
+
export function parseMarkdownBullets(content, relPath) {
|
|
266
|
+
const docs = [];
|
|
267
|
+
const lines = String(content ?? '').split(/\r?\n/);
|
|
268
|
+
let heading = '';
|
|
269
|
+
let fenced = false;
|
|
270
|
+
let learned = false;
|
|
271
|
+
let current = null;
|
|
272
|
+
const flush = () => {
|
|
273
|
+
if (!current) return;
|
|
274
|
+
const text = cleanMarkdown(current.parts.join(' '));
|
|
275
|
+
if (text.length >= MIN_BULLET_CHARS) {
|
|
276
|
+
docs.push({
|
|
277
|
+
id: `md:${relPath}:${current.line}`,
|
|
278
|
+
type: 'markdown',
|
|
279
|
+
source: 'markdown',
|
|
280
|
+
text,
|
|
281
|
+
provenance: `${relPath}:${current.line}`,
|
|
282
|
+
line: current.line,
|
|
283
|
+
heading: current.heading,
|
|
284
|
+
created_at: 0,
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
current = null;
|
|
288
|
+
};
|
|
289
|
+
lines.forEach((line, idx) => {
|
|
290
|
+
if (/^\s*(```|~~~)/.test(line)) {
|
|
291
|
+
fenced = !fenced;
|
|
292
|
+
flush();
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
if (fenced) return;
|
|
296
|
+
if (LEARNED_START_RE.test(line)) { learned = true; flush(); return; }
|
|
297
|
+
if (LEARNED_END_RE.test(line)) { learned = false; return; }
|
|
298
|
+
if (learned) return;
|
|
299
|
+
const headingMatch = /^#{1,6}\s+(.*)$/.exec(line);
|
|
300
|
+
if (headingMatch) {
|
|
301
|
+
flush();
|
|
302
|
+
heading = headingMatch[1].trim();
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
if (/^[-*]\s+/.test(line)) {
|
|
306
|
+
flush();
|
|
307
|
+
current = { line: idx + 1, heading, parts: [line.replace(/^[-*]\s+/, '')] };
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
if (current && /^\s{2,}\S/.test(line)) {
|
|
311
|
+
current.parts.push(line.trim());
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
flush();
|
|
315
|
+
});
|
|
316
|
+
flush();
|
|
317
|
+
return docs;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const markdownCache = new Map(); // abs path -> {size, mtimeMs, docs}
|
|
321
|
+
|
|
322
|
+
export function resetMarkdownCacheForTests() {
|
|
323
|
+
markdownCache.clear();
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// loadMarkdownDocs(projectRoot, sources) -> docs[]; a missing/unreadable/oversized
|
|
327
|
+
// file contributes nothing and is never an error. Async I/O only.
|
|
328
|
+
export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOURCES) {
|
|
329
|
+
if (typeof projectRoot !== 'string' || projectRoot === '') return [];
|
|
330
|
+
const root = path.resolve(projectRoot);
|
|
331
|
+
const docs = [];
|
|
332
|
+
let realRoot = null;
|
|
333
|
+
for (const rel of sources) {
|
|
334
|
+
const abs = path.resolve(root, rel);
|
|
335
|
+
if (abs !== root && !abs.startsWith(root + path.sep)) continue;
|
|
336
|
+
try {
|
|
337
|
+
// A symlinked source must still resolve inside the project.
|
|
338
|
+
realRoot ??= await fsp.realpath(root);
|
|
339
|
+
const real = await fsp.realpath(abs);
|
|
340
|
+
if (!real.startsWith(realRoot + path.sep)) continue;
|
|
341
|
+
const st = await fsp.stat(abs);
|
|
342
|
+
if (!st.isFile() || st.size > MAX_SOURCE_BYTES) continue;
|
|
343
|
+
const cached = markdownCache.get(abs);
|
|
344
|
+
if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) {
|
|
345
|
+
docs.push(...cached.docs);
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
const parsed = parseMarkdownBullets(await fsp.readFile(abs, 'utf8'), rel);
|
|
349
|
+
markdownCache.set(abs, { size: st.size, mtimeMs: st.mtimeMs, docs: parsed });
|
|
350
|
+
docs.push(...parsed);
|
|
351
|
+
} catch {
|
|
352
|
+
markdownCache.delete(abs);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return docs;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const SNIPPET_CHARS = 160;
|
|
359
|
+
|
|
360
|
+
// Shared hook lane: score ALL eligible docs (no newest-N pre-filter) together
|
|
361
|
+
// with the markdown source bullets, then cut. Async I/O only; never throws.
|
|
362
|
+
export async function recallForHook({ projectRoot, query, docs = [], config = null, limit = 2 } = {}) {
|
|
363
|
+
let markdown = [];
|
|
364
|
+
try {
|
|
365
|
+
markdown = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config));
|
|
366
|
+
} catch {
|
|
367
|
+
markdown = [];
|
|
368
|
+
}
|
|
369
|
+
let result;
|
|
370
|
+
try {
|
|
371
|
+
result = rankDocs([...docs, ...markdown], query, { limit });
|
|
372
|
+
} catch {
|
|
373
|
+
result = rankDocs([], '', { limit });
|
|
374
|
+
}
|
|
375
|
+
await emitRecallTelemetry(projectRoot, config, result);
|
|
376
|
+
return result;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// One metadata-only `retrieval.query` event per hook recall, through the
|
|
380
|
+
// shared standalone recorder (observability-emit.mjs, a sibling in the shipped
|
|
381
|
+
// runtime dir). Counts, abstention, top score and record refs only - never the
|
|
382
|
+
// prompt, record text or a raw record/session id. Async, bounded, never throws; the observability stage
|
|
383
|
+
// and the memoryV2 rollout stage are honoured (off/kill-switch => no event).
|
|
384
|
+
const TELEMETRY_DEADLINE_MS = 120;
|
|
385
|
+
|
|
386
|
+
async function emitRecallTelemetry(projectRoot, config, result) {
|
|
387
|
+
try {
|
|
388
|
+
const v2 = config?.memoryV2;
|
|
389
|
+
if (v2?.killSwitch === true || v2?.eligibility?.stage === 'off') return;
|
|
390
|
+
const emitter = await import('./observability-emit.mjs');
|
|
391
|
+
await emitter.emitOutcomeObserved({
|
|
392
|
+
projectRoot,
|
|
393
|
+
config,
|
|
394
|
+
deadlineMs: TELEMETRY_DEADLINE_MS,
|
|
395
|
+
record: {
|
|
396
|
+
semantic_name: 'retrieval.query',
|
|
397
|
+
payload: {
|
|
398
|
+
lane: 'hook',
|
|
399
|
+
engine: RECALL_ENGINE_VERSION,
|
|
400
|
+
query_token_count: result.queryTokenCount,
|
|
401
|
+
candidate_count: result.candidateCount,
|
|
402
|
+
returned_count: result.ranked.length,
|
|
403
|
+
abstained: result.abstained,
|
|
404
|
+
top_score: Number(result.topScore.toFixed(4)),
|
|
405
|
+
// Markdown refs are repo paths; record/session/project ids are hashed,
|
|
406
|
+
// as the CLI lane does (memoryEmit.js), because they can identify.
|
|
407
|
+
returned_refs: result.ranked.map((entry) => (entry.doc.provenance && entry.doc.type === 'markdown'
|
|
408
|
+
? String(entry.doc.provenance).slice(0, 96)
|
|
409
|
+
: crypto.createHash('sha256').update(String(entry.doc.id)).digest('hex'))),
|
|
410
|
+
},
|
|
411
|
+
},
|
|
412
|
+
});
|
|
413
|
+
} catch {
|
|
414
|
+
// Telemetry never affects recall.
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// Cited-data rendering: provenance first so truncation never drops it.
|
|
419
|
+
export function renderMarkdownDoc(doc, maxChars = SNIPPET_CHARS) {
|
|
420
|
+
const text = String(doc?.text ?? '');
|
|
421
|
+
const body = text.length > maxChars ? `${text.slice(0, maxChars - 3)}...` : text;
|
|
422
|
+
return `[${doc?.provenance ?? 'markdown'}] ${body}`;
|
|
423
|
+
}
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
} from './memory-policy.mjs';
|
|
31
31
|
import { resolveRecordFreshness } from './memory-freshness.mjs';
|
|
32
32
|
import { resolveMemoryStage } from './memory-flags.mjs';
|
|
33
|
+
import { recallForHook, renderMarkdownDoc } from './memory-recall.mjs';
|
|
33
34
|
|
|
34
35
|
// Hook-context self-deadline (2.4.1 orphan-leak class): the reinject-context hook passes
|
|
35
36
|
// UKIT_HOOK_DEADLINE_MS so a wedged read can never orphan this process past the hook
|
|
@@ -957,12 +958,6 @@ function normalize(text) {
|
|
|
957
958
|
.trim();
|
|
958
959
|
}
|
|
959
960
|
|
|
960
|
-
function tokenize(text) {
|
|
961
|
-
return normalize(text)
|
|
962
|
-
.split(/\s+/)
|
|
963
|
-
.filter((token) => token && !STOPWORDS.has(token) && token.length > 1);
|
|
964
|
-
}
|
|
965
|
-
|
|
966
961
|
function getMemoryTimestamp(item) {
|
|
967
962
|
return item.content?.updatedAt
|
|
968
963
|
?? item.content?.endedAt
|
|
@@ -1006,29 +1001,6 @@ function buildMemorySegments(item) {
|
|
|
1006
1001
|
];
|
|
1007
1002
|
}
|
|
1008
1003
|
|
|
1009
|
-
function scoreMemoryItem(item, queryTokens) {
|
|
1010
|
-
if (queryTokens.length === 0) {
|
|
1011
|
-
return 0;
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
let score = 0;
|
|
1015
|
-
for (const segment of buildMemorySegments(item)) {
|
|
1016
|
-
const segmentTokens = new Set(tokenize(segment.text));
|
|
1017
|
-
for (const token of queryTokens) {
|
|
1018
|
-
if (segmentTokens.has(token)) {
|
|
1019
|
-
score += segment.weight;
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
if (score === 0) {
|
|
1025
|
-
return 0;
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
const recencyBonus = getMemoryTimestamp(item) > 0 ? Math.min(1, getMemoryTimestamp(item) / Date.now()) : 0;
|
|
1029
|
-
return score + recencyBonus;
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
1004
|
async function readDirectoryJsonItems(dirPath) {
|
|
1033
1005
|
let entries = [];
|
|
1034
1006
|
try {
|
|
@@ -1101,8 +1073,6 @@ function recordLegacyKey(item) {
|
|
|
1101
1073
|
return null;
|
|
1102
1074
|
}
|
|
1103
1075
|
|
|
1104
|
-
const MEMORY_V2_RECORD_POOL_LIMIT = 20;
|
|
1105
|
-
|
|
1106
1076
|
// v2 lane (M01-05): approved records live in a single records.json document.
|
|
1107
1077
|
// Missing/malformed store → [] (never-throw, same as readDirectoryJsonItems).
|
|
1108
1078
|
// Stale snapshots excluded here AND re-checked by eligible() downstream.
|
|
@@ -1125,8 +1095,6 @@ export async function listMemoryV2RecordItems(runtimePathsOrRoot, { config, proj
|
|
|
1125
1095
|
.filter((record) => record && typeof record === 'object')
|
|
1126
1096
|
.filter((record) => record.status === 'active')
|
|
1127
1097
|
.filter((record) => record.valid_until == null || record.valid_until > now)
|
|
1128
|
-
.sort((left, right) => (right.created_at ?? 0) - (left.created_at ?? 0))
|
|
1129
|
-
.slice(0, MEMORY_V2_RECORD_POOL_LIMIT)
|
|
1130
1098
|
.map((record) => ({
|
|
1131
1099
|
id: `record:${record.id}`,
|
|
1132
1100
|
type: 'record',
|
|
@@ -1247,14 +1215,13 @@ async function buildPreviousContextLines(projectRoot, state, config) {
|
|
|
1247
1215
|
|
|
1248
1216
|
const binding = await resolveProjectBinding(projectRoot);
|
|
1249
1217
|
const items = await listMemoryItems(projectRoot, { config, projectId: binding.projectId });
|
|
1250
|
-
const queryTokens = tokenize(taskQuery);
|
|
1251
1218
|
// v2/legacy dedup (M01-05): drop a record whose logical twin
|
|
1252
1219
|
// (meta.legacyId → session:<id>, project_id → project:<id>) exists.
|
|
1253
1220
|
const legacyIds = new Set(
|
|
1254
1221
|
items.filter((item) => item.type !== 'record').map((item) => item.id),
|
|
1255
1222
|
);
|
|
1256
1223
|
const eligibleCtx = { projectId: binding.projectId, projectAliases: binding.aliases, includeUser: false };
|
|
1257
|
-
const
|
|
1224
|
+
const eligibleItems = items
|
|
1258
1225
|
.filter((item) => item.type !== 'user')
|
|
1259
1226
|
.filter((item) => {
|
|
1260
1227
|
if (item.type === 'record') {
|
|
@@ -1266,15 +1233,30 @@ async function buildPreviousContextLines(projectRoot, state, config) {
|
|
|
1266
1233
|
return matchesProjectBinding(item, binding);
|
|
1267
1234
|
}
|
|
1268
1235
|
return true;
|
|
1269
|
-
})
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1236
|
+
});
|
|
1237
|
+
// Score-before-cut (3.5.1): every eligible item plus the markdown source
|
|
1238
|
+
// bullets goes through the shared BM25 engine; only then is the top 2 cut.
|
|
1239
|
+
const itemById = new Map();
|
|
1240
|
+
const docs = eligibleItems.map((item) => {
|
|
1241
|
+
itemById.set(item.id, item);
|
|
1242
|
+
return {
|
|
1243
|
+
id: item.id,
|
|
1244
|
+
type: item.type === 'record' ? item.record.type : item.type,
|
|
1245
|
+
provenance: item.type === 'record' ? item.record.provenance : undefined,
|
|
1246
|
+
// Records go in as their raw text, exactly as the CLI lane passes them: a
|
|
1247
|
+
// type prefix would defeat the engine's start-anchored episode shape checks.
|
|
1248
|
+
text: item.type === 'record'
|
|
1249
|
+
? String(item.record.text ?? '')
|
|
1250
|
+
: buildMemorySegments(item).map((segment) => String(segment.text ?? '')).join(' '),
|
|
1251
|
+
created_at: getMemoryTimestamp(item),
|
|
1252
|
+
};
|
|
1253
|
+
});
|
|
1254
|
+
const recalled = await recallForHook({
|
|
1255
|
+
projectRoot, query: taskQuery, docs, config, limit: 2,
|
|
1256
|
+
});
|
|
1257
|
+
const rankedItems = recalled.ranked.map((entry) => (entry.doc.type === 'markdown'
|
|
1258
|
+
? { id: entry.doc.id, type: 'markdown', doc: entry.doc }
|
|
1259
|
+
: itemById.get(entry.doc.id))).filter(Boolean);
|
|
1278
1260
|
|
|
1279
1261
|
if (rankedItems.length === 0) {
|
|
1280
1262
|
return [];
|
|
@@ -1289,7 +1271,9 @@ async function buildPreviousContextLines(projectRoot, state, config) {
|
|
|
1289
1271
|
} catch { /* absent → unlabeled */ }
|
|
1290
1272
|
}));
|
|
1291
1273
|
|
|
1292
|
-
return [`- Previous context: ${rankedItems.map((item) =>
|
|
1274
|
+
return [`- Previous context: ${rankedItems.map((item) => (item.type === 'markdown'
|
|
1275
|
+
? renderMarkdownDoc(item.doc, 120)
|
|
1276
|
+
: buildPreviousContextSnippet(item, freshnessById.get(item.id)))).join(' | ')}`];
|
|
1293
1277
|
}
|
|
1294
1278
|
|
|
1295
1279
|
// BUG-C23-13: only run main() on direct CLI invocation — an unguarded bottom
|