@ngockhoale/ukit 3.4.84 → 3.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1890,6 +1890,7 @@ function carriedEvidenceLedger(fresh, current) {
1890
1890
  ...fresh,
1891
1891
  taskStartedAt: Number.isFinite(current.taskStartedAt) ? current.taskStartedAt : (fresh.taskStartedAt ?? null),
1892
1892
  parentSessionId: current.parentSessionId || fresh.parentSessionId || null,
1893
+ taskPrompt: current.taskPrompt || fresh.taskPrompt || null,
1893
1894
  operationBank: current.operationBank || {},
1894
1895
  receiptSequence: current.receiptSequence || 0,
1895
1896
  sourceSucceeded: fresh.sourceSucceeded || current.sourceSucceeded === true,
@@ -1989,9 +1990,51 @@ function evidencePromptKey(routeState) {
1989
1990
  }
1990
1991
  export { evidencePromptKey };
1991
1992
 
1993
+ // Episode spool (3.5.1): when a task's ledger is replaced by a different task, the outgoing
1994
+ // task's compact episode inputs are appended to `<ledger>.episodes` (bounded, async, advisory).
1995
+ // The store write stays in the CLI/SessionEnd/backfill consumer (src/core/memory/episodes.js).
1996
+ const EPISODE_SPOOL_MAX = 32;
1997
+ function boundedPrompt(text) {
1998
+ return String(text || '').replace(/\s+/g, ' ').trim().slice(0, 240) || null;
1999
+ }
2000
+ async function spoolOutgoingEpisode(target, current, next) {
2001
+ try {
2002
+ if (!current?.taskInstanceId || current.taskInstanceId === next?.taskInstanceId
2003
+ || next?.adoptedFrom === current.taskInstanceId) return;
2004
+ const receipts = Array.isArray(current.receipts) ? current.receipts : [];
2005
+ const writes = receipts.filter((r) => r?.kind === 'write' && r.success === true && typeof r.file === 'string');
2006
+ if (writes.length === 0 && current.writeSucceeded !== true) return;
2007
+ const lastVerification = receipts.filter((r) => r?.kind === 'verification').slice(-1);
2008
+ const compact = [...writes.slice(-12), ...lastVerification].map((r) => ({
2009
+ kind: r.kind,
2010
+ success: r.success === true,
2011
+ ...(typeof r.file === 'string' ? { file: r.file.slice(0, 300) } : {}),
2012
+ ...(typeof r.command === 'string' ? { command: r.command.slice(0, 300) } : {}),
2013
+ }));
2014
+ const spoolPath = `${target}.episodes`;
2015
+ const list = await readJson(spoolPath, []);
2016
+ const kept = (Array.isArray(list) ? list : []).filter((e) => e?.taskInstanceId !== current.taskInstanceId);
2017
+ kept.push({
2018
+ sessionKey: current.sessionKey,
2019
+ sessionId: current.sessionId || null,
2020
+ taskInstanceId: current.taskInstanceId,
2021
+ taskPrompt: boundedPrompt(current.taskPrompt),
2022
+ startedAt: current.taskStartedAt ?? null,
2023
+ endedAt: Date.now(),
2024
+ writeSucceeded: true,
2025
+ verificationAttempted: current.verificationAttempted === true || lastVerification.length > 0,
2026
+ verificationSucceeded: current.verificationSucceeded === true,
2027
+ blocker: current.blocker ? String(current.blocker.reason || current.blocker.code || current.blocker).slice(0, 80) : null,
2028
+ receipts: compact,
2029
+ });
2030
+ await writeJsonAtomic(spoolPath, kept.slice(-EPISODE_SPOOL_MAX));
2031
+ } catch { /* advisory: a spool failure never changes the ledger event */ }
2032
+ }
2033
+
1992
2034
  function freshLedger(payload, routeState, harness) {
1993
2035
  return {
1994
2036
  version: LEDGER_VERSION,
2037
+ taskPrompt: boundedPrompt(authoritativePromptText(routeState)),
1995
2038
  sessionKey: sessionIdentity(payload),
1996
2039
  sessionId: payload.session_id || payload.sessionId || null,
1997
2040
  transcriptPath: payload.transcript_path || payload.transcriptPath || null,
@@ -3262,6 +3305,7 @@ function reconcilePredecessorLedger(state, ledger, cwd) {
3262
3305
  taskInstanceId: state.taskInstanceId,
3263
3306
  requestKey: state.requestKey || ledger.requestKey || null,
3264
3307
  promptKey: evidencePromptKey(state),
3308
+ taskPrompt: boundedPrompt(authoritativePromptText(state)),
3265
3309
  routeFingerprint: state.fingerprint || ledger.routeFingerprint || null,
3266
3310
  adoptedFrom: ledger.taskInstanceId,
3267
3311
  operationBank: {},
@@ -3448,6 +3492,7 @@ export async function recordLedgerEvent(event, {
3448
3492
  lockedRouteState = state || null;
3449
3493
  const current = await readExecutionLedger(projectRoot, payload);
3450
3494
  let ledger = liveBaseLedger(event, current, payload, state, harness, fallbackLedger, projectRoot);
3495
+ await spoolOutgoingEpisode(target, current, ledger);
3451
3496
  // Reconcile pending journaled events first — inside this caller's acquired lock and
3452
3497
  // before its own evidence, so a timed-out writer's receipt is never stranded.
3453
3498
  const drained = await drainJournal(target, { ledger, payload, signal, deadlineMs, projectRoot });
@@ -0,0 +1,423 @@
1
+ // Memory recall engine (release 3.5.1) - the ONE ranking implementation used by
2
+ // the CLI (src/core/memory/retrieval.js, memoryHit.js) and, through the
3
+ // byte-identical shipped mirror template_project/.claude/ukit/runtime/
4
+ // memory-recall.mjs, by the route-task / reinject-context / skill-router hook
5
+ // lanes. Parity is enforced by tests/consistency/memoryRecallParity.test.js:
6
+ // this file and the mirror must stay byte-identical, so it imports only node:
7
+ // builtins and is fully async on the I/O path (hook deadline rule, MEMORY.md
8
+ // 2026-09-19).
9
+ //
10
+ // Ranking: BM25 with IDF computed over the eligible document set the caller
11
+ // passes in, length normalisation, a multiplicative type prior, no wall-clock
12
+ // term. A hit must clear a
13
+ // floor (matched-token count + matched IDF mass relative to the corpus size);
14
+ // when nothing clears it the result is explicitly empty (abstention).
15
+ // Tokens that come from the pre-3.5.1 episode text template never count.
16
+ //
17
+ // Markdown source: bullets of configured .md files (default docs/MEMORY.md)
18
+ // become read-only documents with provenance `path:line`. They are parsed on
19
+ // demand, cached by file size + mtime, and never copied into records.json.
20
+
21
+ import crypto from 'node:crypto';
22
+ import fsp from 'node:fs/promises';
23
+ import path from 'node:path';
24
+
25
+ export const RECALL_ENGINE_VERSION = 'bm25-1';
26
+ export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md']);
27
+
28
+ const K1 = 1.2;
29
+ const B = 0.75;
30
+ // A document is a hit only when it matches at least this many distinct query
31
+ // tokens (or all of them when the query has fewer) ...
32
+ const MIN_MATCHED_TOKENS = 2;
33
+ const SMALL_CORPUS_DOCS = 8;
34
+ // One matched token is enough only for curated records; markdown bullets and
35
+ // episodes (the noisy sources) must match two distinct tokens.
36
+ const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
37
+ // ... and the IDF mass of the matched tokens reaches this fraction of the
38
+ // largest IDF the corpus can produce (a token present in exactly one doc).
39
+ const MIN_IDF_MASS_RATIO = 0.5;
40
+ // Results scoring below this fraction of the best hit are noise tail.
41
+ const RELATIVE_CUTOFF = 0.35;
42
+ // Query-dependent type prior: multiplies the lexical score, so it can only
43
+ // reorder documents that already match (it never surfaces a non-matching one,
44
+ // unlike the additive type/evidence/recency bonuses removed in 3.5.1). A
45
+ // curated rule or decision outranks an episode with the same lexical match.
46
+ const TYPE_PRIOR = Object.freeze({
47
+ project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
48
+ });
49
+ const MAX_SOURCES = 8;
50
+ const MAX_SOURCE_BYTES = 512 * 1024;
51
+ const MIN_BULLET_CHARS = 12;
52
+
53
+ const STOPWORDS = new Set([
54
+ 'the', 'a', 'an', 'and', 'or', 'to', 'for', 'of', 'with', 'in', 'on', 'is', 'are',
55
+ 'this', 'that', 'it', 'as', 'by', 'be', 'use', 'using', 'implement', 'fix', 'task',
56
+ 'what', 'how', 'why', 'when', 'where', 'which', 'who', 'whom', 'does', 'do', 'did',
57
+ 'should', 'would', 'could', 'can', 'may', 'might', 'will', 'shall', 'from', 'at',
58
+ 'into', 'than', 'then', 'was', 'were', 'been', 'has', 'have', 'had', 'not', 'no',
59
+ 'but', 'if', 'so', 'about', 'after', 'before', 'there', 'their', 'its', 'our',
60
+ 'we', 'you', 'your', 'any', 'all', 'each', 'per', 'via', 'up', 'out', 'over',
61
+ 'only', 'also', 'just', 'too', 'very', 'us', 'me', 'my', 'he', 'she', 'they',
62
+ 'them', 'these', 'those', 'am', 'being', 'both', 'more', 'most', 'some', 'such',
63
+ 'cần', 'và', 'là', 'cho', 'một', 'những', 'dùng',
64
+ ]);
65
+
66
+ // Pre-3.5.1 episode text template ("Session <id>: writeOk, verify=ok, N receipts,
67
+ // last command <cmd>") - template words must never produce a match.
68
+ const EPISODE_TEMPLATE_TOKENS = new Set([
69
+ 'session', 'writeok', 'writefail', 'verify', 'receipts', 'receipt', 'last', 'command',
70
+ 'ok', 'fail', 'run', 'n/a',
71
+ ]);
72
+ const UUID_RE = /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/g;
73
+
74
+ export function normalize(text) {
75
+ return String(text ?? '')
76
+ .toLowerCase()
77
+ .replace(UUID_RE, ' ')
78
+ .replace(/[^\p{L}\p{N}\s./:_-]/gu, ' ')
79
+ .replace(/\s+/g, ' ')
80
+ .trim();
81
+ }
82
+
83
+ const STEM_SUFFIXES = ['ations', 'ation', 'ments', 'ment', 'ings', 'ing', 'ied', 'ed', 'ers', 'er', 'es', 's', 'ly'];
84
+
85
+ // Word forms must meet on one stem: `rule`/`rules`, `file`/`files`,
86
+ // `update`/`updated`, `call`/`called`, `add`/`adding`. So every alphabetic token
87
+ // gets the same final normalisation (one of a doubled final consonant, then a
88
+ // trailing `e`), whether or not a suffix was stripped first.
89
+ function finishStem(base) {
90
+ let out = base;
91
+ if (out.length >= 3 && /(ll|tt|pp|dd|gg|nn|mm|rr)$/.test(out)) out = out.slice(0, -1);
92
+ if (out.length >= 4 && out.endsWith('e')) out = out.slice(0, -1);
93
+ return out;
94
+ }
95
+
96
+ // Forms the suffix rules get wrong: `plane` must not meet `plan`, `written` must
97
+ // meet `write`/`writer`, and `status` must meet `statuses`.
98
+ const STEM_EXCEPTIONS = new Map([
99
+ ['plane', 'plane'], ['planes', 'plane'], ['written', 'writ'], ['status', 'status'],
100
+ ]);
101
+
102
+ function stem(token) {
103
+ if (token.length < 3 || !/^[a-z]+$/.test(token)) return token;
104
+ const fixed = STEM_EXCEPTIONS.get(token);
105
+ if (fixed) return fixed;
106
+ if (/ie[sd]$/.test(token) && token.length > 5) return `${token.slice(0, -3)}y`;
107
+ if (token.length >= 4) {
108
+ for (const suffix of STEM_SUFFIXES) {
109
+ if (token.endsWith(suffix) && token.length - suffix.length >= 3) {
110
+ if (suffix === 's' && token.endsWith('ss')) break;
111
+ return finishStem(token.slice(0, -suffix.length));
112
+ }
113
+ }
114
+ }
115
+ return finishStem(token);
116
+ }
117
+
118
+ // Tokens of `text`: whole tokens plus their separator-split parts
119
+ // (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
120
+ export function tokenize(text) {
121
+ const out = [];
122
+ for (const raw of normalize(text).split(/\s+/)) {
123
+ if (!raw) continue;
124
+ const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
125
+ const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
126
+ for (const candidate of candidates) {
127
+ if (candidate.length < 2 || STOPWORDS.has(candidate)) continue;
128
+ out.push(stem(candidate));
129
+ }
130
+ }
131
+ return out;
132
+ }
133
+
134
+ function episodeBody(text) {
135
+ return String(text ?? '')
136
+ .replace(/^\s*Session\s+\S+?:\s*/i, '')
137
+ .replace(/\b(writeOk|writeFail)\b/g, ' ')
138
+ .replace(/\bverify=(ok|fail|not-run)\b/g, ' ')
139
+ .replace(/\b\d+\s+receipts?\b/g, ' ')
140
+ .replace(/\blast command\b/gi, ' ');
141
+ }
142
+
143
+ export function isTemplateText(doc) {
144
+ return doc?.type === 'episode' || doc?.provenance === 'exec-ledger';
145
+ }
146
+
147
+ // Pre-3.5.1 content-free episode shape. Only this shape gets the template-token
148
+ // filter: a 3.5.1 episode ("Task: … . Wrote: … . Verify: …") carries the user's
149
+ // own words, and dropping `session`/`command`/`run` from it would hide real tasks.
150
+ const LEGACY_EPISODE_RE = /^\s*Session\s+\S+?:\s*(?:writeOk|writeFail)\b/;
151
+ // Field labels of the 3.5.1 episode text; present in every episode, never content.
152
+ const EPISODE_LABEL_RE = /(?:^|\.\s)(?:Task|Wrote|Verify|Unfinished):/g;
153
+
154
+ // Distinct match tokens of a document.
155
+ export function docTokens(doc) {
156
+ if (!isTemplateText(doc)) return tokenize(doc?.text);
157
+ const text = String(doc.text ?? '');
158
+ if (LEGACY_EPISODE_RE.test(text)) {
159
+ return tokenize(episodeBody(text)).filter((token) => !EPISODE_TEMPLATE_TOKENS.has(token));
160
+ }
161
+ return tokenize(text.replace(EPISODE_LABEL_RE, ' '));
162
+ }
163
+
164
+ function idf(totalDocs, docFreq) {
165
+ return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
166
+ }
167
+
168
+ // rankDocs(docs, query, {limit?}) -> {
169
+ // ranked:[{doc, score, matched}], candidateCount, queryTokenCount, topScore,
170
+ // abstained }
171
+ // `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
172
+ // `query` is a string or a pre-tokenized array. Never throws.
173
+ export function rankDocs(docs, query, { limit = 10 } = {}) {
174
+ const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
175
+ const empty = {
176
+ ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
177
+ };
178
+ if (!Array.isArray(docs) || docs.length === 0 || queryTokens.length === 0) return empty;
179
+
180
+ const stats = [];
181
+ const docFreq = new Map();
182
+ let totalLength = 0;
183
+ for (const doc of docs) {
184
+ if (doc == null) continue;
185
+ const tokens = docTokens(doc);
186
+ const tf = new Map();
187
+ for (const token of tokens) tf.set(token, (tf.get(token) ?? 0) + 1);
188
+ for (const token of tf.keys()) docFreq.set(token, (docFreq.get(token) ?? 0) + 1);
189
+ totalLength += tokens.length;
190
+ stats.push({ doc, tf, length: tokens.length });
191
+ }
192
+ const total = stats.length;
193
+ if (total === 0) return empty;
194
+ const avgLength = totalLength / total || 1;
195
+ const idfMax = idf(total, 1);
196
+ // IDF cannot discriminate in a tiny corpus, so the multi-token requirement
197
+ // only applies once there are enough documents for it to mean something.
198
+ const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
199
+
200
+ const scored = [];
201
+ let candidateCount = 0;
202
+ for (const { doc, tf, length } of stats) {
203
+ let score = 0;
204
+ let idfMass = 0;
205
+ let matched = 0;
206
+ for (const token of queryTokens) {
207
+ const freq = tf.get(token);
208
+ if (!freq) continue;
209
+ const weight = idf(total, docFreq.get(token));
210
+ matched += 1;
211
+ idfMass += weight;
212
+ score += weight * ((freq * (K1 + 1)) / (freq + K1 * (1 - B + B * (length / avgLength))));
213
+ }
214
+ if (matched === 0) continue;
215
+ candidateCount += 1;
216
+ if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
217
+ if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
218
+ scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
219
+ }
220
+ // Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
221
+ // entry must not reorder the injected block and invalidate the prompt cache.
222
+ scored.sort((a, b) => b.score - a.score
223
+ || String(a.doc.id ?? '').localeCompare(String(b.doc.id ?? '')));
224
+ const top = scored.length > 0 ? scored[0].score : 0;
225
+ const ranked = scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top).slice(0, limit);
226
+ return {
227
+ ranked,
228
+ candidateCount,
229
+ queryTokenCount: queryTokens.length,
230
+ topScore: top,
231
+ abstained: ranked.length === 0,
232
+ };
233
+ }
234
+
235
+ // ---- Markdown source ------------------------------------------------------
236
+
237
+ export function normalizeMemorySources(config) {
238
+ const raw = config?.memory?.sources;
239
+ const list = Array.isArray(raw) ? raw : DEFAULT_MEMORY_SOURCES;
240
+ const seen = new Set();
241
+ const out = [];
242
+ for (const entry of list) {
243
+ if (typeof entry !== 'string') continue;
244
+ const rel = entry.trim().replace(/\\/g, '/');
245
+ if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
246
+ if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
247
+ seen.add(rel);
248
+ out.push(rel);
249
+ if (out.length >= MAX_SOURCES) break;
250
+ }
251
+ return out;
252
+ }
253
+
254
+ function cleanMarkdown(text) {
255
+ return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
256
+ }
257
+
258
+ // `ukit memory promote` renders active project_rule/procedure records into this
259
+ // block. Those records are already recall documents, so the block is skipped:
260
+ // indexing it too would return the same rule twice (record + bullet).
261
+ const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
262
+ const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
263
+
264
+ // Top-level bullets ("- " / "* ") with their indented continuation lines.
265
+ export function parseMarkdownBullets(content, relPath) {
266
+ const docs = [];
267
+ const lines = String(content ?? '').split(/\r?\n/);
268
+ let heading = '';
269
+ let fenced = false;
270
+ let learned = false;
271
+ let current = null;
272
+ const flush = () => {
273
+ if (!current) return;
274
+ const text = cleanMarkdown(current.parts.join(' '));
275
+ if (text.length >= MIN_BULLET_CHARS) {
276
+ docs.push({
277
+ id: `md:${relPath}:${current.line}`,
278
+ type: 'markdown',
279
+ source: 'markdown',
280
+ text,
281
+ provenance: `${relPath}:${current.line}`,
282
+ line: current.line,
283
+ heading: current.heading,
284
+ created_at: 0,
285
+ });
286
+ }
287
+ current = null;
288
+ };
289
+ lines.forEach((line, idx) => {
290
+ if (/^\s*(```|~~~)/.test(line)) {
291
+ fenced = !fenced;
292
+ flush();
293
+ return;
294
+ }
295
+ if (fenced) return;
296
+ if (LEARNED_START_RE.test(line)) { learned = true; flush(); return; }
297
+ if (LEARNED_END_RE.test(line)) { learned = false; return; }
298
+ if (learned) return;
299
+ const headingMatch = /^#{1,6}\s+(.*)$/.exec(line);
300
+ if (headingMatch) {
301
+ flush();
302
+ heading = headingMatch[1].trim();
303
+ return;
304
+ }
305
+ if (/^[-*]\s+/.test(line)) {
306
+ flush();
307
+ current = { line: idx + 1, heading, parts: [line.replace(/^[-*]\s+/, '')] };
308
+ return;
309
+ }
310
+ if (current && /^\s{2,}\S/.test(line)) {
311
+ current.parts.push(line.trim());
312
+ return;
313
+ }
314
+ flush();
315
+ });
316
+ flush();
317
+ return docs;
318
+ }
319
+
320
+ const markdownCache = new Map(); // abs path -> {size, mtimeMs, docs}
321
+
322
+ export function resetMarkdownCacheForTests() {
323
+ markdownCache.clear();
324
+ }
325
+
326
+ // loadMarkdownDocs(projectRoot, sources) -> docs[]; a missing/unreadable/oversized
327
+ // file contributes nothing and is never an error. Async I/O only.
328
+ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOURCES) {
329
+ if (typeof projectRoot !== 'string' || projectRoot === '') return [];
330
+ const root = path.resolve(projectRoot);
331
+ const docs = [];
332
+ let realRoot = null;
333
+ for (const rel of sources) {
334
+ const abs = path.resolve(root, rel);
335
+ if (abs !== root && !abs.startsWith(root + path.sep)) continue;
336
+ try {
337
+ // A symlinked source must still resolve inside the project.
338
+ realRoot ??= await fsp.realpath(root);
339
+ const real = await fsp.realpath(abs);
340
+ if (!real.startsWith(realRoot + path.sep)) continue;
341
+ const st = await fsp.stat(abs);
342
+ if (!st.isFile() || st.size > MAX_SOURCE_BYTES) continue;
343
+ const cached = markdownCache.get(abs);
344
+ if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) {
345
+ docs.push(...cached.docs);
346
+ continue;
347
+ }
348
+ const parsed = parseMarkdownBullets(await fsp.readFile(abs, 'utf8'), rel);
349
+ markdownCache.set(abs, { size: st.size, mtimeMs: st.mtimeMs, docs: parsed });
350
+ docs.push(...parsed);
351
+ } catch {
352
+ markdownCache.delete(abs);
353
+ }
354
+ }
355
+ return docs;
356
+ }
357
+
358
+ const SNIPPET_CHARS = 160;
359
+
360
+ // Shared hook lane: score ALL eligible docs (no newest-N pre-filter) together
361
+ // with the markdown source bullets, then cut. Async I/O only; never throws.
362
+ export async function recallForHook({ projectRoot, query, docs = [], config = null, limit = 2 } = {}) {
363
+ let markdown = [];
364
+ try {
365
+ markdown = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config));
366
+ } catch {
367
+ markdown = [];
368
+ }
369
+ let result;
370
+ try {
371
+ result = rankDocs([...docs, ...markdown], query, { limit });
372
+ } catch {
373
+ result = rankDocs([], '', { limit });
374
+ }
375
+ await emitRecallTelemetry(projectRoot, config, result);
376
+ return result;
377
+ }
378
+
379
+ // One metadata-only `retrieval.query` event per hook recall, through the
380
+ // shared standalone recorder (observability-emit.mjs, a sibling in the shipped
381
+ // runtime dir). Counts, abstention, top score and record refs only - never the
382
+ // prompt, record text or a raw record/session id. Async, bounded, never throws; the observability stage
383
+ // and the memoryV2 rollout stage are honoured (off/kill-switch => no event).
384
+ const TELEMETRY_DEADLINE_MS = 120;
385
+
386
+ async function emitRecallTelemetry(projectRoot, config, result) {
387
+ try {
388
+ const v2 = config?.memoryV2;
389
+ if (v2?.killSwitch === true || v2?.eligibility?.stage === 'off') return;
390
+ const emitter = await import('./observability-emit.mjs');
391
+ await emitter.emitOutcomeObserved({
392
+ projectRoot,
393
+ config,
394
+ deadlineMs: TELEMETRY_DEADLINE_MS,
395
+ record: {
396
+ semantic_name: 'retrieval.query',
397
+ payload: {
398
+ lane: 'hook',
399
+ engine: RECALL_ENGINE_VERSION,
400
+ query_token_count: result.queryTokenCount,
401
+ candidate_count: result.candidateCount,
402
+ returned_count: result.ranked.length,
403
+ abstained: result.abstained,
404
+ top_score: Number(result.topScore.toFixed(4)),
405
+ // Markdown refs are repo paths; record/session/project ids are hashed,
406
+ // as the CLI lane does (memoryEmit.js), because they can identify.
407
+ returned_refs: result.ranked.map((entry) => (entry.doc.provenance && entry.doc.type === 'markdown'
408
+ ? String(entry.doc.provenance).slice(0, 96)
409
+ : crypto.createHash('sha256').update(String(entry.doc.id)).digest('hex'))),
410
+ },
411
+ },
412
+ });
413
+ } catch {
414
+ // Telemetry never affects recall.
415
+ }
416
+ }
417
+
418
+ // Cited-data rendering: provenance first so truncation never drops it.
419
+ export function renderMarkdownDoc(doc, maxChars = SNIPPET_CHARS) {
420
+ const text = String(doc?.text ?? '');
421
+ const body = text.length > maxChars ? `${text.slice(0, maxChars - 3)}...` : text;
422
+ return `[${doc?.provenance ?? 'markdown'}] ${body}`;
423
+ }
@@ -30,6 +30,7 @@ import {
30
30
  } from './memory-policy.mjs';
31
31
  import { resolveRecordFreshness } from './memory-freshness.mjs';
32
32
  import { resolveMemoryStage } from './memory-flags.mjs';
33
+ import { recallForHook, renderMarkdownDoc } from './memory-recall.mjs';
33
34
 
34
35
  // Hook-context self-deadline (2.4.1 orphan-leak class): the reinject-context hook passes
35
36
  // UKIT_HOOK_DEADLINE_MS so a wedged read can never orphan this process past the hook
@@ -957,12 +958,6 @@ function normalize(text) {
957
958
  .trim();
958
959
  }
959
960
 
960
- function tokenize(text) {
961
- return normalize(text)
962
- .split(/\s+/)
963
- .filter((token) => token && !STOPWORDS.has(token) && token.length > 1);
964
- }
965
-
966
961
  function getMemoryTimestamp(item) {
967
962
  return item.content?.updatedAt
968
963
  ?? item.content?.endedAt
@@ -1006,29 +1001,6 @@ function buildMemorySegments(item) {
1006
1001
  ];
1007
1002
  }
1008
1003
 
1009
- function scoreMemoryItem(item, queryTokens) {
1010
- if (queryTokens.length === 0) {
1011
- return 0;
1012
- }
1013
-
1014
- let score = 0;
1015
- for (const segment of buildMemorySegments(item)) {
1016
- const segmentTokens = new Set(tokenize(segment.text));
1017
- for (const token of queryTokens) {
1018
- if (segmentTokens.has(token)) {
1019
- score += segment.weight;
1020
- }
1021
- }
1022
- }
1023
-
1024
- if (score === 0) {
1025
- return 0;
1026
- }
1027
-
1028
- const recencyBonus = getMemoryTimestamp(item) > 0 ? Math.min(1, getMemoryTimestamp(item) / Date.now()) : 0;
1029
- return score + recencyBonus;
1030
- }
1031
-
1032
1004
  async function readDirectoryJsonItems(dirPath) {
1033
1005
  let entries = [];
1034
1006
  try {
@@ -1101,8 +1073,6 @@ function recordLegacyKey(item) {
1101
1073
  return null;
1102
1074
  }
1103
1075
 
1104
- const MEMORY_V2_RECORD_POOL_LIMIT = 20;
1105
-
1106
1076
  // v2 lane (M01-05): approved records live in a single records.json document.
1107
1077
  // Missing/malformed store → [] (never-throw, same as readDirectoryJsonItems).
1108
1078
  // Stale snapshots excluded here AND re-checked by eligible() downstream.
@@ -1125,8 +1095,6 @@ export async function listMemoryV2RecordItems(runtimePathsOrRoot, { config, proj
1125
1095
  .filter((record) => record && typeof record === 'object')
1126
1096
  .filter((record) => record.status === 'active')
1127
1097
  .filter((record) => record.valid_until == null || record.valid_until > now)
1128
- .sort((left, right) => (right.created_at ?? 0) - (left.created_at ?? 0))
1129
- .slice(0, MEMORY_V2_RECORD_POOL_LIMIT)
1130
1098
  .map((record) => ({
1131
1099
  id: `record:${record.id}`,
1132
1100
  type: 'record',
@@ -1247,14 +1215,13 @@ async function buildPreviousContextLines(projectRoot, state, config) {
1247
1215
 
1248
1216
  const binding = await resolveProjectBinding(projectRoot);
1249
1217
  const items = await listMemoryItems(projectRoot, { config, projectId: binding.projectId });
1250
- const queryTokens = tokenize(taskQuery);
1251
1218
  // v2/legacy dedup (M01-05): drop a record whose logical twin
1252
1219
  // (meta.legacyId → session:<id>, project_id → project:<id>) exists.
1253
1220
  const legacyIds = new Set(
1254
1221
  items.filter((item) => item.type !== 'record').map((item) => item.id),
1255
1222
  );
1256
1223
  const eligibleCtx = { projectId: binding.projectId, projectAliases: binding.aliases, includeUser: false };
1257
- const rankedItems = items
1224
+ const eligibleItems = items
1258
1225
  .filter((item) => item.type !== 'user')
1259
1226
  .filter((item) => {
1260
1227
  if (item.type === 'record') {
@@ -1266,15 +1233,30 @@ async function buildPreviousContextLines(projectRoot, state, config) {
1266
1233
  return matchesProjectBinding(item, binding);
1267
1234
  }
1268
1235
  return true;
1269
- })
1270
- .map((item) => ({ item, score: scoreMemoryItem(item, queryTokens) }))
1271
- .filter((entry) => entry.score > 0)
1272
- .sort((left, right) => (
1273
- right.score - left.score
1274
- || getMemoryTimestamp(right.item) - getMemoryTimestamp(left.item)
1275
- ))
1276
- .slice(0, 2)
1277
- .map((entry) => entry.item);
1236
+ });
1237
+ // Score-before-cut (3.5.1): every eligible item plus the markdown source
1238
+ // bullets goes through the shared BM25 engine; only then is the top 2 cut.
1239
+ const itemById = new Map();
1240
+ const docs = eligibleItems.map((item) => {
1241
+ itemById.set(item.id, item);
1242
+ return {
1243
+ id: item.id,
1244
+ type: item.type === 'record' ? item.record.type : item.type,
1245
+ provenance: item.type === 'record' ? item.record.provenance : undefined,
1246
+ // Records go in as their raw text, exactly as the CLI lane passes them: a
1247
+ // type prefix would defeat the engine's start-anchored episode shape checks.
1248
+ text: item.type === 'record'
1249
+ ? String(item.record.text ?? '')
1250
+ : buildMemorySegments(item).map((segment) => String(segment.text ?? '')).join(' '),
1251
+ created_at: getMemoryTimestamp(item),
1252
+ };
1253
+ });
1254
+ const recalled = await recallForHook({
1255
+ projectRoot, query: taskQuery, docs, config, limit: 2,
1256
+ });
1257
+ const rankedItems = recalled.ranked.map((entry) => (entry.doc.type === 'markdown'
1258
+ ? { id: entry.doc.id, type: 'markdown', doc: entry.doc }
1259
+ : itemById.get(entry.doc.id))).filter(Boolean);
1278
1260
 
1279
1261
  if (rankedItems.length === 0) {
1280
1262
  return [];
@@ -1289,7 +1271,9 @@ async function buildPreviousContextLines(projectRoot, state, config) {
1289
1271
  } catch { /* absent → unlabeled */ }
1290
1272
  }));
1291
1273
 
1292
- return [`- Previous context: ${rankedItems.map((item) => buildPreviousContextSnippet(item, freshnessById.get(item.id))).join(' | ')}`];
1274
+ return [`- Previous context: ${rankedItems.map((item) => (item.type === 'markdown'
1275
+ ? renderMarkdownDoc(item.doc, 120)
1276
+ : buildPreviousContextSnippet(item, freshnessById.get(item.id)))).join(' | ')}`];
1293
1277
  }
1294
1278
 
1295
1279
  // BUG-C23-13: only run main() on direct CLI invocation — an unguarded bottom