@ngockhoale/ukit 3.4.85 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -857
- package/bin/ukit +9 -0
- package/manifests/documentation.yaml +12 -0
- package/package.json +1 -1
- package/src/cli/commands/install.js +12 -0
- package/src/cli/commands/memory.js +30 -3
- package/src/core/memory/episodeArchive.js +147 -0
- package/src/core/memory/episodes.js +199 -37
- package/src/core/memory/memoryHit.js +59 -40
- package/src/core/memory/mutateMemory.js +23 -3
- package/src/core/memory/recallEngine.js +423 -0
- package/src/core/memory/retrieval.js +55 -92
- package/src/core/observability/emit/lifecycle.js +30 -0
- package/src/core/runtimeConfig.js +6 -0
- package/template_project/.claude/hooks/skill-router.sh +82 -51
- package/template_project/.claude/ukit/index/route-task.mjs +29 -48
- package/template_project/.claude/ukit/runtime/execution-ledger.mjs +45 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +423 -0
- package/template_project/.claude/ukit/runtime/reinject-context.mjs +29 -45
|
@@ -6,12 +6,10 @@
|
|
|
6
6
|
// eligibility and the composite score are the only gates.
|
|
7
7
|
|
|
8
8
|
import { eligible } from './policy.js';
|
|
9
|
-
import {
|
|
10
|
-
import { LEGACY_SCOPE_MATCH
|
|
9
|
+
import { rankDocs } from './recallEngine.js';
|
|
10
|
+
import { LEGACY_SCOPE_MATCH } from './retrieval.js';
|
|
11
11
|
|
|
12
12
|
const SNIPPET_MAX = 160;
|
|
13
|
-
const TYPE_WEIGHT_SCALE = 10; // typeWeight bounded to ≤0.7 bonus
|
|
14
|
-
const RECENCY_HALF_LIFE_MS = 30 * 24 * 60 * 60 * 1000; // bounded decay on created_at
|
|
15
13
|
|
|
16
14
|
function firstCitation(evidence) {
|
|
17
15
|
if (!Array.isArray(evidence)) return null;
|
|
@@ -90,55 +88,76 @@ function hasLocatorEvidence(record) {
|
|
|
90
88
|
// → MemoryHit[]
|
|
91
89
|
// Eligibility first (eligible() + requested-scope narrowing), then composite
|
|
92
90
|
// score = lexical + typeBonus + evidenceBonus + recencyBonus. Dedup by id.
|
|
93
|
-
export function rankHits(records, {
|
|
94
|
-
|
|
91
|
+
export function rankHits(records, opts = {}) {
|
|
92
|
+
return rankHitsDetailed(records, opts).hits;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Markdown-source documents (read-only bullets, provenance `path:line`) become
|
|
96
|
+
// hits with a file citation; they are never records.
|
|
97
|
+
export function buildMarkdownHit(doc, { score = 0 } = {}) {
|
|
98
|
+
const text = String(doc?.text ?? '');
|
|
99
|
+
const snippet = text.length > SNIPPET_MAX ? text.slice(0, SNIPPET_MAX) : text;
|
|
100
|
+
return {
|
|
101
|
+
id: doc.id,
|
|
102
|
+
type: 'markdown',
|
|
103
|
+
scope: 'repo',
|
|
104
|
+
projectId: null,
|
|
105
|
+
snippet,
|
|
106
|
+
citation: { kind: 'file', locator: doc.provenance, observedAt: null },
|
|
107
|
+
observedAt: null,
|
|
108
|
+
validFrom: null,
|
|
109
|
+
validUntil: null,
|
|
110
|
+
revision: 1,
|
|
111
|
+
freshness: null,
|
|
112
|
+
reasons: ['lexical', 'source:markdown'],
|
|
113
|
+
relevanceScore: score,
|
|
114
|
+
summary: snippet,
|
|
115
|
+
source: 'markdown',
|
|
116
|
+
timestamp: 0,
|
|
117
|
+
provenance: doc.provenance,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Single ranking engine (recallEngine.js, BM25): eligibility + requested scope
|
|
122
|
+
// narrow the corpus FIRST, then IDF is computed over exactly that eligible set
|
|
123
|
+
// (plus `extraDocs`, e.g. markdown bullets). No wall-clock term, a score floor
|
|
124
|
+
// and an explicit empty result. Type weights, the evidence bonus and the
|
|
125
|
+
// half-life recency bonus were removed from the score: they only ever added a
|
|
126
|
+
// query-independent constant that surfaced content-free episodes.
|
|
127
|
+
export function rankHitsDetailed(records, {
|
|
128
|
+
query = '',
|
|
129
|
+
queryTokens,
|
|
95
130
|
eligibleCtx = {},
|
|
96
131
|
policy = {},
|
|
97
132
|
scope = 'all',
|
|
98
133
|
limit = 10,
|
|
99
134
|
now,
|
|
135
|
+
extraDocs = [],
|
|
100
136
|
} = {}) {
|
|
101
|
-
if (!Array.isArray(records) || records.length === 0) return [];
|
|
102
|
-
if (!Array.isArray(queryTokens) || queryTokens.length === 0) return [];
|
|
103
137
|
const nowMs = typeof now === 'number' ? now : Date.now();
|
|
104
138
|
const ctx = { ...eligibleCtx, now: nowMs };
|
|
105
|
-
|
|
106
139
|
const seen = new Set();
|
|
107
|
-
const
|
|
108
|
-
for (const record of records) {
|
|
140
|
+
const docs = [];
|
|
141
|
+
for (const record of Array.isArray(records) ? records : []) {
|
|
109
142
|
if (record == null || seen.has(record.id)) continue;
|
|
110
143
|
seen.add(record.id);
|
|
111
144
|
if (!eligible(record, ctx, policy).ok) continue;
|
|
112
145
|
if (!matchesRequestedScope(record, scope)) continue;
|
|
113
|
-
|
|
114
|
-
const lexical = scoreRecordTokens(record, queryTokens, nowMs);
|
|
115
|
-
const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
|
|
116
|
-
const typeBonus = typeWeight / TYPE_WEIGHT_SCALE;
|
|
117
|
-
const evidenceBonus = hasLocatorEvidence(record) ? 1 : 0;
|
|
118
|
-
const age = nowMs - (record.created_at ?? 0);
|
|
119
|
-
const recencyBonus = record.created_at > 0 && age >= 0
|
|
120
|
-
? Math.exp(-Math.LN2 * age / RECENCY_HALF_LIFE_MS)
|
|
121
|
-
: 0;
|
|
122
|
-
const score = lexical + typeBonus + evidenceBonus + recencyBonus;
|
|
123
|
-
|
|
124
|
-
const reasons = [];
|
|
125
|
-
if (lexical > 0) reasons.push('lexical');
|
|
126
|
-
reasons.push(`type:${record.type}`);
|
|
127
|
-
if (evidenceBonus > 0) reasons.push('evidence');
|
|
128
|
-
if (recencyBonus > 0) reasons.push('recency');
|
|
129
|
-
|
|
130
|
-
scored.push({ record, score, reasons });
|
|
146
|
+
docs.push(record);
|
|
131
147
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|| String(left.record.text ?? '').localeCompare(String(right.record.text ?? ''))
|
|
138
|
-
))
|
|
139
|
-
.slice(0, limit)
|
|
140
|
-
.map((entry) => buildMemoryHit(entry.record, {
|
|
148
|
+
for (const doc of extraDocs) docs.push(doc);
|
|
149
|
+
const result = rankDocs(docs, Array.isArray(queryTokens) ? queryTokens : query, { limit });
|
|
150
|
+
const hits = result.ranked.map((entry) => (entry.doc.type === 'markdown'
|
|
151
|
+
? buildMarkdownHit(entry.doc, { score: entry.score })
|
|
152
|
+
: buildMemoryHit(entry.doc, {
|
|
141
153
|
score: entry.score,
|
|
142
|
-
reasons: entry.
|
|
143
|
-
}));
|
|
154
|
+
reasons: ['lexical', `type:${entry.doc.type}`, ...(hasLocatorEvidence(entry.doc) ? ['evidence'] : [])],
|
|
155
|
+
})));
|
|
156
|
+
return {
|
|
157
|
+
hits,
|
|
158
|
+
candidateCount: result.candidateCount,
|
|
159
|
+
queryTokenCount: result.queryTokenCount,
|
|
160
|
+
topScore: result.topScore,
|
|
161
|
+
abstained: result.abstained,
|
|
162
|
+
};
|
|
144
163
|
}
|
|
@@ -30,7 +30,7 @@ import { resolveMemoryStage, appendRolloutReceipt, latencyBandForMs } from './me
|
|
|
30
30
|
import { resolveProjectIdentity } from './projectIdentity.js';
|
|
31
31
|
|
|
32
32
|
const JOURNAL_MAX = 256;
|
|
33
|
-
const OPS = new Set(['add', 'update', 'archive', 'forget', 'purge', 'restore']);
|
|
33
|
+
const OPS = new Set(['add', 'update', 'archive', 'forget', 'purge', 'restore', 'relocate']);
|
|
34
34
|
|
|
35
35
|
function sha1(text) {
|
|
36
36
|
return crypto.createHash('sha1').update(String(text)).digest('hex');
|
|
@@ -85,14 +85,18 @@ function findRecord(records, payload) {
|
|
|
85
85
|
}
|
|
86
86
|
|
|
87
87
|
/**
|
|
88
|
-
* mutateMemory(target, envelope, {classifyAdd, guard} = {}) → Promise<result>
|
|
88
|
+
* mutateMemory(target, envelope, {classifyAdd, guard, sink} = {}) → Promise<result>
|
|
89
|
+
* op 'relocate' (payload {ids}): moves records out of the live store without a
|
|
90
|
+
* tombstone, so they stay restorable. `sink(records)` is required and runs
|
|
91
|
+
* inside the lock BEFORE the removal; if it throws, nothing is removed.
|
|
92
|
+
* Result {status:'ok', relocated, remaining}.
|
|
89
93
|
* target = {kind:'project', projectRoot} | {kind:'user', homeDir}
|
|
90
94
|
* envelope = {op, payload?, idempotencyKey?, expectedRevision?,
|
|
91
95
|
* expectedGeneration?, actor?, confirmScope?}
|
|
92
96
|
* result = {status:'ok'|'conflict'|'rejected'|'duplicate'|'not-found'|
|
|
93
97
|
* 'corrupt'|'disabled'|'review-queue-full', record?, reason?, labels?}
|
|
94
98
|
*/
|
|
95
|
-
export async function mutateMemory(target, envelope = {}, { classifyAdd, guard } = {}) {
|
|
99
|
+
export async function mutateMemory(target, envelope = {}, { classifyAdd, guard, sink } = {}) {
|
|
96
100
|
const { recordsPath, paths } = resolveTarget(target);
|
|
97
101
|
const journalPath = path.join(paths.memoryV2Dir, 'mutation-journal.json');
|
|
98
102
|
const { op, payload, idempotencyKey, expectedRevision, expectedGeneration,
|
|
@@ -100,6 +104,9 @@ export async function mutateMemory(target, envelope = {}, { classifyAdd, guard }
|
|
|
100
104
|
if (!OPS.has(op)) {
|
|
101
105
|
return { status: 'rejected', reason: `unknown-op:${String(op)}` };
|
|
102
106
|
}
|
|
107
|
+
if (op === 'relocate' && typeof sink !== 'function') {
|
|
108
|
+
return { status: 'rejected', reason: 'relocate-sink-required' };
|
|
109
|
+
}
|
|
103
110
|
let config = null;
|
|
104
111
|
let receiptStage = null;
|
|
105
112
|
let projectId = null;
|
|
@@ -151,6 +158,19 @@ export async function mutateMemory(target, envelope = {}, { classifyAdd, guard }
|
|
|
151
158
|
if (expectedGeneration !== undefined && expectedGeneration !== prior.generation) {
|
|
152
159
|
return { result: { status: 'conflict', reason: 'generation-mismatch' } };
|
|
153
160
|
}
|
|
161
|
+
if (op === 'relocate') {
|
|
162
|
+
const wanted = new Set(Array.isArray(payload?.ids) ? payload.ids : []);
|
|
163
|
+
const moving = records.filter((r) => wanted.has(r.id));
|
|
164
|
+
if (moving.length === 0) {
|
|
165
|
+
return { result: { status: 'ok', relocated: 0, remaining: records.length } };
|
|
166
|
+
}
|
|
167
|
+
await sink(moving);
|
|
168
|
+
const keep = records.filter((r) => !wanted.has(r.id));
|
|
169
|
+
return {
|
|
170
|
+
result: { status: 'ok', relocated: moving.length, remaining: keep.length },
|
|
171
|
+
records: keep,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
154
174
|
const existing = op === 'add' || op === 'restore' ? undefined : findRecord(records, payload);
|
|
155
175
|
if (op !== 'add' && op !== 'restore' && !existing) {
|
|
156
176
|
return { result: { status: 'not-found' } };
|
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
// Memory recall engine (release 3.5.1) - the ONE ranking implementation used by
|
|
2
|
+
// the CLI (src/core/memory/retrieval.js, memoryHit.js) and, through the
|
|
3
|
+
// byte-identical shipped mirror template_project/.claude/ukit/runtime/
|
|
4
|
+
// memory-recall.mjs, by the route-task / reinject-context / skill-router hook
|
|
5
|
+
// lanes. Parity is enforced by tests/consistency/memoryRecallParity.test.js:
|
|
6
|
+
// this file and the mirror must stay byte-identical, so it imports only node:
|
|
7
|
+
// builtins and is fully async on the I/O path (hook deadline rule, MEMORY.md
|
|
8
|
+
// 2026-09-19).
|
|
9
|
+
//
|
|
10
|
+
// Ranking: BM25 with IDF computed over the eligible document set the caller
|
|
11
|
+
// passes in, length normalisation, a multiplicative type prior, no wall-clock
|
|
12
|
+
// term. A hit must clear a
|
|
13
|
+
// floor (matched-token count + matched IDF mass relative to the corpus size);
|
|
14
|
+
// when nothing clears it the result is explicitly empty (abstention).
|
|
15
|
+
// Tokens that come from the pre-3.5.1 episode text template never count.
|
|
16
|
+
//
|
|
17
|
+
// Markdown source: bullets of configured .md files (default docs/MEMORY.md)
|
|
18
|
+
// become read-only documents with provenance `path:line`. They are parsed on
|
|
19
|
+
// demand, cached by file size + mtime, and never copied into records.json.
|
|
20
|
+
|
|
21
|
+
import crypto from 'node:crypto';
|
|
22
|
+
import fsp from 'node:fs/promises';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
|
|
25
|
+
export const RECALL_ENGINE_VERSION = 'bm25-1';
|
|
26
|
+
export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md']);
|
|
27
|
+
|
|
28
|
+
const K1 = 1.2;
|
|
29
|
+
const B = 0.75;
|
|
30
|
+
// A document is a hit only when it matches at least this many distinct query
|
|
31
|
+
// tokens (or all of them when the query has fewer) ...
|
|
32
|
+
const MIN_MATCHED_TOKENS = 2;
|
|
33
|
+
const SMALL_CORPUS_DOCS = 8;
|
|
34
|
+
// One matched token is enough only for curated records; markdown bullets and
|
|
35
|
+
// episodes (the noisy sources) must match two distinct tokens.
|
|
36
|
+
const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
|
|
37
|
+
// ... and the IDF mass of the matched tokens reaches this fraction of the
|
|
38
|
+
// largest IDF the corpus can produce (a token present in exactly one doc).
|
|
39
|
+
const MIN_IDF_MASS_RATIO = 0.5;
|
|
40
|
+
// Results scoring below this fraction of the best hit are noise tail.
|
|
41
|
+
const RELATIVE_CUTOFF = 0.35;
|
|
42
|
+
// Query-dependent type prior: multiplies the lexical score, so it can only
|
|
43
|
+
// reorder documents that already match (it never surfaces a non-matching one,
|
|
44
|
+
// unlike the additive type/evidence/recency bonuses removed in 3.5.1). A
|
|
45
|
+
// curated rule or decision outranks an episode with the same lexical match.
|
|
46
|
+
const TYPE_PRIOR = Object.freeze({
|
|
47
|
+
project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
|
|
48
|
+
});
|
|
49
|
+
const MAX_SOURCES = 8;
|
|
50
|
+
const MAX_SOURCE_BYTES = 512 * 1024;
|
|
51
|
+
const MIN_BULLET_CHARS = 12;
|
|
52
|
+
|
|
53
|
+
const STOPWORDS = new Set([
|
|
54
|
+
'the', 'a', 'an', 'and', 'or', 'to', 'for', 'of', 'with', 'in', 'on', 'is', 'are',
|
|
55
|
+
'this', 'that', 'it', 'as', 'by', 'be', 'use', 'using', 'implement', 'fix', 'task',
|
|
56
|
+
'what', 'how', 'why', 'when', 'where', 'which', 'who', 'whom', 'does', 'do', 'did',
|
|
57
|
+
'should', 'would', 'could', 'can', 'may', 'might', 'will', 'shall', 'from', 'at',
|
|
58
|
+
'into', 'than', 'then', 'was', 'were', 'been', 'has', 'have', 'had', 'not', 'no',
|
|
59
|
+
'but', 'if', 'so', 'about', 'after', 'before', 'there', 'their', 'its', 'our',
|
|
60
|
+
'we', 'you', 'your', 'any', 'all', 'each', 'per', 'via', 'up', 'out', 'over',
|
|
61
|
+
'only', 'also', 'just', 'too', 'very', 'us', 'me', 'my', 'he', 'she', 'they',
|
|
62
|
+
'them', 'these', 'those', 'am', 'being', 'both', 'more', 'most', 'some', 'such',
|
|
63
|
+
'cần', 'và', 'là', 'cho', 'một', 'những', 'dùng',
|
|
64
|
+
]);
|
|
65
|
+
|
|
66
|
+
// Pre-3.5.1 episode text template ("Session <id>: writeOk, verify=ok, N receipts,
|
|
67
|
+
// last command <cmd>") - template words must never produce a match.
|
|
68
|
+
const EPISODE_TEMPLATE_TOKENS = new Set([
|
|
69
|
+
'session', 'writeok', 'writefail', 'verify', 'receipts', 'receipt', 'last', 'command',
|
|
70
|
+
'ok', 'fail', 'run', 'n/a',
|
|
71
|
+
]);
|
|
72
|
+
const UUID_RE = /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/g;
|
|
73
|
+
|
|
74
|
+
export function normalize(text) {
|
|
75
|
+
return String(text ?? '')
|
|
76
|
+
.toLowerCase()
|
|
77
|
+
.replace(UUID_RE, ' ')
|
|
78
|
+
.replace(/[^\p{L}\p{N}\s./:_-]/gu, ' ')
|
|
79
|
+
.replace(/\s+/g, ' ')
|
|
80
|
+
.trim();
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const STEM_SUFFIXES = ['ations', 'ation', 'ments', 'ment', 'ings', 'ing', 'ied', 'ed', 'ers', 'er', 'es', 's', 'ly'];
|
|
84
|
+
|
|
85
|
+
// Word forms must meet on one stem: `rule`/`rules`, `file`/`files`,
|
|
86
|
+
// `update`/`updated`, `call`/`called`, `add`/`adding`. So every alphabetic token
|
|
87
|
+
// gets the same final normalisation (one of a doubled final consonant, then a
|
|
88
|
+
// trailing `e`), whether or not a suffix was stripped first.
|
|
89
|
+
function finishStem(base) {
|
|
90
|
+
let out = base;
|
|
91
|
+
if (out.length >= 3 && /(ll|tt|pp|dd|gg|nn|mm|rr)$/.test(out)) out = out.slice(0, -1);
|
|
92
|
+
if (out.length >= 4 && out.endsWith('e')) out = out.slice(0, -1);
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Forms the suffix rules get wrong: `plane` must not meet `plan`, `written` must
|
|
97
|
+
// meet `write`/`writer`, and `status` must meet `statuses`.
|
|
98
|
+
const STEM_EXCEPTIONS = new Map([
|
|
99
|
+
['plane', 'plane'], ['planes', 'plane'], ['written', 'writ'], ['status', 'status'],
|
|
100
|
+
]);
|
|
101
|
+
|
|
102
|
+
function stem(token) {
|
|
103
|
+
if (token.length < 3 || !/^[a-z]+$/.test(token)) return token;
|
|
104
|
+
const fixed = STEM_EXCEPTIONS.get(token);
|
|
105
|
+
if (fixed) return fixed;
|
|
106
|
+
if (/ie[sd]$/.test(token) && token.length > 5) return `${token.slice(0, -3)}y`;
|
|
107
|
+
if (token.length >= 4) {
|
|
108
|
+
for (const suffix of STEM_SUFFIXES) {
|
|
109
|
+
if (token.endsWith(suffix) && token.length - suffix.length >= 3) {
|
|
110
|
+
if (suffix === 's' && token.endsWith('ss')) break;
|
|
111
|
+
return finishStem(token.slice(0, -suffix.length));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return finishStem(token);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Tokens of `text`: whole tokens plus their separator-split parts
|
|
119
|
+
// (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
|
|
120
|
+
export function tokenize(text) {
|
|
121
|
+
const out = [];
|
|
122
|
+
for (const raw of normalize(text).split(/\s+/)) {
|
|
123
|
+
if (!raw) continue;
|
|
124
|
+
const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
|
|
125
|
+
const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
|
|
126
|
+
for (const candidate of candidates) {
|
|
127
|
+
if (candidate.length < 2 || STOPWORDS.has(candidate)) continue;
|
|
128
|
+
out.push(stem(candidate));
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return out;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function episodeBody(text) {
|
|
135
|
+
return String(text ?? '')
|
|
136
|
+
.replace(/^\s*Session\s+\S+?:\s*/i, '')
|
|
137
|
+
.replace(/\b(writeOk|writeFail)\b/g, ' ')
|
|
138
|
+
.replace(/\bverify=(ok|fail|not-run)\b/g, ' ')
|
|
139
|
+
.replace(/\b\d+\s+receipts?\b/g, ' ')
|
|
140
|
+
.replace(/\blast command\b/gi, ' ');
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function isTemplateText(doc) {
|
|
144
|
+
return doc?.type === 'episode' || doc?.provenance === 'exec-ledger';
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Pre-3.5.1 content-free episode shape. Only this shape gets the template-token
|
|
148
|
+
// filter: a 3.5.1 episode ("Task: … . Wrote: … . Verify: …") carries the user's
|
|
149
|
+
// own words, and dropping `session`/`command`/`run` from it would hide real tasks.
|
|
150
|
+
const LEGACY_EPISODE_RE = /^\s*Session\s+\S+?:\s*(?:writeOk|writeFail)\b/;
|
|
151
|
+
// Field labels of the 3.5.1 episode text; present in every episode, never content.
|
|
152
|
+
const EPISODE_LABEL_RE = /(?:^|\.\s)(?:Task|Wrote|Verify|Unfinished):/g;
|
|
153
|
+
|
|
154
|
+
// Distinct match tokens of a document.
|
|
155
|
+
export function docTokens(doc) {
|
|
156
|
+
if (!isTemplateText(doc)) return tokenize(doc?.text);
|
|
157
|
+
const text = String(doc.text ?? '');
|
|
158
|
+
if (LEGACY_EPISODE_RE.test(text)) {
|
|
159
|
+
return tokenize(episodeBody(text)).filter((token) => !EPISODE_TEMPLATE_TOKENS.has(token));
|
|
160
|
+
}
|
|
161
|
+
return tokenize(text.replace(EPISODE_LABEL_RE, ' '));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function idf(totalDocs, docFreq) {
|
|
165
|
+
return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// rankDocs(docs, query, {limit?}) -> {
|
|
169
|
+
// ranked:[{doc, score, matched}], candidateCount, queryTokenCount, topScore,
|
|
170
|
+
// abstained }
|
|
171
|
+
// `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
|
|
172
|
+
// `query` is a string or a pre-tokenized array. Never throws.
|
|
173
|
+
export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
174
|
+
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
|
|
175
|
+
const empty = {
|
|
176
|
+
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
177
|
+
};
|
|
178
|
+
if (!Array.isArray(docs) || docs.length === 0 || queryTokens.length === 0) return empty;
|
|
179
|
+
|
|
180
|
+
const stats = [];
|
|
181
|
+
const docFreq = new Map();
|
|
182
|
+
let totalLength = 0;
|
|
183
|
+
for (const doc of docs) {
|
|
184
|
+
if (doc == null) continue;
|
|
185
|
+
const tokens = docTokens(doc);
|
|
186
|
+
const tf = new Map();
|
|
187
|
+
for (const token of tokens) tf.set(token, (tf.get(token) ?? 0) + 1);
|
|
188
|
+
for (const token of tf.keys()) docFreq.set(token, (docFreq.get(token) ?? 0) + 1);
|
|
189
|
+
totalLength += tokens.length;
|
|
190
|
+
stats.push({ doc, tf, length: tokens.length });
|
|
191
|
+
}
|
|
192
|
+
const total = stats.length;
|
|
193
|
+
if (total === 0) return empty;
|
|
194
|
+
const avgLength = totalLength / total || 1;
|
|
195
|
+
const idfMax = idf(total, 1);
|
|
196
|
+
// IDF cannot discriminate in a tiny corpus, so the multi-token requirement
|
|
197
|
+
// only applies once there are enough documents for it to mean something.
|
|
198
|
+
const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
|
|
199
|
+
|
|
200
|
+
const scored = [];
|
|
201
|
+
let candidateCount = 0;
|
|
202
|
+
for (const { doc, tf, length } of stats) {
|
|
203
|
+
let score = 0;
|
|
204
|
+
let idfMass = 0;
|
|
205
|
+
let matched = 0;
|
|
206
|
+
for (const token of queryTokens) {
|
|
207
|
+
const freq = tf.get(token);
|
|
208
|
+
if (!freq) continue;
|
|
209
|
+
const weight = idf(total, docFreq.get(token));
|
|
210
|
+
matched += 1;
|
|
211
|
+
idfMass += weight;
|
|
212
|
+
score += weight * ((freq * (K1 + 1)) / (freq + K1 * (1 - B + B * (length / avgLength))));
|
|
213
|
+
}
|
|
214
|
+
if (matched === 0) continue;
|
|
215
|
+
candidateCount += 1;
|
|
216
|
+
if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
|
|
217
|
+
if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
|
|
218
|
+
scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
|
|
219
|
+
}
|
|
220
|
+
// Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
|
|
221
|
+
// entry must not reorder the injected block and invalidate the prompt cache.
|
|
222
|
+
scored.sort((a, b) => b.score - a.score
|
|
223
|
+
|| String(a.doc.id ?? '').localeCompare(String(b.doc.id ?? '')));
|
|
224
|
+
const top = scored.length > 0 ? scored[0].score : 0;
|
|
225
|
+
const ranked = scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top).slice(0, limit);
|
|
226
|
+
return {
|
|
227
|
+
ranked,
|
|
228
|
+
candidateCount,
|
|
229
|
+
queryTokenCount: queryTokens.length,
|
|
230
|
+
topScore: top,
|
|
231
|
+
abstained: ranked.length === 0,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// ---- Markdown source ------------------------------------------------------
|
|
236
|
+
|
|
237
|
+
export function normalizeMemorySources(config) {
|
|
238
|
+
const raw = config?.memory?.sources;
|
|
239
|
+
const list = Array.isArray(raw) ? raw : DEFAULT_MEMORY_SOURCES;
|
|
240
|
+
const seen = new Set();
|
|
241
|
+
const out = [];
|
|
242
|
+
for (const entry of list) {
|
|
243
|
+
if (typeof entry !== 'string') continue;
|
|
244
|
+
const rel = entry.trim().replace(/\\/g, '/');
|
|
245
|
+
if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
|
|
246
|
+
if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
|
|
247
|
+
seen.add(rel);
|
|
248
|
+
out.push(rel);
|
|
249
|
+
if (out.length >= MAX_SOURCES) break;
|
|
250
|
+
}
|
|
251
|
+
return out;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function cleanMarkdown(text) {
|
|
255
|
+
return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// `ukit memory promote` renders active project_rule/procedure records into this
|
|
259
|
+
// block. Those records are already recall documents, so the block is skipped:
|
|
260
|
+
// indexing it too would return the same rule twice (record + bullet).
|
|
261
|
+
const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
|
|
262
|
+
const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
|
|
263
|
+
|
|
264
|
+
// Top-level bullets ("- " / "* ") with their indented continuation lines.
|
|
265
|
+
export function parseMarkdownBullets(content, relPath) {
|
|
266
|
+
const docs = [];
|
|
267
|
+
const lines = String(content ?? '').split(/\r?\n/);
|
|
268
|
+
let heading = '';
|
|
269
|
+
let fenced = false;
|
|
270
|
+
let learned = false;
|
|
271
|
+
let current = null;
|
|
272
|
+
const flush = () => {
|
|
273
|
+
if (!current) return;
|
|
274
|
+
const text = cleanMarkdown(current.parts.join(' '));
|
|
275
|
+
if (text.length >= MIN_BULLET_CHARS) {
|
|
276
|
+
docs.push({
|
|
277
|
+
id: `md:${relPath}:${current.line}`,
|
|
278
|
+
type: 'markdown',
|
|
279
|
+
source: 'markdown',
|
|
280
|
+
text,
|
|
281
|
+
provenance: `${relPath}:${current.line}`,
|
|
282
|
+
line: current.line,
|
|
283
|
+
heading: current.heading,
|
|
284
|
+
created_at: 0,
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
current = null;
|
|
288
|
+
};
|
|
289
|
+
lines.forEach((line, idx) => {
|
|
290
|
+
if (/^\s*(```|~~~)/.test(line)) {
|
|
291
|
+
fenced = !fenced;
|
|
292
|
+
flush();
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
if (fenced) return;
|
|
296
|
+
if (LEARNED_START_RE.test(line)) { learned = true; flush(); return; }
|
|
297
|
+
if (LEARNED_END_RE.test(line)) { learned = false; return; }
|
|
298
|
+
if (learned) return;
|
|
299
|
+
const headingMatch = /^#{1,6}\s+(.*)$/.exec(line);
|
|
300
|
+
if (headingMatch) {
|
|
301
|
+
flush();
|
|
302
|
+
heading = headingMatch[1].trim();
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
if (/^[-*]\s+/.test(line)) {
|
|
306
|
+
flush();
|
|
307
|
+
current = { line: idx + 1, heading, parts: [line.replace(/^[-*]\s+/, '')] };
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
if (current && /^\s{2,}\S/.test(line)) {
|
|
311
|
+
current.parts.push(line.trim());
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
flush();
|
|
315
|
+
});
|
|
316
|
+
flush();
|
|
317
|
+
return docs;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const markdownCache = new Map(); // abs path -> {size, mtimeMs, docs}
|
|
321
|
+
|
|
322
|
+
export function resetMarkdownCacheForTests() {
|
|
323
|
+
markdownCache.clear();
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// loadMarkdownDocs(projectRoot, sources) -> docs[]; a missing/unreadable/oversized
|
|
327
|
+
// file contributes nothing and is never an error. Async I/O only.
|
|
328
|
+
export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOURCES) {
|
|
329
|
+
if (typeof projectRoot !== 'string' || projectRoot === '') return [];
|
|
330
|
+
const root = path.resolve(projectRoot);
|
|
331
|
+
const docs = [];
|
|
332
|
+
let realRoot = null;
|
|
333
|
+
for (const rel of sources) {
|
|
334
|
+
const abs = path.resolve(root, rel);
|
|
335
|
+
if (abs !== root && !abs.startsWith(root + path.sep)) continue;
|
|
336
|
+
try {
|
|
337
|
+
// A symlinked source must still resolve inside the project.
|
|
338
|
+
realRoot ??= await fsp.realpath(root);
|
|
339
|
+
const real = await fsp.realpath(abs);
|
|
340
|
+
if (!real.startsWith(realRoot + path.sep)) continue;
|
|
341
|
+
const st = await fsp.stat(abs);
|
|
342
|
+
if (!st.isFile() || st.size > MAX_SOURCE_BYTES) continue;
|
|
343
|
+
const cached = markdownCache.get(abs);
|
|
344
|
+
if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) {
|
|
345
|
+
docs.push(...cached.docs);
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
const parsed = parseMarkdownBullets(await fsp.readFile(abs, 'utf8'), rel);
|
|
349
|
+
markdownCache.set(abs, { size: st.size, mtimeMs: st.mtimeMs, docs: parsed });
|
|
350
|
+
docs.push(...parsed);
|
|
351
|
+
} catch {
|
|
352
|
+
markdownCache.delete(abs);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return docs;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const SNIPPET_CHARS = 160;
|
|
359
|
+
|
|
360
|
+
// Shared hook lane: score ALL eligible docs (no newest-N pre-filter) together
|
|
361
|
+
// with the markdown source bullets, then cut. Async I/O only; never throws.
|
|
362
|
+
export async function recallForHook({ projectRoot, query, docs = [], config = null, limit = 2 } = {}) {
|
|
363
|
+
let markdown = [];
|
|
364
|
+
try {
|
|
365
|
+
markdown = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config));
|
|
366
|
+
} catch {
|
|
367
|
+
markdown = [];
|
|
368
|
+
}
|
|
369
|
+
let result;
|
|
370
|
+
try {
|
|
371
|
+
result = rankDocs([...docs, ...markdown], query, { limit });
|
|
372
|
+
} catch {
|
|
373
|
+
result = rankDocs([], '', { limit });
|
|
374
|
+
}
|
|
375
|
+
await emitRecallTelemetry(projectRoot, config, result);
|
|
376
|
+
return result;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// One metadata-only `retrieval.query` event per hook recall, through the
|
|
380
|
+
// shared standalone recorder (observability-emit.mjs, a sibling in the shipped
|
|
381
|
+
// runtime dir). Counts, abstention, top score and record refs only - never the
|
|
382
|
+
// prompt, record text or a raw record/session id. Async, bounded, never throws; the observability stage
|
|
383
|
+
// and the memoryV2 rollout stage are honoured (off/kill-switch => no event).
|
|
384
|
+
const TELEMETRY_DEADLINE_MS = 120;
|
|
385
|
+
|
|
386
|
+
async function emitRecallTelemetry(projectRoot, config, result) {
|
|
387
|
+
try {
|
|
388
|
+
const v2 = config?.memoryV2;
|
|
389
|
+
if (v2?.killSwitch === true || v2?.eligibility?.stage === 'off') return;
|
|
390
|
+
const emitter = await import('./observability-emit.mjs');
|
|
391
|
+
await emitter.emitOutcomeObserved({
|
|
392
|
+
projectRoot,
|
|
393
|
+
config,
|
|
394
|
+
deadlineMs: TELEMETRY_DEADLINE_MS,
|
|
395
|
+
record: {
|
|
396
|
+
semantic_name: 'retrieval.query',
|
|
397
|
+
payload: {
|
|
398
|
+
lane: 'hook',
|
|
399
|
+
engine: RECALL_ENGINE_VERSION,
|
|
400
|
+
query_token_count: result.queryTokenCount,
|
|
401
|
+
candidate_count: result.candidateCount,
|
|
402
|
+
returned_count: result.ranked.length,
|
|
403
|
+
abstained: result.abstained,
|
|
404
|
+
top_score: Number(result.topScore.toFixed(4)),
|
|
405
|
+
// Markdown refs are repo paths; record/session/project ids are hashed,
|
|
406
|
+
// as the CLI lane does (memoryEmit.js), because they can identify.
|
|
407
|
+
returned_refs: result.ranked.map((entry) => (entry.doc.provenance && entry.doc.type === 'markdown'
|
|
408
|
+
? String(entry.doc.provenance).slice(0, 96)
|
|
409
|
+
: crypto.createHash('sha256').update(String(entry.doc.id)).digest('hex'))),
|
|
410
|
+
},
|
|
411
|
+
},
|
|
412
|
+
});
|
|
413
|
+
} catch {
|
|
414
|
+
// Telemetry never affects recall.
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// Cited-data rendering: provenance first so truncation never drops it.
|
|
419
|
+
export function renderMarkdownDoc(doc, maxChars = SNIPPET_CHARS) {
|
|
420
|
+
const text = String(doc?.text ?? '');
|
|
421
|
+
const body = text.length > maxChars ? `${text.slice(0, maxChars - 3)}...` : text;
|
|
422
|
+
return `[${doc?.provenance ?? 'markdown'}] ${body}`;
|
|
423
|
+
}
|