@oscharko-dev/keiko-memory-retrieval 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -0
- package/dist/context.d.ts +16 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +151 -0
- package/dist/decay.d.ts +2 -0
- package/dist/decay.d.ts.map +1 -0
- package/dist/decay.js +22 -0
- package/dist/diversity.d.ts +8 -0
- package/dist/diversity.d.ts.map +1 -0
- package/dist/diversity.js +87 -0
- package/dist/errors.d.ts +9 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +14 -0
- package/dist/graph.d.ts +3 -0
- package/dist/graph.d.ts.map +1 -0
- package/dist/graph.js +33 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/ranking.d.ts +19 -0
- package/dist/ranking.d.ts.map +1 -0
- package/dist/ranking.js +266 -0
- package/dist/recency.d.ts +3 -0
- package/dist/recency.d.ts.map +1 -0
- package/dist/recency.js +17 -0
- package/dist/relevance.d.ts +8 -0
- package/dist/relevance.d.ts.map +1 -0
- package/dist/relevance.js +60 -0
- package/dist/retrieve.d.ts +3 -0
- package/dist/retrieve.d.ts.map +1 -0
- package/dist/retrieve.js +256 -0
- package/dist/strength.d.ts +9 -0
- package/dist/strength.d.ts.map +1 -0
- package/dist/strength.js +51 -0
- package/dist/suppression.d.ts +8 -0
- package/dist/suppression.d.ts.map +1 -0
- package/dist/suppression.js +82 -0
- package/dist/types.d.ts +118 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +40 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +4 -0
- package/package.json +31 -0
package/dist/ranking.js
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
// Ranking orchestration.
|
|
2
|
+
//
|
|
3
|
+
// Hybrid ranker over (relevance, recency, confidence, pinned, correction, graph). Two
|
|
4
|
+
// passes:
|
|
5
|
+
// Pass 1 — baseline subscores per memory (graph contribution = 0). Sort. The top-N of
|
|
6
|
+
// this pass become the "high-rank set" for the graph pass; the graph signal
|
|
7
|
+
// cannot reference itself recursively because the high-rank set is fixed
|
|
8
|
+
// BEFORE the graph contribution is computed.
|
|
9
|
+
// Pass 2 — graph proximity is computed for each memory against the high-rank set, and
|
|
10
|
+
// the entry is rebuilt with the layered subscore. Re-sort and return.
|
|
11
|
+
//
|
|
12
|
+
// Tiebreak order: score desc, updatedAt desc, id asc. Stable across equal-score sets so
|
|
13
|
+
// the same input always produces the same output (determinism is an explicit AC).
|
|
14
|
+
//
|
|
15
|
+
// Inclusion reason names the top contributing WEIGHTED subscore. The threshold for
|
|
16
|
+
// "primarily because of X" is "X contributes more than any other dimension" — no
|
|
17
|
+
// arbitrary cutoff — so the reason text always tracks the actual top contributor.
|
|
18
|
+
import { graphProximityScore } from "./graph.js";
|
|
19
|
+
import { recencyScore } from "./recency.js";
|
|
20
|
+
import { lexicalRelevance } from "./relevance.js";
|
|
21
|
+
const DEFAULT_GRAPH_HIGH_RANK_COUNT = 8;
|
|
22
|
+
// Internal invariant guard. The subscore / rank maps below are built with exactly one entry per
|
|
23
|
+
// memory in `memories`, so a lookup keyed by a memory from that same list is always present. This
|
|
24
|
+
// makes the invariant explicit instead of asserting it away with `!` — a miss is a programmer error,
|
|
25
|
+
// not a recoverable input, so it fails loud rather than silently scoring against a wrong value.
|
|
26
|
+
function requireEntry(map, key) {
|
|
27
|
+
const value = map.get(key);
|
|
28
|
+
if (value === undefined) {
|
|
29
|
+
throw new Error("ranking invariant violated: missing map entry for a known memory");
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
// Frozen source-authority importance (#204, O-F5). A deterministic function of the immutable capture
|
|
34
|
+
// provenance: a fact the user stated outright outranks one passively inferred by the system, at equal
|
|
35
|
+
// relevance. Reproducible and caller-input-free.
|
|
36
|
+
const SOURCE_IMPORTANCE = {
|
|
37
|
+
"explicit-user-instruction": 1,
|
|
38
|
+
"accepted-correction": 0.85,
|
|
39
|
+
"workflow-outcome": 0.6,
|
|
40
|
+
consolidation: 0.5,
|
|
41
|
+
"system-default": 0.4,
|
|
42
|
+
};
|
|
43
|
+
export function sourceImportance(record) {
|
|
44
|
+
// SOURCE_IMPORTANCE is total over MemorySourceKind, so a new source kind in contracts surfaces here
|
|
45
|
+
// as a compile error rather than silently defaulting.
|
|
46
|
+
return SOURCE_IMPORTANCE[record.provenance.sourceKind];
|
|
47
|
+
}
|
|
48
|
+
function baselineSubscores(record, query) {
|
|
49
|
+
return {
|
|
50
|
+
relevance: lexicalRelevance(query.queryText, record),
|
|
51
|
+
recency: recencyScore(record.updatedAt, query.nowMs),
|
|
52
|
+
confidence: record.provenance.confidence,
|
|
53
|
+
pinned: record.pinned ? 1 : 0,
|
|
54
|
+
correction: record.type === "correction" || record.provenance.sourceKind === "accepted-correction"
|
|
55
|
+
? 1
|
|
56
|
+
: 0,
|
|
57
|
+
graph: 0,
|
|
58
|
+
semantic: query.semanticById?.get(record.id) ?? 0,
|
|
59
|
+
strength: query.strengthById?.get(record.id) ?? 0,
|
|
60
|
+
importance: sourceImportance(record),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
function weightedScore(s, w) {
|
|
64
|
+
const raw = s.relevance * w.relevance +
|
|
65
|
+
s.recency * w.recency +
|
|
66
|
+
s.confidence * w.confidence +
|
|
67
|
+
s.pinned * w.pinned +
|
|
68
|
+
s.correction * w.correction +
|
|
69
|
+
s.graph * w.graph +
|
|
70
|
+
s.semantic * w.semantic +
|
|
71
|
+
s.strength * w.strength +
|
|
72
|
+
s.importance * w.importance;
|
|
73
|
+
const totalWeight = w.relevance +
|
|
74
|
+
w.recency +
|
|
75
|
+
w.confidence +
|
|
76
|
+
w.pinned +
|
|
77
|
+
w.correction +
|
|
78
|
+
w.graph +
|
|
79
|
+
w.semantic +
|
|
80
|
+
w.strength +
|
|
81
|
+
w.importance;
|
|
82
|
+
if (totalWeight <= 0)
|
|
83
|
+
return 0;
|
|
84
|
+
return raw / totalWeight;
|
|
85
|
+
}
|
|
86
|
+
function topContributor(s, w) {
|
|
87
|
+
const parts = [
|
|
88
|
+
{ key: "pinned", value: s.pinned * w.pinned },
|
|
89
|
+
{ key: "correction", value: s.correction * w.correction },
|
|
90
|
+
// Semantic before relevance/recency/confidence so the stronger embedding signal wins a tie
|
|
91
|
+
// against the lexical signals; pinned/correction stay above it as today.
|
|
92
|
+
{ key: "semantic", value: s.semantic * w.semantic },
|
|
93
|
+
// Reinforcement sits just below semantic: a heavily-reused memory wins a tie against the lexical
|
|
94
|
+
// signals but not against an explicit pin, a fresh correction, or a strong embedding match.
|
|
95
|
+
{ key: "strength", value: s.strength * w.strength },
|
|
96
|
+
{ key: "relevance", value: s.relevance * w.relevance },
|
|
97
|
+
{ key: "recency", value: s.recency * w.recency },
|
|
98
|
+
{ key: "confidence", value: s.confidence * w.confidence },
|
|
99
|
+
{ key: "importance", value: s.importance * w.importance },
|
|
100
|
+
{ key: "graph", value: s.graph * w.graph },
|
|
101
|
+
];
|
|
102
|
+
let bestKey = "recency";
|
|
103
|
+
let bestValue = -1;
|
|
104
|
+
for (const p of parts) {
|
|
105
|
+
if (p.value > bestValue) {
|
|
106
|
+
bestKey = p.key;
|
|
107
|
+
bestValue = p.value;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return inclusionReasonText(bestKey, bestValue);
|
|
111
|
+
}
|
|
112
|
+
function inclusionReasonText(key, value) {
|
|
113
|
+
if (value <= 0)
|
|
114
|
+
return "included by default ranking";
|
|
115
|
+
const label = {
|
|
116
|
+
relevance: "lexical relevance to query",
|
|
117
|
+
recency: "recent update",
|
|
118
|
+
confidence: "high provenance confidence",
|
|
119
|
+
pinned: "pinned memory",
|
|
120
|
+
correction: "recent correction overrides older facts",
|
|
121
|
+
graph: "graph proximity to other top memories",
|
|
122
|
+
semantic: "semantic similarity to query",
|
|
123
|
+
strength: "frequently recalled (reinforced)",
|
|
124
|
+
importance: "authoritative source",
|
|
125
|
+
};
|
|
126
|
+
return `top signal: ${label[key]}`;
|
|
127
|
+
}
|
|
128
|
+
function entryFor(record, subscores, weights) {
|
|
129
|
+
const score = weightedScore(subscores, weights);
|
|
130
|
+
return {
|
|
131
|
+
memoryId: record.id,
|
|
132
|
+
score,
|
|
133
|
+
subscores,
|
|
134
|
+
inclusionReason: topContributor(subscores, weights),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
function compareEntries(aEntry, bEntry, aRecord, bRecord) {
|
|
138
|
+
if (aEntry.score !== bEntry.score)
|
|
139
|
+
return bEntry.score - aEntry.score;
|
|
140
|
+
if (aRecord.updatedAt !== bRecord.updatedAt)
|
|
141
|
+
return bRecord.updatedAt - aRecord.updatedAt;
|
|
142
|
+
if (aEntry.memoryId < bEntry.memoryId)
|
|
143
|
+
return -1;
|
|
144
|
+
if (aEntry.memoryId > bEntry.memoryId)
|
|
145
|
+
return 1;
|
|
146
|
+
return 0;
|
|
147
|
+
}
|
|
148
|
+
function sortByRank(entries, recordById) {
|
|
149
|
+
return [...entries].sort((a, b) => {
|
|
150
|
+
const aRecord = recordById.get(a.memoryId);
|
|
151
|
+
const bRecord = recordById.get(b.memoryId);
|
|
152
|
+
if (aRecord === undefined || bRecord === undefined)
|
|
153
|
+
return 0;
|
|
154
|
+
return compareEntries(a, b, aRecord, bRecord);
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
// Byte-identity guarantee (#204): when the caller supplied NO per-memory semantic scores, the
|
|
158
|
+
// semantic weight is forced to 0 so it leaves the weighted sum AND its denominator untouched —
|
|
159
|
+
// every score, reason, and ordering is identical to the pre-semantic lexical ranker. Only when
|
|
160
|
+
// `semanticById` is present does the configured semantic weight participate.
|
|
161
|
+
function effectiveWeights(query) {
|
|
162
|
+
// Each optional signal zeroes its own weight when the caller supplied no scores for it, so the
|
|
163
|
+
// weighted sum AND its denominator are untouched and the output is byte-identical to the behaviour
|
|
164
|
+
// before that signal existed. The two conditions are independent.
|
|
165
|
+
let weights = query.weights;
|
|
166
|
+
if (query.semanticById === undefined) {
|
|
167
|
+
weights = { ...weights, semantic: 0 };
|
|
168
|
+
}
|
|
169
|
+
if (query.strengthById === undefined) {
|
|
170
|
+
weights = { ...weights, strength: 0 };
|
|
171
|
+
}
|
|
172
|
+
return weights;
|
|
173
|
+
}
|
|
174
|
+
// Reciprocal Rank Fusion constant (Cormack et al. 2009). 60 is the field-standard k; larger flattens
|
|
175
|
+
// the rank advantage, smaller sharpens it.
|
|
176
|
+
export const RRF_K = 60;
|
|
177
|
+
const SUBSCORE_KEYS = [
|
|
178
|
+
"relevance",
|
|
179
|
+
"recency",
|
|
180
|
+
"confidence",
|
|
181
|
+
"pinned",
|
|
182
|
+
"correction",
|
|
183
|
+
"graph",
|
|
184
|
+
"semantic",
|
|
185
|
+
"strength",
|
|
186
|
+
"importance",
|
|
187
|
+
];
|
|
188
|
+
// Final subscores per memory: baseline + (when edges are supplied) the graph layer. The graph
|
|
189
|
+
// high-rank set is the WEIGHTED-SUM baseline top-N, so graph proximity is computed identically
|
|
190
|
+
// regardless of the final fusion mode (and the weighted-sum path stays byte-identical to before).
|
|
191
|
+
function computeFinalSubscores(memories, query, weights, options, recordById) {
|
|
192
|
+
const map = new Map();
|
|
193
|
+
for (const m of memories)
|
|
194
|
+
map.set(m.id, baselineSubscores(m, query));
|
|
195
|
+
if (options.edgesByMemory === undefined)
|
|
196
|
+
return map;
|
|
197
|
+
const baselineSorted = sortByRank(memories.map((m) => entryFor(m, requireEntry(map, m.id), weights)), recordById);
|
|
198
|
+
const highRankCount = options.graphHighRankCount ?? DEFAULT_GRAPH_HIGH_RANK_COUNT;
|
|
199
|
+
const highRankIds = new Set(baselineSorted.slice(0, highRankCount).map((e) => e.memoryId));
|
|
200
|
+
const edges = options.edgesByMemory;
|
|
201
|
+
for (const m of memories) {
|
|
202
|
+
const base = requireEntry(map, m.id);
|
|
203
|
+
map.set(m.id, { ...base, graph: graphProximityScore(m.id, edges, highRankIds) });
|
|
204
|
+
}
|
|
205
|
+
return map;
|
|
206
|
+
}
|
|
207
|
+
// Reciprocal Rank Fusion (#204, O-F2): for each positive-weight signal, rank the memories by that
|
|
208
|
+
// subscore (desc, id tiebreak) and fuse score = Σ w/(RRF_K + rank). Rank-based, so heterogeneous
|
|
209
|
+
// score scales (Jaccard ~[0,0.3] vs cosine [0,1]) need no normalization, and agreement across signals
|
|
210
|
+
// compounds. The fused value is normalized to [0,1] (best possible = rank 1 in every signal) to
|
|
211
|
+
// honour the documented score range; ordering uses the shared (score desc, updatedAt desc, id asc) sort.
|
|
212
|
+
function rrfRank(memories, subscoresById, weights, recordById) {
|
|
213
|
+
const signals = SUBSCORE_KEYS.filter((k) => weights[k] > 0);
|
|
214
|
+
const firstSignal = signals[0];
|
|
215
|
+
if (firstSignal === undefined) {
|
|
216
|
+
return sortByRank(memories.map((m) => entryFor(m, requireEntry(subscoresById, m.id), weights)), recordById);
|
|
217
|
+
}
|
|
218
|
+
const rankBySignal = new Map();
|
|
219
|
+
for (const sig of signals) {
|
|
220
|
+
const ordered = [...memories].sort((a, b) => {
|
|
221
|
+
const av = requireEntry(subscoresById, a.id)[sig];
|
|
222
|
+
const bv = requireEntry(subscoresById, b.id)[sig];
|
|
223
|
+
if (av !== bv)
|
|
224
|
+
return bv - av;
|
|
225
|
+
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
226
|
+
});
|
|
227
|
+
const ranks = new Map();
|
|
228
|
+
ordered.forEach((m, i) => ranks.set(m.id, i + 1));
|
|
229
|
+
rankBySignal.set(sig, ranks);
|
|
230
|
+
}
|
|
231
|
+
const maxFused = signals.reduce((sum, sig) => sum + weights[sig] / (RRF_K + 1), 0);
|
|
232
|
+
const entries = memories.map((m) => {
|
|
233
|
+
let fused = 0;
|
|
234
|
+
let bestSig = firstSignal;
|
|
235
|
+
let bestContrib = -1;
|
|
236
|
+
for (const sig of signals) {
|
|
237
|
+
const rank = requireEntry(requireEntry(rankBySignal, sig), m.id);
|
|
238
|
+
const contrib = weights[sig] / (RRF_K + rank);
|
|
239
|
+
fused += contrib;
|
|
240
|
+
if (contrib > bestContrib) {
|
|
241
|
+
bestContrib = contrib;
|
|
242
|
+
bestSig = sig;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
return {
|
|
246
|
+
memoryId: m.id,
|
|
247
|
+
score: maxFused > 0 ? fused / maxFused : 0,
|
|
248
|
+
subscores: requireEntry(subscoresById, m.id),
|
|
249
|
+
inclusionReason: inclusionReasonText(bestSig, bestContrib),
|
|
250
|
+
};
|
|
251
|
+
});
|
|
252
|
+
return sortByRank(entries, recordById);
|
|
253
|
+
}
|
|
254
|
+
export function rankMemories(memories, query, options = {}) {
|
|
255
|
+
if (memories.length === 0)
|
|
256
|
+
return [];
|
|
257
|
+
const recordById = new Map();
|
|
258
|
+
for (const m of memories)
|
|
259
|
+
recordById.set(m.id, m);
|
|
260
|
+
const weights = effectiveWeights(query);
|
|
261
|
+
const subscoresById = computeFinalSubscores(memories, query, weights, options, recordById);
|
|
262
|
+
if (query.fusion === "rrf") {
|
|
263
|
+
return rrfRank(memories, subscoresById, weights, recordById);
|
|
264
|
+
}
|
|
265
|
+
return sortByRank(memories.map((m) => entryFor(m, requireEntry(subscoresById, m.id), weights)), recordById);
|
|
266
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency.d.ts","sourceRoot":"","sources":["../src/recency.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,oBAAoB,QAAiB,CAAC;AAEnD,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAIrE"}
|
package/dist/recency.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Recency-decay primitive. Exponential half-life model: a memory is worth 1.0 at
|
|
2
|
+
// updatedAt = nowMs, 0.5 at one half-life of age, 0.25 at two half-lives, etc. Future-dated
|
|
3
|
+
// records are clamped to 1.0 so a clock-skew or capture-source-ahead-of-orchestrator does
|
|
4
|
+
// not produce a negative or super-unit score that would poison the weighted sum.
|
|
5
|
+
//
|
|
6
|
+
// Half-life is 7 days. This matches the conversational-memory expectation that "recent"
|
|
7
|
+
// runs the previous week; consolidation (#208) will collapse older records into semantic
|
|
8
|
+
// facts whose recency continues to refresh on every update. The constant is exported so
|
|
9
|
+
// callers (e.g. an audit dashboard) can reproduce the score deterministically.
|
|
10
|
+
import { exponentialDecay } from "./decay.js";
|
|
11
|
+
const MS_PER_DAY = 86_400_000;
|
|
12
|
+
export const RECENCY_HALF_LIFE_MS = 7 * MS_PER_DAY;
|
|
13
|
+
export function recencyScore(updatedAt, nowMs) {
|
|
14
|
+
// Edit-recency: decays since the record was last UPDATED. (Reuse-recency, which decays since last
|
|
15
|
+
// ACCESS at a different half-life, lives in strength.ts — both share the kernel in decay.ts.)
|
|
16
|
+
return exponentialDecay(nowMs - updatedAt, RECENCY_HALF_LIFE_MS);
|
|
17
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { MemoryRecord } from "@oscharko-dev/keiko-contracts/memory";
|
|
2
|
+
export declare function tokenize(text: string): readonly string[];
|
|
3
|
+
/**
|
|
4
|
+
* Jaccard similarity between the query token set and the record token set (body + tags).
|
|
5
|
+
* Returns 0 for empty/undefined queries; returns 1 when the two sets are equal.
|
|
6
|
+
*/
|
|
7
|
+
export declare function lexicalRelevance(queryText: string | undefined, record: MemoryRecord): number;
|
|
8
|
+
//# sourceMappingURL=relevance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"relevance.d.ts","sourceRoot":"","sources":["../src/relevance.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAOzE,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAaxD;AAWD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAc5F"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Lexical-relevance primitive. Deterministic Jaccard over (body + tags) tokens.
|
|
2
|
+
// No IDF, no document-frequency state — keeps the function pure and the score reproducible
|
|
3
|
+
// across runs without a corpus dependency. The brief explicitly permits "normalized token
|
|
4
|
+
// Jaccard or simple BM25-lite without IDF — keep deterministic"; Jaccard is the simplest
|
|
5
|
+
// fully-deterministic choice and the cheapest to reason about for explainability.
|
|
6
|
+
// Word boundary uses a Unicode-aware non-word matcher: any character that is not a letter,
|
|
7
|
+
// digit, or underscore (in any script) becomes a separator. This is a fixed-length pattern
|
|
8
|
+
// with no alternation or backreferences — not ReDoS-prone.
|
|
9
|
+
const NON_WORD = /[^\p{L}\p{N}_]+/u;
|
|
10
|
+
export function tokenize(text) {
|
|
11
|
+
if (text === "")
|
|
12
|
+
return [];
|
|
13
|
+
const lowered = text.toLowerCase();
|
|
14
|
+
const split = lowered.split(NON_WORD);
|
|
15
|
+
const seen = new Set();
|
|
16
|
+
const out = [];
|
|
17
|
+
for (const piece of split) {
|
|
18
|
+
if (piece === "")
|
|
19
|
+
continue;
|
|
20
|
+
if (seen.has(piece))
|
|
21
|
+
continue;
|
|
22
|
+
seen.add(piece);
|
|
23
|
+
out.push(piece);
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
function recordTokens(record) {
|
|
28
|
+
const tokens = new Set();
|
|
29
|
+
for (const t of tokenize(record.body))
|
|
30
|
+
tokens.add(t);
|
|
31
|
+
for (const tag of record.tags) {
|
|
32
|
+
for (const t of tokenize(tag))
|
|
33
|
+
tokens.add(t);
|
|
34
|
+
}
|
|
35
|
+
return tokens;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Jaccard similarity between the query token set and the record token set (body + tags).
|
|
39
|
+
* Returns 0 for empty/undefined queries; returns 1 when the two sets are equal.
|
|
40
|
+
*/
|
|
41
|
+
export function lexicalRelevance(queryText, record) {
|
|
42
|
+
if (queryText === undefined || queryText === "")
|
|
43
|
+
return 0;
|
|
44
|
+
const queryTokens = new Set(tokenize(queryText));
|
|
45
|
+
if (queryTokens.size === 0)
|
|
46
|
+
return 0;
|
|
47
|
+
const docTokens = recordTokens(record);
|
|
48
|
+
if (docTokens.size === 0)
|
|
49
|
+
return 0;
|
|
50
|
+
let intersection = 0;
|
|
51
|
+
for (const t of queryTokens) {
|
|
52
|
+
if (docTokens.has(t))
|
|
53
|
+
intersection += 1;
|
|
54
|
+
}
|
|
55
|
+
// Union = |A| + |B| - |A ∩ B|
|
|
56
|
+
const union = queryTokens.size + docTokens.size - intersection;
|
|
57
|
+
if (union === 0)
|
|
58
|
+
return 0;
|
|
59
|
+
return intersection / union;
|
|
60
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { type MemoryQueryPort, type MemoryRetrievalRequest, type MemoryRetrievalResult } from "./types.js";
|
|
2
|
+
export declare function retrieveMemoryContext(request: MemoryRetrievalRequest, port: MemoryQueryPort): MemoryRetrievalResult;
|
|
3
|
+
//# sourceMappingURL=retrieve.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retrieve.d.ts","sourceRoot":"","sources":["../src/retrieve.ts"],"names":[],"mappings":"AAoCA,OAAO,EAOL,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAG3B,MAAM,YAAY,CAAC;AA0PpB,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,sBAAsB,EAC/B,IAAI,EAAE,eAAe,GACpB,qBAAqB,CA0CvB"}
|
package/dist/retrieve.js
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
// Top-level scoped-memory retrieval orchestrator.
|
|
2
|
+
//
|
|
3
|
+
// Cross-scope isolation is structural: the function ONLY iterates `request.scopes` when
|
|
4
|
+
// calling port.listByScope. A caller cannot trick this layer into surfacing records from
|
|
5
|
+
// a scope it did not authorise because no other code path reaches the port.
|
|
6
|
+
//
|
|
7
|
+
// Pipeline:
|
|
8
|
+
// 1. Validate request (non-empty scopes, finite non-negative weights, finite integer
|
|
9
|
+
// budget/maxIncluded).
|
|
10
|
+
// 2. For each scope in request.scopes -> port.listByScope(scope, {maxResults, include*}).
|
|
11
|
+
// Wrap any port throw as RetrievalError('port-failure', cause: original).
|
|
12
|
+
// 3. Dedupe by memoryId (a record reachable from multiple scopes appears once).
|
|
13
|
+
// 4. Apply suppression (status / validity / confidence) -> "suppressed-by-status".
|
|
14
|
+
// Apply type filter when request.types is set -> "type-filtered".
|
|
15
|
+
// 5. Build an edges-by-memory map for the candidate set if the port exposes
|
|
16
|
+
// listOutgoingEdges/listIncomingEdges (bounded fetch — only candidates we still hold).
|
|
17
|
+
// 6. Rank with the hybrid ranker; assemble with the token-budgeted greedy assembler.
|
|
18
|
+
// 7. Attach request to the assembler's result and return.
|
|
19
|
+
//
|
|
20
|
+
// Determinism: every step is pure given the port's return values, so identical port
|
|
21
|
+
// responses + identical request -> identical output. The cross-scope isolation test pins
|
|
22
|
+
// this with a spy port that records every listByScope call.
|
|
23
|
+
import { assembleContextBlock } from "./context.js";
|
|
24
|
+
import { DEFAULT_MMR_LAMBDA, reorderByMmr } from "./diversity.js";
|
|
25
|
+
import { RetrievalError } from "./errors.js";
|
|
26
|
+
import { tokenize } from "./relevance.js";
|
|
27
|
+
import { rankMemories } from "./ranking.js";
|
|
28
|
+
import { isMemorySuppressed } from "./suppression.js";
|
|
29
|
+
import { DEFAULT_BUDGET_TOKENS, DEFAULT_LIST_BY_SCOPE_MAX_RESULTS, DEFAULT_MAX_INCLUDED, DEFAULT_RANKING_WEIGHTS, DEFAULT_STALE_CONFIDENCE_THRESHOLD, } from "./types.js";
|
|
30
|
+
function emptyResult(request, budgetTokens) {
|
|
31
|
+
return {
|
|
32
|
+
contextBlock: {
|
|
33
|
+
text: "",
|
|
34
|
+
memories: [],
|
|
35
|
+
},
|
|
36
|
+
included: [],
|
|
37
|
+
omitted: [],
|
|
38
|
+
budget: {
|
|
39
|
+
tokens: budgetTokens,
|
|
40
|
+
used: 0,
|
|
41
|
+
},
|
|
42
|
+
request,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function resolveWeights(request) {
|
|
46
|
+
return {
|
|
47
|
+
relevance: request.relevanceWeight ?? DEFAULT_RANKING_WEIGHTS.relevance,
|
|
48
|
+
recency: request.recencyWeight ?? DEFAULT_RANKING_WEIGHTS.recency,
|
|
49
|
+
confidence: request.confidenceWeight ?? DEFAULT_RANKING_WEIGHTS.confidence,
|
|
50
|
+
pinned: request.pinnedBoost ?? DEFAULT_RANKING_WEIGHTS.pinned,
|
|
51
|
+
correction: request.correctionBoost ?? DEFAULT_RANKING_WEIGHTS.correction,
|
|
52
|
+
graph: request.graphProximityBoost ?? DEFAULT_RANKING_WEIGHTS.graph,
|
|
53
|
+
semantic: request.semanticWeight ?? DEFAULT_RANKING_WEIGHTS.semantic,
|
|
54
|
+
strength: request.strengthWeight ?? DEFAULT_RANKING_WEIGHTS.strength,
|
|
55
|
+
importance: request.importanceWeight ?? DEFAULT_RANKING_WEIGHTS.importance,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function assertNonNegativeWeights(weights) {
|
|
59
|
+
for (const [name, value] of Object.entries(weights)) {
|
|
60
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
61
|
+
throw new RetrievalError("invalid-weight", `weight ${name} must be a finite number >= 0 (got ${String(value)})`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
function assertNonNegativeBudget(budgetTokens, maxIncluded) {
|
|
66
|
+
if (!Number.isFinite(budgetTokens) || !Number.isInteger(budgetTokens) || budgetTokens < 0) {
|
|
67
|
+
throw new RetrievalError("invalid-budget", `budgetTokens must be a finite integer >= 0 (got ${String(budgetTokens)})`);
|
|
68
|
+
}
|
|
69
|
+
if (!Number.isFinite(maxIncluded) || !Number.isInteger(maxIncluded) || maxIncluded < 0) {
|
|
70
|
+
throw new RetrievalError("invalid-budget", `maxIncluded must be a finite integer >= 0 (got ${String(maxIncluded)})`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function assertValidThreshold(staleConfidenceThreshold) {
|
|
74
|
+
if (Number.isFinite(staleConfidenceThreshold) &&
|
|
75
|
+
staleConfidenceThreshold >= 0 &&
|
|
76
|
+
staleConfidenceThreshold <= 1) {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
throw new RetrievalError("invalid-threshold", `staleConfidenceThreshold must be a finite number in [0, 1] (got ${String(staleConfidenceThreshold)})`);
|
|
80
|
+
}
|
|
81
|
+
function validateAndResolve(request) {
|
|
82
|
+
if (request.scopes.length === 0) {
|
|
83
|
+
throw new RetrievalError("empty-scopes", "request.scopes must contain at least one scope");
|
|
84
|
+
}
|
|
85
|
+
const budgetTokens = request.budgetTokens ?? DEFAULT_BUDGET_TOKENS;
|
|
86
|
+
const maxIncluded = request.maxIncluded ?? DEFAULT_MAX_INCLUDED;
|
|
87
|
+
assertNonNegativeBudget(budgetTokens, maxIncluded);
|
|
88
|
+
const weights = resolveWeights(request);
|
|
89
|
+
assertNonNegativeWeights(weights);
|
|
90
|
+
const staleConfidenceThreshold = request.staleConfidenceThreshold ?? DEFAULT_STALE_CONFIDENCE_THRESHOLD;
|
|
91
|
+
assertValidThreshold(staleConfidenceThreshold);
|
|
92
|
+
return {
|
|
93
|
+
budgetTokens,
|
|
94
|
+
maxIncluded,
|
|
95
|
+
weights,
|
|
96
|
+
staleConfidenceThreshold,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function fetchScoped(port, scopes) {
|
|
100
|
+
const all = [];
|
|
101
|
+
for (const scope of scopes) {
|
|
102
|
+
try {
|
|
103
|
+
const batch = port.listByScope(scope, {
|
|
104
|
+
includeForgotten: true,
|
|
105
|
+
includeArchived: true,
|
|
106
|
+
includeExpired: true,
|
|
107
|
+
maxResults: DEFAULT_LIST_BY_SCOPE_MAX_RESULTS,
|
|
108
|
+
});
|
|
109
|
+
for (const r of batch)
|
|
110
|
+
all.push(r);
|
|
111
|
+
}
|
|
112
|
+
catch (cause) {
|
|
113
|
+
throw new RetrievalError("port-failure", `listByScope threw for scope.kind=${scope.kind}`, {
|
|
114
|
+
cause,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return all;
|
|
119
|
+
}
|
|
120
|
+
function dedupeById(records) {
|
|
121
|
+
const seen = new Set();
|
|
122
|
+
const out = [];
|
|
123
|
+
for (const r of records) {
|
|
124
|
+
if (seen.has(r.id))
|
|
125
|
+
continue;
|
|
126
|
+
seen.add(r.id);
|
|
127
|
+
out.push(r);
|
|
128
|
+
}
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
131
|
+
function applyFilters(records, request, resolved) {
|
|
132
|
+
const typeFilter = request.types;
|
|
133
|
+
const candidates = [];
|
|
134
|
+
const omitted = [];
|
|
135
|
+
for (const r of records) {
|
|
136
|
+
if (typeFilter !== undefined && !typeFilter.includes(r.type)) {
|
|
137
|
+
omitted.push({ memoryId: r.id, reason: "type-filtered" });
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
if (r.status === "superseded" && request.includeSuperseded !== true) {
|
|
141
|
+
omitted.push({
|
|
142
|
+
memoryId: r.id,
|
|
143
|
+
reason: "suppressed-by-status",
|
|
144
|
+
suppressionDetail: "superseded",
|
|
145
|
+
});
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const sup = isMemorySuppressed(r, request.nowMs, resolved.staleConfidenceThreshold);
|
|
149
|
+
if (sup.suppressed) {
|
|
150
|
+
// sup.reason is optional on SuppressionResult; under exactOptionalPropertyTypes we
|
|
151
|
+
// must conditionally add the field so we never write `suppressionDetail: undefined`.
|
|
152
|
+
omitted.push(sup.reason === undefined
|
|
153
|
+
? { memoryId: r.id, reason: "suppressed-by-status" }
|
|
154
|
+
: { memoryId: r.id, reason: "suppressed-by-status", suppressionDetail: sup.reason });
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
candidates.push(r);
|
|
158
|
+
}
|
|
159
|
+
return { candidates, omitted };
|
|
160
|
+
}
|
|
161
|
+
function buildEdgesIndex(port, candidates) {
|
|
162
|
+
if (port.listOutgoingEdges === undefined && port.listIncomingEdges === undefined)
|
|
163
|
+
return undefined;
|
|
164
|
+
const map = new Map();
|
|
165
|
+
for (const c of candidates) {
|
|
166
|
+
try {
|
|
167
|
+
// Call through the port object directly so `this` binds correctly on a class-based
|
|
168
|
+
// port implementation (avoids the @typescript-eslint/unbound-method trap).
|
|
169
|
+
const edges = dedupeEdges([
|
|
170
|
+
...(port.listOutgoingEdges?.(c.id) ?? []),
|
|
171
|
+
...(port.listIncomingEdges?.(c.id) ?? []),
|
|
172
|
+
]);
|
|
173
|
+
if (edges.length > 0)
|
|
174
|
+
map.set(c.id, edges);
|
|
175
|
+
}
|
|
176
|
+
catch (cause) {
|
|
177
|
+
throw new RetrievalError("port-failure", `listEdges threw for ${c.id}`, { cause });
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return map;
|
|
181
|
+
}
|
|
182
|
+
function dedupeEdges(edges) {
|
|
183
|
+
const seen = new Set();
|
|
184
|
+
const out = [];
|
|
185
|
+
for (const edge of edges) {
|
|
186
|
+
if (seen.has(edge.id))
|
|
187
|
+
continue;
|
|
188
|
+
seen.add(edge.id);
|
|
189
|
+
out.push(edge);
|
|
190
|
+
}
|
|
191
|
+
return out;
|
|
192
|
+
}
|
|
193
|
+
function hasPositiveSemanticSignal(semanticById) {
|
|
194
|
+
if (semanticById === undefined)
|
|
195
|
+
return false;
|
|
196
|
+
for (const score of semanticById.values()) {
|
|
197
|
+
if (score > 0)
|
|
198
|
+
return true;
|
|
199
|
+
}
|
|
200
|
+
return false;
|
|
201
|
+
}
|
|
202
|
+
function hasQuerySignal(request) {
|
|
203
|
+
const lexicalSignal = request.queryText !== undefined && tokenize(request.queryText).length > 0;
|
|
204
|
+
return lexicalSignal || hasPositiveSemanticSignal(request.semanticById);
|
|
205
|
+
}
|
|
206
|
+
function applyRelevanceFloor(ranked, request) {
|
|
207
|
+
if (!hasQuerySignal(request)) {
|
|
208
|
+
return { ranked, omitted: [] };
|
|
209
|
+
}
|
|
210
|
+
const kept = [];
|
|
211
|
+
const omitted = [];
|
|
212
|
+
for (const entry of ranked) {
|
|
213
|
+
if (entry.subscores.relevance === 0 &&
|
|
214
|
+
entry.subscores.semantic === 0 &&
|
|
215
|
+
entry.subscores.graph === 0) {
|
|
216
|
+
omitted.push({ memoryId: entry.memoryId, reason: "below-threshold" });
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
kept.push(entry);
|
|
220
|
+
}
|
|
221
|
+
return { ranked: kept, omitted };
|
|
222
|
+
}
|
|
223
|
+
export function retrieveMemoryContext(request, port) {
|
|
224
|
+
const resolved = validateAndResolve(request);
|
|
225
|
+
if (resolved.maxIncluded === 0 || resolved.budgetTokens === 0) {
|
|
226
|
+
return emptyResult(request, resolved.budgetTokens);
|
|
227
|
+
}
|
|
228
|
+
const fetched = fetchScoped(port, request.scopes);
|
|
229
|
+
const deduped = dedupeById(fetched);
|
|
230
|
+
const filtered = applyFilters(deduped, request, resolved);
|
|
231
|
+
const edgesByMemory = buildEdgesIndex(port, filtered.candidates);
|
|
232
|
+
const rankQuery = {
|
|
233
|
+
nowMs: request.nowMs,
|
|
234
|
+
weights: resolved.weights,
|
|
235
|
+
...(request.queryText === undefined ? {} : { queryText: request.queryText }),
|
|
236
|
+
...(request.semanticById === undefined ? {} : { semanticById: request.semanticById }),
|
|
237
|
+
...(request.strengthById === undefined ? {} : { strengthById: request.strengthById }),
|
|
238
|
+
...(request.fusion === undefined ? {} : { fusion: request.fusion }),
|
|
239
|
+
};
|
|
240
|
+
const ranked = rankMemories(filtered.candidates, rankQuery, edgesByMemory === undefined ? {} : { edgesByMemory });
|
|
241
|
+
const thresholded = applyRelevanceFloor(ranked, request);
|
|
242
|
+
// MMR diversity (#204, O-F3): re-order the ranked candidates so near-duplicates do not all consume
|
|
243
|
+
// the token budget. Inert (byte-identical greedy-by-rank) when the caller supplies no embeddings.
|
|
244
|
+
const selectionOrder = request.embeddingById === undefined
|
|
245
|
+
? thresholded.ranked
|
|
246
|
+
: reorderByMmr(thresholded.ranked, request.embeddingById, request.mmrLambda ?? DEFAULT_MMR_LAMBDA);
|
|
247
|
+
const assembled = assembleContextBlock(selectionOrder, filtered.candidates, {
|
|
248
|
+
budgetTokens: resolved.budgetTokens,
|
|
249
|
+
maxIncluded: resolved.maxIncluded,
|
|
250
|
+
});
|
|
251
|
+
return {
|
|
252
|
+
...assembled,
|
|
253
|
+
omitted: [...filtered.omitted, ...thresholded.omitted, ...assembled.omitted],
|
|
254
|
+
request,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const STRENGTH_HALF_LIFE_MS: number;
|
|
2
|
+
export declare const STRENGTH_FREQUENCY_SATURATION = 8;
|
|
3
|
+
export interface MemoryAccessStat {
|
|
4
|
+
readonly accessCount: number;
|
|
5
|
+
readonly lastAccessedAt: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function reinforcementStrength(stat: MemoryAccessStat | undefined, nowMs: number): number;
|
|
8
|
+
export declare function buildStrengthById<K>(statsById: ReadonlyMap<K, MemoryAccessStat>, nowMs: number): ReadonlyMap<K, number>;
|
|
9
|
+
//# sourceMappingURL=strength.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strength.d.ts","sourceRoot":"","sources":["../src/strength.ts"],"names":[],"mappings":"AAsBA,eAAO,MAAM,qBAAqB,QAAkB,CAAC;AAKrD,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAI/C,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAGD,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,gBAAgB,GAAG,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAW/F;AAID,wBAAgB,iBAAiB,CAAC,CAAC,EACjC,SAAS,EAAE,WAAW,CAAC,CAAC,EAAE,gBAAgB,CAAC,EAC3C,KAAK,EAAE,MAAM,GACZ,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAOxB"}
|