open-memex 0.9.0 → 0.10.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/README.md +19 -7
- package/README.zh-CN.md +6 -2
- package/dist/.build-stamp +1 -1
- package/dist/cli.js +24 -2
- package/dist/config.js +2 -0
- package/dist/mcp.js +1 -1
- package/dist/retrieve/feedback.js +95 -0
- package/dist/retrieve/search.js +56 -12
- package/dist/store/usage.js +23 -0
- package/dist/tools/memory.js +1 -1
- package/dist/tools/ops.js +2 -1
- package/docs/V2-DESIGN.md +2 -2
- package/docs/design/decisions.md +3 -0
- package/package.json +66 -66
- package/scripts/retrieval-eval/fixture.ts +67 -3
- package/scripts/retrieval-eval.ts +100 -24
- package/scripts/smoke-pure.ts +44 -0
- package/src/cli.ts +32 -5
- package/src/config.ts +9 -0
- package/src/mcp.ts +1 -1
- package/src/retrieve/feedback.ts +128 -0
- package/src/retrieve/search.ts +98 -4
- package/src/store/usage.ts +25 -0
- package/src/tools/memory.ts +1 -1
- package/src/tools/ops.ts +2 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* D79 (first slice): reuse-signal ranking.
|
|
3
|
+
*
|
|
4
|
+
* D77-L2 already records `retrieved_count` (exposure) and the useful split
|
|
5
|
+
* (explicit confirmations only). This module turns that telemetry into a
|
|
6
|
+
* deliberately small ranking nudge: inside a near-tied BM25 band, memories
|
|
7
|
+
* with a high `reuse_rate` move ahead of equally relevant memories that were
|
|
8
|
+
* only ever retrieved.
|
|
9
|
+
*
|
|
10
|
+
* Guardrails (D77, non-negotiable):
|
|
11
|
+
* - the score itself is never modified — feedback only reorders a band, so
|
|
12
|
+
* `search --explain` still shows the raw BM25 number;
|
|
13
|
+
* - only memories with `retrieved_count >= FEEDBACK_MIN_RETRIEVED` participate;
|
|
14
|
+
* new/rare memories keep their exact BM25 slot (no cold-start penalty),
|
|
15
|
+
* and an eligible run never jumps over one of them;
|
|
16
|
+
* - `reuse_rate` is signal strength, not a probability (it may exceed 1.0);
|
|
17
|
+
* it is never used as a hard prune threshold;
|
|
18
|
+
* - band membership is anchored to the band head's score, so a large score gap
|
|
19
|
+
* cannot be crossed. Review tiers are reordered separately by search.ts —
|
|
20
|
+
* feedback never lifts a draft over reviewed knowledge.
|
|
21
|
+
*
|
|
22
|
+
* Pure by construction: smoke-pure.ts tests it without opening SQLite.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** Minimum exposure before a memory may be reordered by feedback. */
|
|
26
|
+
export const FEEDBACK_MIN_RETRIEVED = 3;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* A hit is in the same band as the band head while its score is at least
|
|
30
|
+
* this fraction of the head's. 0.75 = within 25%, i.e. a near-tie, not a
|
|
31
|
+
* relevance gap. Measured on the D79 fixture (see docs/design/decisions.md).
|
|
32
|
+
*/
|
|
33
|
+
export const DEFAULT_FEEDBACK_BAND_RATIO = 0.75;
|
|
34
|
+
|
|
35
|
+
export interface FeedbackSignal {
|
|
36
|
+
retrieved: number;
|
|
37
|
+
useful: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface FeedbackHitInfo {
|
|
41
|
+
retrieved: number;
|
|
42
|
+
useful: number;
|
|
43
|
+
/** useful / retrieved; null when never retrieved. May exceed 1.0. */
|
|
44
|
+
rate: number | null;
|
|
45
|
+
/** Has enough exposure to participate (useful may still be 0). */
|
|
46
|
+
eligible: boolean;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface FeedbackRankOptions {
|
|
50
|
+
minRetrieved?: number;
|
|
51
|
+
bandRatio?: number;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface FeedbackRankResult<T> {
|
|
55
|
+
ordered: T[];
|
|
56
|
+
info: Map<string, FeedbackHitInfo>;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Reorder one review tier. Callers pass a tier's hits in BM25 order. Within
|
|
61
|
+
* each near-tied band, only contiguous runs of eligible hits are re-sorted by
|
|
62
|
+
* reuse_rate (descending, stable); ineligible hits keep their exact slot. So a
|
|
63
|
+
* memory with too little telemetry is never pushed down by this mechanism.
|
|
64
|
+
*/
|
|
65
|
+
export function applyFeedbackRanking<T extends { id: string; score: number }>(
|
|
66
|
+
hits: readonly T[],
|
|
67
|
+
usage: ReadonlyMap<string, FeedbackSignal>,
|
|
68
|
+
opts: FeedbackRankOptions = {},
|
|
69
|
+
): FeedbackRankResult<T> {
|
|
70
|
+
const minRetrieved = opts.minRetrieved ?? FEEDBACK_MIN_RETRIEVED;
|
|
71
|
+
const bandRatio = opts.bandRatio ?? DEFAULT_FEEDBACK_BAND_RATIO;
|
|
72
|
+
const info = new Map<string, FeedbackHitInfo>();
|
|
73
|
+
for (const h of hits) {
|
|
74
|
+
const u = usage.get(h.id);
|
|
75
|
+
const retrieved = Math.max(0, Number(u?.retrieved) || 0);
|
|
76
|
+
const useful = Math.max(0, Number(u?.useful) || 0);
|
|
77
|
+
const rate = retrieved > 0 ? useful / retrieved : null;
|
|
78
|
+
info.set(h.id, {
|
|
79
|
+
retrieved,
|
|
80
|
+
useful,
|
|
81
|
+
rate,
|
|
82
|
+
eligible: retrieved >= minRetrieved,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const ordered: T[] = [];
|
|
87
|
+
let i = 0;
|
|
88
|
+
while (i < hits.length) {
|
|
89
|
+
const head = hits[i]!;
|
|
90
|
+
const threshold = head.score > 0 ? head.score * bandRatio : head.score;
|
|
91
|
+
let j = i + 1;
|
|
92
|
+
while (j < hits.length && hits[j]!.score >= threshold) j++;
|
|
93
|
+
const band = hits.slice(i, j);
|
|
94
|
+
ordered.push(...reorderBand(band, info));
|
|
95
|
+
i = j;
|
|
96
|
+
}
|
|
97
|
+
return { ordered, info };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function reorderBand<T extends { id: string }>(
|
|
101
|
+
band: readonly T[],
|
|
102
|
+
info: ReadonlyMap<string, FeedbackHitInfo>,
|
|
103
|
+
): T[] {
|
|
104
|
+
if (band.length < 2) return [...band];
|
|
105
|
+
const out = [...band];
|
|
106
|
+
let i = 0;
|
|
107
|
+
while (i < band.length) {
|
|
108
|
+
if (!info.get(band[i]!.id)!.eligible) {
|
|
109
|
+
i++;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
let j = i + 1;
|
|
113
|
+
while (j < band.length && info.get(band[j]!.id)!.eligible) j++;
|
|
114
|
+
if (j - i > 1) {
|
|
115
|
+
// Stable sort of the run only. Ineligible hits are barriers, so a run
|
|
116
|
+
// can never jump over a memory with too little telemetry.
|
|
117
|
+
const run = band.slice(i, j).map((h, k) => ({ h, k }));
|
|
118
|
+
run.sort((a, b) => {
|
|
119
|
+
const ra = info.get(a.h.id)!.rate ?? 0;
|
|
120
|
+
const rb = info.get(b.h.id)!.rate ?? 0;
|
|
121
|
+
return ra !== rb ? rb - ra : a.k - b.k;
|
|
122
|
+
});
|
|
123
|
+
run.forEach((x, k) => (out[i + k] = x.h));
|
|
124
|
+
}
|
|
125
|
+
i = j;
|
|
126
|
+
}
|
|
127
|
+
return out;
|
|
128
|
+
}
|
package/src/retrieve/search.ts
CHANGED
|
@@ -6,7 +6,14 @@ import {
|
|
|
6
6
|
planAliasExpansions,
|
|
7
7
|
mergeRounds,
|
|
8
8
|
} from "./alias-memory.ts";
|
|
9
|
-
import { recordRetrieved } from "../store/usage.ts";
|
|
9
|
+
import { recordRetrieved, loadUsageForIds, usefulCount } from "../store/usage.ts";
|
|
10
|
+
import {
|
|
11
|
+
applyFeedbackRanking,
|
|
12
|
+
FEEDBACK_MIN_RETRIEVED,
|
|
13
|
+
DEFAULT_FEEDBACK_BAND_RATIO,
|
|
14
|
+
type FeedbackHitInfo,
|
|
15
|
+
type FeedbackSignal,
|
|
16
|
+
} from "./feedback.ts";
|
|
10
17
|
|
|
11
18
|
/** D83: which indexed column a hit's match came from. */
|
|
12
19
|
export type MatchedVia = "body" | "alias" | "expansion" | "other";
|
|
@@ -32,6 +39,12 @@ export interface SearchHit {
|
|
|
32
39
|
/** D68: provenance + creation time for the plain-language inventory. */
|
|
33
40
|
source: string;
|
|
34
41
|
created_at: number;
|
|
42
|
+
/**
|
|
43
|
+
* D79: present only when feedback weighting ran for this search. `promoted`
|
|
44
|
+
* is how many places this hit moved up (0 = stayed put); a negative value
|
|
45
|
+
* would mean it moved down, which the band sort never does.
|
|
46
|
+
*/
|
|
47
|
+
feedback?: FeedbackHitInfo & { promoted: number };
|
|
35
48
|
}
|
|
36
49
|
|
|
37
50
|
interface RawRow {
|
|
@@ -138,6 +151,19 @@ export interface SearchStats {
|
|
|
138
151
|
hiddenExcluded: number;
|
|
139
152
|
/** D81: query-time alias expansions actually run (rounds 2+, raw text). */
|
|
140
153
|
aliasExpansions: string[];
|
|
154
|
+
/** D79: feedback-weighting state and the hits it moved up. */
|
|
155
|
+
feedback?: {
|
|
156
|
+
enabled: boolean;
|
|
157
|
+
minRetrieved: number;
|
|
158
|
+
bandRatio: number;
|
|
159
|
+
promotions: Array<{
|
|
160
|
+
id: string;
|
|
161
|
+
promoted: number;
|
|
162
|
+
rate: number | null;
|
|
163
|
+
retrieved: number;
|
|
164
|
+
useful: number;
|
|
165
|
+
}>;
|
|
166
|
+
};
|
|
141
167
|
}
|
|
142
168
|
|
|
143
169
|
/**
|
|
@@ -148,7 +174,17 @@ export interface SearchStats {
|
|
|
148
174
|
* drafts/rejected rank after unreviewed content;
|
|
149
175
|
* - deprecated stays visible as a warning but ranks after active.
|
|
150
176
|
*/
|
|
151
|
-
|
|
177
|
+
interface FeedbackOptions {
|
|
178
|
+
minRetrieved?: number;
|
|
179
|
+
bandRatio?: number;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function resolveVisible(
|
|
183
|
+
rows: RawRow[],
|
|
184
|
+
limit: number,
|
|
185
|
+
stats?: SearchStats,
|
|
186
|
+
feedback?: FeedbackOptions,
|
|
187
|
+
): SearchHit[] {
|
|
152
188
|
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
153
189
|
const fullRow = (id: string): RawRow | undefined => {
|
|
154
190
|
const cached = byId.get(id);
|
|
@@ -198,7 +234,47 @@ function resolveVisible(rows: RawRow[], limit: number, stats?: SearchStats): Sea
|
|
|
198
234
|
if (target.status === "deprecated") deprecated.push(hit);
|
|
199
235
|
else tiers[reviewTier(target)].push(hit);
|
|
200
236
|
}
|
|
201
|
-
|
|
237
|
+
// D79: tiers are reordered independently, so feedback can never lift a
|
|
238
|
+
// draft over reviewed knowledge. Without feedback this is exactly the old
|
|
239
|
+
// concatenation.
|
|
240
|
+
const before = [...tiers[0], ...tiers[1], ...tiers[2], ...deprecated];
|
|
241
|
+
if (!feedback) return before.slice(0, limit);
|
|
242
|
+
|
|
243
|
+
const usageById = loadUsageForIds(before.map((h) => h.id));
|
|
244
|
+
const signals = new Map<string, FeedbackSignal>();
|
|
245
|
+
for (const [id, u] of usageById) {
|
|
246
|
+
signals.set(id, { retrieved: u.retrieved_count, useful: usefulCount(u) });
|
|
247
|
+
}
|
|
248
|
+
const beforeRank = new Map(before.map((h, i) => [h.id, i]));
|
|
249
|
+
const infoById = new Map<string, FeedbackHitInfo>();
|
|
250
|
+
const ordered: SearchHit[] = [];
|
|
251
|
+
for (const tier of [...tiers, deprecated]) {
|
|
252
|
+
const r = applyFeedbackRanking(tier, signals, feedback);
|
|
253
|
+
ordered.push(...r.ordered);
|
|
254
|
+
for (const [id, info] of r.info) infoById.set(id, info);
|
|
255
|
+
}
|
|
256
|
+
ordered.forEach((h, i) => {
|
|
257
|
+
const info = infoById.get(h.id);
|
|
258
|
+
if (!info) return;
|
|
259
|
+
h.feedback = { ...info, promoted: (beforeRank.get(h.id) ?? i) - i };
|
|
260
|
+
});
|
|
261
|
+
if (stats) {
|
|
262
|
+
stats.feedback = {
|
|
263
|
+
enabled: true,
|
|
264
|
+
minRetrieved: feedback.minRetrieved ?? FEEDBACK_MIN_RETRIEVED,
|
|
265
|
+
bandRatio: feedback.bandRatio ?? DEFAULT_FEEDBACK_BAND_RATIO,
|
|
266
|
+
promotions: ordered
|
|
267
|
+
.filter((h) => (h.feedback?.promoted ?? 0) > 0)
|
|
268
|
+
.map((h) => ({
|
|
269
|
+
id: h.id,
|
|
270
|
+
promoted: h.feedback!.promoted,
|
|
271
|
+
rate: h.feedback!.rate,
|
|
272
|
+
retrieved: h.feedback!.retrieved,
|
|
273
|
+
useful: h.feedback!.useful,
|
|
274
|
+
})),
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
return ordered.slice(0, limit);
|
|
202
278
|
}
|
|
203
279
|
|
|
204
280
|
/**
|
|
@@ -230,6 +306,15 @@ export function search(
|
|
|
230
306
|
* leave it off so fixture runs never pollute real telemetry.
|
|
231
307
|
*/
|
|
232
308
|
record?: boolean;
|
|
309
|
+
/**
|
|
310
|
+
* D79: apply the D77 reuse-signal nudge inside near-tied BM25 bands.
|
|
311
|
+
* Default off — callers pass `cfg.feedbackWeighting`; the eval script
|
|
312
|
+
* passes both values to measure the delta. The score is never changed.
|
|
313
|
+
*/
|
|
314
|
+
feedback?: boolean;
|
|
315
|
+
/** D79 ablation knobs; shipped defaults come from feedback.ts. */
|
|
316
|
+
feedbackMinRetrieved?: number;
|
|
317
|
+
feedbackBandRatio?: number;
|
|
233
318
|
} = {},
|
|
234
319
|
): SearchHit[] {
|
|
235
320
|
const qflags = { stopwords: opts.stopwords, orJoin: opts.orJoin, prefix: opts.prefix };
|
|
@@ -240,6 +325,12 @@ export function search(
|
|
|
240
325
|
opts.stats.hiddenSuperseded = 0;
|
|
241
326
|
opts.stats.hiddenExcluded = 0;
|
|
242
327
|
opts.stats.aliasExpansions = [];
|
|
328
|
+
opts.stats.feedback = {
|
|
329
|
+
enabled: opts.feedback === true,
|
|
330
|
+
minRetrieved: opts.feedbackMinRetrieved ?? FEEDBACK_MIN_RETRIEVED,
|
|
331
|
+
bandRatio: opts.feedbackBandRatio ?? DEFAULT_FEEDBACK_BAND_RATIO,
|
|
332
|
+
promotions: [],
|
|
333
|
+
};
|
|
243
334
|
}
|
|
244
335
|
if (!q) return [];
|
|
245
336
|
const limit = Math.max(1, Math.min(opts.limit ?? 8, 50));
|
|
@@ -337,7 +428,10 @@ export function search(
|
|
|
337
428
|
// expands variants into the indexed aliases column at sync time), so
|
|
338
429
|
// without a user alias the query side stays a single FTS round.
|
|
339
430
|
if (opts.stats) opts.stats.candidates = merged.length;
|
|
340
|
-
const
|
|
431
|
+
const feedback = opts.feedback === true
|
|
432
|
+
? { minRetrieved: opts.feedbackMinRetrieved, bandRatio: opts.feedbackBandRatio }
|
|
433
|
+
: undefined;
|
|
434
|
+
const hits = resolveVisible(merged, limit, opts.stats, feedback);
|
|
341
435
|
// D77-L2: record only the ids actually returned — the candidate pool is
|
|
342
436
|
// not an exposure, and #53 guarantees the cut below is the final result.
|
|
343
437
|
if (opts.record && hits.length > 0) recordRetrieved(hits.map((h) => h.id));
|
package/src/store/usage.ts
CHANGED
|
@@ -122,6 +122,31 @@ export function recordUseful(
|
|
|
122
122
|
return getUsage(id)!;
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
/**
|
|
126
|
+
* D79: batch-read the counters a search ranking needs, keyed by id. One
|
|
127
|
+
* query per retrieval pass, not one per candidate; missing ids are simply
|
|
128
|
+
* absent (no exposure yet), never synthesized.
|
|
129
|
+
*/
|
|
130
|
+
export function loadUsageForIds(ids: readonly string[]): Map<string, MemoryUsage> {
|
|
131
|
+
const unique = [...new Set(ids)];
|
|
132
|
+
if (unique.length === 0) return new Map();
|
|
133
|
+
const rows = db()
|
|
134
|
+
.prepare(
|
|
135
|
+
`SELECT id, retrieved_count, useful_self, useful_team,
|
|
136
|
+
last_retrieved_at, last_useful_at, updated_at
|
|
137
|
+
FROM memory_usage WHERE id IN (${unique.map(() => "?").join(",")})`,
|
|
138
|
+
)
|
|
139
|
+
.all(...(unique as unknown[])) as MemoryUsage[];
|
|
140
|
+
const map = new Map<string, MemoryUsage>();
|
|
141
|
+
for (const row of rows) {
|
|
142
|
+
row.retrieved_count = Number(row.retrieved_count) || 0;
|
|
143
|
+
row.useful_self = Number(row.useful_self) || 0;
|
|
144
|
+
row.useful_team = Number(row.useful_team) || 0;
|
|
145
|
+
map.set(row.id, row);
|
|
146
|
+
}
|
|
147
|
+
return map;
|
|
148
|
+
}
|
|
149
|
+
|
|
125
150
|
export function getUsage(id: string): MemoryUsage | null {
|
|
126
151
|
const row = db()
|
|
127
152
|
.prepare(
|
package/src/tools/memory.ts
CHANGED
package/src/tools/ops.ts
CHANGED
|
@@ -444,6 +444,7 @@ export async function addMemory(
|
|
|
444
444
|
|
|
445
445
|
export async function searchMemories(
|
|
446
446
|
getScope: () => Scope,
|
|
447
|
+
cfg: MyOMemoryConfig,
|
|
447
448
|
args: MemorySearchArgs,
|
|
448
449
|
): Promise<ToolResult> {
|
|
449
450
|
const project = getScope();
|
|
@@ -459,6 +460,7 @@ export async function searchMemories(
|
|
|
459
460
|
limit: args.limit,
|
|
460
461
|
type: args.type,
|
|
461
462
|
record: true,
|
|
463
|
+
feedback: cfg.feedbackWeighting,
|
|
462
464
|
});
|
|
463
465
|
if (hits.length === 0) {
|
|
464
466
|
return {
|