open-memex 0.11.0-alpha.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CHANGELOG.md +1 -3
- package/dist/.build-stamp +1 -1
- package/dist/retrieve/embedding.js +1 -1
- package/dist/store/embeddings.js +2 -2
- package/package.json +1 -1
- package/scripts/bench-semantic.ts +463 -463
- package/server.json +2 -2
- package/src/retrieve/embedding.ts +214 -214
- package/src/retrieve/rerank.ts +96 -96
- package/src/store/embeddings.ts +82 -82
|
@@ -1,463 +1,463 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Semantic-ranking gate bench (D77 origin, re-measured for the D96 gate).
|
|
3
|
-
*
|
|
4
|
-
* Question the gate answers: does a local multilingual embedding layer beat
|
|
5
|
-
* the measured lexical ceiling, and at what tax (model size, load time,
|
|
6
|
-
* per-query latency, disk cache)?
|
|
7
|
-
*
|
|
8
|
-
* Target: pooled LongMemEval-S R@5 40.6% (the lexical ceiling, D77).
|
|
9
|
-
* Product slice: the checked-in retrieval fixture's EN<->ZH cross-lingual
|
|
10
|
-
* queries — the only slice where a multilingual model can plausibly help.
|
|
11
|
-
*
|
|
12
|
-
* Model: any `@huggingface/transformers` feature-extraction model. Default is
|
|
13
|
-
* `Xenova/paraphrase-multilingual-MiniLM-L12-v2` (the 2026-10-07 choice).
|
|
14
|
-
* Local only, no API. Embeddings are cached to disk; cosine is brute-forced
|
|
15
|
-
* in JS. Vectors never leave the machine.
|
|
16
|
-
*
|
|
17
|
-
* Arms:
|
|
18
|
-
* bm25 the shipped lexical pipeline (no model needed)
|
|
19
|
-
* vec vector-only ranking over the whole corpus (upper bound)
|
|
20
|
-
* rrf naive RRF fusion of BM25 + vector (§4.7 measured this as harmful)
|
|
21
|
-
* rerank BM25-first, embeddings only reorder cross-lingual hits inside the
|
|
22
|
-
* BM25 candidate pool (the only shape this project would ship)
|
|
23
|
-
*
|
|
24
|
-
* Usage:
|
|
25
|
-
* node --experimental-strip-types scripts/bench-semantic.ts --arms bm25
|
|
26
|
-
* npm i --no-save @huggingface/transformers # once, for the model arms
|
|
27
|
-
* node --experimental-strip-types scripts/bench-semantic.ts --cache-dir <dir>
|
|
28
|
-
* node --experimental-strip-types scripts/bench-semantic.ts --pooled \
|
|
29
|
-
* --data <longmemeval_s_cleaned.json> --cache-dir <dir>
|
|
30
|
-
*
|
|
31
|
-
* `--cache-dir` (or OPEN_MEMEX_BENCH_DATA) holds the vector caches and the
|
|
32
|
-
* Hugging Face model download (`<cache-dir>/hf`). It defaults to
|
|
33
|
-
* `~/workspace/bench-data` — the 2026-10-07 location. Nothing is written into
|
|
34
|
-
* the repo.
|
|
35
|
-
*/
|
|
36
|
-
import fs from "node:fs";
|
|
37
|
-
import os from "node:os";
|
|
38
|
-
import path from "node:path";
|
|
39
|
-
|
|
40
|
-
const args = process.argv.slice(2);
|
|
41
|
-
const opt = (name: string): string | undefined => {
|
|
42
|
-
const i = args.indexOf(name);
|
|
43
|
-
return i === -1 ? undefined : args[i + 1];
|
|
44
|
-
};
|
|
45
|
-
const POOLED = args.includes("--pooled");
|
|
46
|
-
const LIMIT = Number(opt("--limit") ?? Infinity);
|
|
47
|
-
const DEPTH = Number(opt("--depth") ?? 50);
|
|
48
|
-
const MODEL = opt("--model") ?? "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
49
|
-
/** ONNX precision. `fp32` is the 2026-10-07 parity run; `q8` is what a
|
|
50
|
-
* download-on-first-use feature would plausibly ship. */
|
|
51
|
-
const DTYPE = opt("--dtype") ?? "fp32";
|
|
52
|
-
const CACHE_DIR =
|
|
53
|
-
opt("--cache-dir") ?? process.env.OPEN_MEMEX_BENCH_DATA ?? path.join(os.homedir(), "workspace", "bench-data");
|
|
54
|
-
const ARMS = new Set((opt("--arms") ?? "bm25,vec,rrf,rerank").split(",").map((s) => s.trim()));
|
|
55
|
-
const JSON_OUT = opt("--json");
|
|
56
|
-
/** Similarity floors swept by the `rerank` arm. */
|
|
57
|
-
const FLOORS = (opt("--floors") ?? "0.2,0.3,0.4,0.5,0.6").split(",").map(Number);
|
|
58
|
-
const K = 5;
|
|
59
|
-
|
|
60
|
-
function median(xs: number[]): number {
|
|
61
|
-
if (xs.length === 0) return 0;
|
|
62
|
-
const s = [...xs].sort((a, b) => a - b);
|
|
63
|
-
return s[Math.floor(s.length / 2)]!;
|
|
64
|
-
}
|
|
65
|
-
function p95(xs: number[]): number {
|
|
66
|
-
if (xs.length === 0) return 0;
|
|
67
|
-
const s = [...xs].sort((a, b) => a - b);
|
|
68
|
-
return s[Math.min(s.length - 1, Math.floor(s.length * 0.95))]!;
|
|
69
|
-
}
|
|
70
|
-
function dirBytes(dir: string): number {
|
|
71
|
-
if (!fs.existsSync(dir)) return 0;
|
|
72
|
-
let total = 0;
|
|
73
|
-
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
74
|
-
const p = path.join(dir, e.name);
|
|
75
|
-
if (e.isDirectory()) total += dirBytes(p);
|
|
76
|
-
else if (e.isFile()) total += fs.statSync(p).size;
|
|
77
|
-
}
|
|
78
|
-
return total;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
const HOME = fs.mkdtempSync(path.join(os.tmpdir(), "sem-bench-"));
|
|
82
|
-
process.env.OPEN_MEMEX_HOME = HOME;
|
|
83
|
-
const START = Date.now();
|
|
84
|
-
|
|
85
|
-
import { serialize, normalizeFrontmatter } from "../src/store/markdown.ts";
|
|
86
|
-
import { syncScope } from "../src/store/sync.ts";
|
|
87
|
-
import { search } from "../src/retrieve/search.ts";
|
|
88
|
-
import { hasCjk } from "../src/retrieve/cjk.ts";
|
|
89
|
-
import { rerankCrossLingual } from "../src/retrieve/rerank.ts";
|
|
90
|
-
import { memoriesDirPath } from "../src/paths.ts";
|
|
91
|
-
|
|
92
|
-
interface Doc {
|
|
93
|
-
id: string;
|
|
94
|
-
/** Text the embedding model sees (body + captured aliases + tags). */
|
|
95
|
-
text: string;
|
|
96
|
-
/** Markdown body written to the index — fixture mode keeps this identical
|
|
97
|
-
* to retrieval-eval so the BM25 arm reproduces the published baseline. */
|
|
98
|
-
body: string;
|
|
99
|
-
/** EN/ZH by script; the cross-lingual trigger is query script != doc script. */
|
|
100
|
-
lang: "en" | "zh";
|
|
101
|
-
}
|
|
102
|
-
interface Q {
|
|
103
|
-
text: string;
|
|
104
|
-
gold: Set<string>;
|
|
105
|
-
note: string;
|
|
106
|
-
lang: "en" | "zh";
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const docs: Doc[] = [];
|
|
110
|
-
const questions: Q[] = [];
|
|
111
|
-
/** Fixture frontmatter, written verbatim so the BM25 arm reproduces the
|
|
112
|
-
* checked-in retrieval-eval baseline exactly (aliases/tags in their real
|
|
113
|
-
* FTS columns, not folded into the body). */
|
|
114
|
-
let fixtureMemories: Array<Record<string, unknown>> | null = null;
|
|
115
|
-
|
|
116
|
-
if (POOLED) {
|
|
117
|
-
const dataPath = opt("--data") ?? "";
|
|
118
|
-
if (!dataPath) throw new Error("missing --data");
|
|
119
|
-
const items = JSON.parse(fs.readFileSync(dataPath, "utf8"));
|
|
120
|
-
let qi = 0;
|
|
121
|
-
for (const item of items) {
|
|
122
|
-
if (item.question_id.endsWith("_abs")) continue;
|
|
123
|
-
if (qi >= LIMIT) break;
|
|
124
|
-
const sids: string[] = item.haystack_session_ids;
|
|
125
|
-
const gold = new Set<string>();
|
|
126
|
-
for (const g of item.answer_session_ids) {
|
|
127
|
-
const i = sids.indexOf(g);
|
|
128
|
-
if (i !== -1) gold.add(`q${qi}-s${i}`);
|
|
129
|
-
}
|
|
130
|
-
if (gold.size === 0) continue;
|
|
131
|
-
item.haystack_sessions.forEach((turns: Array<{ role: string; content: string }>, i: number) => {
|
|
132
|
-
const text = turns.map((t) => `${t.role}: ${t.content}`).join("\n").slice(0, 2000);
|
|
133
|
-
docs.push({ id: `q${qi}-s${i}`, text, body: text, lang: hasCjk(text) ? "zh" : "en" });
|
|
134
|
-
});
|
|
135
|
-
const text: string = item.question;
|
|
136
|
-
questions.push({ text, gold, note: item.question_type, lang: hasCjk(text) ? "zh" : "en" });
|
|
137
|
-
qi++;
|
|
138
|
-
}
|
|
139
|
-
} else {
|
|
140
|
-
const { MEMORIES, QUERIES } = await import("./retrieval-eval/fixture.ts");
|
|
141
|
-
fixtureMemories = MEMORIES;
|
|
142
|
-
for (const m of MEMORIES) {
|
|
143
|
-
const text = [m.body, ...(m.aliases ?? []), ...(m.tags ?? [])].join("\n").slice(0, 2000);
|
|
144
|
-
docs.push({ id: m.id, text, body: m.body, lang: hasCjk(m.body) ? "zh" : "en" });
|
|
145
|
-
}
|
|
146
|
-
for (const q of QUERIES.slice(0, LIMIT)) {
|
|
147
|
-
questions.push({
|
|
148
|
-
text: q.query,
|
|
149
|
-
gold: new Set(q.gold),
|
|
150
|
-
note: q.note,
|
|
151
|
-
lang: hasCjk(q.query) ? "zh" : "en",
|
|
152
|
-
});
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
console.log(`docs: ${docs.length}, questions: ${questions.length}, model: ${MODEL} (${DTYPE})`);
|
|
157
|
-
console.log(`cache dir: ${CACHE_DIR}`);
|
|
158
|
-
const byId = new Map(docs.map((d) => [d.id, d]));
|
|
159
|
-
|
|
160
|
-
// Seed the FTS index so the BM25 arm runs on the same corpus.
|
|
161
|
-
{
|
|
162
|
-
const dir = memoriesDirPath("personal");
|
|
163
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
164
|
-
const B = 2000;
|
|
165
|
-
for (let i = 0; i < docs.length; i += B) {
|
|
166
|
-
for (const d of docs.slice(i, i + B)) {
|
|
167
|
-
// Fixture mode reproduces retrieval-eval's frontmatter exactly, so the
|
|
168
|
-
// BM25 arm equals the published baseline; pooled mode has none.
|
|
169
|
-
const fmDef = fixtureMemories?.find((m) => (m as { id: string }).id === d.id) as
|
|
170
|
-
| {
|
|
171
|
-
type?: string;
|
|
172
|
-
tags?: string[];
|
|
173
|
-
aliases?: string[];
|
|
174
|
-
alias?: string;
|
|
175
|
-
target?: string;
|
|
176
|
-
}
|
|
177
|
-
| undefined;
|
|
178
|
-
const fm = normalizeFrontmatter({
|
|
179
|
-
id: d.id,
|
|
180
|
-
scope: "personal",
|
|
181
|
-
type: fmDef?.type ?? "fact",
|
|
182
|
-
tags: fmDef?.tags ?? [],
|
|
183
|
-
aliases: fmDef?.aliases,
|
|
184
|
-
alias: fmDef?.alias,
|
|
185
|
-
target: fmDef?.target,
|
|
186
|
-
source: fmDef ? "retrieval-eval fixture" : undefined,
|
|
187
|
-
});
|
|
188
|
-
fs.writeFileSync(path.join(dir, `${d.id}.md`), serialize(fm, d.body), "utf8");
|
|
189
|
-
}
|
|
190
|
-
if (docs.length > B) console.log(` seeded ${Math.min(i + B, docs.length)}/${docs.length}`);
|
|
191
|
-
}
|
|
192
|
-
syncScope("personal");
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// ---- BM25 arm (always measured: it is both a result and the rerank pool) ----
|
|
196
|
-
const bm25Ranks = new Map<string, number>();
|
|
197
|
-
const bm25Raw = new Map<string, string[]>();
|
|
198
|
-
const bm25Ms: number[] = [];
|
|
199
|
-
for (const q of questions) {
|
|
200
|
-
const t0 = performance.now();
|
|
201
|
-
const ids = search(q.text, { scopeKeys: ["personal"], limit: DEPTH, synonyms: false }).map((h) => h.id);
|
|
202
|
-
bm25Ms.push(performance.now() - t0);
|
|
203
|
-
bm25Raw.set(q.text, ids);
|
|
204
|
-
let rank = -1;
|
|
205
|
-
for (const g of q.gold) {
|
|
206
|
-
const i = ids.indexOf(g);
|
|
207
|
-
if (i !== -1 && (rank === -1 || i < rank)) rank = i;
|
|
208
|
-
}
|
|
209
|
-
bm25Ranks.set(q.text, rank);
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
interface Metrics {
|
|
213
|
-
n: number;
|
|
214
|
-
r1: number;
|
|
215
|
-
r5: number;
|
|
216
|
-
mrr: number;
|
|
217
|
-
}
|
|
218
|
-
function metrics(ranks: Map<string, number>, qs: Q[]): Metrics {
|
|
219
|
-
let r1 = 0;
|
|
220
|
-
let r5 = 0;
|
|
221
|
-
let mrr = 0;
|
|
222
|
-
for (const q of qs) {
|
|
223
|
-
const rank = ranks.get(q.text) ?? -1;
|
|
224
|
-
if (rank === 0) r1++;
|
|
225
|
-
if (rank !== -1 && rank < K) r5++;
|
|
226
|
-
if (rank !== -1) mrr += 1 / (rank + 1);
|
|
227
|
-
}
|
|
228
|
-
return { n: qs.length, r1, r5, mrr };
|
|
229
|
-
}
|
|
230
|
-
function fmt(m: Metrics): string {
|
|
231
|
-
return `R@1 ${(m.r1 / m.n * 100).toFixed(1)}% R@5 ${(m.r5 / m.n * 100).toFixed(1)}% MRR ${(m.mrr / m.n).toFixed(3)} (n=${m.n})`;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/**
|
|
235
|
-
* The product shape: BM25 retrieves, embeddings may only reorder contiguous
|
|
236
|
-
* runs of cross-lingual hits whose cosine clears `floor`. This calls the
|
|
237
|
-
* SHIPPED pure rule (`src/retrieve/rerank.ts`), so the bench cannot drift
|
|
238
|
-
* from the product's ordering; the arm only supplies cosines.
|
|
239
|
-
*/
|
|
240
|
-
function rerankOrder(ids: string[], queryLang: "en" | "zh", sim: (id: string) => number, floor: number): string[] {
|
|
241
|
-
const cosines = new Map<string, number>();
|
|
242
|
-
for (const id of ids) {
|
|
243
|
-
const d = byId.get(id);
|
|
244
|
-
if (d && d.lang !== queryLang) cosines.set(id, sim(id));
|
|
245
|
-
}
|
|
246
|
-
return rerankCrossLingual(ids.map((id) => ({ id })), cosines, { minCosine: floor }).ordered.map((h) => h.id);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
function rankOf(ids: string[], gold: Set<string>): number {
|
|
250
|
-
let rank = -1;
|
|
251
|
-
for (const g of gold) {
|
|
252
|
-
const i = ids.indexOf(g);
|
|
253
|
-
if (i !== -1 && (rank === -1 || i < rank)) rank = i;
|
|
254
|
-
}
|
|
255
|
-
return rank;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
const results: Record<string, Metrics> = {};
|
|
259
|
-
const ranksByArm = new Map<string, Map<string, number>>();
|
|
260
|
-
function record(arm: string, ranks: Map<string, number>) {
|
|
261
|
-
ranksByArm.set(arm, ranks);
|
|
262
|
-
results[arm] = metrics(ranks, questions);
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
record("bm25", bm25Ranks);
|
|
266
|
-
|
|
267
|
-
const crossQs = questions.filter((q) => {
|
|
268
|
-
const goldLangs = [...q.gold].map((g) => byId.get(g)?.lang).filter(Boolean);
|
|
269
|
-
return goldLangs.length > 0 && goldLangs.some((l) => l !== q.lang);
|
|
270
|
-
});
|
|
271
|
-
const sameQs = questions.filter((q) => !crossQs.includes(q));
|
|
272
|
-
|
|
273
|
-
let tax: {
|
|
274
|
-
modelLoadMs: number;
|
|
275
|
-
modelBytes: number;
|
|
276
|
-
docEmbedMs: number;
|
|
277
|
-
docEmbedPerDocMs: number;
|
|
278
|
-
queryEmbedMedianMs: number;
|
|
279
|
-
queryEmbedP95Ms: number;
|
|
280
|
-
cacheBytes: number;
|
|
281
|
-
rssMb: number;
|
|
282
|
-
dim: number;
|
|
283
|
-
} | null = null;
|
|
284
|
-
|
|
285
|
-
if (ARMS.size > 0 && !(ARMS.size === 1 && ARMS.has("bm25"))) {
|
|
286
|
-
const { pipeline, env } = await import("@huggingface/transformers");
|
|
287
|
-
env.cacheDir = path.join(CACHE_DIR, "hf");
|
|
288
|
-
fs.mkdirSync(env.cacheDir, { recursive: true });
|
|
289
|
-
|
|
290
|
-
const slug = MODEL.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
291
|
-
const cachePath = path.join(CACHE_DIR, `emb-${slug}-${DTYPE}-${POOLED ? "pooled" : "fixture"}.json`);
|
|
292
|
-
|
|
293
|
-
let vectors: number[][] | null = null;
|
|
294
|
-
if (fs.existsSync(cachePath)) {
|
|
295
|
-
try {
|
|
296
|
-
const cached = JSON.parse(fs.readFileSync(cachePath, "utf8")) as {
|
|
297
|
-
model: string;
|
|
298
|
-
dim: number;
|
|
299
|
-
ids: string[];
|
|
300
|
-
vectors: number[][];
|
|
301
|
-
};
|
|
302
|
-
if (cached.model === `${MODEL}::${DTYPE}` && cached.ids.length === docs.length && cached.ids.every((id, i) => id === docs[i]!.id)) {
|
|
303
|
-
vectors = cached.vectors;
|
|
304
|
-
console.log(`vectors: cache hit ${cachePath} (${cached.dim}-dim)`);
|
|
305
|
-
} else {
|
|
306
|
-
console.log("vectors: cache is stale (model or corpus changed) — recomputing");
|
|
307
|
-
}
|
|
308
|
-
} catch {
|
|
309
|
-
console.log("vectors: cache unreadable — recomputing");
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
console.log("loading model… (first run downloads it into the cache dir)");
|
|
314
|
-
const tLoad = performance.now();
|
|
315
|
-
const extractor: any = await pipeline("feature-extraction", MODEL, { dtype: DTYPE as any });
|
|
316
|
-
const modelLoadMs = performance.now() - tLoad;
|
|
317
|
-
|
|
318
|
-
let docEmbedMs = 0;
|
|
319
|
-
if (!vectors) {
|
|
320
|
-
console.log(`embedding ${docs.length} docs…`);
|
|
321
|
-
const tDocs = performance.now();
|
|
322
|
-
vectors = [];
|
|
323
|
-
const B = 32;
|
|
324
|
-
for (let i = 0; i < docs.length; i += B) {
|
|
325
|
-
const batch = docs.map((d) => d.text).slice(i, i + B);
|
|
326
|
-
const out: any = await extractor(batch, { pooling: "mean", normalize: true });
|
|
327
|
-
for (const v of out.tolist()) vectors.push(v);
|
|
328
|
-
if (docs.length > 320 && i % 320 === 0) console.log(` embedded ${i}/${docs.length}`);
|
|
329
|
-
}
|
|
330
|
-
docEmbedMs = performance.now() - tDocs;
|
|
331
|
-
fs.mkdirSync(path.dirname(cachePath), { recursive: true });
|
|
332
|
-
fs.writeFileSync(
|
|
333
|
-
cachePath,
|
|
334
|
-
JSON.stringify({ model: `${MODEL}::${DTYPE}`, dim: vectors[0]!.length, ids: docs.map((d) => d.id), vectors }),
|
|
335
|
-
);
|
|
336
|
-
console.log(`vectors cached: ${cachePath}`);
|
|
337
|
-
}
|
|
338
|
-
const DIM = vectors[0]!.length;
|
|
339
|
-
|
|
340
|
-
const cos = (qv: number[], v: number[]): number => {
|
|
341
|
-
let s = 0;
|
|
342
|
-
for (let d = 0; d < DIM; d++) s += qv[d]! * v[d]!;
|
|
343
|
-
return s;
|
|
344
|
-
};
|
|
345
|
-
const vecById = new Map<string, number[]>();
|
|
346
|
-
docs.forEach((d, i) => vecById.set(d.id, vectors![i]!));
|
|
347
|
-
|
|
348
|
-
const vecRanks = new Map<string, number>();
|
|
349
|
-
const rrfRanks = new Map<string, number>();
|
|
350
|
-
const rerankRanks = new Map<number, Map<string, number>>();
|
|
351
|
-
for (const f of FLOORS) rerankRanks.set(f, new Map());
|
|
352
|
-
const queryEmbedMs: number[] = [];
|
|
353
|
-
|
|
354
|
-
let qi = 0;
|
|
355
|
-
for (const q of questions) {
|
|
356
|
-
const t0 = performance.now();
|
|
357
|
-
const qv: number[] = (await extractor(q.text, { pooling: "mean", normalize: true })).tolist()[0];
|
|
358
|
-
queryEmbedMs.push(performance.now() - t0);
|
|
359
|
-
|
|
360
|
-
const vecOrder = docs
|
|
361
|
-
.map((d) => ({ id: d.id, s: cos(qv, vecById.get(d.id)!) }))
|
|
362
|
-
.sort((a, b) => b.s - a.s)
|
|
363
|
-
.map((x) => x.id);
|
|
364
|
-
vecRanks.set(q.text, rankOf(vecOrder, q.gold));
|
|
365
|
-
|
|
366
|
-
const rrf = (lists: string[][], k = 60): string[] => {
|
|
367
|
-
const scores = new Map<string, number>();
|
|
368
|
-
for (const list of lists) list.forEach((id, rank) => scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank + 1)));
|
|
369
|
-
return [...scores.entries()].sort((a, b) => b[1] - a[1]).map(([id]) => id);
|
|
370
|
-
};
|
|
371
|
-
rrfRanks.set(q.text, rankOf(rrf([bm25Raw.get(q.text)!, vecOrder]), q.gold));
|
|
372
|
-
|
|
373
|
-
const simById = new Map<string, number>();
|
|
374
|
-
for (const id of bm25Raw.get(q.text)!) simById.set(id, cos(qv, vecById.get(id)!));
|
|
375
|
-
for (const f of FLOORS) {
|
|
376
|
-
const order = rerankOrder(bm25Raw.get(q.text)!, q.lang, (id) => simById.get(id) ?? -1, f);
|
|
377
|
-
rerankRanks.get(f)!.set(q.text, rankOf(order, q.gold));
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
qi++;
|
|
381
|
-
if (questions.length > 100 && qi % 100 === 0) console.log(` ... ${qi}/${questions.length}`);
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
record("vec", vecRanks);
|
|
385
|
-
record("rrf", rrfRanks);
|
|
386
|
-
for (const f of FLOORS) record(`rerank@${f}`, rerankRanks.get(f)!);
|
|
387
|
-
|
|
388
|
-
tax = {
|
|
389
|
-
modelLoadMs,
|
|
390
|
-
modelBytes: dirBytes(path.join(env.cacheDir!, MODEL)),
|
|
391
|
-
docEmbedMs,
|
|
392
|
-
docEmbedPerDocMs: docs.length ? docEmbedMs / docs.length : 0,
|
|
393
|
-
queryEmbedMedianMs: median(queryEmbedMs),
|
|
394
|
-
queryEmbedP95Ms: p95(queryEmbedMs),
|
|
395
|
-
cacheBytes: fs.existsSync(cachePath) ? fs.statSync(cachePath).size : 0,
|
|
396
|
-
rssMb: process.memoryUsage().rss / 1024 / 1024,
|
|
397
|
-
dim: DIM,
|
|
398
|
-
};
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
// ---- report ----
|
|
402
|
-
const armOrder = ["bm25", ...Object.keys(results).filter((a) => a !== "bm25")];
|
|
403
|
-
console.log(`\n==== semantic gate (${POOLED ? "pooled LongMemEval-S" : "retrieval fixture"}) ====`);
|
|
404
|
-
console.log(`corpus : ${docs.length} docs, ${questions.length} questions, BM25 depth ${DEPTH}`);
|
|
405
|
-
for (const arm of armOrder) console.log(`${arm.padEnd(12)} ${fmt(results[arm]!)}`);
|
|
406
|
-
|
|
407
|
-
if (crossQs.length > 0) {
|
|
408
|
-
console.log(`\ncross-lingual slice (${crossQs.length} queries, gold script != query script):`);
|
|
409
|
-
for (const arm of armOrder) console.log(`${arm.padEnd(12)} ${fmt(metrics(ranksByArm.get(arm)!, crossQs))}`);
|
|
410
|
-
console.log(`\nsame-language slice (${sameQs.length} queries — rerank must not change these):`);
|
|
411
|
-
for (const arm of armOrder) console.log(`${arm.padEnd(12)} ${fmt(metrics(ranksByArm.get(arm)!, sameQs))}`);
|
|
412
|
-
|
|
413
|
-
console.log("\ncross-lingual per-query ranks (1-based, - = miss):");
|
|
414
|
-
const show = armOrder.filter((a) => a === "bm25" || a === "vec" || a === "rrf" || a.startsWith("rerank@"));
|
|
415
|
-
for (const q of crossQs) {
|
|
416
|
-
const cells = show.map((a) => {
|
|
417
|
-
const r = ranksByArm.get(a)!.get(q.text) ?? -1;
|
|
418
|
-
return `${a.replace("rerank@", "r@")}=${r === -1 ? "-" : r + 1}`;
|
|
419
|
-
});
|
|
420
|
-
console.log(` ${JSON.stringify(q.text).padEnd(46)} [${q.note}] ${cells.join(" ")}`);
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
if (tax) {
|
|
425
|
-
const mb = (b: number) => (b / 1024 / 1024).toFixed(1);
|
|
426
|
-
console.log("\n---- tax (measured on this machine) ----");
|
|
427
|
-
console.log(`model files : ${mb(tax.modelBytes)} MB (${path.join(CACHE_DIR, "hf", MODEL)} @ ${DTYPE})`);
|
|
428
|
-
console.log(`model load : ${(tax.modelLoadMs / 1000).toFixed(2)} s (cold, one process)`);
|
|
429
|
-
console.log(`doc embedding : ${(tax.docEmbedMs / 1000).toFixed(2)} s total, ${tax.docEmbedPerDocMs.toFixed(1)} ms/doc (${docs.length} docs)`);
|
|
430
|
-
console.log(`query embedding : ${tax.queryEmbedMedianMs.toFixed(1)} ms median, ${tax.queryEmbedP95Ms.toFixed(1)} ms p95`);
|
|
431
|
-
console.log(`vector cache : ${mb(tax.cacheBytes)} MB on disk (${tax.dim}-dim)`);
|
|
432
|
-
console.log(`process RSS : ${tax.rssMb.toFixed(0)} MB after load`);
|
|
433
|
-
console.log(`total wall : ${((Date.now() - START) / 1000).toFixed(1)} s`);
|
|
434
|
-
} else {
|
|
435
|
-
console.log("\n(tax not measured — model arms not requested)");
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
console.log(`\nbm25 latency: ${median(bm25Ms).toFixed(2)} ms median, ${p95(bm25Ms).toFixed(2)} ms p95`);
|
|
439
|
-
|
|
440
|
-
if (JSON_OUT) {
|
|
441
|
-
const payload = {
|
|
442
|
-
mode: POOLED ? "pooled" : "fixture",
|
|
443
|
-
model: MODEL,
|
|
444
|
-
dtype: DTYPE,
|
|
445
|
-
docs: docs.length,
|
|
446
|
-
questions: questions.length,
|
|
447
|
-
depth: DEPTH,
|
|
448
|
-
arms: Object.fromEntries(
|
|
449
|
-
armOrder.map((a) => [
|
|
450
|
-
a,
|
|
451
|
-
{
|
|
452
|
-
all: results[a],
|
|
453
|
-
crossLingual: crossQs.length ? metrics(ranksByArm.get(a)!, crossQs) : null,
|
|
454
|
-
sameLanguage: sameQs.length ? metrics(ranksByArm.get(a)!, sameQs) : null,
|
|
455
|
-
},
|
|
456
|
-
]),
|
|
457
|
-
),
|
|
458
|
-
tax,
|
|
459
|
-
bm25Latency: { medianMs: median(bm25Ms), p95Ms: p95(bm25Ms) },
|
|
460
|
-
};
|
|
461
|
-
fs.writeFileSync(JSON_OUT, JSON.stringify(payload, null, 2));
|
|
462
|
-
console.log(`json: ${JSON_OUT}`);
|
|
463
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Semantic-ranking gate bench (D77 origin, re-measured for the D96 gate).
|
|
3
|
+
*
|
|
4
|
+
* Question the gate answers: does a local multilingual embedding layer beat
|
|
5
|
+
* the measured lexical ceiling, and at what tax (model size, load time,
|
|
6
|
+
* per-query latency, disk cache)?
|
|
7
|
+
*
|
|
8
|
+
* Target: pooled LongMemEval-S R@5 40.6% (the lexical ceiling, D77).
|
|
9
|
+
* Product slice: the checked-in retrieval fixture's EN<->ZH cross-lingual
|
|
10
|
+
* queries — the only slice where a multilingual model can plausibly help.
|
|
11
|
+
*
|
|
12
|
+
* Model: any `@huggingface/transformers` feature-extraction model. Default is
|
|
13
|
+
* `Xenova/paraphrase-multilingual-MiniLM-L12-v2` (the 2026-10-07 choice).
|
|
14
|
+
* Local only, no API. Embeddings are cached to disk; cosine is brute-forced
|
|
15
|
+
* in JS. Vectors never leave the machine.
|
|
16
|
+
*
|
|
17
|
+
* Arms:
|
|
18
|
+
* bm25 the shipped lexical pipeline (no model needed)
|
|
19
|
+
* vec vector-only ranking over the whole corpus (upper bound)
|
|
20
|
+
* rrf naive RRF fusion of BM25 + vector (§4.7 measured this as harmful)
|
|
21
|
+
* rerank BM25-first, embeddings only reorder cross-lingual hits inside the
|
|
22
|
+
* BM25 candidate pool (the only shape this project would ship)
|
|
23
|
+
*
|
|
24
|
+
* Usage:
|
|
25
|
+
* node --experimental-strip-types scripts/bench-semantic.ts --arms bm25
|
|
26
|
+
* npm i --no-save @huggingface/transformers # once, for the model arms
|
|
27
|
+
* node --experimental-strip-types scripts/bench-semantic.ts --cache-dir <dir>
|
|
28
|
+
* node --experimental-strip-types scripts/bench-semantic.ts --pooled \
|
|
29
|
+
* --data <longmemeval_s_cleaned.json> --cache-dir <dir>
|
|
30
|
+
*
|
|
31
|
+
* `--cache-dir` (or OPEN_MEMEX_BENCH_DATA) holds the vector caches and the
|
|
32
|
+
* Hugging Face model download (`<cache-dir>/hf`). It defaults to
|
|
33
|
+
* `~/workspace/bench-data` — the 2026-10-07 location. Nothing is written into
|
|
34
|
+
* the repo.
|
|
35
|
+
*/
|
|
36
|
+
import fs from "node:fs";
|
|
37
|
+
import os from "node:os";
|
|
38
|
+
import path from "node:path";
|
|
39
|
+
|
|
40
|
+
const args = process.argv.slice(2);
|
|
41
|
+
const opt = (name: string): string | undefined => {
|
|
42
|
+
const i = args.indexOf(name);
|
|
43
|
+
return i === -1 ? undefined : args[i + 1];
|
|
44
|
+
};
|
|
45
|
+
const POOLED = args.includes("--pooled");
|
|
46
|
+
const LIMIT = Number(opt("--limit") ?? Infinity);
|
|
47
|
+
const DEPTH = Number(opt("--depth") ?? 50);
|
|
48
|
+
const MODEL = opt("--model") ?? "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
49
|
+
/** ONNX precision. `fp32` is the 2026-10-07 parity run; `q8` is what a
|
|
50
|
+
* download-on-first-use feature would plausibly ship. */
|
|
51
|
+
const DTYPE = opt("--dtype") ?? "fp32";
|
|
52
|
+
const CACHE_DIR =
|
|
53
|
+
opt("--cache-dir") ?? process.env.OPEN_MEMEX_BENCH_DATA ?? path.join(os.homedir(), "workspace", "bench-data");
|
|
54
|
+
const ARMS = new Set((opt("--arms") ?? "bm25,vec,rrf,rerank").split(",").map((s) => s.trim()));
|
|
55
|
+
const JSON_OUT = opt("--json");
|
|
56
|
+
/** Similarity floors swept by the `rerank` arm. */
|
|
57
|
+
const FLOORS = (opt("--floors") ?? "0.2,0.3,0.4,0.5,0.6").split(",").map(Number);
|
|
58
|
+
const K = 5;
|
|
59
|
+
|
|
60
|
+
function median(xs: number[]): number {
|
|
61
|
+
if (xs.length === 0) return 0;
|
|
62
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
63
|
+
return s[Math.floor(s.length / 2)]!;
|
|
64
|
+
}
|
|
65
|
+
function p95(xs: number[]): number {
|
|
66
|
+
if (xs.length === 0) return 0;
|
|
67
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
68
|
+
return s[Math.min(s.length - 1, Math.floor(s.length * 0.95))]!;
|
|
69
|
+
}
|
|
70
|
+
function dirBytes(dir: string): number {
|
|
71
|
+
if (!fs.existsSync(dir)) return 0;
|
|
72
|
+
let total = 0;
|
|
73
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
74
|
+
const p = path.join(dir, e.name);
|
|
75
|
+
if (e.isDirectory()) total += dirBytes(p);
|
|
76
|
+
else if (e.isFile()) total += fs.statSync(p).size;
|
|
77
|
+
}
|
|
78
|
+
return total;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const HOME = fs.mkdtempSync(path.join(os.tmpdir(), "sem-bench-"));
|
|
82
|
+
process.env.OPEN_MEMEX_HOME = HOME;
|
|
83
|
+
const START = Date.now();
|
|
84
|
+
|
|
85
|
+
import { serialize, normalizeFrontmatter } from "../src/store/markdown.ts";
|
|
86
|
+
import { syncScope } from "../src/store/sync.ts";
|
|
87
|
+
import { search } from "../src/retrieve/search.ts";
|
|
88
|
+
import { hasCjk } from "../src/retrieve/cjk.ts";
|
|
89
|
+
import { rerankCrossLingual } from "../src/retrieve/rerank.ts";
|
|
90
|
+
import { memoriesDirPath } from "../src/paths.ts";
|
|
91
|
+
|
|
92
|
+
interface Doc {
|
|
93
|
+
id: string;
|
|
94
|
+
/** Text the embedding model sees (body + captured aliases + tags). */
|
|
95
|
+
text: string;
|
|
96
|
+
/** Markdown body written to the index — fixture mode keeps this identical
|
|
97
|
+
* to retrieval-eval so the BM25 arm reproduces the published baseline. */
|
|
98
|
+
body: string;
|
|
99
|
+
/** EN/ZH by script; the cross-lingual trigger is query script != doc script. */
|
|
100
|
+
lang: "en" | "zh";
|
|
101
|
+
}
|
|
102
|
+
interface Q {
|
|
103
|
+
text: string;
|
|
104
|
+
gold: Set<string>;
|
|
105
|
+
note: string;
|
|
106
|
+
lang: "en" | "zh";
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const docs: Doc[] = [];
|
|
110
|
+
const questions: Q[] = [];
|
|
111
|
+
/** Fixture frontmatter, written verbatim so the BM25 arm reproduces the
|
|
112
|
+
* checked-in retrieval-eval baseline exactly (aliases/tags in their real
|
|
113
|
+
* FTS columns, not folded into the body). */
|
|
114
|
+
let fixtureMemories: Array<Record<string, unknown>> | null = null;
|
|
115
|
+
|
|
116
|
+
if (POOLED) {
|
|
117
|
+
const dataPath = opt("--data") ?? "";
|
|
118
|
+
if (!dataPath) throw new Error("missing --data");
|
|
119
|
+
const items = JSON.parse(fs.readFileSync(dataPath, "utf8"));
|
|
120
|
+
let qi = 0;
|
|
121
|
+
for (const item of items) {
|
|
122
|
+
if (item.question_id.endsWith("_abs")) continue;
|
|
123
|
+
if (qi >= LIMIT) break;
|
|
124
|
+
const sids: string[] = item.haystack_session_ids;
|
|
125
|
+
const gold = new Set<string>();
|
|
126
|
+
for (const g of item.answer_session_ids) {
|
|
127
|
+
const i = sids.indexOf(g);
|
|
128
|
+
if (i !== -1) gold.add(`q${qi}-s${i}`);
|
|
129
|
+
}
|
|
130
|
+
if (gold.size === 0) continue;
|
|
131
|
+
item.haystack_sessions.forEach((turns: Array<{ role: string; content: string }>, i: number) => {
|
|
132
|
+
const text = turns.map((t) => `${t.role}: ${t.content}`).join("\n").slice(0, 2000);
|
|
133
|
+
docs.push({ id: `q${qi}-s${i}`, text, body: text, lang: hasCjk(text) ? "zh" : "en" });
|
|
134
|
+
});
|
|
135
|
+
const text: string = item.question;
|
|
136
|
+
questions.push({ text, gold, note: item.question_type, lang: hasCjk(text) ? "zh" : "en" });
|
|
137
|
+
qi++;
|
|
138
|
+
}
|
|
139
|
+
} else {
|
|
140
|
+
const { MEMORIES, QUERIES } = await import("./retrieval-eval/fixture.ts");
|
|
141
|
+
fixtureMemories = MEMORIES;
|
|
142
|
+
for (const m of MEMORIES) {
|
|
143
|
+
const text = [m.body, ...(m.aliases ?? []), ...(m.tags ?? [])].join("\n").slice(0, 2000);
|
|
144
|
+
docs.push({ id: m.id, text, body: m.body, lang: hasCjk(m.body) ? "zh" : "en" });
|
|
145
|
+
}
|
|
146
|
+
for (const q of QUERIES.slice(0, LIMIT)) {
|
|
147
|
+
questions.push({
|
|
148
|
+
text: q.query,
|
|
149
|
+
gold: new Set(q.gold),
|
|
150
|
+
note: q.note,
|
|
151
|
+
lang: hasCjk(q.query) ? "zh" : "en",
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
console.log(`docs: ${docs.length}, questions: ${questions.length}, model: ${MODEL} (${DTYPE})`);
|
|
157
|
+
console.log(`cache dir: ${CACHE_DIR}`);
|
|
158
|
+
const byId = new Map(docs.map((d) => [d.id, d]));
|
|
159
|
+
|
|
160
|
+
// Seed the FTS index so the BM25 arm runs on the same corpus.
|
|
161
|
+
{
|
|
162
|
+
const dir = memoriesDirPath("personal");
|
|
163
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
164
|
+
const B = 2000;
|
|
165
|
+
for (let i = 0; i < docs.length; i += B) {
|
|
166
|
+
for (const d of docs.slice(i, i + B)) {
|
|
167
|
+
// Fixture mode reproduces retrieval-eval's frontmatter exactly, so the
|
|
168
|
+
// BM25 arm equals the published baseline; pooled mode has none.
|
|
169
|
+
const fmDef = fixtureMemories?.find((m) => (m as { id: string }).id === d.id) as
|
|
170
|
+
| {
|
|
171
|
+
type?: string;
|
|
172
|
+
tags?: string[];
|
|
173
|
+
aliases?: string[];
|
|
174
|
+
alias?: string;
|
|
175
|
+
target?: string;
|
|
176
|
+
}
|
|
177
|
+
| undefined;
|
|
178
|
+
const fm = normalizeFrontmatter({
|
|
179
|
+
id: d.id,
|
|
180
|
+
scope: "personal",
|
|
181
|
+
type: fmDef?.type ?? "fact",
|
|
182
|
+
tags: fmDef?.tags ?? [],
|
|
183
|
+
aliases: fmDef?.aliases,
|
|
184
|
+
alias: fmDef?.alias,
|
|
185
|
+
target: fmDef?.target,
|
|
186
|
+
source: fmDef ? "retrieval-eval fixture" : undefined,
|
|
187
|
+
});
|
|
188
|
+
fs.writeFileSync(path.join(dir, `${d.id}.md`), serialize(fm, d.body), "utf8");
|
|
189
|
+
}
|
|
190
|
+
if (docs.length > B) console.log(` seeded ${Math.min(i + B, docs.length)}/${docs.length}`);
|
|
191
|
+
}
|
|
192
|
+
syncScope("personal");
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// ---- BM25 arm (always measured: it is both a result and the rerank pool) ----
|
|
196
|
+
const bm25Ranks = new Map<string, number>();
|
|
197
|
+
const bm25Raw = new Map<string, string[]>();
|
|
198
|
+
const bm25Ms: number[] = [];
|
|
199
|
+
for (const q of questions) {
|
|
200
|
+
const t0 = performance.now();
|
|
201
|
+
const ids = search(q.text, { scopeKeys: ["personal"], limit: DEPTH, synonyms: false }).map((h) => h.id);
|
|
202
|
+
bm25Ms.push(performance.now() - t0);
|
|
203
|
+
bm25Raw.set(q.text, ids);
|
|
204
|
+
let rank = -1;
|
|
205
|
+
for (const g of q.gold) {
|
|
206
|
+
const i = ids.indexOf(g);
|
|
207
|
+
if (i !== -1 && (rank === -1 || i < rank)) rank = i;
|
|
208
|
+
}
|
|
209
|
+
bm25Ranks.set(q.text, rank);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
interface Metrics {
|
|
213
|
+
n: number;
|
|
214
|
+
r1: number;
|
|
215
|
+
r5: number;
|
|
216
|
+
mrr: number;
|
|
217
|
+
}
|
|
218
|
+
function metrics(ranks: Map<string, number>, qs: Q[]): Metrics {
|
|
219
|
+
let r1 = 0;
|
|
220
|
+
let r5 = 0;
|
|
221
|
+
let mrr = 0;
|
|
222
|
+
for (const q of qs) {
|
|
223
|
+
const rank = ranks.get(q.text) ?? -1;
|
|
224
|
+
if (rank === 0) r1++;
|
|
225
|
+
if (rank !== -1 && rank < K) r5++;
|
|
226
|
+
if (rank !== -1) mrr += 1 / (rank + 1);
|
|
227
|
+
}
|
|
228
|
+
return { n: qs.length, r1, r5, mrr };
|
|
229
|
+
}
|
|
230
|
+
function fmt(m: Metrics): string {
|
|
231
|
+
return `R@1 ${(m.r1 / m.n * 100).toFixed(1)}% R@5 ${(m.r5 / m.n * 100).toFixed(1)}% MRR ${(m.mrr / m.n).toFixed(3)} (n=${m.n})`;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* The product shape: BM25 retrieves, embeddings may only reorder contiguous
|
|
236
|
+
* runs of cross-lingual hits whose cosine clears `floor`. This calls the
|
|
237
|
+
* SHIPPED pure rule (`src/retrieve/rerank.ts`), so the bench cannot drift
|
|
238
|
+
* from the product's ordering; the arm only supplies cosines.
|
|
239
|
+
*/
|
|
240
|
+
function rerankOrder(ids: string[], queryLang: "en" | "zh", sim: (id: string) => number, floor: number): string[] {
|
|
241
|
+
const cosines = new Map<string, number>();
|
|
242
|
+
for (const id of ids) {
|
|
243
|
+
const d = byId.get(id);
|
|
244
|
+
if (d && d.lang !== queryLang) cosines.set(id, sim(id));
|
|
245
|
+
}
|
|
246
|
+
return rerankCrossLingual(ids.map((id) => ({ id })), cosines, { minCosine: floor }).ordered.map((h) => h.id);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function rankOf(ids: string[], gold: Set<string>): number {
|
|
250
|
+
let rank = -1;
|
|
251
|
+
for (const g of gold) {
|
|
252
|
+
const i = ids.indexOf(g);
|
|
253
|
+
if (i !== -1 && (rank === -1 || i < rank)) rank = i;
|
|
254
|
+
}
|
|
255
|
+
return rank;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const results: Record<string, Metrics> = {};
|
|
259
|
+
const ranksByArm = new Map<string, Map<string, number>>();
|
|
260
|
+
function record(arm: string, ranks: Map<string, number>) {
|
|
261
|
+
ranksByArm.set(arm, ranks);
|
|
262
|
+
results[arm] = metrics(ranks, questions);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
record("bm25", bm25Ranks);
|
|
266
|
+
|
|
267
|
+
const crossQs = questions.filter((q) => {
|
|
268
|
+
const goldLangs = [...q.gold].map((g) => byId.get(g)?.lang).filter(Boolean);
|
|
269
|
+
return goldLangs.length > 0 && goldLangs.some((l) => l !== q.lang);
|
|
270
|
+
});
|
|
271
|
+
const sameQs = questions.filter((q) => !crossQs.includes(q));
|
|
272
|
+
|
|
273
|
+
let tax: {
|
|
274
|
+
modelLoadMs: number;
|
|
275
|
+
modelBytes: number;
|
|
276
|
+
docEmbedMs: number;
|
|
277
|
+
docEmbedPerDocMs: number;
|
|
278
|
+
queryEmbedMedianMs: number;
|
|
279
|
+
queryEmbedP95Ms: number;
|
|
280
|
+
cacheBytes: number;
|
|
281
|
+
rssMb: number;
|
|
282
|
+
dim: number;
|
|
283
|
+
} | null = null;
|
|
284
|
+
|
|
285
|
+
if (ARMS.size > 0 && !(ARMS.size === 1 && ARMS.has("bm25"))) {
|
|
286
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
287
|
+
env.cacheDir = path.join(CACHE_DIR, "hf");
|
|
288
|
+
fs.mkdirSync(env.cacheDir, { recursive: true });
|
|
289
|
+
|
|
290
|
+
const slug = MODEL.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
291
|
+
const cachePath = path.join(CACHE_DIR, `emb-${slug}-${DTYPE}-${POOLED ? "pooled" : "fixture"}.json`);
|
|
292
|
+
|
|
293
|
+
let vectors: number[][] | null = null;
|
|
294
|
+
if (fs.existsSync(cachePath)) {
|
|
295
|
+
try {
|
|
296
|
+
const cached = JSON.parse(fs.readFileSync(cachePath, "utf8")) as {
|
|
297
|
+
model: string;
|
|
298
|
+
dim: number;
|
|
299
|
+
ids: string[];
|
|
300
|
+
vectors: number[][];
|
|
301
|
+
};
|
|
302
|
+
if (cached.model === `${MODEL}::${DTYPE}` && cached.ids.length === docs.length && cached.ids.every((id, i) => id === docs[i]!.id)) {
|
|
303
|
+
vectors = cached.vectors;
|
|
304
|
+
console.log(`vectors: cache hit ${cachePath} (${cached.dim}-dim)`);
|
|
305
|
+
} else {
|
|
306
|
+
console.log("vectors: cache is stale (model or corpus changed) — recomputing");
|
|
307
|
+
}
|
|
308
|
+
} catch {
|
|
309
|
+
console.log("vectors: cache unreadable — recomputing");
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
console.log("loading model… (first run downloads it into the cache dir)");
|
|
314
|
+
const tLoad = performance.now();
|
|
315
|
+
const extractor: any = await pipeline("feature-extraction", MODEL, { dtype: DTYPE as any });
|
|
316
|
+
const modelLoadMs = performance.now() - tLoad;
|
|
317
|
+
|
|
318
|
+
let docEmbedMs = 0;
|
|
319
|
+
if (!vectors) {
|
|
320
|
+
console.log(`embedding ${docs.length} docs…`);
|
|
321
|
+
const tDocs = performance.now();
|
|
322
|
+
vectors = [];
|
|
323
|
+
const B = 32;
|
|
324
|
+
for (let i = 0; i < docs.length; i += B) {
|
|
325
|
+
const batch = docs.map((d) => d.text).slice(i, i + B);
|
|
326
|
+
const out: any = await extractor(batch, { pooling: "mean", normalize: true });
|
|
327
|
+
for (const v of out.tolist()) vectors.push(v);
|
|
328
|
+
if (docs.length > 320 && i % 320 === 0) console.log(` embedded ${i}/${docs.length}`);
|
|
329
|
+
}
|
|
330
|
+
docEmbedMs = performance.now() - tDocs;
|
|
331
|
+
fs.mkdirSync(path.dirname(cachePath), { recursive: true });
|
|
332
|
+
fs.writeFileSync(
|
|
333
|
+
cachePath,
|
|
334
|
+
JSON.stringify({ model: `${MODEL}::${DTYPE}`, dim: vectors[0]!.length, ids: docs.map((d) => d.id), vectors }),
|
|
335
|
+
);
|
|
336
|
+
console.log(`vectors cached: ${cachePath}`);
|
|
337
|
+
}
|
|
338
|
+
const DIM = vectors[0]!.length;
|
|
339
|
+
|
|
340
|
+
const cos = (qv: number[], v: number[]): number => {
|
|
341
|
+
let s = 0;
|
|
342
|
+
for (let d = 0; d < DIM; d++) s += qv[d]! * v[d]!;
|
|
343
|
+
return s;
|
|
344
|
+
};
|
|
345
|
+
const vecById = new Map<string, number[]>();
|
|
346
|
+
docs.forEach((d, i) => vecById.set(d.id, vectors![i]!));
|
|
347
|
+
|
|
348
|
+
const vecRanks = new Map<string, number>();
|
|
349
|
+
const rrfRanks = new Map<string, number>();
|
|
350
|
+
const rerankRanks = new Map<number, Map<string, number>>();
|
|
351
|
+
for (const f of FLOORS) rerankRanks.set(f, new Map());
|
|
352
|
+
const queryEmbedMs: number[] = [];
|
|
353
|
+
|
|
354
|
+
let qi = 0;
|
|
355
|
+
for (const q of questions) {
|
|
356
|
+
const t0 = performance.now();
|
|
357
|
+
const qv: number[] = (await extractor(q.text, { pooling: "mean", normalize: true })).tolist()[0];
|
|
358
|
+
queryEmbedMs.push(performance.now() - t0);
|
|
359
|
+
|
|
360
|
+
const vecOrder = docs
|
|
361
|
+
.map((d) => ({ id: d.id, s: cos(qv, vecById.get(d.id)!) }))
|
|
362
|
+
.sort((a, b) => b.s - a.s)
|
|
363
|
+
.map((x) => x.id);
|
|
364
|
+
vecRanks.set(q.text, rankOf(vecOrder, q.gold));
|
|
365
|
+
|
|
366
|
+
const rrf = (lists: string[][], k = 60): string[] => {
|
|
367
|
+
const scores = new Map<string, number>();
|
|
368
|
+
for (const list of lists) list.forEach((id, rank) => scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank + 1)));
|
|
369
|
+
return [...scores.entries()].sort((a, b) => b[1] - a[1]).map(([id]) => id);
|
|
370
|
+
};
|
|
371
|
+
rrfRanks.set(q.text, rankOf(rrf([bm25Raw.get(q.text)!, vecOrder]), q.gold));
|
|
372
|
+
|
|
373
|
+
const simById = new Map<string, number>();
|
|
374
|
+
for (const id of bm25Raw.get(q.text)!) simById.set(id, cos(qv, vecById.get(id)!));
|
|
375
|
+
for (const f of FLOORS) {
|
|
376
|
+
const order = rerankOrder(bm25Raw.get(q.text)!, q.lang, (id) => simById.get(id) ?? -1, f);
|
|
377
|
+
rerankRanks.get(f)!.set(q.text, rankOf(order, q.gold));
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
qi++;
|
|
381
|
+
if (questions.length > 100 && qi % 100 === 0) console.log(` ... ${qi}/${questions.length}`);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
record("vec", vecRanks);
|
|
385
|
+
record("rrf", rrfRanks);
|
|
386
|
+
for (const f of FLOORS) record(`rerank@${f}`, rerankRanks.get(f)!);
|
|
387
|
+
|
|
388
|
+
tax = {
|
|
389
|
+
modelLoadMs,
|
|
390
|
+
modelBytes: dirBytes(path.join(env.cacheDir!, MODEL)),
|
|
391
|
+
docEmbedMs,
|
|
392
|
+
docEmbedPerDocMs: docs.length ? docEmbedMs / docs.length : 0,
|
|
393
|
+
queryEmbedMedianMs: median(queryEmbedMs),
|
|
394
|
+
queryEmbedP95Ms: p95(queryEmbedMs),
|
|
395
|
+
cacheBytes: fs.existsSync(cachePath) ? fs.statSync(cachePath).size : 0,
|
|
396
|
+
rssMb: process.memoryUsage().rss / 1024 / 1024,
|
|
397
|
+
dim: DIM,
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// ---- report ----
|
|
402
|
+
const armOrder = ["bm25", ...Object.keys(results).filter((a) => a !== "bm25")];
|
|
403
|
+
console.log(`\n==== semantic gate (${POOLED ? "pooled LongMemEval-S" : "retrieval fixture"}) ====`);
|
|
404
|
+
console.log(`corpus : ${docs.length} docs, ${questions.length} questions, BM25 depth ${DEPTH}`);
|
|
405
|
+
for (const arm of armOrder) console.log(`${arm.padEnd(12)} ${fmt(results[arm]!)}`);
|
|
406
|
+
|
|
407
|
+
if (crossQs.length > 0) {
|
|
408
|
+
console.log(`\ncross-lingual slice (${crossQs.length} queries, gold script != query script):`);
|
|
409
|
+
for (const arm of armOrder) console.log(`${arm.padEnd(12)} ${fmt(metrics(ranksByArm.get(arm)!, crossQs))}`);
|
|
410
|
+
console.log(`\nsame-language slice (${sameQs.length} queries — rerank must not change these):`);
|
|
411
|
+
for (const arm of armOrder) console.log(`${arm.padEnd(12)} ${fmt(metrics(ranksByArm.get(arm)!, sameQs))}`);
|
|
412
|
+
|
|
413
|
+
console.log("\ncross-lingual per-query ranks (1-based, - = miss):");
|
|
414
|
+
const show = armOrder.filter((a) => a === "bm25" || a === "vec" || a === "rrf" || a.startsWith("rerank@"));
|
|
415
|
+
for (const q of crossQs) {
|
|
416
|
+
const cells = show.map((a) => {
|
|
417
|
+
const r = ranksByArm.get(a)!.get(q.text) ?? -1;
|
|
418
|
+
return `${a.replace("rerank@", "r@")}=${r === -1 ? "-" : r + 1}`;
|
|
419
|
+
});
|
|
420
|
+
console.log(` ${JSON.stringify(q.text).padEnd(46)} [${q.note}] ${cells.join(" ")}`);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
if (tax) {
|
|
425
|
+
const mb = (b: number) => (b / 1024 / 1024).toFixed(1);
|
|
426
|
+
console.log("\n---- tax (measured on this machine) ----");
|
|
427
|
+
console.log(`model files : ${mb(tax.modelBytes)} MB (${path.join(CACHE_DIR, "hf", MODEL)} @ ${DTYPE})`);
|
|
428
|
+
console.log(`model load : ${(tax.modelLoadMs / 1000).toFixed(2)} s (cold, one process)`);
|
|
429
|
+
console.log(`doc embedding : ${(tax.docEmbedMs / 1000).toFixed(2)} s total, ${tax.docEmbedPerDocMs.toFixed(1)} ms/doc (${docs.length} docs)`);
|
|
430
|
+
console.log(`query embedding : ${tax.queryEmbedMedianMs.toFixed(1)} ms median, ${tax.queryEmbedP95Ms.toFixed(1)} ms p95`);
|
|
431
|
+
console.log(`vector cache : ${mb(tax.cacheBytes)} MB on disk (${tax.dim}-dim)`);
|
|
432
|
+
console.log(`process RSS : ${tax.rssMb.toFixed(0)} MB after load`);
|
|
433
|
+
console.log(`total wall : ${((Date.now() - START) / 1000).toFixed(1)} s`);
|
|
434
|
+
} else {
|
|
435
|
+
console.log("\n(tax not measured — model arms not requested)");
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
console.log(`\nbm25 latency: ${median(bm25Ms).toFixed(2)} ms median, ${p95(bm25Ms).toFixed(2)} ms p95`);
|
|
439
|
+
|
|
440
|
+
if (JSON_OUT) {
|
|
441
|
+
const payload = {
|
|
442
|
+
mode: POOLED ? "pooled" : "fixture",
|
|
443
|
+
model: MODEL,
|
|
444
|
+
dtype: DTYPE,
|
|
445
|
+
docs: docs.length,
|
|
446
|
+
questions: questions.length,
|
|
447
|
+
depth: DEPTH,
|
|
448
|
+
arms: Object.fromEntries(
|
|
449
|
+
armOrder.map((a) => [
|
|
450
|
+
a,
|
|
451
|
+
{
|
|
452
|
+
all: results[a],
|
|
453
|
+
crossLingual: crossQs.length ? metrics(ranksByArm.get(a)!, crossQs) : null,
|
|
454
|
+
sameLanguage: sameQs.length ? metrics(ranksByArm.get(a)!, sameQs) : null,
|
|
455
|
+
},
|
|
456
|
+
]),
|
|
457
|
+
),
|
|
458
|
+
tax,
|
|
459
|
+
bm25Latency: { medianMs: median(bm25Ms), p95Ms: p95(bm25Ms) },
|
|
460
|
+
};
|
|
461
|
+
fs.writeFileSync(JSON_OUT, JSON.stringify(payload, null, 2));
|
|
462
|
+
console.log(`json: ${JSON_OUT}`);
|
|
463
|
+
}
|