open-memex 0.11.0-alpha.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CHANGELOG.md +1 -3
- package/dist/.build-stamp +1 -1
- package/dist/retrieve/embedding.js +1 -1
- package/dist/store/embeddings.js +2 -2
- package/package.json +1 -1
- package/scripts/bench-semantic.ts +463 -463
- package/server.json +2 -2
- package/src/retrieve/embedding.ts +214 -214
- package/src/retrieve/rerank.ts +96 -96
- package/src/store/embeddings.ts +82 -82
package/server.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"name": "io.github.stoneskin/open-memex",
|
|
4
4
|
"title": "open-memex",
|
|
5
5
|
"description": "Local-first memory for AI coding agents — Markdown + SQLite FTS5, shared across editors via MCP.",
|
|
6
|
-
"version": "0.
|
|
6
|
+
"version": "0.11.0",
|
|
7
7
|
"repository": {
|
|
8
8
|
"url": "https://github.com/stoneskin/open-memex",
|
|
9
9
|
"source": "github"
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
{
|
|
14
14
|
"registryType": "npm",
|
|
15
15
|
"identifier": "open-memex",
|
|
16
|
-
"version": "0.
|
|
16
|
+
"version": "0.11.0",
|
|
17
17
|
"runtimeHint": "npx",
|
|
18
18
|
"packageArguments": [
|
|
19
19
|
{
|
|
@@ -1,214 +1,214 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* D96: local embedding runtime for the opt-in cross-lingual rerank.
|
|
3
|
-
*
|
|
4
|
-
* Local-first, no API: the model runs through `@huggingface/transformers`
|
|
5
|
-
* (ONNX, CPU) and is downloaded into `<data root>/models/` on first use.
|
|
6
|
-
* Vectors are cached in SQLite (`embedding_vectors`, keyed by a hash of the
|
|
7
|
-
* embedded text), so a warm store never re-embeds a memory.
|
|
8
|
-
*
|
|
9
|
-
* The package is deliberately NOT a dependency of open-memex: it pulls
|
|
10
|
-
* ~100-500 MB of model + onnxruntime, and the default install must stay
|
|
11
|
-
* zero-download. When the flag is on and the package is missing, the rerank
|
|
12
|
-
* degrades to plain BM25 with one stderr warning — never an error.
|
|
13
|
-
*
|
|
14
|
-
* Nothing here loads a model unless the caller already decided that a
|
|
15
|
-
* cross-lingual candidate exists; a same-language search pays zero cost.
|
|
16
|
-
*/
|
|
17
|
-
import fs from "node:fs";
|
|
18
|
-
import path from "node:path";
|
|
19
|
-
import crypto from "node:crypto";
|
|
20
|
-
|
|
21
|
-
import { dataRootPath } from "../paths.ts";
|
|
22
|
-
import { db } from "../store/db.ts";
|
|
23
|
-
import { loadVectors, saveVector } from "../store/embeddings.ts";
|
|
24
|
-
import { hasCjk } from "./cjk.ts";
|
|
25
|
-
import { rerankCrossLingual, type RerankHitInfo } from "./rerank.ts";
|
|
26
|
-
import { hitReviewTier, type SearchHit } from "./search.ts";
|
|
27
|
-
|
|
28
|
-
/** The model the D96 gate measured. User-configurable via `embeddingModel`. */
|
|
29
|
-
export const DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
30
|
-
|
|
31
|
-
/** Package name, spelled once so the dynamic import stays non-literal (it is optional). */
|
|
32
|
-
const PACKAGE = "@huggingface/transformers";
|
|
33
|
-
|
|
34
|
-
type Embedder = (
|
|
35
|
-
texts: string[],
|
|
36
|
-
opts: { pooling: "mean"; normalize: true },
|
|
37
|
-
) => Promise<{ tolist(): number[][] }>;
|
|
38
|
-
|
|
39
|
-
interface Runtime {
|
|
40
|
-
embedder: Embedder;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
let runtimeCache: { model: string; promise: Promise<Runtime | null> } | null = null;
|
|
44
|
-
const warned = new Set<string>();
|
|
45
|
-
|
|
46
|
-
function warnOnce(key: string, message: string): void {
|
|
47
|
-
if (warned.has(key)) return;
|
|
48
|
-
warned.add(key);
|
|
49
|
-
// stdout is the MCP protocol channel — diagnostics go to stderr.
|
|
50
|
-
process.stderr.write(`[open-memex] ${message}\n`);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/** Cache root the ONNX model files are downloaded into (never the repo). */
|
|
54
|
-
export function embeddingModelDir(): string {
|
|
55
|
-
return path.join(dataRootPath(), "models");
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
async function loadRuntime(model: string): Promise<Runtime | null> {
|
|
59
|
-
try {
|
|
60
|
-
const spec = PACKAGE;
|
|
61
|
-
const mod: any = await import(spec);
|
|
62
|
-
const cacheDir = embeddingModelDir();
|
|
63
|
-
fs.mkdirSync(cacheDir, { recursive: true });
|
|
64
|
-
mod.env.cacheDir = cacheDir;
|
|
65
|
-
const pipeline = mod.pipeline;
|
|
66
|
-
// D96: the gate measured q8 identical to fp32 on the fixture at ~1/4 the
|
|
67
|
-
// download (113 MB vs 449 MB). If a user override lacks a quantized file,
|
|
68
|
-
// the load fails and we degrade to BM25 with a warning.
|
|
69
|
-
const extractor = await pipeline("feature-extraction", model, { dtype: "q8" });
|
|
70
|
-
return { embedder: extractor as Embedder };
|
|
71
|
-
} catch (err) {
|
|
72
|
-
warnOnce(
|
|
73
|
-
`load:${model}`,
|
|
74
|
-
`cross-lingual rerank is on but the local embedding runtime is unavailable (${(err as Error).message}). ` +
|
|
75
|
-
`Install it with: npm install -g ${PACKAGE}. Search falls back to BM25.`,
|
|
76
|
-
);
|
|
77
|
-
return null;
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
function runtime(model: string): Promise<Runtime | null> {
|
|
82
|
-
if (runtimeCache?.model !== model) runtimeCache = { model, promise: loadRuntime(model) };
|
|
83
|
-
return runtimeCache.promise;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function hashText(text: string): string {
|
|
87
|
-
return crypto.createHash("sha256").update(text, "utf8").digest("hex").slice(0, 32);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/** The text a memory contributes to the embedding — the same fields the bench embedded. */
|
|
91
|
-
function memoryText(content: string, aliases: string, tags: string): string {
|
|
92
|
-
const split = (s: string): string[] =>
|
|
93
|
-
s
|
|
94
|
-
.split(",")
|
|
95
|
-
.map((x) => x.trim())
|
|
96
|
-
.filter(Boolean);
|
|
97
|
-
return [content, ...split(aliases), ...split(tags)]
|
|
98
|
-
.filter((s) => s.trim().length > 0)
|
|
99
|
-
.join("\n")
|
|
100
|
-
.slice(0, 2000);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
function cosine(a: number[], b: number[]): number {
|
|
104
|
-
const n = Math.min(a.length, b.length);
|
|
105
|
-
let s = 0;
|
|
106
|
-
for (let i = 0; i < n; i++) s += a[i]! * b[i]!;
|
|
107
|
-
return s;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
export interface RerankOutcome {
|
|
111
|
-
hits: SearchHit[];
|
|
112
|
-
/** Why the rerank did not run: "off" | "no-cross-lingual-hits" | "model-unavailable". */
|
|
113
|
-
skipped?: "off" | "no-cross-lingual-hits" | "model-unavailable";
|
|
114
|
-
/** How many returned hits were cross-lingual candidates. */
|
|
115
|
-
candidates: number;
|
|
116
|
-
info: Map<string, RerankHitInfo>;
|
|
117
|
-
promotions: Array<{ id: string; promoted: number; cosine: number }>;
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* Rerank an already-retrieved list for EN<->ZH queries only.
|
|
122
|
-
*
|
|
123
|
-
* Called after `search()`, so BM25 has already chosen the set and the review
|
|
124
|
-
* tiers are already ordered; this only reorders within each contiguous tier
|
|
125
|
-
* block. A query with no cross-lingual hit returns immediately and never
|
|
126
|
-
* loads a model.
|
|
127
|
-
*/
|
|
128
|
-
export async function applyCrossLingualRerank(
|
|
129
|
-
query: string,
|
|
130
|
-
hits: SearchHit[],
|
|
131
|
-
cfg: { crossLingualRerank: boolean; embeddingModel: string },
|
|
132
|
-
): Promise<RerankOutcome> {
|
|
133
|
-
const untouched: RerankOutcome = { hits, candidates: 0, info: new Map(), promotions: [] };
|
|
134
|
-
if (!cfg.crossLingualRerank) return { ...untouched, skipped: "off" };
|
|
135
|
-
if (hits.length === 0) return { ...untouched, skipped: "no-cross-lingual-hits" };
|
|
136
|
-
|
|
137
|
-
const queryCjk = hasCjk(query);
|
|
138
|
-
const ids = hits.map((h) => h.id);
|
|
139
|
-
const rows = db()
|
|
140
|
-
.prepare(
|
|
141
|
-
`SELECT id, content, COALESCE(aliases, '') AS aliases, COALESCE(tags, '') AS tags
|
|
142
|
-
FROM memories WHERE id IN (${ids.map(() => "?").join(",")})`,
|
|
143
|
-
)
|
|
144
|
-
.all(...ids) as Array<{ id: string; content: string; aliases: string; tags: string }>;
|
|
145
|
-
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
146
|
-
|
|
147
|
-
// Cross-lingual = the memory's script differs from the query's. Everything
|
|
148
|
-
// else is a barrier that keeps its exact BM25 slot.
|
|
149
|
-
const texts = new Map<string, string>();
|
|
150
|
-
for (const h of hits) {
|
|
151
|
-
const r = byId.get(h.id);
|
|
152
|
-
if (!r) continue;
|
|
153
|
-
if (hasCjk(r.content) === queryCjk) continue;
|
|
154
|
-
texts.set(h.id, memoryText(r.content, r.aliases, r.tags));
|
|
155
|
-
}
|
|
156
|
-
if (texts.size === 0) return { ...untouched, skipped: "no-cross-lingual-hits" };
|
|
157
|
-
|
|
158
|
-
const rt = await runtime(cfg.embeddingModel);
|
|
159
|
-
if (!rt) return { ...untouched, skipped: "model-unavailable" };
|
|
160
|
-
|
|
161
|
-
const hashes = new Map<string, string>();
|
|
162
|
-
for (const [id, text] of texts) hashes.set(id, hashText(text));
|
|
163
|
-
let cached: Map<string, number[]>;
|
|
164
|
-
try {
|
|
165
|
-
cached = loadVectors(cfg.embeddingModel, [...new Set(hashes.values())]);
|
|
166
|
-
} catch {
|
|
167
|
-
cached = new Map();
|
|
168
|
-
}
|
|
169
|
-
const pending = [...texts].filter(([id]) => !cached.has(hashes.get(id)!));
|
|
170
|
-
if (pending.length > 0) {
|
|
171
|
-
const vectors = await rt.embedder(pending.map(([, text]) => text), {
|
|
172
|
-
pooling: "mean",
|
|
173
|
-
normalize: true,
|
|
174
|
-
});
|
|
175
|
-
const list = vectors.tolist();
|
|
176
|
-
pending.forEach(([id], i) => {
|
|
177
|
-
const vec = list[i]!;
|
|
178
|
-
cached.set(hashes.get(id)!, vec);
|
|
179
|
-
try {
|
|
180
|
-
saveVector(cfg.embeddingModel, hashes.get(id)!, vec);
|
|
181
|
-
} catch (err) {
|
|
182
|
-
// A cache that cannot be written is a slowdown, not a failure.
|
|
183
|
-
warnOnce("vector-cache-write", `could not cache the embedding (${(err as Error).message})`);
|
|
184
|
-
}
|
|
185
|
-
});
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
const queryVector = (await rt.embedder([query], { pooling: "mean", normalize: true })).tolist()[0]!;
|
|
189
|
-
const cosines = new Map<string, number>();
|
|
190
|
-
for (const id of texts.keys()) {
|
|
191
|
-
const v = cached.get(hashes.get(id)!);
|
|
192
|
-
if (v) cosines.set(id, cosine(queryVector, v));
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// Rerank inside each contiguous review-tier block — a draft can never be
|
|
196
|
-
// lifted over reviewed knowledge (D30).
|
|
197
|
-
const ordered: SearchHit[] = [];
|
|
198
|
-
const info = new Map<string, RerankHitInfo>();
|
|
199
|
-
const promotions: RerankOutcome["promotions"] = [];
|
|
200
|
-
let i = 0;
|
|
201
|
-
while (i < hits.length) {
|
|
202
|
-
const tier = hitReviewTier(hits[i]!);
|
|
203
|
-
let j = i + 1;
|
|
204
|
-
while (j < hits.length && hitReviewTier(hits[j]!) === tier) j++;
|
|
205
|
-
const block = rerankCrossLingual(hits.slice(i, j), cosines);
|
|
206
|
-
for (const [id, x] of block.info) {
|
|
207
|
-
info.set(id, { ...x, from: x.from + i, to: x.to + i });
|
|
208
|
-
}
|
|
209
|
-
for (const p of block.promotions) promotions.push(p);
|
|
210
|
-
ordered.push(...block.ordered);
|
|
211
|
-
i = j;
|
|
212
|
-
}
|
|
213
|
-
return { hits: ordered, candidates: texts.size, info, promotions };
|
|
214
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* D96: local embedding runtime for the opt-in cross-lingual rerank.
|
|
3
|
+
*
|
|
4
|
+
* Local-first, no API: the model runs through `@huggingface/transformers`
|
|
5
|
+
* (ONNX, CPU) and is downloaded into `<data root>/models/` on first use.
|
|
6
|
+
* Vectors are cached in SQLite (`embedding_vectors`, keyed by a hash of the
|
|
7
|
+
* embedded text), so a warm store never re-embeds a memory.
|
|
8
|
+
*
|
|
9
|
+
* The package is deliberately NOT a dependency of open-memex: it pulls
|
|
10
|
+
* ~100-500 MB of model + onnxruntime, and the default install must stay
|
|
11
|
+
* zero-download. When the flag is on and the package is missing, the rerank
|
|
12
|
+
* degrades to plain BM25 with one stderr warning — never an error.
|
|
13
|
+
*
|
|
14
|
+
* Nothing here loads a model unless the caller already decided that a
|
|
15
|
+
* cross-lingual candidate exists; a same-language search pays zero cost.
|
|
16
|
+
*/
|
|
17
|
+
import fs from "node:fs";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import crypto from "node:crypto";
|
|
20
|
+
|
|
21
|
+
import { dataRootPath } from "../paths.ts";
|
|
22
|
+
import { db } from "../store/db.ts";
|
|
23
|
+
import { loadVectors, saveVector } from "../store/embeddings.ts";
|
|
24
|
+
import { hasCjk } from "./cjk.ts";
|
|
25
|
+
import { rerankCrossLingual, type RerankHitInfo } from "./rerank.ts";
|
|
26
|
+
import { hitReviewTier, type SearchHit } from "./search.ts";
|
|
27
|
+
|
|
28
|
+
/** The model the D96 gate measured. User-configurable via `embeddingModel`. */
|
|
29
|
+
export const DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
30
|
+
|
|
31
|
+
/** Package name, spelled once so the dynamic import stays non-literal (it is optional). */
|
|
32
|
+
const PACKAGE = "@huggingface/transformers";
|
|
33
|
+
|
|
34
|
+
type Embedder = (
|
|
35
|
+
texts: string[],
|
|
36
|
+
opts: { pooling: "mean"; normalize: true },
|
|
37
|
+
) => Promise<{ tolist(): number[][] }>;
|
|
38
|
+
|
|
39
|
+
interface Runtime {
|
|
40
|
+
embedder: Embedder;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
let runtimeCache: { model: string; promise: Promise<Runtime | null> } | null = null;
|
|
44
|
+
const warned = new Set<string>();
|
|
45
|
+
|
|
46
|
+
function warnOnce(key: string, message: string): void {
|
|
47
|
+
if (warned.has(key)) return;
|
|
48
|
+
warned.add(key);
|
|
49
|
+
// stdout is the MCP protocol channel — diagnostics go to stderr.
|
|
50
|
+
process.stderr.write(`[open-memex] ${message}\n`);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Cache root the ONNX model files are downloaded into (never the repo). */
|
|
54
|
+
export function embeddingModelDir(): string {
|
|
55
|
+
return path.join(dataRootPath(), "models");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function loadRuntime(model: string): Promise<Runtime | null> {
|
|
59
|
+
try {
|
|
60
|
+
const spec = PACKAGE;
|
|
61
|
+
const mod: any = await import(spec);
|
|
62
|
+
const cacheDir = embeddingModelDir();
|
|
63
|
+
fs.mkdirSync(cacheDir, { recursive: true });
|
|
64
|
+
mod.env.cacheDir = cacheDir;
|
|
65
|
+
const pipeline = mod.pipeline;
|
|
66
|
+
// D96: the gate measured q8 identical to fp32 on the fixture at ~1/4 the
|
|
67
|
+
// download (113 MB vs 449 MB). If a user override lacks a quantized file,
|
|
68
|
+
// the load fails and we degrade to BM25 with a warning.
|
|
69
|
+
const extractor = await pipeline("feature-extraction", model, { dtype: "q8" });
|
|
70
|
+
return { embedder: extractor as Embedder };
|
|
71
|
+
} catch (err) {
|
|
72
|
+
warnOnce(
|
|
73
|
+
`load:${model}`,
|
|
74
|
+
`cross-lingual rerank is on but the local embedding runtime is unavailable (${(err as Error).message}). ` +
|
|
75
|
+
`Install it with: npm install -g ${PACKAGE}. Search falls back to BM25.`,
|
|
76
|
+
);
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function runtime(model: string): Promise<Runtime | null> {
|
|
82
|
+
if (runtimeCache?.model !== model) runtimeCache = { model, promise: loadRuntime(model) };
|
|
83
|
+
return runtimeCache.promise;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function hashText(text: string): string {
|
|
87
|
+
return crypto.createHash("sha256").update(text, "utf8").digest("hex").slice(0, 32);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The text a memory contributes to the embedding — the same fields the bench embedded. */
|
|
91
|
+
function memoryText(content: string, aliases: string, tags: string): string {
|
|
92
|
+
const split = (s: string): string[] =>
|
|
93
|
+
s
|
|
94
|
+
.split(",")
|
|
95
|
+
.map((x) => x.trim())
|
|
96
|
+
.filter(Boolean);
|
|
97
|
+
return [content, ...split(aliases), ...split(tags)]
|
|
98
|
+
.filter((s) => s.trim().length > 0)
|
|
99
|
+
.join("\n")
|
|
100
|
+
.slice(0, 2000);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function cosine(a: number[], b: number[]): number {
|
|
104
|
+
const n = Math.min(a.length, b.length);
|
|
105
|
+
let s = 0;
|
|
106
|
+
for (let i = 0; i < n; i++) s += a[i]! * b[i]!;
|
|
107
|
+
return s;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export interface RerankOutcome {
|
|
111
|
+
hits: SearchHit[];
|
|
112
|
+
/** Why the rerank did not run: "off" | "no-cross-lingual-hits" | "model-unavailable". */
|
|
113
|
+
skipped?: "off" | "no-cross-lingual-hits" | "model-unavailable";
|
|
114
|
+
/** How many returned hits were cross-lingual candidates. */
|
|
115
|
+
candidates: number;
|
|
116
|
+
info: Map<string, RerankHitInfo>;
|
|
117
|
+
promotions: Array<{ id: string; promoted: number; cosine: number }>;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Rerank an already-retrieved list for EN<->ZH queries only.
|
|
122
|
+
*
|
|
123
|
+
* Called after `search()`, so BM25 has already chosen the set and the review
|
|
124
|
+
* tiers are already ordered; this only reorders within each contiguous tier
|
|
125
|
+
* block. A query with no cross-lingual hit returns immediately and never
|
|
126
|
+
* loads a model.
|
|
127
|
+
*/
|
|
128
|
+
export async function applyCrossLingualRerank(
|
|
129
|
+
query: string,
|
|
130
|
+
hits: SearchHit[],
|
|
131
|
+
cfg: { crossLingualRerank: boolean; embeddingModel: string },
|
|
132
|
+
): Promise<RerankOutcome> {
|
|
133
|
+
const untouched: RerankOutcome = { hits, candidates: 0, info: new Map(), promotions: [] };
|
|
134
|
+
if (!cfg.crossLingualRerank) return { ...untouched, skipped: "off" };
|
|
135
|
+
if (hits.length === 0) return { ...untouched, skipped: "no-cross-lingual-hits" };
|
|
136
|
+
|
|
137
|
+
const queryCjk = hasCjk(query);
|
|
138
|
+
const ids = hits.map((h) => h.id);
|
|
139
|
+
const rows = db()
|
|
140
|
+
.prepare(
|
|
141
|
+
`SELECT id, content, COALESCE(aliases, '') AS aliases, COALESCE(tags, '') AS tags
|
|
142
|
+
FROM memories WHERE id IN (${ids.map(() => "?").join(",")})`,
|
|
143
|
+
)
|
|
144
|
+
.all(...ids) as Array<{ id: string; content: string; aliases: string; tags: string }>;
|
|
145
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
146
|
+
|
|
147
|
+
// Cross-lingual = the memory's script differs from the query's. Everything
|
|
148
|
+
// else is a barrier that keeps its exact BM25 slot.
|
|
149
|
+
const texts = new Map<string, string>();
|
|
150
|
+
for (const h of hits) {
|
|
151
|
+
const r = byId.get(h.id);
|
|
152
|
+
if (!r) continue;
|
|
153
|
+
if (hasCjk(r.content) === queryCjk) continue;
|
|
154
|
+
texts.set(h.id, memoryText(r.content, r.aliases, r.tags));
|
|
155
|
+
}
|
|
156
|
+
if (texts.size === 0) return { ...untouched, skipped: "no-cross-lingual-hits" };
|
|
157
|
+
|
|
158
|
+
const rt = await runtime(cfg.embeddingModel);
|
|
159
|
+
if (!rt) return { ...untouched, skipped: "model-unavailable" };
|
|
160
|
+
|
|
161
|
+
const hashes = new Map<string, string>();
|
|
162
|
+
for (const [id, text] of texts) hashes.set(id, hashText(text));
|
|
163
|
+
let cached: Map<string, number[]>;
|
|
164
|
+
try {
|
|
165
|
+
cached = loadVectors(cfg.embeddingModel, [...new Set(hashes.values())]);
|
|
166
|
+
} catch {
|
|
167
|
+
cached = new Map();
|
|
168
|
+
}
|
|
169
|
+
const pending = [...texts].filter(([id]) => !cached.has(hashes.get(id)!));
|
|
170
|
+
if (pending.length > 0) {
|
|
171
|
+
const vectors = await rt.embedder(pending.map(([, text]) => text), {
|
|
172
|
+
pooling: "mean",
|
|
173
|
+
normalize: true,
|
|
174
|
+
});
|
|
175
|
+
const list = vectors.tolist();
|
|
176
|
+
pending.forEach(([id], i) => {
|
|
177
|
+
const vec = list[i]!;
|
|
178
|
+
cached.set(hashes.get(id)!, vec);
|
|
179
|
+
try {
|
|
180
|
+
saveVector(cfg.embeddingModel, hashes.get(id)!, vec);
|
|
181
|
+
} catch (err) {
|
|
182
|
+
// A cache that cannot be written is a slowdown, not a failure.
|
|
183
|
+
warnOnce("vector-cache-write", `could not cache the embedding (${(err as Error).message})`);
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const queryVector = (await rt.embedder([query], { pooling: "mean", normalize: true })).tolist()[0]!;
|
|
189
|
+
const cosines = new Map<string, number>();
|
|
190
|
+
for (const id of texts.keys()) {
|
|
191
|
+
const v = cached.get(hashes.get(id)!);
|
|
192
|
+
if (v) cosines.set(id, cosine(queryVector, v));
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// Rerank inside each contiguous review-tier block — a draft can never be
|
|
196
|
+
// lifted over reviewed knowledge (D30).
|
|
197
|
+
const ordered: SearchHit[] = [];
|
|
198
|
+
const info = new Map<string, RerankHitInfo>();
|
|
199
|
+
const promotions: RerankOutcome["promotions"] = [];
|
|
200
|
+
let i = 0;
|
|
201
|
+
while (i < hits.length) {
|
|
202
|
+
const tier = hitReviewTier(hits[i]!);
|
|
203
|
+
let j = i + 1;
|
|
204
|
+
while (j < hits.length && hitReviewTier(hits[j]!) === tier) j++;
|
|
205
|
+
const block = rerankCrossLingual(hits.slice(i, j), cosines);
|
|
206
|
+
for (const [id, x] of block.info) {
|
|
207
|
+
info.set(id, { ...x, from: x.from + i, to: x.to + i });
|
|
208
|
+
}
|
|
209
|
+
for (const p of block.promotions) promotions.push(p);
|
|
210
|
+
ordered.push(...block.ordered);
|
|
211
|
+
i = j;
|
|
212
|
+
}
|
|
213
|
+
return { hits: ordered, candidates: texts.size, info, promotions };
|
|
214
|
+
}
|
package/src/retrieve/rerank.ts
CHANGED
|
@@ -1,96 +1,96 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* D96: opt-in cross-lingual rerank (EN<->ZH only).
|
|
3
|
-
*
|
|
4
|
-
* Cold-start position: BM25 is the retriever, embeddings only ever *reorder*
|
|
5
|
-
* what BM25 already returned. The layout is identical to D79's reuse band —
|
|
6
|
-
* a hit is eligible only when the query and the memory are written in
|
|
7
|
-
* different scripts (one side has CJK, the other does not) *and* its cosine
|
|
8
|
-
* clears `DEFAULT_RERANK_MIN_COSINE`. Eligible hits are re-sorted by cosine
|
|
9
|
-
* inside their contiguous run; everything else — same-language hits and
|
|
10
|
-
* cross-lingual hits with too little signal — keeps its exact BM25 slot and
|
|
11
|
-
* acts as a barrier, so a run can never leapfrog them.
|
|
12
|
-
*
|
|
13
|
-
* Guardrails (D79, reused):
|
|
14
|
-
* - the BM25 score is never modified, only the order inside a run;
|
|
15
|
-
* - the caller has already split review tiers, so a rerank can never lift a
|
|
16
|
-
* draft over reviewed knowledge (search.ts resolves tiers before any
|
|
17
|
-
* caller-side reordering);
|
|
18
|
-
* - a low-similarity cross-lingual hit is a "no signal" hit, not a loser:
|
|
19
|
-
* it stays exactly where BM25 put it.
|
|
20
|
-
*
|
|
21
|
-
* Pure by construction: smoke-pure.ts tests it without SQLite or a model.
|
|
22
|
-
*/
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Minimum cosine for a cross-lingual pair to participate. Set by the D96
|
|
26
|
-
* fixture sweep (0.2 = the highest floor that still captures the measured
|
|
27
|
-
* cross-lingual gains; 0.3 keeps one of six, 0.4+ is inert). Lower it only
|
|
28
|
-
* with a new measurement.
|
|
29
|
-
*/
|
|
30
|
-
export const DEFAULT_RERANK_MIN_COSINE = 0.2;
|
|
31
|
-
|
|
32
|
-
export interface RerankHitInfo {
|
|
33
|
-
cosine: number;
|
|
34
|
-
/** 0-based BM25 slot before the rerank. */
|
|
35
|
-
from: number;
|
|
36
|
-
/** 0-based slot after the rerank. */
|
|
37
|
-
to: number;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export interface RerankResult<T> {
|
|
41
|
-
ordered: T[];
|
|
42
|
-
info: Map<string, RerankHitInfo>;
|
|
43
|
-
/** Hits that actually moved up, in their new order. */
|
|
44
|
-
promotions: Array<{ id: string; promoted: number; cosine: number }>;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Reorder one already-retrieved list. `cosines` carries only the hits the
|
|
49
|
-
* caller considers cross-lingual (same-language hits must be absent, not
|
|
50
|
-
* present with a low value — absence, not score, is what marks a barrier).
|
|
51
|
-
*/
|
|
52
|
-
export function rerankCrossLingual<T extends { id: string }>(
|
|
53
|
-
hits: readonly T[],
|
|
54
|
-
cosines: ReadonlyMap<string, number>,
|
|
55
|
-
opts: { minCosine?: number } = {},
|
|
56
|
-
): RerankResult<T> {
|
|
57
|
-
const minCosine = opts.minCosine ?? DEFAULT_RERANK_MIN_COSINE;
|
|
58
|
-
const info = new Map<string, RerankHitInfo>();
|
|
59
|
-
const ordered: T[] = [];
|
|
60
|
-
const eligible = (h: T): boolean => {
|
|
61
|
-
const c = cosines.get(h.id);
|
|
62
|
-
return c !== undefined && Number.isFinite(c) && c >= minCosine;
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
let i = 0;
|
|
66
|
-
while (i < hits.length) {
|
|
67
|
-
if (!eligible(hits[i]!)) {
|
|
68
|
-
info.set(hits[i]!.id, { cosine: cosines.get(hits[i]!.id) ?? NaN, from: i, to: i });
|
|
69
|
-
ordered.push(hits[i]!);
|
|
70
|
-
i++;
|
|
71
|
-
continue;
|
|
72
|
-
}
|
|
73
|
-
let j = i + 1;
|
|
74
|
-
while (j < hits.length && eligible(hits[j]!)) j++;
|
|
75
|
-
const run = hits.slice(i, j).map((h, k) => ({ h, k }));
|
|
76
|
-
run.sort((a, b) => {
|
|
77
|
-
const ca = cosines.get(a.h.id)!;
|
|
78
|
-
const cb = cosines.get(b.h.id)!;
|
|
79
|
-
return ca !== cb ? cb - ca : a.k - b.k;
|
|
80
|
-
});
|
|
81
|
-
run.forEach((x, k) => {
|
|
82
|
-
info.set(x.h.id, { cosine: cosines.get(x.h.id)!, from: i + x.k, to: i + k });
|
|
83
|
-
ordered.push(x.h);
|
|
84
|
-
});
|
|
85
|
-
i = j;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
const promotions: RerankResult<T>["promotions"] = [];
|
|
89
|
-
for (const h of ordered) {
|
|
90
|
-
const i2 = info.get(h.id)!;
|
|
91
|
-
if (i2.to < i2.from) {
|
|
92
|
-
promotions.push({ id: h.id, promoted: i2.from - i2.to, cosine: i2.cosine });
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
return { ordered, info, promotions };
|
|
96
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* D96: opt-in cross-lingual rerank (EN<->ZH only).
|
|
3
|
+
*
|
|
4
|
+
* Cold-start position: BM25 is the retriever, embeddings only ever *reorder*
|
|
5
|
+
* what BM25 already returned. The layout is identical to D79's reuse band —
|
|
6
|
+
* a hit is eligible only when the query and the memory are written in
|
|
7
|
+
* different scripts (one side has CJK, the other does not) *and* its cosine
|
|
8
|
+
* clears `DEFAULT_RERANK_MIN_COSINE`. Eligible hits are re-sorted by cosine
|
|
9
|
+
* inside their contiguous run; everything else — same-language hits and
|
|
10
|
+
* cross-lingual hits with too little signal — keeps its exact BM25 slot and
|
|
11
|
+
* acts as a barrier, so a run can never leapfrog them.
|
|
12
|
+
*
|
|
13
|
+
* Guardrails (D79, reused):
|
|
14
|
+
* - the BM25 score is never modified, only the order inside a run;
|
|
15
|
+
* - the caller has already split review tiers, so a rerank can never lift a
|
|
16
|
+
* draft over reviewed knowledge (search.ts resolves tiers before any
|
|
17
|
+
* caller-side reordering);
|
|
18
|
+
* - a low-similarity cross-lingual hit is a "no signal" hit, not a loser:
|
|
19
|
+
* it stays exactly where BM25 put it.
|
|
20
|
+
*
|
|
21
|
+
* Pure by construction: smoke-pure.ts tests it without SQLite or a model.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Minimum cosine for a cross-lingual pair to participate. Set by the D96
|
|
26
|
+
* fixture sweep (0.2 = the highest floor that still captures the measured
|
|
27
|
+
* cross-lingual gains; 0.3 keeps one of six, 0.4+ is inert). Lower it only
|
|
28
|
+
* with a new measurement.
|
|
29
|
+
*/
|
|
30
|
+
export const DEFAULT_RERANK_MIN_COSINE = 0.2;
|
|
31
|
+
|
|
32
|
+
export interface RerankHitInfo {
|
|
33
|
+
cosine: number;
|
|
34
|
+
/** 0-based BM25 slot before the rerank. */
|
|
35
|
+
from: number;
|
|
36
|
+
/** 0-based slot after the rerank. */
|
|
37
|
+
to: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface RerankResult<T> {
|
|
41
|
+
ordered: T[];
|
|
42
|
+
info: Map<string, RerankHitInfo>;
|
|
43
|
+
/** Hits that actually moved up, in their new order. */
|
|
44
|
+
promotions: Array<{ id: string; promoted: number; cosine: number }>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Reorder one already-retrieved list. `cosines` carries only the hits the
|
|
49
|
+
* caller considers cross-lingual (same-language hits must be absent, not
|
|
50
|
+
* present with a low value — absence, not score, is what marks a barrier).
|
|
51
|
+
*/
|
|
52
|
+
export function rerankCrossLingual<T extends { id: string }>(
|
|
53
|
+
hits: readonly T[],
|
|
54
|
+
cosines: ReadonlyMap<string, number>,
|
|
55
|
+
opts: { minCosine?: number } = {},
|
|
56
|
+
): RerankResult<T> {
|
|
57
|
+
const minCosine = opts.minCosine ?? DEFAULT_RERANK_MIN_COSINE;
|
|
58
|
+
const info = new Map<string, RerankHitInfo>();
|
|
59
|
+
const ordered: T[] = [];
|
|
60
|
+
const eligible = (h: T): boolean => {
|
|
61
|
+
const c = cosines.get(h.id);
|
|
62
|
+
return c !== undefined && Number.isFinite(c) && c >= minCosine;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
let i = 0;
|
|
66
|
+
while (i < hits.length) {
|
|
67
|
+
if (!eligible(hits[i]!)) {
|
|
68
|
+
info.set(hits[i]!.id, { cosine: cosines.get(hits[i]!.id) ?? NaN, from: i, to: i });
|
|
69
|
+
ordered.push(hits[i]!);
|
|
70
|
+
i++;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
let j = i + 1;
|
|
74
|
+
while (j < hits.length && eligible(hits[j]!)) j++;
|
|
75
|
+
const run = hits.slice(i, j).map((h, k) => ({ h, k }));
|
|
76
|
+
run.sort((a, b) => {
|
|
77
|
+
const ca = cosines.get(a.h.id)!;
|
|
78
|
+
const cb = cosines.get(b.h.id)!;
|
|
79
|
+
return ca !== cb ? cb - ca : a.k - b.k;
|
|
80
|
+
});
|
|
81
|
+
run.forEach((x, k) => {
|
|
82
|
+
info.set(x.h.id, { cosine: cosines.get(x.h.id)!, from: i + x.k, to: i + k });
|
|
83
|
+
ordered.push(x.h);
|
|
84
|
+
});
|
|
85
|
+
i = j;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const promotions: RerankResult<T>["promotions"] = [];
|
|
89
|
+
for (const h of ordered) {
|
|
90
|
+
const i2 = info.get(h.id)!;
|
|
91
|
+
if (i2.to < i2.from) {
|
|
92
|
+
promotions.push({ id: h.id, promoted: i2.from - i2.to, cosine: i2.cosine });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return { ordered, info, promotions };
|
|
96
|
+
}
|