@modusensus/dsh-mneme 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +419 -399
- package/lib/client.js +1302 -1302
- package/lib/hot-memory.js +53 -53
- package/lib/reranker.js +218 -218
- package/lib/service.js +1489 -1486
- package/lib/store.js +6 -2
- package/lib/summarize.js +5 -1
- package/lib/tools.js +7 -2
- package/package.json +1 -1
- package/src/hot-memory.js +53 -53
- package/src/reranker.js +218 -218
- package/src/service.js +1489 -1486
- package/src/store.js +6 -2
- package/src/summarize.js +5 -1
- package/src/tools.js +7 -2
- package/test/hot-memory.test.js +174 -174
- package/test/provenance.test.js +103 -0
- package/test/reranker.test.js +240 -240
- package/test/service-search.test.js +199 -199
package/lib/store.js
CHANGED
|
@@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
12
12
|
forgotten INTEGER NOT NULL DEFAULT 0,
|
|
13
13
|
archived INTEGER NOT NULL DEFAULT 0,
|
|
14
14
|
source TEXT,
|
|
15
|
+
session_id TEXT,
|
|
15
16
|
content_history TEXT,
|
|
16
17
|
embedding TEXT,
|
|
17
18
|
epistemic_status TEXT NOT NULL DEFAULT 'subjective',
|
|
@@ -321,6 +322,7 @@ function toRow(row) {
|
|
|
321
322
|
forgotten: row.forgotten === 1,
|
|
322
323
|
archived: row.archived === 1,
|
|
323
324
|
source: row.source ?? undefined,
|
|
325
|
+
session_id: row.session_id ?? undefined,
|
|
324
326
|
content_history: parseJsonArray(row.content_history),
|
|
325
327
|
quality_score: row.quality_score !== null && row.quality_score !== undefined ? Number(row.quality_score) : undefined,
|
|
326
328
|
epistemic_status: row.epistemic_status ?? "subjective",
|
|
@@ -571,6 +573,7 @@ export function createStore(path) {
|
|
|
571
573
|
addColumn("memories", "epistemic_status", "ALTER TABLE memories ADD COLUMN epistemic_status TEXT NOT NULL DEFAULT 'subjective'");
|
|
572
574
|
addColumn("memories", "content_history", "ALTER TABLE memories ADD COLUMN content_history TEXT");
|
|
573
575
|
addColumn("memories", "quality_score", "ALTER TABLE memories ADD COLUMN quality_score REAL");
|
|
576
|
+
addColumn("memories", "session_id", "ALTER TABLE memories ADD COLUMN session_id TEXT");
|
|
574
577
|
|
|
575
578
|
// Legacy dream_runs without policy_epoch → backfill with the default epoch.
|
|
576
579
|
addColumn("dream_runs", "policy_epoch", "ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0");
|
|
@@ -657,8 +660,8 @@ export function createStore(path) {
|
|
|
657
660
|
: inferEpistemicStatus(memory);
|
|
658
661
|
runAtomically(() => {
|
|
659
662
|
db.prepare(
|
|
660
|
-
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, content_history, quality_score, embedding, epistemic_status, created_at, updated_at)
|
|
661
|
-
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
663
|
+
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, session_id, content_history, quality_score, embedding, epistemic_status, created_at, updated_at)
|
|
664
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
662
665
|
).run(
|
|
663
666
|
id,
|
|
664
667
|
type,
|
|
@@ -668,6 +671,7 @@ export function createStore(path) {
|
|
|
668
671
|
importance,
|
|
669
672
|
memory.archived ? 1 : 0,
|
|
670
673
|
memory.source ?? null,
|
|
674
|
+
memory.session_id ?? null,
|
|
671
675
|
JSON.stringify(memory.content_history ?? []),
|
|
672
676
|
Number.isFinite(memory.quality_score) ? memory.quality_score : null,
|
|
673
677
|
embedding,
|
package/lib/summarize.js
CHANGED
|
@@ -182,7 +182,11 @@ export function createSummarizer(ctx, service, config) {
|
|
|
182
182
|
.join("");
|
|
183
183
|
const entries = parseSummaryJson(text || assembledText);
|
|
184
184
|
for (const entry of entries) {
|
|
185
|
-
|
|
185
|
+
// Provenance: the summarizer runs on a real session (turn/end hook), so
|
|
186
|
+
// session.id is always available here — it rides both the human-readable
|
|
187
|
+
// source label and the structured session_id column (v0.5.x memory
|
|
188
|
+
// provenance, the raw material for v0.6.0 reasoning-path / drift analysis).
|
|
189
|
+
service.saveWithDedupe({ ...entry, source: `session:${session.id}`, session_id: session.id });
|
|
186
190
|
}
|
|
187
191
|
} finally {
|
|
188
192
|
if (audit) {
|
package/lib/tools.js
CHANGED
|
@@ -16,6 +16,7 @@ const MEMORY_ITEM_SCHEMA = {
|
|
|
16
16
|
tags: { type: "array", items: { type: "string" } },
|
|
17
17
|
importance: { type: "integer", required: true },
|
|
18
18
|
source: { type: "string" },
|
|
19
|
+
session_id: { type: "string" },
|
|
19
20
|
created_at: { type: "string", required: true },
|
|
20
21
|
updated_at: { type: "string", required: true }
|
|
21
22
|
}
|
|
@@ -48,14 +49,18 @@ export function createTools(ctx, service, config, embedder) {
|
|
|
48
49
|
},
|
|
49
50
|
render: (_args, value) => TEXT_OUTPUT(`memory ${value.action}: ${value.id}`)
|
|
50
51
|
},
|
|
51
|
-
async execute(args) {
|
|
52
|
+
async execute(args, exec) {
|
|
52
53
|
const { action, memory } = service.saveWithDedupe({
|
|
53
54
|
type: args.type,
|
|
54
55
|
title: args.title,
|
|
55
56
|
content: args.content,
|
|
56
57
|
tags: args.tags ?? [],
|
|
57
58
|
importance: args.importance ?? 3,
|
|
58
|
-
source: args.source ?? "tool"
|
|
59
|
+
source: args.source ?? "tool",
|
|
60
|
+
// Provenance: the session that issued the tool call. exec.agent is
|
|
61
|
+
// set by the agent loop (undefined in unit tests / direct calls) —
|
|
62
|
+
// absent a session, session_id stays null rather than fabricating one.
|
|
63
|
+
session_id: exec?.agent?.session?.id ?? undefined
|
|
59
64
|
});
|
|
60
65
|
return { action, id: memory.id };
|
|
61
66
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modusensus/dsh-mneme",
|
|
3
3
|
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 7 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
|
|
4
|
-
"version": "0.5.
|
|
4
|
+
"version": "0.5.2",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
package/src/hot-memory.js
CHANGED
|
@@ -1,53 +1,53 @@
|
|
|
1
|
-
// Session-scoped hot memory (v0.5.0 召回率优化 1.3): a short-term buffer of
|
|
2
|
-
// the latest dialogue rounds, kept strictly apart from the long-term memory
|
|
3
|
-
// store. The injector renders it ahead of the long-term recall block so the
|
|
4
|
-
// agent sees "what we were just talking about" without those rounds ever
|
|
5
|
-
// being persisted as memories. Bounded two ways: maxRounds (count) and
|
|
6
|
-
// maxTokens (budget) — whichever evicts first.
|
|
7
|
-
|
|
8
|
-
// CJK-aware token estimate: one Chinese character ≈ 0.6 tokens (clustering
|
|
9
|
-
// behavior of mainstream tokenizers), one ASCII char ≈ 0.25.
|
|
10
|
-
export function estimateTokens(text) {
|
|
11
|
-
const s = String(text ?? "");
|
|
12
|
-
let cjk = 0;
|
|
13
|
-
for (const ch of s) if (ch >= "\u4e00" && ch <= "\u9fff") cjk++;
|
|
14
|
-
return Math.ceil(cjk * 0.6 + (s.length - cjk) * 0.25);
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* @param {{maxRounds?: number, maxTokens?: number}} opts
|
|
19
|
-
* @returns {{add(round: {query: string, response?: string}): void,
|
|
20
|
-
* getContext(): string,
|
|
21
|
-
* rounds(): Array, clear(): void}}
|
|
22
|
-
*/
|
|
23
|
-
export function createHotMemory({ maxRounds = 5, maxTokens = 2000 } = {}) {
|
|
24
|
-
// Entry defense: a non-positive or non-integer maxRounds (0, -1, 1.5, NaN,
|
|
25
|
-
// null, "2") would make the eviction while-loop unbounded — the buffer can
|
|
26
|
-
// never shrink below `buffer.length > maxRounds`, so `add` would spin forever.
|
|
27
|
-
// Fall back to the defaults so a hostile/buggy caller can never wedge the
|
|
28
|
-
// hot-memory buffer in an infinite loop.
|
|
29
|
-
maxRounds = (Number.isInteger(maxRounds) && maxRounds > 0) ? maxRounds : 5;
|
|
30
|
-
maxTokens = (Number.isFinite(maxTokens) && maxTokens > 0) ? maxTokens : 2000;
|
|
31
|
-
const buffer = [];
|
|
32
|
-
|
|
33
|
-
function totalTokens() {
|
|
34
|
-
return buffer.reduce(
|
|
35
|
-
(sum, r) => sum + estimateTokens(`Q: ${r.query}\nA: ${r.response ?? ""}`),
|
|
36
|
-
0
|
|
37
|
-
);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
return {
|
|
41
|
-
add(round) {
|
|
42
|
-
if (!round?.query) return;
|
|
43
|
-
buffer.push({ query: String(round.query), response: String(round.response ?? "") });
|
|
44
|
-
while (buffer.length > maxRounds) buffer.shift();
|
|
45
|
-
while (buffer.length > 1 && totalTokens() > maxTokens) buffer.shift();
|
|
46
|
-
},
|
|
47
|
-
getContext() {
|
|
48
|
-
return buffer.map((r) => `Q: ${r.query}\nA: ${r.response ?? ""}`).join("\n\n");
|
|
49
|
-
},
|
|
50
|
-
rounds: () => [...buffer],
|
|
51
|
-
clear() { buffer.length = 0; }
|
|
52
|
-
};
|
|
53
|
-
}
|
|
1
|
+
// Session-scoped hot memory (v0.5.0 召回率优化 1.3): a short-term buffer of
|
|
2
|
+
// the latest dialogue rounds, kept strictly apart from the long-term memory
|
|
3
|
+
// store. The injector renders it ahead of the long-term recall block so the
|
|
4
|
+
// agent sees "what we were just talking about" without those rounds ever
|
|
5
|
+
// being persisted as memories. Bounded two ways: maxRounds (count) and
|
|
6
|
+
// maxTokens (budget) — whichever evicts first.
|
|
7
|
+
|
|
8
|
+
// CJK-aware token estimate: one Chinese character ≈ 0.6 tokens (clustering
|
|
9
|
+
// behavior of mainstream tokenizers), one ASCII char ≈ 0.25.
|
|
10
|
+
export function estimateTokens(text) {
|
|
11
|
+
const s = String(text ?? "");
|
|
12
|
+
let cjk = 0;
|
|
13
|
+
for (const ch of s) if (ch >= "\u4e00" && ch <= "\u9fff") cjk++;
|
|
14
|
+
return Math.ceil(cjk * 0.6 + (s.length - cjk) * 0.25);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {{maxRounds?: number, maxTokens?: number}} opts
|
|
19
|
+
* @returns {{add(round: {query: string, response?: string}): void,
|
|
20
|
+
* getContext(): string,
|
|
21
|
+
* rounds(): Array, clear(): void}}
|
|
22
|
+
*/
|
|
23
|
+
export function createHotMemory({ maxRounds = 5, maxTokens = 2000 } = {}) {
|
|
24
|
+
// Entry defense: a non-positive or non-integer maxRounds (0, -1, 1.5, NaN,
|
|
25
|
+
// null, "2") would make the eviction while-loop unbounded — the buffer can
|
|
26
|
+
// never shrink below `buffer.length > maxRounds`, so `add` would spin forever.
|
|
27
|
+
// Fall back to the defaults so a hostile/buggy caller can never wedge the
|
|
28
|
+
// hot-memory buffer in an infinite loop.
|
|
29
|
+
maxRounds = (Number.isInteger(maxRounds) && maxRounds > 0) ? maxRounds : 5;
|
|
30
|
+
maxTokens = (Number.isFinite(maxTokens) && maxTokens > 0) ? maxTokens : 2000;
|
|
31
|
+
const buffer = [];
|
|
32
|
+
|
|
33
|
+
function totalTokens() {
|
|
34
|
+
return buffer.reduce(
|
|
35
|
+
(sum, r) => sum + estimateTokens(`Q: ${r.query}\nA: ${r.response ?? ""}`),
|
|
36
|
+
0
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
add(round) {
|
|
42
|
+
if (!round?.query) return;
|
|
43
|
+
buffer.push({ query: String(round.query), response: String(round.response ?? "") });
|
|
44
|
+
while (buffer.length > maxRounds) buffer.shift();
|
|
45
|
+
while (buffer.length > 1 && totalTokens() > maxTokens) buffer.shift();
|
|
46
|
+
},
|
|
47
|
+
getContext() {
|
|
48
|
+
return buffer.map((r) => `Q: ${r.query}\nA: ${r.response ?? ""}`).join("\n\n");
|
|
49
|
+
},
|
|
50
|
+
rounds: () => [...buffer],
|
|
51
|
+
clear() { buffer.length = 0; }
|
|
52
|
+
};
|
|
53
|
+
}
|
package/src/reranker.js
CHANGED
|
@@ -1,218 +1,218 @@
|
|
|
1
|
-
import os from "node:os";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
|
|
4
|
-
// Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
|
|
5
|
-
// bge-reranker-base through transformers.js: tries the native `rerank` task
|
|
6
|
-
// first, then the sequence-classification head (sigmoid on the logit delta),
|
|
7
|
-
// and finally feature-extraction over concatenated query+passage (cosine).
|
|
8
|
-
// Every strategy funnels into scorePair(query, passage) so tests can inject a
|
|
9
|
-
// fake scorer and never download a model. Failures throw — the caller degrades
|
|
10
|
-
// back to the original candidate order.
|
|
11
|
-
function hashString(s) {
|
|
12
|
-
let h = 5381;
|
|
13
|
-
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
14
|
-
return h.toString(16);
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/** Same provider#hash fingerprint convention as the embedders. */
|
|
18
|
-
function modelHash(model) {
|
|
19
|
-
return `${model}#${hashString(model)}`;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
/** Lazy default pipeline factory: dynamic import keeps module load cheap. */
|
|
23
|
-
async function defaultPipelineLoader(task, model, options) {
|
|
24
|
-
const { env, pipeline } = await import("@huggingface/transformers");
|
|
25
|
-
// issue #13: mirror cache_dir onto env.cacheDir so the tokenizer pre-check
|
|
26
|
-
// resolves locally too (same fix as local-embedder.js).
|
|
27
|
-
if (options?.cache_dir) env.cacheDir = options.cache_dir;
|
|
28
|
-
return pipeline(task, model, options);
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/** Flatten a transformers.js Tensor [batch, seq, dim] into number[][] rows. */
|
|
32
|
-
function tensorToRows(tensor) {
|
|
33
|
-
const { data, dims } = tensor;
|
|
34
|
-
const dim = dims[dims.length - 1] || 0;
|
|
35
|
-
const rows = [];
|
|
36
|
-
for (let i = 0; i < data.length; i += dim) rows.push(Array.from(data.subarray(i, i + dim)));
|
|
37
|
-
if (rows.length === 0 && dim > 0) rows.push(Array.from(data));
|
|
38
|
-
return rows;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
function cosine(a, b) {
|
|
42
|
-
let dot = 0;
|
|
43
|
-
let na = 0;
|
|
44
|
-
let nb = 0;
|
|
45
|
-
for (let i = 0; i < a.length; i++) {
|
|
46
|
-
dot += a[i] * b[i];
|
|
47
|
-
na += a[i] * a[i];
|
|
48
|
-
nb += b[i] * b[i];
|
|
49
|
-
}
|
|
50
|
-
if (na === 0 || nb === 0) return 0;
|
|
51
|
-
return dot / Math.sqrt(na * nb);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function clamp01(x) {
|
|
55
|
-
return Math.max(0, Math.min(1, x));
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
// Pipeline strategies, best to worst. `rerank` is unsupported by the bundled
|
|
59
|
-
// transformers.js (v4.2.0) and rejects cheaply before any model download, so
|
|
60
|
-
// the cascade normally lands on the classification head.
|
|
61
|
-
const STRATEGIES = [
|
|
62
|
-
["rerank", "rerank"],
|
|
63
|
-
["text-classification", "tc"],
|
|
64
|
-
["feature-extraction", "fe"]
|
|
65
|
-
];
|
|
66
|
-
|
|
67
|
-
export class LocalReranker {
|
|
68
|
-
constructor(opts = {}) {
|
|
69
|
-
this.model = opts.model || "Xenova/bge-reranker-base";
|
|
70
|
-
this.batchSize = opts.batchSize || 8;
|
|
71
|
-
this.maxCandidates = opts.maxCandidates || 30;
|
|
72
|
-
this.scoreThreshold = opts.scoreThreshold ?? 0.1;
|
|
73
|
-
this.device = opts.device || "cpu";
|
|
74
|
-
this.cacheDir =
|
|
75
|
-
String(opts.cacheDir ?? "").trim() ||
|
|
76
|
-
path.join(os.homedir(), ".dsh", "mneme", "models");
|
|
77
|
-
this.logger = opts.logger ?? null;
|
|
78
|
-
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
79
|
-
// Injectable seam: async (query, passage) => number. When set, init()
|
|
80
|
-
// skips model loading entirely so tests never hit the network.
|
|
81
|
-
this.scorePair = opts.scorePair ?? null;
|
|
82
|
-
this.pipeline = null;
|
|
83
|
-
this._batchScorer = null;
|
|
84
|
-
this._queryVec = null;
|
|
85
|
-
this._queryKey = null;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/** Load the model; throws when no strategy can be bound. */
|
|
89
|
-
async init() {
|
|
90
|
-
if (this.scorePair) return this;
|
|
91
|
-
let lastErr = null;
|
|
92
|
-
for (const [task, strategy] of STRATEGIES) {
|
|
93
|
-
try {
|
|
94
|
-
this.pipeline = await this.engineFactory(task, this.model, this._engineOptions());
|
|
95
|
-
this._bindBatchScorer(strategy);
|
|
96
|
-
this.logger?.info?.(
|
|
97
|
-
`[dsh-mneme] local reranker ready: ${this.model} (strategy=${strategy}, device=${this.device})`
|
|
98
|
-
);
|
|
99
|
-
return this;
|
|
100
|
-
} catch (err) {
|
|
101
|
-
lastErr = err;
|
|
102
|
-
this.logger?.warn?.(
|
|
103
|
-
`[dsh-mneme] reranker task "${task}" unavailable for ${this.model}: ${String(err?.message ?? err)}`
|
|
104
|
-
);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
throw new Error(`LocalReranker failed to load ${this.model}: ${String(lastErr?.message ?? lastErr)}`);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
_engineOptions() {
|
|
111
|
-
const options = { device: this.device };
|
|
112
|
-
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
113
|
-
return options;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/** Bind a batch scorer for the chosen strategy. */
|
|
117
|
-
_bindBatchScorer(strategy) {
|
|
118
|
-
if (strategy === "rerank") {
|
|
119
|
-
// Native cross-encoder task (transformers.js >= 4.6): { query, documents }
|
|
120
|
-
// returns one score per document.
|
|
121
|
-
this._batchScorer = async (query, passages) => {
|
|
122
|
-
const out = await this.pipeline({ query, documents: passages });
|
|
123
|
-
if (!Array.isArray(out)) throw new Error("rerank pipeline returned unexpected output");
|
|
124
|
-
return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
|
|
125
|
-
};
|
|
126
|
-
} else if (strategy === "tc") {
|
|
127
|
-
// Classification head: tokenize (query, passage) pairs, score with
|
|
128
|
-
// sigmoid(l1 - l0) so relevance lands in [0, 1].
|
|
129
|
-
this._batchScorer = async (query, passages) => {
|
|
130
|
-
const { tokenizer, model } = this.pipeline;
|
|
131
|
-
const inputs = tokenizer(passages.map(() => query), {
|
|
132
|
-
text_pair: passages,
|
|
133
|
-
padding: true,
|
|
134
|
-
truncation: true
|
|
135
|
-
});
|
|
136
|
-
const logits = await model(inputs).then((o) => o.logits);
|
|
137
|
-
const dims = logits.dims;
|
|
138
|
-
const cols = dims[dims.length - 1] || 2;
|
|
139
|
-
const rows = [];
|
|
140
|
-
for (let i = 0; i < dims[0]; i++) {
|
|
141
|
-
const base = i * cols;
|
|
142
|
-
const l0 = logits.data[base];
|
|
143
|
-
const l1 = cols > 1 ? logits.data[base + 1] : l0;
|
|
144
|
-
// sigmoid(l1 - l0) == softmax probability of the positive class.
|
|
145
|
-
rows.push(clamp01(1 / (1 + Math.exp(l0 - l1))));
|
|
146
|
-
}
|
|
147
|
-
return rows;
|
|
148
|
-
};
|
|
149
|
-
} else {
|
|
150
|
-
// Feature extraction: mean-pool the concatenated pair and compare with
|
|
151
|
-
// the query embedding via cosine. Degraded but model-agnostic.
|
|
152
|
-
// The query vector is cached per query string, so a new query always
|
|
153
|
-
// recomputes it instead of reusing a stale vector from the previous call.
|
|
154
|
-
this._queryVec = null;
|
|
155
|
-
this._queryKey = null;
|
|
156
|
-
this._batchScorer = async (query, passages) => {
|
|
157
|
-
if (this._queryKey !== query) {
|
|
158
|
-
const t = await this.pipeline([query], { pooling: "mean", normalize: true });
|
|
159
|
-
this._queryVec = tensorToRows(t)[0];
|
|
160
|
-
this._queryKey = query;
|
|
161
|
-
}
|
|
162
|
-
const t = await this.pipeline(passages.map((p) => `${query}\n${p}`), {
|
|
163
|
-
pooling: "mean",
|
|
164
|
-
normalize: true
|
|
165
|
-
});
|
|
166
|
-
return tensorToRows(t).map((v) => clamp01(cosine(this._queryVec, v)));
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
/** Score one batch of passages; injected scorePair scores pair by pair. */
|
|
172
|
-
async _scores(query, passages) {
|
|
173
|
-
if (this._batchScorer) return this._batchScorer(query, passages);
|
|
174
|
-
return Promise.all(passages.map((p) => (p ? this.scorePair(query, p) : 0)));
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* Re-rank recall candidates. Returns [{ id, score }] filtered by
|
|
179
|
-
* scoreThreshold and sorted by descending score. Throws on engine failure —
|
|
180
|
-
* the caller degrades to the original candidate order.
|
|
181
|
-
*/
|
|
182
|
-
async rerank(query, candidates) {
|
|
183
|
-
if (!Array.isArray(candidates)) throw new TypeError("rerank expects an array of candidates");
|
|
184
|
-
const list = candidates.slice(0, this.maxCandidates);
|
|
185
|
-
if (!list.length) return [];
|
|
186
|
-
const q = String(query ?? "");
|
|
187
|
-
const results = [];
|
|
188
|
-
for (let i = 0; i < list.length; i += this.batchSize) {
|
|
189
|
-
const chunk = list.slice(i, i + this.batchSize);
|
|
190
|
-
const passages = chunk.map((c) => [c.title, c.content].filter(Boolean).join("\n"));
|
|
191
|
-
const scores = await this._scores(q, passages);
|
|
192
|
-
if (!Array.isArray(scores) || scores.length !== chunk.length) {
|
|
193
|
-
throw new Error("reranker scorer returned mismatched scores");
|
|
194
|
-
}
|
|
195
|
-
for (let j = 0; j < chunk.length; j++) {
|
|
196
|
-
results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
return results
|
|
200
|
-
.filter((r) => r.score >= this.scoreThreshold)
|
|
201
|
-
.sort((a, b) => b.score - a.score);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
get modelHash() {
|
|
205
|
-
return modelHash(this.model);
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
dispose() {
|
|
209
|
-
try {
|
|
210
|
-
this.pipeline?.dispose?.();
|
|
211
|
-
} catch {
|
|
212
|
-
// best-effort: some engines free resources on GC
|
|
213
|
-
}
|
|
214
|
-
this.pipeline = null;
|
|
215
|
-
this._queryVec = null;
|
|
216
|
-
this._queryKey = null;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
1
|
+
import os from "node:os";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
// Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
|
|
5
|
+
// bge-reranker-base through transformers.js: tries the native `rerank` task
|
|
6
|
+
// first, then the sequence-classification head (sigmoid on the logit delta),
|
|
7
|
+
// and finally feature-extraction over concatenated query+passage (cosine).
|
|
8
|
+
// Every strategy funnels into scorePair(query, passage) so tests can inject a
|
|
9
|
+
// fake scorer and never download a model. Failures throw — the caller degrades
|
|
10
|
+
// back to the original candidate order.
|
|
11
|
+
function hashString(s) {
|
|
12
|
+
let h = 5381;
|
|
13
|
+
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
14
|
+
return h.toString(16);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** Same provider#hash fingerprint convention as the embedders. */
|
|
18
|
+
function modelHash(model) {
|
|
19
|
+
return `${model}#${hashString(model)}`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Lazy default pipeline factory: dynamic import keeps module load cheap. */
|
|
23
|
+
async function defaultPipelineLoader(task, model, options) {
|
|
24
|
+
const { env, pipeline } = await import("@huggingface/transformers");
|
|
25
|
+
// issue #13: mirror cache_dir onto env.cacheDir so the tokenizer pre-check
|
|
26
|
+
// resolves locally too (same fix as local-embedder.js).
|
|
27
|
+
if (options?.cache_dir) env.cacheDir = options.cache_dir;
|
|
28
|
+
return pipeline(task, model, options);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Flatten a transformers.js Tensor [batch, seq, dim] into number[][] rows. */
|
|
32
|
+
function tensorToRows(tensor) {
|
|
33
|
+
const { data, dims } = tensor;
|
|
34
|
+
const dim = dims[dims.length - 1] || 0;
|
|
35
|
+
const rows = [];
|
|
36
|
+
for (let i = 0; i < data.length; i += dim) rows.push(Array.from(data.subarray(i, i + dim)));
|
|
37
|
+
if (rows.length === 0 && dim > 0) rows.push(Array.from(data));
|
|
38
|
+
return rows;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function cosine(a, b) {
|
|
42
|
+
let dot = 0;
|
|
43
|
+
let na = 0;
|
|
44
|
+
let nb = 0;
|
|
45
|
+
for (let i = 0; i < a.length; i++) {
|
|
46
|
+
dot += a[i] * b[i];
|
|
47
|
+
na += a[i] * a[i];
|
|
48
|
+
nb += b[i] * b[i];
|
|
49
|
+
}
|
|
50
|
+
if (na === 0 || nb === 0) return 0;
|
|
51
|
+
return dot / Math.sqrt(na * nb);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function clamp01(x) {
|
|
55
|
+
return Math.max(0, Math.min(1, x));
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Pipeline strategies, best to worst. `rerank` is unsupported by the bundled
|
|
59
|
+
// transformers.js (v4.2.0) and rejects cheaply before any model download, so
|
|
60
|
+
// the cascade normally lands on the classification head.
|
|
61
|
+
const STRATEGIES = [
|
|
62
|
+
["rerank", "rerank"],
|
|
63
|
+
["text-classification", "tc"],
|
|
64
|
+
["feature-extraction", "fe"]
|
|
65
|
+
];
|
|
66
|
+
|
|
67
|
+
export class LocalReranker {
|
|
68
|
+
constructor(opts = {}) {
|
|
69
|
+
this.model = opts.model || "Xenova/bge-reranker-base";
|
|
70
|
+
this.batchSize = opts.batchSize || 8;
|
|
71
|
+
this.maxCandidates = opts.maxCandidates || 30;
|
|
72
|
+
this.scoreThreshold = opts.scoreThreshold ?? 0.1;
|
|
73
|
+
this.device = opts.device || "cpu";
|
|
74
|
+
this.cacheDir =
|
|
75
|
+
String(opts.cacheDir ?? "").trim() ||
|
|
76
|
+
path.join(os.homedir(), ".dsh", "mneme", "models");
|
|
77
|
+
this.logger = opts.logger ?? null;
|
|
78
|
+
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
79
|
+
// Injectable seam: async (query, passage) => number. When set, init()
|
|
80
|
+
// skips model loading entirely so tests never hit the network.
|
|
81
|
+
this.scorePair = opts.scorePair ?? null;
|
|
82
|
+
this.pipeline = null;
|
|
83
|
+
this._batchScorer = null;
|
|
84
|
+
this._queryVec = null;
|
|
85
|
+
this._queryKey = null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Load the model; throws when no strategy can be bound. */
|
|
89
|
+
async init() {
|
|
90
|
+
if (this.scorePair) return this;
|
|
91
|
+
let lastErr = null;
|
|
92
|
+
for (const [task, strategy] of STRATEGIES) {
|
|
93
|
+
try {
|
|
94
|
+
this.pipeline = await this.engineFactory(task, this.model, this._engineOptions());
|
|
95
|
+
this._bindBatchScorer(strategy);
|
|
96
|
+
this.logger?.info?.(
|
|
97
|
+
`[dsh-mneme] local reranker ready: ${this.model} (strategy=${strategy}, device=${this.device})`
|
|
98
|
+
);
|
|
99
|
+
return this;
|
|
100
|
+
} catch (err) {
|
|
101
|
+
lastErr = err;
|
|
102
|
+
this.logger?.warn?.(
|
|
103
|
+
`[dsh-mneme] reranker task "${task}" unavailable for ${this.model}: ${String(err?.message ?? err)}`
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
throw new Error(`LocalReranker failed to load ${this.model}: ${String(lastErr?.message ?? lastErr)}`);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
_engineOptions() {
|
|
111
|
+
const options = { device: this.device };
|
|
112
|
+
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
113
|
+
return options;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Bind a batch scorer for the chosen strategy. */
|
|
117
|
+
_bindBatchScorer(strategy) {
|
|
118
|
+
if (strategy === "rerank") {
|
|
119
|
+
// Native cross-encoder task (transformers.js >= 4.6): { query, documents }
|
|
120
|
+
// returns one score per document.
|
|
121
|
+
this._batchScorer = async (query, passages) => {
|
|
122
|
+
const out = await this.pipeline({ query, documents: passages });
|
|
123
|
+
if (!Array.isArray(out)) throw new Error("rerank pipeline returned unexpected output");
|
|
124
|
+
return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
|
|
125
|
+
};
|
|
126
|
+
} else if (strategy === "tc") {
|
|
127
|
+
// Classification head: tokenize (query, passage) pairs, score with
|
|
128
|
+
// sigmoid(l1 - l0) so relevance lands in [0, 1].
|
|
129
|
+
this._batchScorer = async (query, passages) => {
|
|
130
|
+
const { tokenizer, model } = this.pipeline;
|
|
131
|
+
const inputs = tokenizer(passages.map(() => query), {
|
|
132
|
+
text_pair: passages,
|
|
133
|
+
padding: true,
|
|
134
|
+
truncation: true
|
|
135
|
+
});
|
|
136
|
+
const logits = await model(inputs).then((o) => o.logits);
|
|
137
|
+
const dims = logits.dims;
|
|
138
|
+
const cols = dims[dims.length - 1] || 2;
|
|
139
|
+
const rows = [];
|
|
140
|
+
for (let i = 0; i < dims[0]; i++) {
|
|
141
|
+
const base = i * cols;
|
|
142
|
+
const l0 = logits.data[base];
|
|
143
|
+
const l1 = cols > 1 ? logits.data[base + 1] : l0;
|
|
144
|
+
// sigmoid(l1 - l0) == softmax probability of the positive class.
|
|
145
|
+
rows.push(clamp01(1 / (1 + Math.exp(l0 - l1))));
|
|
146
|
+
}
|
|
147
|
+
return rows;
|
|
148
|
+
};
|
|
149
|
+
} else {
|
|
150
|
+
// Feature extraction: mean-pool the concatenated pair and compare with
|
|
151
|
+
// the query embedding via cosine. Degraded but model-agnostic.
|
|
152
|
+
// The query vector is cached per query string, so a new query always
|
|
153
|
+
// recomputes it instead of reusing a stale vector from the previous call.
|
|
154
|
+
this._queryVec = null;
|
|
155
|
+
this._queryKey = null;
|
|
156
|
+
this._batchScorer = async (query, passages) => {
|
|
157
|
+
if (this._queryKey !== query) {
|
|
158
|
+
const t = await this.pipeline([query], { pooling: "mean", normalize: true });
|
|
159
|
+
this._queryVec = tensorToRows(t)[0];
|
|
160
|
+
this._queryKey = query;
|
|
161
|
+
}
|
|
162
|
+
const t = await this.pipeline(passages.map((p) => `${query}\n${p}`), {
|
|
163
|
+
pooling: "mean",
|
|
164
|
+
normalize: true
|
|
165
|
+
});
|
|
166
|
+
return tensorToRows(t).map((v) => clamp01(cosine(this._queryVec, v)));
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Score one batch of passages; injected scorePair scores pair by pair. */
|
|
172
|
+
async _scores(query, passages) {
|
|
173
|
+
if (this._batchScorer) return this._batchScorer(query, passages);
|
|
174
|
+
return Promise.all(passages.map((p) => (p ? this.scorePair(query, p) : 0)));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Re-rank recall candidates. Returns [{ id, score }] filtered by
|
|
179
|
+
* scoreThreshold and sorted by descending score. Throws on engine failure —
|
|
180
|
+
* the caller degrades to the original candidate order.
|
|
181
|
+
*/
|
|
182
|
+
async rerank(query, candidates) {
|
|
183
|
+
if (!Array.isArray(candidates)) throw new TypeError("rerank expects an array of candidates");
|
|
184
|
+
const list = candidates.slice(0, this.maxCandidates);
|
|
185
|
+
if (!list.length) return [];
|
|
186
|
+
const q = String(query ?? "");
|
|
187
|
+
const results = [];
|
|
188
|
+
for (let i = 0; i < list.length; i += this.batchSize) {
|
|
189
|
+
const chunk = list.slice(i, i + this.batchSize);
|
|
190
|
+
const passages = chunk.map((c) => [c.title, c.content].filter(Boolean).join("\n"));
|
|
191
|
+
const scores = await this._scores(q, passages);
|
|
192
|
+
if (!Array.isArray(scores) || scores.length !== chunk.length) {
|
|
193
|
+
throw new Error("reranker scorer returned mismatched scores");
|
|
194
|
+
}
|
|
195
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
196
|
+
results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return results
|
|
200
|
+
.filter((r) => r.score >= this.scoreThreshold)
|
|
201
|
+
.sort((a, b) => b.score - a.score);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
get modelHash() {
|
|
205
|
+
return modelHash(this.model);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
dispose() {
|
|
209
|
+
try {
|
|
210
|
+
this.pipeline?.dispose?.();
|
|
211
|
+
} catch {
|
|
212
|
+
// best-effort: some engines free resources on GC
|
|
213
|
+
}
|
|
214
|
+
this.pipeline = null;
|
|
215
|
+
this._queryVec = null;
|
|
216
|
+
this._queryKey = null;
|
|
217
|
+
}
|
|
218
|
+
}
|