@modusensus/dsh-mneme 0.8.3 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +718 -636
- package/bin/cli.mjs +0 -0
- package/bin/dsh-mneme-mcp.mjs +502 -0
- package/cordis.patch.yml +21 -21
- package/lib/api-standalone.js +495 -316
- package/lib/api.js +1382 -1316
- package/lib/bootstrap.js +211 -0
- package/lib/client.js +4475 -4366
- package/lib/commands.js +95 -95
- package/lib/config.js +533 -510
- package/lib/dream/clustering.js +118 -118
- package/lib/dream/decisions.js +658 -658
- package/lib/dream/narratives.js +44 -0
- package/lib/dream/sleep.js +700 -700
- package/lib/dream.js +1319 -1216
- package/lib/embedding.js +177 -177
- package/lib/entities/extractor.js +249 -249
- package/lib/heat.js +142 -136
- package/lib/hot-memory.js +46 -46
- package/lib/index.js +515 -515
- package/lib/inject.js +299 -295
- package/lib/lang.js +202 -179
- package/lib/local-embedder.js +320 -320
- package/lib/mirror.js +198 -198
- package/lib/quality-filter.js +126 -126
- package/lib/recall-stats.js +103 -0
- package/lib/reranker.js +269 -269
- package/lib/runtime/adopt.js +176 -176
- package/lib/runtime/closure.js +172 -172
- package/lib/runtime/download.js +247 -247
- package/lib/runtime/layout.js +235 -235
- package/lib/runtime/loader.js +249 -249
- package/lib/runtime/model-fetch.js +361 -361
- package/lib/runtime/provision.js +237 -237
- package/lib/runtime/tarball.js +130 -130
- package/lib/runtime/verify.js +191 -191
- package/lib/scope.js +118 -118
- package/lib/search/adaptive.js +22 -22
- package/lib/search/bm25.js +96 -96
- package/lib/service.js +2171 -2087
- package/lib/settings.js +430 -420
- package/lib/store.js +2448 -2367
- package/lib/summarize.js +575 -449
- package/lib/tools.js +574 -574
- package/lib/vector-index.js +110 -110
- package/lib/version-check.js +122 -122
- package/package.json +109 -108
- package/runtime-manifest.json +596 -596
- package/scripts/benchmark-embed.js +201 -201
- package/scripts/benchmark-recall.js +199 -199
- package/scripts/benchmark-rerank.js +166 -166
- package/scripts/build-runtime-manifest.mjs +217 -217
- package/scripts/check-sync.js +42 -42
- package/scripts/e2e-dsh.js +220 -220
- package/scripts/mneme-runtime.mjs +277 -277
- package/scripts/stress-dsh.js +255 -255
- package/scripts/sync-lib.js +47 -47
- package/scripts/sync-test-badge.mjs +61 -61
- package/src/api-standalone.js +495 -316
- package/src/api.js +1382 -1316
- package/src/bootstrap.js +211 -0
- package/src/commands.js +95 -95
- package/src/config.js +533 -510
- package/src/dream/clustering.js +118 -118
- package/src/dream/decisions.js +658 -658
- package/src/dream/narratives.js +44 -0
- package/src/dream/sleep.js +700 -700
- package/src/dream.js +1319 -1216
- package/src/embedding.js +177 -177
- package/src/entities/extractor.js +249 -249
- package/src/heat.js +142 -136
- package/src/hot-memory.js +46 -46
- package/src/index.js +515 -515
- package/src/inject.js +299 -295
- package/src/lang.js +202 -179
- package/src/local-embedder.js +320 -320
- package/src/mirror.js +198 -198
- package/src/quality-filter.js +126 -126
- package/src/recall-stats.js +103 -0
- package/src/reranker.js +269 -269
- package/src/runtime/adopt.js +176 -176
- package/src/runtime/closure.js +172 -172
- package/src/runtime/download.js +247 -247
- package/src/runtime/layout.js +235 -235
- package/src/runtime/loader.js +249 -249
- package/src/runtime/model-fetch.js +361 -361
- package/src/runtime/provision.js +237 -237
- package/src/runtime/tarball.js +130 -130
- package/src/runtime/verify.js +191 -191
- package/src/scope.js +118 -118
- package/src/search/adaptive.js +22 -22
- package/src/search/bm25.js +96 -96
- package/src/service.js +2171 -2087
- package/src/settings.js +430 -420
- package/src/store.js +2448 -2367
- package/src/summarize.js +575 -449
- package/src/tools.js +574 -574
- package/src/vector-index.js +110 -110
- package/src/version-check.js +122 -122
- package/test/api-routes-gaps.test.js +218 -218
- package/test/api.test.js +1578 -1482
- package/test/audit.test.js +448 -448
- package/test/benchmark.test.js +94 -94
- package/test/bootstrap.test.js +220 -0
- package/test/client.test.js +799 -736
- package/test/clustering.test.js +100 -100
- package/test/commands.test.js +106 -106
- package/test/config.test.js +66 -66
- package/test/conflict-freeze.test.js +300 -300
- package/test/dream-archive-guardrail.test.js +116 -116
- package/test/dream-candidate-hybrid.test.js +281 -281
- package/test/dream-coverage-degrade.test.js +86 -86
- package/test/dream.test.js +1257 -1254
- package/test/embedding-ready.test.js +196 -196
- package/test/entities.test.js +547 -547
- package/test/entity-recall.test.js +179 -0
- package/test/epistemic.test.js +298 -298
- package/test/fnew-0112.test.js +311 -311
- package/test/fnew-03.test.js +422 -422
- package/test/graph-api.test.js +175 -175
- package/test/heat.test.js +175 -148
- package/test/helpers/dream-mock.js +82 -82
- package/test/helpers/peer-worker.mjs +25 -25
- package/test/helpers/runtime-source.js +60 -60
- package/test/helpers/tar-builder.js +80 -80
- package/test/hot-memory.test.js +145 -145
- package/test/inject-content-cap.test.js +89 -0
- package/test/inject-first-round-bm25.test.js +89 -89
- package/test/inject-heat.test.js +61 -0
- package/test/inject-rotation.test.js +161 -161
- package/test/inject.test.js +249 -249
- package/test/lang.test.js +87 -87
- package/test/lib-smoke.test.js +96 -96
- package/test/llm-audit.test.js +309 -309
- package/test/llm-message-source.test.js +177 -177
- package/test/local-embedder.test.js +249 -249
- package/test/mcp-stdio.test.js +385 -0
- package/test/mirror-dirty.test.js +424 -424
- package/test/mirror-edit-digest.test.js +187 -187
- package/test/mirror-generation.test.js +499 -499
- package/test/mirror.test.js +249 -249
- package/test/narrative-bars.test.js +162 -0
- package/test/peer-blockers.test.js +193 -193
- package/test/policy-epoch.test.js +259 -259
- package/test/quality-filter.test.js +186 -186
- package/test/reasoning-effort.test.js +746 -746
- package/test/recall-evals.test.js +235 -235
- package/test/recall-layer.test.js +326 -326
- package/test/recall-stats.test.js +117 -0
- package/test/receipt-chain.test.js +451 -451
- package/test/reflection.test.js +226 -226
- package/test/reindex-backfill.test.js +62 -62
- package/test/reranker-single-logit.test.js +73 -73
- package/test/reranker.test.js +197 -197
- package/test/resident-status.test.js +83 -0
- package/test/runtime-adopt.test.js +206 -206
- package/test/runtime-cli.test.js +89 -89
- package/test/runtime-closure.test.js +193 -193
- package/test/runtime-download.test.js +302 -302
- package/test/runtime-layout.test.js +163 -163
- package/test/runtime-loader.test.js +237 -237
- package/test/runtime-manifest.test.js +158 -158
- package/test/runtime-model-fetch.test.js +348 -348
- package/test/runtime-provision.test.js +101 -101
- package/test/runtime-tarball.test.js +81 -81
- package/test/runtime-verify.test.js +214 -214
- package/test/scope-candidates.test.js +239 -239
- package/test/scope-explicit.test.js +452 -452
- package/test/scope-retrieval.test.js +197 -197
- package/test/scope-storage.test.js +255 -255
- package/test/scope-strict.test.js +231 -231
- package/test/search-fusion.test.js +90 -90
- package/test/semantic.test.js +124 -124
- package/test/service-search.test.js +174 -174
- package/test/service.test.js +405 -405
- package/test/settings.test.js +261 -261
- package/test/skip-invalid-restore.test.js +262 -262
- package/test/sleep-action-set.test.js +422 -422
- package/test/sleep-heat.test.js +126 -125
- package/test/sleep-scheduler-rearm.test.js +89 -89
- package/test/sleep.test.js +373 -373
- package/test/standalone-api.test.js +544 -359
- package/test/store-vector-cache.test.js +98 -98
- package/test/store.test.js +383 -383
- package/test/stress.test.js +209 -209
- package/test/summarize.test.js +884 -575
- package/test/tools-memory-get-schema.test.js +65 -65
- package/test/tools.test.js +341 -341
- package/test/updated-at-semantics.test.js +113 -113
- package/test/vector-index.test.js +221 -221
- package/test/version-check.test.js +152 -152
- package/test/webServer-optional.test.js +82 -82
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// dsh-mneme/src/recall-stats.js
|
|
2
|
+
// 记忆复用统计(#217):对 recall_runs + memories 做只读聚合,面板「记忆复用」
|
|
3
|
+
// 卡与 GET /api/dsh-mneme/recall-stats 的数据源。
|
|
4
|
+
// 独立成模块的原因:service.js 已过 2000 行参考线(AGENTS.md 尺寸约定,#221),
|
|
5
|
+
// 新内聚块从第一行就落在自己的文件里,service.js 保留 barrel 出口、调用方零改动。
|
|
6
|
+
// 纯读不写,绝不触发任何 write hook。
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* 口径(issue #217 评论 2026-09-18,锚 5bd2dab):
|
|
10
|
+
* - Top-N:窗口内 recall_runs.candidates(最终返回集)按 id 计数,join
|
|
11
|
+
* memories 补 type/source;记忆已删的 type=null(标题/来源回退候选值)。
|
|
12
|
+
* candidates 只存最终集,「入池未中」语义(B)现有表拿不到——v1 只报
|
|
13
|
+
* 零曝光(A),B 待 recordRecall 记录形状扩展。
|
|
14
|
+
* - 僵尸率:活跃(未归档未遗忘)且窗口内零候选出现;分母剔除写入不足
|
|
15
|
+
* 豁免期的记忆(按当前时刻计,exemptCount 单独报数)。
|
|
16
|
+
* - 注入命中率:注入侧无留痕,待 #217 mode='inject' 口径拍板后接入。
|
|
17
|
+
* - coverage:recallRecordDefault 开启前的窗口算不到,earliestRunAt 为
|
|
18
|
+
* null(窗口内无回执)或早于窗口起点时,前端标注可信度;扫描行数有
|
|
19
|
+
* 上限,超出标 truncated(degraded 口径),不静默少算。
|
|
20
|
+
*
|
|
21
|
+
* @param {object} store - createStore 产物(只调用 listRecallRunsSince / all)
|
|
22
|
+
* @param {{ windowDays?: number, exemptDays?: number }} [options]
|
|
23
|
+
*/
|
|
24
|
+
export function recallStats(store, options = {}) {
|
|
25
|
+
const rawWindow = Number(options.windowDays ?? 30);
|
|
26
|
+
const windowDays = Number.isInteger(rawWindow) && rawWindow >= 1 && rawWindow <= 365 ? rawWindow : 30;
|
|
27
|
+
// 豁免期天数(issue #217 建议 7 天):option 化是给测试归零用 + 维护者
|
|
28
|
+
// 调口径不留硬编码;越界值静默回默认(与 windowDays 同款宽容)。
|
|
29
|
+
const rawExempt = Number(options.exemptDays ?? 7);
|
|
30
|
+
const exemptDays = Number.isInteger(rawExempt) && rawExempt >= 0 && rawExempt <= 365 ? rawExempt : 7;
|
|
31
|
+
const now = Date.now();
|
|
32
|
+
const since = new Date(now - windowDays * 86400000).toISOString();
|
|
33
|
+
|
|
34
|
+
const { rows: runs, total } = store.listRecallRunsSince(since);
|
|
35
|
+
const hits = new Map(); // id -> { count, title, source }
|
|
36
|
+
for (const run of runs) {
|
|
37
|
+
for (const cand of run.candidates ?? []) {
|
|
38
|
+
if (!cand?.id) continue;
|
|
39
|
+
const cur = hits.get(cand.id) ?? { count: 0, title: cand.title ?? null, source: cand.source ?? null };
|
|
40
|
+
cur.count += 1;
|
|
41
|
+
hits.set(cand.id, cur);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const memories = store.all();
|
|
46
|
+
const memoryIndex = new Map(memories.map((m) => [m.id, m]));
|
|
47
|
+
const EXEMPT_MS = exemptDays * 86400000;
|
|
48
|
+
const byType = {};
|
|
49
|
+
const bySource = {};
|
|
50
|
+
let activeCount = 0;
|
|
51
|
+
let zombieCount = 0;
|
|
52
|
+
let exemptCount = 0;
|
|
53
|
+
for (const m of memories) {
|
|
54
|
+
if (m.archived || m.forgotten) continue;
|
|
55
|
+
const createdMs = Date.parse(m.created_at ?? "");
|
|
56
|
+
// created_at 解析不了时无法证明已过机会期 → 归入豁免,宁漏勿误伤
|
|
57
|
+
if (!Number.isFinite(createdMs) || now - createdMs < EXEMPT_MS) {
|
|
58
|
+
exemptCount += 1;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
activeCount += 1;
|
|
62
|
+
const zombied = !hits.has(m.id);
|
|
63
|
+
if (zombied) zombieCount += 1;
|
|
64
|
+
const type = m.type ?? "unknown";
|
|
65
|
+
const source = m.source ?? "unknown";
|
|
66
|
+
if (!byType[type]) byType[type] = { active: 0, zombie: 0 };
|
|
67
|
+
const t = byType[type];
|
|
68
|
+
t.active += 1;
|
|
69
|
+
if (zombied) t.zombie += 1;
|
|
70
|
+
if (!bySource[source]) bySource[source] = { active: 0, zombie: 0 };
|
|
71
|
+
const s = bySource[source];
|
|
72
|
+
s.active += 1;
|
|
73
|
+
if (zombied) s.zombie += 1;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const topRecalled = [...hits.entries()]
|
|
77
|
+
.map(([id, h]) => {
|
|
78
|
+
const m = memoryIndex.get(id);
|
|
79
|
+
return { id, title: m?.title ?? h.title, type: m?.type ?? null, source: m?.source ?? h.source, count: h.count };
|
|
80
|
+
})
|
|
81
|
+
.sort((a, b) => b.count - a.count || a.id.localeCompare(b.id))
|
|
82
|
+
.slice(0, 10);
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
windowDays,
|
|
86
|
+
generatedAt: new Date(now).toISOString(),
|
|
87
|
+
coverage: {
|
|
88
|
+
runsScanned: runs.length,
|
|
89
|
+
runsTotal: total,
|
|
90
|
+
truncated: total > runs.length,
|
|
91
|
+
earliestRunAt: runs.length ? runs[0].created_at : null
|
|
92
|
+
},
|
|
93
|
+
topRecalled,
|
|
94
|
+
zombie: {
|
|
95
|
+
activeCount,
|
|
96
|
+
zombieCount,
|
|
97
|
+
exemptCount,
|
|
98
|
+
rate: activeCount > 0 ? zombieCount / activeCount : null,
|
|
99
|
+
byType,
|
|
100
|
+
bySource
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
}
|
package/lib/reranker.js
CHANGED
|
@@ -1,269 +1,269 @@
|
|
|
1
|
-
import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
|
|
2
|
-
import { installResilientFetch } from "./runtime/model-fetch.js";
|
|
3
|
-
import { loadTransformers } from "./runtime/loader.js";
|
|
4
|
-
|
|
5
|
-
// Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
|
|
6
|
-
// bge-reranker-base through transformers.js: tries the native `rerank` task
|
|
7
|
-
// first, then the sequence-classification head (sigmoid on the logit delta),
|
|
8
|
-
// and finally feature-extraction over concatenated query+passage (cosine).
|
|
9
|
-
// Every strategy funnels into scorePair(query, passage) so tests can inject a
|
|
10
|
-
// fake scorer and never download a model. Failures throw — the caller degrades
|
|
11
|
-
// back to the original candidate order.
|
|
12
|
-
function hashString(s) {
|
|
13
|
-
let h = 5381;
|
|
14
|
-
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
15
|
-
return h.toString(16);
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
/** Same provider#hash fingerprint convention as the embedders. */
|
|
19
|
-
function modelHash(model) {
|
|
20
|
-
return `${model}#${hashString(model)}`;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* Lazy default pipeline factory: dynamic import keeps module load cheap.
|
|
25
|
-
*
|
|
26
|
-
* 与 local-embedder 一样经 runtime/loader.js 的三层解析取运行时(自管 payload 优先,
|
|
27
|
-
* 其次宿主裸 specifier),这样 PR-B 把依赖从 profile 摘掉后 rerank 也还能用。
|
|
28
|
-
* runtimeDir 取走后不混进 transformers 的选项。
|
|
29
|
-
*/
|
|
30
|
-
async function defaultPipelineLoader(task, model, options) {
|
|
31
|
-
const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
|
|
32
|
-
const { module } = await loadTransformers({
|
|
33
|
-
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
34
|
-
});
|
|
35
|
-
const { env, pipeline } = module;
|
|
36
|
-
// issue #13 同款处理:transformers.js 在预检 tokenizer_config.json 元数据时会丢掉调用方
|
|
37
|
-
// 的 cache_dir,回退到 env.cacheDir —— 即使模型已完全缓存在本地也会去请求网络。
|
|
38
|
-
// embedder 侧早就镜像了,reranker 侧一直漏着,于是「已缓存 + 离线」时重排初始化会失败。
|
|
39
|
-
if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
|
|
40
|
-
// issue #188 补充:embedModelMirror 此前是死配置——transformers.js 不认
|
|
41
|
-
// HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
|
|
42
|
-
// 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
|
|
43
|
-
if (remoteHost) env.remoteHost = remoteHost;
|
|
44
|
-
// issue #194:与 embedder 同款断点续传。开关关 = env.fetch 一字不动。
|
|
45
|
-
installResilientFetch(env, {
|
|
46
|
-
enabled: resilientModelDownload === true,
|
|
47
|
-
cacheDir: pipelineOptions.cache_dir,
|
|
48
|
-
logger
|
|
49
|
-
});
|
|
50
|
-
return pipeline(task, model, pipelineOptions);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/** Flatten a transformers.js Tensor [batch, seq, dim] into number[][] rows. */
|
|
54
|
-
function tensorToRows(tensor) {
|
|
55
|
-
const { data, dims } = tensor;
|
|
56
|
-
const dim = dims[dims.length - 1] || 0;
|
|
57
|
-
const rows = [];
|
|
58
|
-
for (let i = 0; i < data.length; i += dim) rows.push(Array.from(data.subarray(i, i + dim)));
|
|
59
|
-
if (rows.length === 0 && dim > 0) rows.push(Array.from(data));
|
|
60
|
-
return rows;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
function cosine(a, b) {
|
|
64
|
-
let dot = 0;
|
|
65
|
-
let na = 0;
|
|
66
|
-
let nb = 0;
|
|
67
|
-
for (let i = 0; i < a.length; i++) {
|
|
68
|
-
dot += a[i] * b[i];
|
|
69
|
-
na += a[i] * a[i];
|
|
70
|
-
nb += b[i] * b[i];
|
|
71
|
-
}
|
|
72
|
-
if (na === 0 || nb === 0) return 0;
|
|
73
|
-
return dot / Math.sqrt(na * nb);
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function clamp01(x) {
|
|
77
|
-
return Math.max(0, Math.min(1, x));
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
// Pipeline strategies, best to worst. `rerank` is unsupported by the bundled
|
|
81
|
-
// transformers.js (v4.2.0) and rejects cheaply before any model download, so
|
|
82
|
-
// the cascade normally lands on the classification head.
|
|
83
|
-
const STRATEGIES = [
|
|
84
|
-
["rerank", "rerank"],
|
|
85
|
-
["text-classification", "tc"],
|
|
86
|
-
["feature-extraction", "fe"]
|
|
87
|
-
];
|
|
88
|
-
|
|
89
|
-
export class LocalReranker {
|
|
90
|
-
constructor(opts = {}) {
|
|
91
|
-
this.model = opts.model || "Xenova/bge-reranker-base";
|
|
92
|
-
this.batchSize = opts.batchSize || 8;
|
|
93
|
-
this.maxCandidates = opts.maxCandidates || 30;
|
|
94
|
-
this.scoreThreshold = opts.scoreThreshold ?? 0.1;
|
|
95
|
-
this.device = opts.device || "cpu";
|
|
96
|
-
this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
|
|
97
|
-
// 自管运行时根目录(issue #131):空表示用默认位置,解析交给 loader。
|
|
98
|
-
this.runtimeDir = String(opts.runtimeDir ?? "").trim();
|
|
99
|
-
// 与嵌入层同约定:默认 q8(4 倍小的 model_quantized.onnx),"fp32" 可关。
|
|
100
|
-
// 此前不传 dtype,即使缓存里有量化文件也会去要 1GB 级的 model.onnx(#188)。
|
|
101
|
-
this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
|
|
102
|
-
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
103
|
-
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
104
|
-
// issue #194:模型文件下载断点续传。只在显式传 true 时启用,与 embedder 同规。
|
|
105
|
-
this.resilientModelDownload = opts.resilientModelDownload === true;
|
|
106
|
-
this.logger = opts.logger ?? null;
|
|
107
|
-
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
108
|
-
// Injectable seam: async (query, passage) => number. When set, init()
|
|
109
|
-
// skips model loading entirely so tests never hit the network.
|
|
110
|
-
this.scorePair = opts.scorePair ?? null;
|
|
111
|
-
this.pipeline = null;
|
|
112
|
-
this._batchScorer = null;
|
|
113
|
-
this._queryVec = null;
|
|
114
|
-
this._queryKey = null;
|
|
115
|
-
this._constantScoreWarned = false;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/** Load the model; throws when no strategy can be bound. */
|
|
119
|
-
async init() {
|
|
120
|
-
if (this.scorePair) return this;
|
|
121
|
-
let lastErr = null;
|
|
122
|
-
for (const [task, strategy] of STRATEGIES) {
|
|
123
|
-
try {
|
|
124
|
-
this.pipeline = await this.engineFactory(task, this.model, this._engineOptions());
|
|
125
|
-
this._bindBatchScorer(strategy);
|
|
126
|
-
this.logger?.info?.(
|
|
127
|
-
`[dsh-mneme] local reranker ready: ${this.model} (strategy=${strategy}, device=${this.device})`
|
|
128
|
-
);
|
|
129
|
-
return this;
|
|
130
|
-
} catch (err) {
|
|
131
|
-
lastErr = err;
|
|
132
|
-
this.logger?.warn?.(
|
|
133
|
-
`[dsh-mneme] reranker task "${task}" unavailable for ${this.model}: ${String(err?.message ?? err)}`
|
|
134
|
-
);
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
throw new Error(`LocalReranker failed to load ${this.model}: ${String(lastErr?.message ?? lastErr)}`);
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
_engineOptions() {
|
|
141
|
-
const options = { device: this.device, dtype: this.useDtype, resilientModelDownload: this.resilientModelDownload, logger: this.logger };
|
|
142
|
-
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
143
|
-
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
144
|
-
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
145
|
-
return options;
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/** Bind a batch scorer for the chosen strategy. */
|
|
149
|
-
_bindBatchScorer(strategy) {
|
|
150
|
-
if (strategy === "rerank") {
|
|
151
|
-
// Native cross-encoder task (transformers.js >= 4.6): { query, documents }
|
|
152
|
-
// returns one score per document.
|
|
153
|
-
this._batchScorer = async (query, passages) => {
|
|
154
|
-
const out = await this.pipeline({ query, documents: passages });
|
|
155
|
-
if (!Array.isArray(out)) throw new Error("rerank pipeline returned unexpected output");
|
|
156
|
-
return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
|
|
157
|
-
};
|
|
158
|
-
} else if (strategy === "tc") {
|
|
159
|
-
// Classification head: score so relevance lands in [0, 1]. Branch on the
|
|
160
|
-
// output width (issue #188): a dual-logit head (num_labels=2 exports,
|
|
161
|
-
// e.g. bge-reranker-v2-m3) gives the positive class as the softmax
|
|
162
|
-
// probability sigmoid(l1 - l0); a SINGLE-logit head (e.g.
|
|
163
|
-
// Xenova/bge-reranker-base) emits one raw relevance logit per pair, and
|
|
164
|
-
// the positive class is simply sigmoid(logit) — reading l1 as l0 there
|
|
165
|
-
// collapses every score to sigmoid(0) = 0.5 and silently disables
|
|
166
|
-
// reranking (no error, status still "ready").
|
|
167
|
-
this._batchScorer = async (query, passages) => {
|
|
168
|
-
const { tokenizer, model } = this.pipeline;
|
|
169
|
-
const inputs = tokenizer(passages.map(() => query), {
|
|
170
|
-
text_pair: passages,
|
|
171
|
-
padding: true,
|
|
172
|
-
truncation: true
|
|
173
|
-
});
|
|
174
|
-
const logits = await model(inputs).then((o) => o.logits);
|
|
175
|
-
const dims = logits.dims;
|
|
176
|
-
const cols = dims[dims.length - 1] || 2;
|
|
177
|
-
const rows = [];
|
|
178
|
-
for (let i = 0; i < dims[0]; i++) {
|
|
179
|
-
const base = i * cols;
|
|
180
|
-
const score = cols > 1
|
|
181
|
-
? 1 / (1 + Math.exp(logits.data[base] - logits.data[base + 1]))
|
|
182
|
-
: 1 / (1 + Math.exp(-logits.data[base]));
|
|
183
|
-
rows.push(clamp01(score));
|
|
184
|
-
}
|
|
185
|
-
return rows;
|
|
186
|
-
};
|
|
187
|
-
} else {
|
|
188
|
-
// Feature extraction: mean-pool the concatenated pair and compare with
|
|
189
|
-
// the query embedding via cosine. Degraded but model-agnostic.
|
|
190
|
-
// The query vector is cached per query string, so a new query always
|
|
191
|
-
// recomputes it instead of reusing a stale vector from the previous call.
|
|
192
|
-
this._queryVec = null;
|
|
193
|
-
this._queryKey = null;
|
|
194
|
-
this._batchScorer = async (query, passages) => {
|
|
195
|
-
if (this._queryKey !== query) {
|
|
196
|
-
const t = await this.pipeline([query], { pooling: "mean", normalize: true });
|
|
197
|
-
this._queryVec = tensorToRows(t)[0];
|
|
198
|
-
this._queryKey = query;
|
|
199
|
-
}
|
|
200
|
-
const t = await this.pipeline(passages.map((p) => `${query}\n${p}`), {
|
|
201
|
-
pooling: "mean",
|
|
202
|
-
normalize: true
|
|
203
|
-
});
|
|
204
|
-
return tensorToRows(t).map((v) => clamp01(cosine(this._queryVec, v)));
|
|
205
|
-
};
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
/** Score one batch of passages; injected scorePair scores pair by pair. */
|
|
210
|
-
async _scores(query, passages) {
|
|
211
|
-
if (this._batchScorer) return this._batchScorer(query, passages);
|
|
212
|
-
return Promise.all(passages.map((p) => (p ? this.scorePair(query, p) : 0)));
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
/**
|
|
216
|
-
* Re-rank recall candidates. Returns [{ id, score }] filtered by
|
|
217
|
-
* scoreThreshold and sorted by descending score. Throws on engine failure —
|
|
218
|
-
* the caller degrades to the original candidate order.
|
|
219
|
-
*/
|
|
220
|
-
async rerank(query, candidates) {
|
|
221
|
-
if (!Array.isArray(candidates)) throw new TypeError("rerank expects an array of candidates");
|
|
222
|
-
const list = candidates.slice(0, this.maxCandidates);
|
|
223
|
-
if (!list.length) return [];
|
|
224
|
-
const q = String(query ?? "");
|
|
225
|
-
const results = [];
|
|
226
|
-
for (let i = 0; i < list.length; i += this.batchSize) {
|
|
227
|
-
const chunk = list.slice(i, i + this.batchSize);
|
|
228
|
-
const passages = chunk.map((c) => [c.title, c.content].filter(Boolean).join("\n"));
|
|
229
|
-
const scores = await this._scores(q, passages);
|
|
230
|
-
if (!Array.isArray(scores) || scores.length !== chunk.length) {
|
|
231
|
-
throw new Error("reranker scorer returned mismatched scores");
|
|
232
|
-
}
|
|
233
|
-
for (let j = 0; j < chunk.length; j++) {
|
|
234
|
-
results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
// Silent-failure tripwire (issue #188): a reranker whose every candidate
|
|
238
|
-
// scores identically is not ranking anything — e.g. a single-logit head
|
|
239
|
-
// read through the dual-logit formula. Warn once so the degradation is
|
|
240
|
-
// observable instead of a constant 0.5 column hiding behind a "ready"
|
|
241
|
-
// status forever.
|
|
242
|
-
if (results.length > 1 && results.every((r) => r.score === results[0].score)) {
|
|
243
|
-
if (!this._constantScoreWarned) {
|
|
244
|
-
this._constantScoreWarned = true;
|
|
245
|
-
this.logger?.warn?.(
|
|
246
|
-
`[dsh-mneme] reranker scores degenerated to a constant (${results[0].score}) — check the model's output dims (single vs dual logit head)`
|
|
247
|
-
);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
return results
|
|
251
|
-
.filter((r) => r.score >= this.scoreThreshold)
|
|
252
|
-
.sort((a, b) => b.score - a.score);
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
get modelHash() {
|
|
256
|
-
return modelHash(this.model);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
dispose() {
|
|
260
|
-
try {
|
|
261
|
-
this.pipeline?.dispose?.();
|
|
262
|
-
} catch {
|
|
263
|
-
// best-effort: some engines free resources on GC
|
|
264
|
-
}
|
|
265
|
-
this.pipeline = null;
|
|
266
|
-
this._queryVec = null;
|
|
267
|
-
this._queryKey = null;
|
|
268
|
-
}
|
|
269
|
-
}
|
|
1
|
+
import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
|
|
2
|
+
import { installResilientFetch } from "./runtime/model-fetch.js";
|
|
3
|
+
import { loadTransformers } from "./runtime/loader.js";
|
|
4
|
+
|
|
5
|
+
// Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
|
|
6
|
+
// bge-reranker-base through transformers.js: tries the native `rerank` task
|
|
7
|
+
// first, then the sequence-classification head (sigmoid on the logit delta),
|
|
8
|
+
// and finally feature-extraction over concatenated query+passage (cosine).
|
|
9
|
+
// Every strategy funnels into scorePair(query, passage) so tests can inject a
|
|
10
|
+
// fake scorer and never download a model. Failures throw — the caller degrades
|
|
11
|
+
// back to the original candidate order.
|
|
12
|
+
function hashString(s) {
|
|
13
|
+
let h = 5381;
|
|
14
|
+
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
15
|
+
return h.toString(16);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Same provider#hash fingerprint convention as the embedders. */
|
|
19
|
+
function modelHash(model) {
|
|
20
|
+
return `${model}#${hashString(model)}`;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Lazy default pipeline factory: dynamic import keeps module load cheap.
|
|
25
|
+
*
|
|
26
|
+
* 与 local-embedder 一样经 runtime/loader.js 的三层解析取运行时(自管 payload 优先,
|
|
27
|
+
* 其次宿主裸 specifier),这样 PR-B 把依赖从 profile 摘掉后 rerank 也还能用。
|
|
28
|
+
* runtimeDir 取走后不混进 transformers 的选项。
|
|
29
|
+
*/
|
|
30
|
+
async function defaultPipelineLoader(task, model, options) {
|
|
31
|
+
const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
|
|
32
|
+
const { module } = await loadTransformers({
|
|
33
|
+
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
34
|
+
});
|
|
35
|
+
const { env, pipeline } = module;
|
|
36
|
+
// issue #13 同款处理:transformers.js 在预检 tokenizer_config.json 元数据时会丢掉调用方
|
|
37
|
+
// 的 cache_dir,回退到 env.cacheDir —— 即使模型已完全缓存在本地也会去请求网络。
|
|
38
|
+
// embedder 侧早就镜像了,reranker 侧一直漏着,于是「已缓存 + 离线」时重排初始化会失败。
|
|
39
|
+
if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
|
|
40
|
+
// issue #188 补充:embedModelMirror 此前是死配置——transformers.js 不认
|
|
41
|
+
// HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
|
|
42
|
+
// 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
|
|
43
|
+
if (remoteHost) env.remoteHost = remoteHost;
|
|
44
|
+
// issue #194:与 embedder 同款断点续传。开关关 = env.fetch 一字不动。
|
|
45
|
+
installResilientFetch(env, {
|
|
46
|
+
enabled: resilientModelDownload === true,
|
|
47
|
+
cacheDir: pipelineOptions.cache_dir,
|
|
48
|
+
logger
|
|
49
|
+
});
|
|
50
|
+
return pipeline(task, model, pipelineOptions);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Flatten a transformers.js Tensor [batch, seq, dim] into number[][] rows. */
|
|
54
|
+
function tensorToRows(tensor) {
|
|
55
|
+
const { data, dims } = tensor;
|
|
56
|
+
const dim = dims[dims.length - 1] || 0;
|
|
57
|
+
const rows = [];
|
|
58
|
+
for (let i = 0; i < data.length; i += dim) rows.push(Array.from(data.subarray(i, i + dim)));
|
|
59
|
+
if (rows.length === 0 && dim > 0) rows.push(Array.from(data));
|
|
60
|
+
return rows;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function cosine(a, b) {
|
|
64
|
+
let dot = 0;
|
|
65
|
+
let na = 0;
|
|
66
|
+
let nb = 0;
|
|
67
|
+
for (let i = 0; i < a.length; i++) {
|
|
68
|
+
dot += a[i] * b[i];
|
|
69
|
+
na += a[i] * a[i];
|
|
70
|
+
nb += b[i] * b[i];
|
|
71
|
+
}
|
|
72
|
+
if (na === 0 || nb === 0) return 0;
|
|
73
|
+
return dot / Math.sqrt(na * nb);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function clamp01(x) {
|
|
77
|
+
return Math.max(0, Math.min(1, x));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Pipeline strategies, best to worst. `rerank` is unsupported by the bundled
|
|
81
|
+
// transformers.js (v4.2.0) and rejects cheaply before any model download, so
|
|
82
|
+
// the cascade normally lands on the classification head.
|
|
83
|
+
const STRATEGIES = [
|
|
84
|
+
["rerank", "rerank"],
|
|
85
|
+
["text-classification", "tc"],
|
|
86
|
+
["feature-extraction", "fe"]
|
|
87
|
+
];
|
|
88
|
+
|
|
89
|
+
export class LocalReranker {
|
|
90
|
+
constructor(opts = {}) {
|
|
91
|
+
this.model = opts.model || "Xenova/bge-reranker-base";
|
|
92
|
+
this.batchSize = opts.batchSize || 8;
|
|
93
|
+
this.maxCandidates = opts.maxCandidates || 30;
|
|
94
|
+
this.scoreThreshold = opts.scoreThreshold ?? 0.1;
|
|
95
|
+
this.device = opts.device || "cpu";
|
|
96
|
+
this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
|
|
97
|
+
// 自管运行时根目录(issue #131):空表示用默认位置,解析交给 loader。
|
|
98
|
+
this.runtimeDir = String(opts.runtimeDir ?? "").trim();
|
|
99
|
+
// 与嵌入层同约定:默认 q8(4 倍小的 model_quantized.onnx),"fp32" 可关。
|
|
100
|
+
// 此前不传 dtype,即使缓存里有量化文件也会去要 1GB 级的 model.onnx(#188)。
|
|
101
|
+
this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
|
|
102
|
+
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
103
|
+
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
104
|
+
// issue #194:模型文件下载断点续传。只在显式传 true 时启用,与 embedder 同规。
|
|
105
|
+
this.resilientModelDownload = opts.resilientModelDownload === true;
|
|
106
|
+
this.logger = opts.logger ?? null;
|
|
107
|
+
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
108
|
+
// Injectable seam: async (query, passage) => number. When set, init()
|
|
109
|
+
// skips model loading entirely so tests never hit the network.
|
|
110
|
+
this.scorePair = opts.scorePair ?? null;
|
|
111
|
+
this.pipeline = null;
|
|
112
|
+
this._batchScorer = null;
|
|
113
|
+
this._queryVec = null;
|
|
114
|
+
this._queryKey = null;
|
|
115
|
+
this._constantScoreWarned = false;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Load the model; throws when no strategy can be bound. */
|
|
119
|
+
async init() {
|
|
120
|
+
if (this.scorePair) return this;
|
|
121
|
+
let lastErr = null;
|
|
122
|
+
for (const [task, strategy] of STRATEGIES) {
|
|
123
|
+
try {
|
|
124
|
+
this.pipeline = await this.engineFactory(task, this.model, this._engineOptions());
|
|
125
|
+
this._bindBatchScorer(strategy);
|
|
126
|
+
this.logger?.info?.(
|
|
127
|
+
`[dsh-mneme] local reranker ready: ${this.model} (strategy=${strategy}, device=${this.device})`
|
|
128
|
+
);
|
|
129
|
+
return this;
|
|
130
|
+
} catch (err) {
|
|
131
|
+
lastErr = err;
|
|
132
|
+
this.logger?.warn?.(
|
|
133
|
+
`[dsh-mneme] reranker task "${task}" unavailable for ${this.model}: ${String(err?.message ?? err)}`
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
throw new Error(`LocalReranker failed to load ${this.model}: ${String(lastErr?.message ?? lastErr)}`);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
_engineOptions() {
|
|
141
|
+
const options = { device: this.device, dtype: this.useDtype, resilientModelDownload: this.resilientModelDownload, logger: this.logger };
|
|
142
|
+
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
143
|
+
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
144
|
+
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
145
|
+
return options;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Bind a batch scorer for the chosen strategy. */
|
|
149
|
+
_bindBatchScorer(strategy) {
|
|
150
|
+
if (strategy === "rerank") {
|
|
151
|
+
// Native cross-encoder task (transformers.js >= 4.6): { query, documents }
|
|
152
|
+
// returns one score per document.
|
|
153
|
+
this._batchScorer = async (query, passages) => {
|
|
154
|
+
const out = await this.pipeline({ query, documents: passages });
|
|
155
|
+
if (!Array.isArray(out)) throw new Error("rerank pipeline returned unexpected output");
|
|
156
|
+
return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
|
|
157
|
+
};
|
|
158
|
+
} else if (strategy === "tc") {
|
|
159
|
+
// Classification head: score so relevance lands in [0, 1]. Branch on the
|
|
160
|
+
// output width (issue #188): a dual-logit head (num_labels=2 exports,
|
|
161
|
+
// e.g. bge-reranker-v2-m3) gives the positive class as the softmax
|
|
162
|
+
// probability sigmoid(l1 - l0); a SINGLE-logit head (e.g.
|
|
163
|
+
// Xenova/bge-reranker-base) emits one raw relevance logit per pair, and
|
|
164
|
+
// the positive class is simply sigmoid(logit) — reading l1 as l0 there
|
|
165
|
+
// collapses every score to sigmoid(0) = 0.5 and silently disables
|
|
166
|
+
// reranking (no error, status still "ready").
|
|
167
|
+
this._batchScorer = async (query, passages) => {
|
|
168
|
+
const { tokenizer, model } = this.pipeline;
|
|
169
|
+
const inputs = tokenizer(passages.map(() => query), {
|
|
170
|
+
text_pair: passages,
|
|
171
|
+
padding: true,
|
|
172
|
+
truncation: true
|
|
173
|
+
});
|
|
174
|
+
const logits = await model(inputs).then((o) => o.logits);
|
|
175
|
+
const dims = logits.dims;
|
|
176
|
+
const cols = dims[dims.length - 1] || 2;
|
|
177
|
+
const rows = [];
|
|
178
|
+
for (let i = 0; i < dims[0]; i++) {
|
|
179
|
+
const base = i * cols;
|
|
180
|
+
const score = cols > 1
|
|
181
|
+
? 1 / (1 + Math.exp(logits.data[base] - logits.data[base + 1]))
|
|
182
|
+
: 1 / (1 + Math.exp(-logits.data[base]));
|
|
183
|
+
rows.push(clamp01(score));
|
|
184
|
+
}
|
|
185
|
+
return rows;
|
|
186
|
+
};
|
|
187
|
+
} else {
|
|
188
|
+
// Feature extraction: mean-pool the concatenated pair and compare with
|
|
189
|
+
// the query embedding via cosine. Degraded but model-agnostic.
|
|
190
|
+
// The query vector is cached per query string, so a new query always
|
|
191
|
+
// recomputes it instead of reusing a stale vector from the previous call.
|
|
192
|
+
this._queryVec = null;
|
|
193
|
+
this._queryKey = null;
|
|
194
|
+
this._batchScorer = async (query, passages) => {
|
|
195
|
+
if (this._queryKey !== query) {
|
|
196
|
+
const t = await this.pipeline([query], { pooling: "mean", normalize: true });
|
|
197
|
+
this._queryVec = tensorToRows(t)[0];
|
|
198
|
+
this._queryKey = query;
|
|
199
|
+
}
|
|
200
|
+
const t = await this.pipeline(passages.map((p) => `${query}\n${p}`), {
|
|
201
|
+
pooling: "mean",
|
|
202
|
+
normalize: true
|
|
203
|
+
});
|
|
204
|
+
return tensorToRows(t).map((v) => clamp01(cosine(this._queryVec, v)));
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Score one batch of passages; injected scorePair scores pair by pair. */
|
|
210
|
+
async _scores(query, passages) {
|
|
211
|
+
if (this._batchScorer) return this._batchScorer(query, passages);
|
|
212
|
+
return Promise.all(passages.map((p) => (p ? this.scorePair(query, p) : 0)));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Re-rank recall candidates. Returns [{ id, score }] filtered by
|
|
217
|
+
* scoreThreshold and sorted by descending score. Throws on engine failure —
|
|
218
|
+
* the caller degrades to the original candidate order.
|
|
219
|
+
*/
|
|
220
|
+
async rerank(query, candidates) {
|
|
221
|
+
if (!Array.isArray(candidates)) throw new TypeError("rerank expects an array of candidates");
|
|
222
|
+
const list = candidates.slice(0, this.maxCandidates);
|
|
223
|
+
if (!list.length) return [];
|
|
224
|
+
const q = String(query ?? "");
|
|
225
|
+
const results = [];
|
|
226
|
+
for (let i = 0; i < list.length; i += this.batchSize) {
|
|
227
|
+
const chunk = list.slice(i, i + this.batchSize);
|
|
228
|
+
const passages = chunk.map((c) => [c.title, c.content].filter(Boolean).join("\n"));
|
|
229
|
+
const scores = await this._scores(q, passages);
|
|
230
|
+
if (!Array.isArray(scores) || scores.length !== chunk.length) {
|
|
231
|
+
throw new Error("reranker scorer returned mismatched scores");
|
|
232
|
+
}
|
|
233
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
234
|
+
results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
// Silent-failure tripwire (issue #188): a reranker whose every candidate
|
|
238
|
+
// scores identically is not ranking anything — e.g. a single-logit head
|
|
239
|
+
// read through the dual-logit formula. Warn once so the degradation is
|
|
240
|
+
// observable instead of a constant 0.5 column hiding behind a "ready"
|
|
241
|
+
// status forever.
|
|
242
|
+
if (results.length > 1 && results.every((r) => r.score === results[0].score)) {
|
|
243
|
+
if (!this._constantScoreWarned) {
|
|
244
|
+
this._constantScoreWarned = true;
|
|
245
|
+
this.logger?.warn?.(
|
|
246
|
+
`[dsh-mneme] reranker scores degenerated to a constant (${results[0].score}) — check the model's output dims (single vs dual logit head)`
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return results
|
|
251
|
+
.filter((r) => r.score >= this.scoreThreshold)
|
|
252
|
+
.sort((a, b) => b.score - a.score);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
get modelHash() {
|
|
256
|
+
return modelHash(this.model);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
dispose() {
|
|
260
|
+
try {
|
|
261
|
+
this.pipeline?.dispose?.();
|
|
262
|
+
} catch {
|
|
263
|
+
// best-effort: some engines free resources on GC
|
|
264
|
+
}
|
|
265
|
+
this.pipeline = null;
|
|
266
|
+
this._queryVec = null;
|
|
267
|
+
this._queryKey = null;
|
|
268
|
+
}
|
|
269
|
+
}
|