@modusensus/dsh-mneme 0.8.3 → 0.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +718 -636
  3. package/bin/cli.mjs +0 -0
  4. package/bin/dsh-mneme-mcp.mjs +502 -0
  5. package/cordis.patch.yml +21 -21
  6. package/lib/api-standalone.js +495 -316
  7. package/lib/api.js +1382 -1316
  8. package/lib/bootstrap.js +211 -0
  9. package/lib/client.js +4475 -4366
  10. package/lib/commands.js +95 -95
  11. package/lib/config.js +533 -510
  12. package/lib/dream/clustering.js +118 -118
  13. package/lib/dream/decisions.js +658 -658
  14. package/lib/dream/narratives.js +44 -0
  15. package/lib/dream/sleep.js +700 -700
  16. package/lib/dream.js +1319 -1216
  17. package/lib/embedding.js +177 -177
  18. package/lib/entities/extractor.js +249 -249
  19. package/lib/heat.js +142 -136
  20. package/lib/hot-memory.js +46 -46
  21. package/lib/index.js +515 -515
  22. package/lib/inject.js +299 -295
  23. package/lib/lang.js +202 -179
  24. package/lib/local-embedder.js +320 -320
  25. package/lib/mirror.js +198 -198
  26. package/lib/quality-filter.js +126 -126
  27. package/lib/recall-stats.js +103 -0
  28. package/lib/reranker.js +269 -269
  29. package/lib/runtime/adopt.js +176 -176
  30. package/lib/runtime/closure.js +172 -172
  31. package/lib/runtime/download.js +247 -247
  32. package/lib/runtime/layout.js +235 -235
  33. package/lib/runtime/loader.js +249 -249
  34. package/lib/runtime/model-fetch.js +361 -361
  35. package/lib/runtime/provision.js +237 -237
  36. package/lib/runtime/tarball.js +130 -130
  37. package/lib/runtime/verify.js +191 -191
  38. package/lib/scope.js +118 -118
  39. package/lib/search/adaptive.js +22 -22
  40. package/lib/search/bm25.js +96 -96
  41. package/lib/service.js +2171 -2087
  42. package/lib/settings.js +430 -420
  43. package/lib/store.js +2448 -2367
  44. package/lib/summarize.js +575 -449
  45. package/lib/tools.js +574 -574
  46. package/lib/vector-index.js +110 -110
  47. package/lib/version-check.js +122 -122
  48. package/package.json +109 -108
  49. package/runtime-manifest.json +596 -596
  50. package/scripts/benchmark-embed.js +201 -201
  51. package/scripts/benchmark-recall.js +199 -199
  52. package/scripts/benchmark-rerank.js +166 -166
  53. package/scripts/build-runtime-manifest.mjs +217 -217
  54. package/scripts/check-sync.js +42 -42
  55. package/scripts/e2e-dsh.js +220 -220
  56. package/scripts/mneme-runtime.mjs +277 -277
  57. package/scripts/stress-dsh.js +255 -255
  58. package/scripts/sync-lib.js +47 -47
  59. package/scripts/sync-test-badge.mjs +61 -61
  60. package/src/api-standalone.js +495 -316
  61. package/src/api.js +1382 -1316
  62. package/src/bootstrap.js +211 -0
  63. package/src/commands.js +95 -95
  64. package/src/config.js +533 -510
  65. package/src/dream/clustering.js +118 -118
  66. package/src/dream/decisions.js +658 -658
  67. package/src/dream/narratives.js +44 -0
  68. package/src/dream/sleep.js +700 -700
  69. package/src/dream.js +1319 -1216
  70. package/src/embedding.js +177 -177
  71. package/src/entities/extractor.js +249 -249
  72. package/src/heat.js +142 -136
  73. package/src/hot-memory.js +46 -46
  74. package/src/index.js +515 -515
  75. package/src/inject.js +299 -295
  76. package/src/lang.js +202 -179
  77. package/src/local-embedder.js +320 -320
  78. package/src/mirror.js +198 -198
  79. package/src/quality-filter.js +126 -126
  80. package/src/recall-stats.js +103 -0
  81. package/src/reranker.js +269 -269
  82. package/src/runtime/adopt.js +176 -176
  83. package/src/runtime/closure.js +172 -172
  84. package/src/runtime/download.js +247 -247
  85. package/src/runtime/layout.js +235 -235
  86. package/src/runtime/loader.js +249 -249
  87. package/src/runtime/model-fetch.js +361 -361
  88. package/src/runtime/provision.js +237 -237
  89. package/src/runtime/tarball.js +130 -130
  90. package/src/runtime/verify.js +191 -191
  91. package/src/scope.js +118 -118
  92. package/src/search/adaptive.js +22 -22
  93. package/src/search/bm25.js +96 -96
  94. package/src/service.js +2171 -2087
  95. package/src/settings.js +430 -420
  96. package/src/store.js +2448 -2367
  97. package/src/summarize.js +575 -449
  98. package/src/tools.js +574 -574
  99. package/src/vector-index.js +110 -110
  100. package/src/version-check.js +122 -122
  101. package/test/api-routes-gaps.test.js +218 -218
  102. package/test/api.test.js +1578 -1482
  103. package/test/audit.test.js +448 -448
  104. package/test/benchmark.test.js +94 -94
  105. package/test/bootstrap.test.js +220 -0
  106. package/test/client.test.js +799 -736
  107. package/test/clustering.test.js +100 -100
  108. package/test/commands.test.js +106 -106
  109. package/test/config.test.js +66 -66
  110. package/test/conflict-freeze.test.js +300 -300
  111. package/test/dream-archive-guardrail.test.js +116 -116
  112. package/test/dream-candidate-hybrid.test.js +281 -281
  113. package/test/dream-coverage-degrade.test.js +86 -86
  114. package/test/dream.test.js +1257 -1254
  115. package/test/embedding-ready.test.js +196 -196
  116. package/test/entities.test.js +547 -547
  117. package/test/entity-recall.test.js +179 -0
  118. package/test/epistemic.test.js +298 -298
  119. package/test/fnew-0112.test.js +311 -311
  120. package/test/fnew-03.test.js +422 -422
  121. package/test/graph-api.test.js +175 -175
  122. package/test/heat.test.js +175 -148
  123. package/test/helpers/dream-mock.js +82 -82
  124. package/test/helpers/peer-worker.mjs +25 -25
  125. package/test/helpers/runtime-source.js +60 -60
  126. package/test/helpers/tar-builder.js +80 -80
  127. package/test/hot-memory.test.js +145 -145
  128. package/test/inject-content-cap.test.js +89 -0
  129. package/test/inject-first-round-bm25.test.js +89 -89
  130. package/test/inject-heat.test.js +61 -0
  131. package/test/inject-rotation.test.js +161 -161
  132. package/test/inject.test.js +249 -249
  133. package/test/lang.test.js +87 -87
  134. package/test/lib-smoke.test.js +96 -96
  135. package/test/llm-audit.test.js +309 -309
  136. package/test/llm-message-source.test.js +177 -177
  137. package/test/local-embedder.test.js +249 -249
  138. package/test/mcp-stdio.test.js +385 -0
  139. package/test/mirror-dirty.test.js +424 -424
  140. package/test/mirror-edit-digest.test.js +187 -187
  141. package/test/mirror-generation.test.js +499 -499
  142. package/test/mirror.test.js +249 -249
  143. package/test/narrative-bars.test.js +162 -0
  144. package/test/peer-blockers.test.js +193 -193
  145. package/test/policy-epoch.test.js +259 -259
  146. package/test/quality-filter.test.js +186 -186
  147. package/test/reasoning-effort.test.js +746 -746
  148. package/test/recall-evals.test.js +235 -235
  149. package/test/recall-layer.test.js +326 -326
  150. package/test/recall-stats.test.js +117 -0
  151. package/test/receipt-chain.test.js +451 -451
  152. package/test/reflection.test.js +226 -226
  153. package/test/reindex-backfill.test.js +62 -62
  154. package/test/reranker-single-logit.test.js +73 -73
  155. package/test/reranker.test.js +197 -197
  156. package/test/resident-status.test.js +83 -0
  157. package/test/runtime-adopt.test.js +206 -206
  158. package/test/runtime-cli.test.js +89 -89
  159. package/test/runtime-closure.test.js +193 -193
  160. package/test/runtime-download.test.js +302 -302
  161. package/test/runtime-layout.test.js +163 -163
  162. package/test/runtime-loader.test.js +237 -237
  163. package/test/runtime-manifest.test.js +158 -158
  164. package/test/runtime-model-fetch.test.js +348 -348
  165. package/test/runtime-provision.test.js +101 -101
  166. package/test/runtime-tarball.test.js +81 -81
  167. package/test/runtime-verify.test.js +214 -214
  168. package/test/scope-candidates.test.js +239 -239
  169. package/test/scope-explicit.test.js +452 -452
  170. package/test/scope-retrieval.test.js +197 -197
  171. package/test/scope-storage.test.js +255 -255
  172. package/test/scope-strict.test.js +231 -231
  173. package/test/search-fusion.test.js +90 -90
  174. package/test/semantic.test.js +124 -124
  175. package/test/service-search.test.js +174 -174
  176. package/test/service.test.js +405 -405
  177. package/test/settings.test.js +261 -261
  178. package/test/skip-invalid-restore.test.js +262 -262
  179. package/test/sleep-action-set.test.js +422 -422
  180. package/test/sleep-heat.test.js +126 -125
  181. package/test/sleep-scheduler-rearm.test.js +89 -89
  182. package/test/sleep.test.js +373 -373
  183. package/test/standalone-api.test.js +544 -359
  184. package/test/store-vector-cache.test.js +98 -98
  185. package/test/store.test.js +383 -383
  186. package/test/stress.test.js +209 -209
  187. package/test/summarize.test.js +884 -575
  188. package/test/tools-memory-get-schema.test.js +65 -65
  189. package/test/tools.test.js +341 -341
  190. package/test/updated-at-semantics.test.js +113 -113
  191. package/test/vector-index.test.js +221 -221
  192. package/test/version-check.test.js +152 -152
  193. package/test/webServer-optional.test.js +82 -82
@@ -0,0 +1,103 @@
1
+ // dsh-mneme/src/recall-stats.js
2
+ // 记忆复用统计(#217):对 recall_runs + memories 做只读聚合,面板「记忆复用」
3
+ // 卡与 GET /api/dsh-mneme/recall-stats 的数据源。
4
+ // 独立成模块的原因:service.js 已过 2000 行参考线(AGENTS.md 尺寸约定,#221),
5
+ // 新内聚块从第一行就落在自己的文件里,service.js 保留 barrel 出口、调用方零改动。
6
+ // 纯读不写,绝不触发任何 write hook。
7
+
8
+ /**
9
+ * 口径(issue #217 评论 2026-09-18,锚 5bd2dab):
10
+ * - Top-N:窗口内 recall_runs.candidates(最终返回集)按 id 计数,join
11
+ * memories 补 type/source;记忆已删的 type=null(标题/来源回退候选值)。
12
+ * candidates 只存最终集,「入池未中」语义(B)现有表拿不到——v1 只报
13
+ * 零曝光(A),B 待 recordRecall 记录形状扩展。
14
+ * - 僵尸率:活跃(未归档未遗忘)且窗口内零候选出现;分母剔除写入不足
15
+ * 豁免期的记忆(按当前时刻计,exemptCount 单独报数)。
16
+ * - 注入命中率:注入侧无留痕,待 #217 mode='inject' 口径拍板后接入。
17
+ * - coverage:recallRecordDefault 开启前的窗口算不到,earliestRunAt 为
18
+ * null(窗口内无回执)或早于窗口起点时,前端标注可信度;扫描行数有
19
+ * 上限,超出标 truncated(degraded 口径),不静默少算。
20
+ *
21
+ * @param {object} store - createStore 产物(只调用 listRecallRunsSince / all)
22
+ * @param {{ windowDays?: number, exemptDays?: number }} [options]
23
+ */
24
+ export function recallStats(store, options = {}) {
25
+ const rawWindow = Number(options.windowDays ?? 30);
26
+ const windowDays = Number.isInteger(rawWindow) && rawWindow >= 1 && rawWindow <= 365 ? rawWindow : 30;
27
+ // 豁免期天数(issue #217 建议 7 天):option 化是给测试归零用 + 维护者
28
+ // 调口径不留硬编码;越界值静默回默认(与 windowDays 同款宽容)。
29
+ const rawExempt = Number(options.exemptDays ?? 7);
30
+ const exemptDays = Number.isInteger(rawExempt) && rawExempt >= 0 && rawExempt <= 365 ? rawExempt : 7;
31
+ const now = Date.now();
32
+ const since = new Date(now - windowDays * 86400000).toISOString();
33
+
34
+ const { rows: runs, total } = store.listRecallRunsSince(since);
35
+ const hits = new Map(); // id -> { count, title, source }
36
+ for (const run of runs) {
37
+ for (const cand of run.candidates ?? []) {
38
+ if (!cand?.id) continue;
39
+ const cur = hits.get(cand.id) ?? { count: 0, title: cand.title ?? null, source: cand.source ?? null };
40
+ cur.count += 1;
41
+ hits.set(cand.id, cur);
42
+ }
43
+ }
44
+
45
+ const memories = store.all();
46
+ const memoryIndex = new Map(memories.map((m) => [m.id, m]));
47
+ const EXEMPT_MS = exemptDays * 86400000;
48
+ const byType = {};
49
+ const bySource = {};
50
+ let activeCount = 0;
51
+ let zombieCount = 0;
52
+ let exemptCount = 0;
53
+ for (const m of memories) {
54
+ if (m.archived || m.forgotten) continue;
55
+ const createdMs = Date.parse(m.created_at ?? "");
56
+ // created_at 解析不了时无法证明已过机会期 → 归入豁免,宁漏勿误伤
57
+ if (!Number.isFinite(createdMs) || now - createdMs < EXEMPT_MS) {
58
+ exemptCount += 1;
59
+ continue;
60
+ }
61
+ activeCount += 1;
62
+ const zombied = !hits.has(m.id);
63
+ if (zombied) zombieCount += 1;
64
+ const type = m.type ?? "unknown";
65
+ const source = m.source ?? "unknown";
66
+ if (!byType[type]) byType[type] = { active: 0, zombie: 0 };
67
+ const t = byType[type];
68
+ t.active += 1;
69
+ if (zombied) t.zombie += 1;
70
+ if (!bySource[source]) bySource[source] = { active: 0, zombie: 0 };
71
+ const s = bySource[source];
72
+ s.active += 1;
73
+ if (zombied) s.zombie += 1;
74
+ }
75
+
76
+ const topRecalled = [...hits.entries()]
77
+ .map(([id, h]) => {
78
+ const m = memoryIndex.get(id);
79
+ return { id, title: m?.title ?? h.title, type: m?.type ?? null, source: m?.source ?? h.source, count: h.count };
80
+ })
81
+ .sort((a, b) => b.count - a.count || a.id.localeCompare(b.id))
82
+ .slice(0, 10);
83
+
84
+ return {
85
+ windowDays,
86
+ generatedAt: new Date(now).toISOString(),
87
+ coverage: {
88
+ runsScanned: runs.length,
89
+ runsTotal: total,
90
+ truncated: total > runs.length,
91
+ earliestRunAt: runs.length ? runs[0].created_at : null
92
+ },
93
+ topRecalled,
94
+ zombie: {
95
+ activeCount,
96
+ zombieCount,
97
+ exemptCount,
98
+ rate: activeCount > 0 ? zombieCount / activeCount : null,
99
+ byType,
100
+ bySource
101
+ }
102
+ };
103
+ }
package/lib/reranker.js CHANGED
@@ -1,269 +1,269 @@
1
- import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
2
- import { installResilientFetch } from "./runtime/model-fetch.js";
3
- import { loadTransformers } from "./runtime/loader.js";
4
-
5
- // Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
6
- // bge-reranker-base through transformers.js: tries the native `rerank` task
7
- // first, then the sequence-classification head (sigmoid on the logit delta),
8
- // and finally feature-extraction over concatenated query+passage (cosine).
9
- // Every strategy funnels into scorePair(query, passage) so tests can inject a
10
- // fake scorer and never download a model. Failures throw — the caller degrades
11
- // back to the original candidate order.
12
- function hashString(s) {
13
- let h = 5381;
14
- for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
15
- return h.toString(16);
16
- }
17
-
18
- /** Same provider#hash fingerprint convention as the embedders. */
19
- function modelHash(model) {
20
- return `${model}#${hashString(model)}`;
21
- }
22
-
23
- /**
24
- * Lazy default pipeline factory: dynamic import keeps module load cheap.
25
- *
26
- * 与 local-embedder 一样经 runtime/loader.js 的三层解析取运行时(自管 payload 优先,
27
- * 其次宿主裸 specifier),这样 PR-B 把依赖从 profile 摘掉后 rerank 也还能用。
28
- * runtimeDir 取走后不混进 transformers 的选项。
29
- */
30
- async function defaultPipelineLoader(task, model, options) {
31
- const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
32
- const { module } = await loadTransformers({
33
- runtimeDir: runtimeDir || defaultRuntimeDir()
34
- });
35
- const { env, pipeline } = module;
36
- // issue #13 同款处理:transformers.js 在预检 tokenizer_config.json 元数据时会丢掉调用方
37
- // 的 cache_dir,回退到 env.cacheDir —— 即使模型已完全缓存在本地也会去请求网络。
38
- // embedder 侧早就镜像了,reranker 侧一直漏着,于是「已缓存 + 离线」时重排初始化会失败。
39
- if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
40
- // issue #188 补充:embedModelMirror 此前是死配置——transformers.js 不认
41
- // HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
42
- // 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
43
- if (remoteHost) env.remoteHost = remoteHost;
44
- // issue #194:与 embedder 同款断点续传。开关关 = env.fetch 一字不动。
45
- installResilientFetch(env, {
46
- enabled: resilientModelDownload === true,
47
- cacheDir: pipelineOptions.cache_dir,
48
- logger
49
- });
50
- return pipeline(task, model, pipelineOptions);
51
- }
52
-
53
- /** Flatten a transformers.js Tensor [batch, seq, dim] into number[][] rows. */
54
- function tensorToRows(tensor) {
55
- const { data, dims } = tensor;
56
- const dim = dims[dims.length - 1] || 0;
57
- const rows = [];
58
- for (let i = 0; i < data.length; i += dim) rows.push(Array.from(data.subarray(i, i + dim)));
59
- if (rows.length === 0 && dim > 0) rows.push(Array.from(data));
60
- return rows;
61
- }
62
-
63
- function cosine(a, b) {
64
- let dot = 0;
65
- let na = 0;
66
- let nb = 0;
67
- for (let i = 0; i < a.length; i++) {
68
- dot += a[i] * b[i];
69
- na += a[i] * a[i];
70
- nb += b[i] * b[i];
71
- }
72
- if (na === 0 || nb === 0) return 0;
73
- return dot / Math.sqrt(na * nb);
74
- }
75
-
76
- function clamp01(x) {
77
- return Math.max(0, Math.min(1, x));
78
- }
79
-
80
- // Pipeline strategies, best to worst. `rerank` is unsupported by the bundled
81
- // transformers.js (v4.2.0) and rejects cheaply before any model download, so
82
- // the cascade normally lands on the classification head.
83
- const STRATEGIES = [
84
- ["rerank", "rerank"],
85
- ["text-classification", "tc"],
86
- ["feature-extraction", "fe"]
87
- ];
88
-
89
- export class LocalReranker {
90
- constructor(opts = {}) {
91
- this.model = opts.model || "Xenova/bge-reranker-base";
92
- this.batchSize = opts.batchSize || 8;
93
- this.maxCandidates = opts.maxCandidates || 30;
94
- this.scoreThreshold = opts.scoreThreshold ?? 0.1;
95
- this.device = opts.device || "cpu";
96
- this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
97
- // 自管运行时根目录(issue #131):空表示用默认位置,解析交给 loader。
98
- this.runtimeDir = String(opts.runtimeDir ?? "").trim();
99
- // 与嵌入层同约定:默认 q8(4 倍小的 model_quantized.onnx),"fp32" 可关。
100
- // 此前不传 dtype,即使缓存里有量化文件也会去要 1GB 级的 model.onnx(#188)。
101
- this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
102
- // 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
103
- this.remoteHost = String(opts.remoteHost ?? "").trim();
104
- // issue #194:模型文件下载断点续传。只在显式传 true 时启用,与 embedder 同规。
105
- this.resilientModelDownload = opts.resilientModelDownload === true;
106
- this.logger = opts.logger ?? null;
107
- this.engineFactory = opts.engineFactory || defaultPipelineLoader;
108
- // Injectable seam: async (query, passage) => number. When set, init()
109
- // skips model loading entirely so tests never hit the network.
110
- this.scorePair = opts.scorePair ?? null;
111
- this.pipeline = null;
112
- this._batchScorer = null;
113
- this._queryVec = null;
114
- this._queryKey = null;
115
- this._constantScoreWarned = false;
116
- }
117
-
118
- /** Load the model; throws when no strategy can be bound. */
119
- async init() {
120
- if (this.scorePair) return this;
121
- let lastErr = null;
122
- for (const [task, strategy] of STRATEGIES) {
123
- try {
124
- this.pipeline = await this.engineFactory(task, this.model, this._engineOptions());
125
- this._bindBatchScorer(strategy);
126
- this.logger?.info?.(
127
- `[dsh-mneme] local reranker ready: ${this.model} (strategy=${strategy}, device=${this.device})`
128
- );
129
- return this;
130
- } catch (err) {
131
- lastErr = err;
132
- this.logger?.warn?.(
133
- `[dsh-mneme] reranker task "${task}" unavailable for ${this.model}: ${String(err?.message ?? err)}`
134
- );
135
- }
136
- }
137
- throw new Error(`LocalReranker failed to load ${this.model}: ${String(lastErr?.message ?? lastErr)}`);
138
- }
139
-
140
- _engineOptions() {
141
- const options = { device: this.device, dtype: this.useDtype, resilientModelDownload: this.resilientModelDownload, logger: this.logger };
142
- if (this.cacheDir) options.cache_dir = this.cacheDir;
143
- if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
144
- if (this.remoteHost) options.remoteHost = this.remoteHost;
145
- return options;
146
- }
147
-
148
- /** Bind a batch scorer for the chosen strategy. */
149
- _bindBatchScorer(strategy) {
150
- if (strategy === "rerank") {
151
- // Native cross-encoder task (transformers.js >= 4.6): { query, documents }
152
- // returns one score per document.
153
- this._batchScorer = async (query, passages) => {
154
- const out = await this.pipeline({ query, documents: passages });
155
- if (!Array.isArray(out)) throw new Error("rerank pipeline returned unexpected output");
156
- return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
157
- };
158
- } else if (strategy === "tc") {
159
- // Classification head: score so relevance lands in [0, 1]. Branch on the
160
- // output width (issue #188): a dual-logit head (num_labels=2 exports,
161
- // e.g. bge-reranker-v2-m3) gives the positive class as the softmax
162
- // probability sigmoid(l1 - l0); a SINGLE-logit head (e.g.
163
- // Xenova/bge-reranker-base) emits one raw relevance logit per pair, and
164
- // the positive class is simply sigmoid(logit) — reading l1 as l0 there
165
- // collapses every score to sigmoid(0) = 0.5 and silently disables
166
- // reranking (no error, status still "ready").
167
- this._batchScorer = async (query, passages) => {
168
- const { tokenizer, model } = this.pipeline;
169
- const inputs = tokenizer(passages.map(() => query), {
170
- text_pair: passages,
171
- padding: true,
172
- truncation: true
173
- });
174
- const logits = await model(inputs).then((o) => o.logits);
175
- const dims = logits.dims;
176
- const cols = dims[dims.length - 1] || 2;
177
- const rows = [];
178
- for (let i = 0; i < dims[0]; i++) {
179
- const base = i * cols;
180
- const score = cols > 1
181
- ? 1 / (1 + Math.exp(logits.data[base] - logits.data[base + 1]))
182
- : 1 / (1 + Math.exp(-logits.data[base]));
183
- rows.push(clamp01(score));
184
- }
185
- return rows;
186
- };
187
- } else {
188
- // Feature extraction: mean-pool the concatenated pair and compare with
189
- // the query embedding via cosine. Degraded but model-agnostic.
190
- // The query vector is cached per query string, so a new query always
191
- // recomputes it instead of reusing a stale vector from the previous call.
192
- this._queryVec = null;
193
- this._queryKey = null;
194
- this._batchScorer = async (query, passages) => {
195
- if (this._queryKey !== query) {
196
- const t = await this.pipeline([query], { pooling: "mean", normalize: true });
197
- this._queryVec = tensorToRows(t)[0];
198
- this._queryKey = query;
199
- }
200
- const t = await this.pipeline(passages.map((p) => `${query}\n${p}`), {
201
- pooling: "mean",
202
- normalize: true
203
- });
204
- return tensorToRows(t).map((v) => clamp01(cosine(this._queryVec, v)));
205
- };
206
- }
207
- }
208
-
209
- /** Score one batch of passages; injected scorePair scores pair by pair. */
210
- async _scores(query, passages) {
211
- if (this._batchScorer) return this._batchScorer(query, passages);
212
- return Promise.all(passages.map((p) => (p ? this.scorePair(query, p) : 0)));
213
- }
214
-
215
- /**
216
- * Re-rank recall candidates. Returns [{ id, score }] filtered by
217
- * scoreThreshold and sorted by descending score. Throws on engine failure —
218
- * the caller degrades to the original candidate order.
219
- */
220
- async rerank(query, candidates) {
221
- if (!Array.isArray(candidates)) throw new TypeError("rerank expects an array of candidates");
222
- const list = candidates.slice(0, this.maxCandidates);
223
- if (!list.length) return [];
224
- const q = String(query ?? "");
225
- const results = [];
226
- for (let i = 0; i < list.length; i += this.batchSize) {
227
- const chunk = list.slice(i, i + this.batchSize);
228
- const passages = chunk.map((c) => [c.title, c.content].filter(Boolean).join("\n"));
229
- const scores = await this._scores(q, passages);
230
- if (!Array.isArray(scores) || scores.length !== chunk.length) {
231
- throw new Error("reranker scorer returned mismatched scores");
232
- }
233
- for (let j = 0; j < chunk.length; j++) {
234
- results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
235
- }
236
- }
237
- // Silent-failure tripwire (issue #188): a reranker whose every candidate
238
- // scores identically is not ranking anything — e.g. a single-logit head
239
- // read through the dual-logit formula. Warn once so the degradation is
240
- // observable instead of a constant 0.5 column hiding behind a "ready"
241
- // status forever.
242
- if (results.length > 1 && results.every((r) => r.score === results[0].score)) {
243
- if (!this._constantScoreWarned) {
244
- this._constantScoreWarned = true;
245
- this.logger?.warn?.(
246
- `[dsh-mneme] reranker scores degenerated to a constant (${results[0].score}) — check the model's output dims (single vs dual logit head)`
247
- );
248
- }
249
- }
250
- return results
251
- .filter((r) => r.score >= this.scoreThreshold)
252
- .sort((a, b) => b.score - a.score);
253
- }
254
-
255
- get modelHash() {
256
- return modelHash(this.model);
257
- }
258
-
259
- dispose() {
260
- try {
261
- this.pipeline?.dispose?.();
262
- } catch {
263
- // best-effort: some engines free resources on GC
264
- }
265
- this.pipeline = null;
266
- this._queryVec = null;
267
- this._queryKey = null;
268
- }
269
- }
1
+ import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
2
+ import { installResilientFetch } from "./runtime/model-fetch.js";
3
+ import { loadTransformers } from "./runtime/loader.js";
4
+
5
+ // Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
6
+ // bge-reranker-base through transformers.js: tries the native `rerank` task
7
+ // first, then the sequence-classification head (sigmoid on the logit delta),
8
+ // and finally feature-extraction over concatenated query+passage (cosine).
9
+ // Every strategy funnels into scorePair(query, passage) so tests can inject a
10
+ // fake scorer and never download a model. Failures throw — the caller degrades
11
+ // back to the original candidate order.
12
+ function hashString(s) {
13
+ let h = 5381;
14
+ for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
15
+ return h.toString(16);
16
+ }
17
+
18
+ /** Same provider#hash fingerprint convention as the embedders. */
19
+ function modelHash(model) {
20
+ return `${model}#${hashString(model)}`;
21
+ }
22
+
23
+ /**
24
+ * Lazy default pipeline factory: dynamic import keeps module load cheap.
25
+ *
26
+ * 与 local-embedder 一样经 runtime/loader.js 的三层解析取运行时(自管 payload 优先,
27
+ * 其次宿主裸 specifier),这样 PR-B 把依赖从 profile 摘掉后 rerank 也还能用。
28
+ * runtimeDir 取走后不混进 transformers 的选项。
29
+ */
30
+ async function defaultPipelineLoader(task, model, options) {
31
+ const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
32
+ const { module } = await loadTransformers({
33
+ runtimeDir: runtimeDir || defaultRuntimeDir()
34
+ });
35
+ const { env, pipeline } = module;
36
+ // issue #13 同款处理:transformers.js 在预检 tokenizer_config.json 元数据时会丢掉调用方
37
+ // 的 cache_dir,回退到 env.cacheDir —— 即使模型已完全缓存在本地也会去请求网络。
38
+ // embedder 侧早就镜像了,reranker 侧一直漏着,于是「已缓存 + 离线」时重排初始化会失败。
39
+ if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
40
+ // issue #188 补充:embedModelMirror 此前是死配置——transformers.js 不认
41
+ // HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
42
+ // 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
43
+ if (remoteHost) env.remoteHost = remoteHost;
44
+ // issue #194:与 embedder 同款断点续传。开关关 = env.fetch 一字不动。
45
+ installResilientFetch(env, {
46
+ enabled: resilientModelDownload === true,
47
+ cacheDir: pipelineOptions.cache_dir,
48
+ logger
49
+ });
50
+ return pipeline(task, model, pipelineOptions);
51
+ }
52
+
53
+ /** Flatten a transformers.js Tensor [batch, seq, dim] into number[][] rows. */
54
+ function tensorToRows(tensor) {
55
+ const { data, dims } = tensor;
56
+ const dim = dims[dims.length - 1] || 0;
57
+ const rows = [];
58
+ for (let i = 0; i < data.length; i += dim) rows.push(Array.from(data.subarray(i, i + dim)));
59
+ if (rows.length === 0 && dim > 0) rows.push(Array.from(data));
60
+ return rows;
61
+ }
62
+
63
+ function cosine(a, b) {
64
+ let dot = 0;
65
+ let na = 0;
66
+ let nb = 0;
67
+ for (let i = 0; i < a.length; i++) {
68
+ dot += a[i] * b[i];
69
+ na += a[i] * a[i];
70
+ nb += b[i] * b[i];
71
+ }
72
+ if (na === 0 || nb === 0) return 0;
73
+ return dot / Math.sqrt(na * nb);
74
+ }
75
+
76
+ function clamp01(x) {
77
+ return Math.max(0, Math.min(1, x));
78
+ }
79
+
80
+ // Pipeline strategies, best to worst. `rerank` is unsupported by the bundled
81
+ // transformers.js (v4.2.0) and rejects cheaply before any model download, so
82
+ // the cascade normally lands on the classification head.
83
+ const STRATEGIES = [
84
+ ["rerank", "rerank"],
85
+ ["text-classification", "tc"],
86
+ ["feature-extraction", "fe"]
87
+ ];
88
+
89
+ export class LocalReranker {
90
+ constructor(opts = {}) {
91
+ this.model = opts.model || "Xenova/bge-reranker-base";
92
+ this.batchSize = opts.batchSize || 8;
93
+ this.maxCandidates = opts.maxCandidates || 30;
94
+ this.scoreThreshold = opts.scoreThreshold ?? 0.1;
95
+ this.device = opts.device || "cpu";
96
+ this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
97
+ // 自管运行时根目录(issue #131):空表示用默认位置,解析交给 loader。
98
+ this.runtimeDir = String(opts.runtimeDir ?? "").trim();
99
+ // 与嵌入层同约定:默认 q8(4 倍小的 model_quantized.onnx),"fp32" 可关。
100
+ // 此前不传 dtype,即使缓存里有量化文件也会去要 1GB 级的 model.onnx(#188)。
101
+ this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
102
+ // 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
103
+ this.remoteHost = String(opts.remoteHost ?? "").trim();
104
+ // issue #194:模型文件下载断点续传。只在显式传 true 时启用,与 embedder 同规。
105
+ this.resilientModelDownload = opts.resilientModelDownload === true;
106
+ this.logger = opts.logger ?? null;
107
+ this.engineFactory = opts.engineFactory || defaultPipelineLoader;
108
+ // Injectable seam: async (query, passage) => number. When set, init()
109
+ // skips model loading entirely so tests never hit the network.
110
+ this.scorePair = opts.scorePair ?? null;
111
+ this.pipeline = null;
112
+ this._batchScorer = null;
113
+ this._queryVec = null;
114
+ this._queryKey = null;
115
+ this._constantScoreWarned = false;
116
+ }
117
+
118
+ /** Load the model; throws when no strategy can be bound. */
119
+ async init() {
120
+ if (this.scorePair) return this;
121
+ let lastErr = null;
122
+ for (const [task, strategy] of STRATEGIES) {
123
+ try {
124
+ this.pipeline = await this.engineFactory(task, this.model, this._engineOptions());
125
+ this._bindBatchScorer(strategy);
126
+ this.logger?.info?.(
127
+ `[dsh-mneme] local reranker ready: ${this.model} (strategy=${strategy}, device=${this.device})`
128
+ );
129
+ return this;
130
+ } catch (err) {
131
+ lastErr = err;
132
+ this.logger?.warn?.(
133
+ `[dsh-mneme] reranker task "${task}" unavailable for ${this.model}: ${String(err?.message ?? err)}`
134
+ );
135
+ }
136
+ }
137
+ throw new Error(`LocalReranker failed to load ${this.model}: ${String(lastErr?.message ?? lastErr)}`);
138
+ }
139
+
140
+ _engineOptions() {
141
+ const options = { device: this.device, dtype: this.useDtype, resilientModelDownload: this.resilientModelDownload, logger: this.logger };
142
+ if (this.cacheDir) options.cache_dir = this.cacheDir;
143
+ if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
144
+ if (this.remoteHost) options.remoteHost = this.remoteHost;
145
+ return options;
146
+ }
147
+
148
+ /** Bind a batch scorer for the chosen strategy. */
149
+ _bindBatchScorer(strategy) {
150
+ if (strategy === "rerank") {
151
+ // Native cross-encoder task (transformers.js >= 4.6): { query, documents }
152
+ // returns one score per document.
153
+ this._batchScorer = async (query, passages) => {
154
+ const out = await this.pipeline({ query, documents: passages });
155
+ if (!Array.isArray(out)) throw new Error("rerank pipeline returned unexpected output");
156
+ return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
157
+ };
158
+ } else if (strategy === "tc") {
159
+ // Classification head: score so relevance lands in [0, 1]. Branch on the
160
+ // output width (issue #188): a dual-logit head (num_labels=2 exports,
161
+ // e.g. bge-reranker-v2-m3) gives the positive class as the softmax
162
+ // probability sigmoid(l1 - l0); a SINGLE-logit head (e.g.
163
+ // Xenova/bge-reranker-base) emits one raw relevance logit per pair, and
164
+ // the positive class is simply sigmoid(logit) — reading l1 as l0 there
165
+ // collapses every score to sigmoid(0) = 0.5 and silently disables
166
+ // reranking (no error, status still "ready").
167
+ this._batchScorer = async (query, passages) => {
168
+ const { tokenizer, model } = this.pipeline;
169
+ const inputs = tokenizer(passages.map(() => query), {
170
+ text_pair: passages,
171
+ padding: true,
172
+ truncation: true
173
+ });
174
+ const logits = await model(inputs).then((o) => o.logits);
175
+ const dims = logits.dims;
176
+ const cols = dims[dims.length - 1] || 2;
177
+ const rows = [];
178
+ for (let i = 0; i < dims[0]; i++) {
179
+ const base = i * cols;
180
+ const score = cols > 1
181
+ ? 1 / (1 + Math.exp(logits.data[base] - logits.data[base + 1]))
182
+ : 1 / (1 + Math.exp(-logits.data[base]));
183
+ rows.push(clamp01(score));
184
+ }
185
+ return rows;
186
+ };
187
+ } else {
188
+ // Feature extraction: mean-pool the concatenated pair and compare with
189
+ // the query embedding via cosine. Degraded but model-agnostic.
190
+ // The query vector is cached per query string, so a new query always
191
+ // recomputes it instead of reusing a stale vector from the previous call.
192
+ this._queryVec = null;
193
+ this._queryKey = null;
194
+ this._batchScorer = async (query, passages) => {
195
+ if (this._queryKey !== query) {
196
+ const t = await this.pipeline([query], { pooling: "mean", normalize: true });
197
+ this._queryVec = tensorToRows(t)[0];
198
+ this._queryKey = query;
199
+ }
200
+ const t = await this.pipeline(passages.map((p) => `${query}\n${p}`), {
201
+ pooling: "mean",
202
+ normalize: true
203
+ });
204
+ return tensorToRows(t).map((v) => clamp01(cosine(this._queryVec, v)));
205
+ };
206
+ }
207
+ }
208
+
209
+ /** Score one batch of passages; injected scorePair scores pair by pair. */
210
+ async _scores(query, passages) {
211
+ if (this._batchScorer) return this._batchScorer(query, passages);
212
+ return Promise.all(passages.map((p) => (p ? this.scorePair(query, p) : 0)));
213
+ }
214
+
215
+ /**
216
+ * Re-rank recall candidates. Returns [{ id, score }] filtered by
217
+ * scoreThreshold and sorted by descending score. Throws on engine failure —
218
+ * the caller degrades to the original candidate order.
219
+ */
220
+ async rerank(query, candidates) {
221
+ if (!Array.isArray(candidates)) throw new TypeError("rerank expects an array of candidates");
222
+ const list = candidates.slice(0, this.maxCandidates);
223
+ if (!list.length) return [];
224
+ const q = String(query ?? "");
225
+ const results = [];
226
+ for (let i = 0; i < list.length; i += this.batchSize) {
227
+ const chunk = list.slice(i, i + this.batchSize);
228
+ const passages = chunk.map((c) => [c.title, c.content].filter(Boolean).join("\n"));
229
+ const scores = await this._scores(q, passages);
230
+ if (!Array.isArray(scores) || scores.length !== chunk.length) {
231
+ throw new Error("reranker scorer returned mismatched scores");
232
+ }
233
+ for (let j = 0; j < chunk.length; j++) {
234
+ results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
235
+ }
236
+ }
237
+ // Silent-failure tripwire (issue #188): a reranker whose every candidate
238
+ // scores identically is not ranking anything — e.g. a single-logit head
239
+ // read through the dual-logit formula. Warn once so the degradation is
240
+ // observable instead of a constant 0.5 column hiding behind a "ready"
241
+ // status forever.
242
+ if (results.length > 1 && results.every((r) => r.score === results[0].score)) {
243
+ if (!this._constantScoreWarned) {
244
+ this._constantScoreWarned = true;
245
+ this.logger?.warn?.(
246
+ `[dsh-mneme] reranker scores degenerated to a constant (${results[0].score}) — check the model's output dims (single vs dual logit head)`
247
+ );
248
+ }
249
+ }
250
+ return results
251
+ .filter((r) => r.score >= this.scoreThreshold)
252
+ .sort((a, b) => b.score - a.score);
253
+ }
254
+
255
+ get modelHash() {
256
+ return modelHash(this.model);
257
+ }
258
+
259
+ dispose() {
260
+ try {
261
+ this.pipeline?.dispose?.();
262
+ } catch {
263
+ // best-effort: some engines free resources on GC
264
+ }
265
+ this.pipeline = null;
266
+ this._queryVec = null;
267
+ this._queryKey = null;
268
+ }
269
+ }