@modusensus/dsh-mneme 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/lib/client.js +5 -1
- package/lib/config.js +10 -0
- package/lib/index.js +2 -0
- package/lib/inject.js +43 -1
- package/lib/local-embedder.js +13 -1
- package/lib/reranker.js +11 -2
- package/lib/runtime/model-fetch.js +361 -0
- package/lib/service.js +22 -6
- package/lib/settings.js +1 -0
- package/package.json +1 -1
- package/src/config.js +10 -0
- package/src/index.js +2 -0
- package/src/inject.js +43 -1
- package/src/local-embedder.js +13 -1
- package/src/reranker.js +11 -2
- package/src/runtime/model-fetch.js +361 -0
- package/src/service.js +22 -6
- package/src/settings.js +1 -0
- package/test/api.test.js +5 -4
- package/test/inject-rotation.test.js +161 -0
- package/test/local-embedder.test.js +20 -0
- package/test/runtime-model-fetch.test.js +348 -0
package/lib/service.js
CHANGED
|
@@ -1235,7 +1235,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1235
1235
|
* fills + dedupes the remaining slots. Empty query / no cached recall /
|
|
1236
1236
|
* hybridInject off → pure legacy rule-based selection.
|
|
1237
1237
|
*/
|
|
1238
|
-
function injectCandidates({ query = "", maxItems = 5, threshold = 3, queryVector, scope = null } = {}) {
|
|
1238
|
+
function injectCandidates({ query = "", maxItems = 5, threshold = 3, queryVector, scope = null, rotate = null, rotateWindow = 0 } = {}) {
|
|
1239
1239
|
const q = String(query ?? "").trim();
|
|
1240
1240
|
// codingRetrospect 读取侧门控:编码记忆(rejected_solution / pitfall /
|
|
1241
1241
|
// constraint)只在编码任务时注入,防噪声污染其他业务;编码任务时按
|
|
@@ -1246,7 +1246,15 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1246
1246
|
// (quality_score null) count as 100 (weight 1), so legacy stores keep their
|
|
1247
1247
|
// exact summary>preference>importance ordering.
|
|
1248
1248
|
const qualityWeight = (m) => (m.quality_score != null ? m.quality_score / 100 : 1);
|
|
1249
|
-
|
|
1249
|
+
// Issue #205 补测:候选池必须比轮换窗口大——窗口 N 需要 maxItems×(N+1) 张
|
|
1250
|
+
// 不同的牌,池子只有 maxItems×2 时窗口一开就被吃光、只剩回填(重复率的
|
|
1251
|
+
// 结构性下限由池子大小决定)。无轮换时保持既有 maxItems×2 不变;规则路
|
|
1252
|
+
// 的 200 条上限同样给轮换让路(上限翻倍也只是多读一次 SQL LIMIT)。
|
|
1253
|
+
const rotateWindowN = Number.isInteger(rotateWindow) && rotateWindow > 0 ? rotateWindow : 0;
|
|
1254
|
+
const poolSize = rotateWindowN > 0
|
|
1255
|
+
? Math.max(maxItems * 2, maxItems * (rotateWindowN + 1))
|
|
1256
|
+
: maxItems * 2;
|
|
1257
|
+
const items = store.list({ limit: Math.max(200, poolSize), includeForgotten: false })
|
|
1250
1258
|
.filter((m) => !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
|
|
1251
1259
|
codingGate(m) &&
|
|
1252
1260
|
(m.type === "summary" || m.type === "preference" || m.importance >= threshold))
|
|
@@ -1277,7 +1285,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1277
1285
|
const semanticItems = [];
|
|
1278
1286
|
if (Array.isArray(queryVector) && queryVector.length && vectorIndex) {
|
|
1279
1287
|
try {
|
|
1280
|
-
const hits = vectorIndex.search(queryVector, { limit:
|
|
1288
|
+
const hits = vectorIndex.search(queryVector, { limit: poolSize, threshold: 0 });
|
|
1281
1289
|
for (const m of hits) {
|
|
1282
1290
|
if (m && !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
|
|
1283
1291
|
codingGate(m) &&
|
|
@@ -1300,7 +1308,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1300
1308
|
// 足以把当前话题顶到前排;门控与向量路径一致(含 importance 阈值)。
|
|
1301
1309
|
// 语义向量命中时本分支不参与,行为不变。
|
|
1302
1310
|
if (!semanticItems.length) {
|
|
1303
|
-
for (const hit of bm25Recall(q,
|
|
1311
|
+
for (const hit of bm25Recall(q, poolSize)) {
|
|
1304
1312
|
if (hit && !hit.archived && INJECT_TYPES.has(hit.type) && !hit.forgotten &&
|
|
1305
1313
|
codingGate(hit) &&
|
|
1306
1314
|
(hit.type === "summary" || hit.type === "preference" || hit.importance >= threshold)) {
|
|
@@ -1318,10 +1326,10 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1318
1326
|
};
|
|
1319
1327
|
for (const m of semanticItems) {
|
|
1320
1328
|
push(m);
|
|
1321
|
-
if (merged.length >=
|
|
1329
|
+
if (merged.length >= poolSize) break;
|
|
1322
1330
|
}
|
|
1323
1331
|
for (const m of items) {
|
|
1324
|
-
if (merged.length >=
|
|
1332
|
+
if (merged.length >= poolSize) break;
|
|
1325
1333
|
push(m);
|
|
1326
1334
|
}
|
|
1327
1335
|
candidates = merged;
|
|
@@ -1346,6 +1354,14 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1346
1354
|
if (config?.strictScope === true && scope) {
|
|
1347
1355
|
candidates = candidates.filter((m) => isVisibleInScope(m, scope));
|
|
1348
1356
|
}
|
|
1357
|
+
// Issue #205:注入位跨轮轮换。rotate = 最近 N 轮注入过的 id 集合(由注入层
|
|
1358
|
+
// 按会话维护并传入):这些条目本轮不再优先——新鲜者前置(各自内部相对次序
|
|
1359
|
+
// 保持),不足时按原序回填,槽位数与 touchRecall 语义均不变。rotate 为空时
|
|
1360
|
+
// 行为与既有排序完全一致。
|
|
1361
|
+
if (rotate && rotate.size > 0 && candidates.length > 0) {
|
|
1362
|
+
const fresh = candidates.filter((m) => !rotate.has(m.id));
|
|
1363
|
+
if (fresh.length > 0) candidates = [...fresh, ...candidates.filter((m) => rotate.has(m.id))];
|
|
1364
|
+
}
|
|
1349
1365
|
const selected = candidates.slice(0, maxItems);
|
|
1350
1366
|
touchRecalled(selected);
|
|
1351
1367
|
return selected;
|
package/lib/settings.js
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modusensus/dsh-mneme",
|
|
3
3
|
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 7 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
|
|
4
|
-
"version": "0.8.
|
|
4
|
+
"version": "0.8.3",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
package/src/config.js
CHANGED
|
@@ -37,6 +37,11 @@ export const Config = z.object({
|
|
|
37
37
|
distillRateLimitRetries: z.natural().min(0).max(10).default(3),
|
|
38
38
|
distillRateLimitBaseDelayMs: z.natural().min(100).max(60000).default(1000),
|
|
39
39
|
maxInjectedItems: z.natural().min(1).max(20).default(5),
|
|
40
|
+
// Issue #205:注入位跨轮轮换。同一会话里,最近 N 个不同用户查询轮次注入过
|
|
41
|
+
// 的记忆本轮不再优先(新鲜优先、原序回填,槽位数不变);长会话中避免相邻
|
|
42
|
+
// 轮次反复注入同一条(实测 15.5h 会话 86 次注入 93% 重复)。0 = 关闭(默认,
|
|
43
|
+
// 保持既有行为)。会话边界自动重置:新会话从零开始。
|
|
44
|
+
injectRotationTurns: z.natural().min(0).max(20).default(0),
|
|
40
45
|
importanceThreshold: z.natural().min(1).max(5).default(3),
|
|
41
46
|
// 编码记忆蒸馏(codingRetrospect,opt-in,默认关)。开启时,turn/end 蒸馏
|
|
42
47
|
// 额外提取三类编码专属记忆:rejected_solution(被否决方案)/ pitfall(踩坑)/
|
|
@@ -180,6 +185,11 @@ export const Config = z.object({
|
|
|
180
185
|
embedModelCacheDir: z.string().default(""),
|
|
181
186
|
embedModelMirror: z.string().default("https://hf-mirror.com"),
|
|
182
187
|
|
|
188
|
+
// issue #194:模型文件下载的断点续传与重试。开启时 transformers.js 的 env.fetch 被
|
|
189
|
+
// 替换为带断点续传的实现(已收字节落盘、Range/If-Range 续传、空闲看门狗),大文件
|
|
190
|
+
// 中断后不必整份重来;关闭 = env.fetch 保持原样(线上回滚开关)。
|
|
191
|
+
resilientModelDownload: z.boolean().default(true),
|
|
192
|
+
|
|
183
193
|
// 自管运行时目录(issue #131)。空 = ~/.dsh/mneme/runtime:里面放收编或下载来的
|
|
184
194
|
// transformers + onnxruntime 闭包,插件经 src/runtime/loader.js 的三层解析加载它。
|
|
185
195
|
// 目的就是让这条重依赖不必留在宿主 profile 的依赖图里——profile 是所有插件共用的
|
package/src/index.js
CHANGED
|
@@ -277,6 +277,7 @@ export const apply = (ctx, config) => {
|
|
|
277
277
|
runtimeDir: cfg.runtimeDir,
|
|
278
278
|
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
279
279
|
remoteHost: cfg.embedModelMirror,
|
|
280
|
+
resilientModelDownload: cfg.resilientModelDownload,
|
|
280
281
|
baseUrl: cfg.ollamaBaseUrl,
|
|
281
282
|
logger: ctx.logger
|
|
282
283
|
});
|
|
@@ -333,6 +334,7 @@ export const apply = (ctx, config) => {
|
|
|
333
334
|
// embedModelMirror 此前是死配置,现接成 transformers 的下载镜像。
|
|
334
335
|
useDtype: cfg.rerankDtype,
|
|
335
336
|
remoteHost: cfg.embedModelMirror,
|
|
337
|
+
resilientModelDownload: cfg.resilientModelDownload,
|
|
336
338
|
logger: ctx.logger
|
|
337
339
|
});
|
|
338
340
|
service.setReranker(reranker);
|
package/src/inject.js
CHANGED
|
@@ -94,6 +94,41 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
94
94
|
const language = langOf(config);
|
|
95
95
|
const maxItems = config.maxInjectedItems ?? 5;
|
|
96
96
|
const threshold = config.importanceThreshold ?? 3;
|
|
97
|
+
// Issue #205:注入位跨轮轮换。rotationTurns = 最近 N 个「不同用户查询」轮次
|
|
98
|
+
// 注入过的记忆本轮不再优先(0 = 关闭,保持既有行为)。历史按会话维护——
|
|
99
|
+
// 新会话从零开始;同一查询的多次渲染(工具调用轮)视为同一轮,不推进窗口。
|
|
100
|
+
const rotationTurns = Math.max(0, Math.floor(config.injectRotationTurns ?? 0));
|
|
101
|
+
const rotationHistory = new Map(); // sessionId -> [{ query, ids: Set }]
|
|
102
|
+
const ROTATION_HISTORY_MAX = 32; // 覆盖 max(20) 配置档仍有余量
|
|
103
|
+
|
|
104
|
+
function recentInjectedIds(sessionId, query) {
|
|
105
|
+
if (rotationTurns <= 0) return null;
|
|
106
|
+
const deque = rotationHistory.get(sessionId);
|
|
107
|
+
if (!deque || deque.length === 0) return null;
|
|
108
|
+
// 只看「之前的轮次」:排除与当前查询相同的条目——同一查询的重复渲染
|
|
109
|
+
// (工具调用轮)不该拿本轮自己的集合来转自己。
|
|
110
|
+
const prior = query ? deque.filter((e) => e.query !== query) : deque;
|
|
111
|
+
if (prior.length === 0) return null;
|
|
112
|
+
const recent = new Set();
|
|
113
|
+
for (const entry of prior.slice(-rotationTurns)) {
|
|
114
|
+
for (const id of entry.ids) recent.add(id);
|
|
115
|
+
}
|
|
116
|
+
return recent.size > 0 ? recent : null;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function recordInjection(sessionId, query, candidates) {
|
|
120
|
+
if (rotationTurns <= 0 || !query) return;
|
|
121
|
+
const deque = rotationHistory.get(sessionId) ?? [];
|
|
122
|
+
const ids = new Set(candidates.map((c) => c.id));
|
|
123
|
+
const last = deque[deque.length - 1];
|
|
124
|
+
if (last && last.query === query) {
|
|
125
|
+
last.ids = ids; // 同一查询的重复渲染:覆盖本轮内容,不推进窗口
|
|
126
|
+
} else {
|
|
127
|
+
deque.push({ query, ids });
|
|
128
|
+
if (deque.length > ROTATION_HISTORY_MAX) deque.shift();
|
|
129
|
+
rotationHistory.set(sessionId, deque);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
97
132
|
// v0.8.0 A3(issue #17):注入路径的会话 scope 解析——strictScope 硬过滤
|
|
98
133
|
// 需要。渲染 ctx 与工具 exec 同形(agent.session),解析器直接复用。
|
|
99
134
|
// logger 透传:registry 反查失败时 warnOnce 才有出口。
|
|
@@ -226,7 +261,13 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
226
261
|
// v0.8.0 A3:scope 随请求解析(strict 开启时 injectCandidates 内硬过滤;
|
|
227
262
|
// flag 关闭时解析器返回 null,注入行为不变)。
|
|
228
263
|
const scope = resolveSessionScope(ctx);
|
|
229
|
-
|
|
264
|
+
// Issue #205:跨轮轮换——最近 N 个查询轮次注入过的 id 本轮不再优先。
|
|
265
|
+
const sessionId = ctx?.agent?.session?.id ?? "_";
|
|
266
|
+
const rotate = recentInjectedIds(sessionId, query);
|
|
267
|
+
// rotateWindow 随集合一起传入:候选池按 maxItems×(N+1) 扩容——窗口要的
|
|
268
|
+
// 牌比既有 maxItems×2 池多时,轮换才有新鲜牌可换(#205 补测)。
|
|
269
|
+
const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold, scope, rotate, rotateWindow: rotationTurns });
|
|
270
|
+
recordInjection(sessionId, query, candidates);
|
|
230
271
|
// Hot memory (v0.5.0 1.3) leads the single memory block: the agent
|
|
231
272
|
// sees the short-term rounds first, then the cross-session recall —
|
|
232
273
|
// the documented injection order 1→2. Folding it here (instead of a
|
|
@@ -246,6 +287,7 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
246
287
|
|
|
247
288
|
return () => {
|
|
248
289
|
queryVectorCache.clear();
|
|
290
|
+
rotationHistory.clear();
|
|
249
291
|
for (const dispose of disposers) {
|
|
250
292
|
if (typeof dispose === "function") dispose();
|
|
251
293
|
}
|
package/src/local-embedder.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// can pick a backend by provider name and degrade gracefully on failure.
|
|
5
5
|
// Methods throw on error — the caller decides the fallback chain.
|
|
6
6
|
import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
|
|
7
|
+
import { installResilientFetch } from "./runtime/model-fetch.js";
|
|
7
8
|
import { loadTransformers } from "./runtime/loader.js";
|
|
8
9
|
|
|
9
10
|
const DEFAULT_TIMEOUT_MS = 15000;
|
|
@@ -30,7 +31,7 @@ function modelHash(model) {
|
|
|
30
31
|
* transformers 的选项里。
|
|
31
32
|
*/
|
|
32
33
|
async function defaultPipelineLoader(task, model, options) {
|
|
33
|
-
const { runtimeDir, remoteHost, ...pipelineOptions } = options ?? {};
|
|
34
|
+
const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
|
|
34
35
|
const { module } = await loadTransformers({
|
|
35
36
|
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
36
37
|
});
|
|
@@ -46,6 +47,12 @@ async function defaultPipelineLoader(task, model, options) {
|
|
|
46
47
|
// 模型下载源由 env.remoteHost 决定——把镜像接到真正的开关上,否则
|
|
47
48
|
// huggingface.co 不可达的网络连模型都下不下来。
|
|
48
49
|
if (remoteHost) env.remoteHost = remoteHost;
|
|
50
|
+
// issue #194:模型文件(几百 MB~1GB)下载中断后从断点续传。开关关 = env.fetch 一字不动。
|
|
51
|
+
installResilientFetch(env, {
|
|
52
|
+
enabled: resilientModelDownload === true,
|
|
53
|
+
cacheDir: pipelineOptions.cache_dir,
|
|
54
|
+
logger
|
|
55
|
+
});
|
|
49
56
|
return pipeline(task, model, pipelineOptions);
|
|
50
57
|
}
|
|
51
58
|
|
|
@@ -76,6 +83,9 @@ export class LocalEmbedder {
|
|
|
76
83
|
this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
|
|
77
84
|
// 自管运行时根目录(issue #131):空表示用默认位置,具体解析交给 loader。
|
|
78
85
|
this.runtimeDir = String(opts.runtimeDir ?? "").trim();
|
|
86
|
+
// issue #194:模型文件下载断点续传。只在显式传 true 时启用——直连构造的旧调用方
|
|
87
|
+
// (含测试)保持裸 fetch 的现状,配置默认值由 index.js 传进来。
|
|
88
|
+
this.resilientModelDownload = opts.resilientModelDownload === true;
|
|
79
89
|
this.useDtype = opts.useDtype || "q8";
|
|
80
90
|
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
81
91
|
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
@@ -99,6 +109,8 @@ export class LocalEmbedder {
|
|
|
99
109
|
// 传给注入的 engineFactory,由 defaultPipelineLoader 取走后交给三层解析。
|
|
100
110
|
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
101
111
|
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
112
|
+
options.resilientModelDownload = this.resilientModelDownload;
|
|
113
|
+
options.logger = this.logger;
|
|
102
114
|
this.extractor = await this.engineFactory("feature-extraction", this.model, options);
|
|
103
115
|
this.ready = true; // service reads this to flush queued re-embeds
|
|
104
116
|
this.logger?.info?.(
|
package/src/reranker.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
|
|
2
|
+
import { installResilientFetch } from "./runtime/model-fetch.js";
|
|
2
3
|
import { loadTransformers } from "./runtime/loader.js";
|
|
3
4
|
|
|
4
5
|
// Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
|
|
@@ -27,7 +28,7 @@ function modelHash(model) {
|
|
|
27
28
|
* runtimeDir 取走后不混进 transformers 的选项。
|
|
28
29
|
*/
|
|
29
30
|
async function defaultPipelineLoader(task, model, options) {
|
|
30
|
-
const { runtimeDir, remoteHost, ...pipelineOptions } = options ?? {};
|
|
31
|
+
const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
|
|
31
32
|
const { module } = await loadTransformers({
|
|
32
33
|
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
33
34
|
});
|
|
@@ -40,6 +41,12 @@ async function defaultPipelineLoader(task, model, options) {
|
|
|
40
41
|
// HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
|
|
41
42
|
// 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
|
|
42
43
|
if (remoteHost) env.remoteHost = remoteHost;
|
|
44
|
+
// issue #194:与 embedder 同款断点续传。开关关 = env.fetch 一字不动。
|
|
45
|
+
installResilientFetch(env, {
|
|
46
|
+
enabled: resilientModelDownload === true,
|
|
47
|
+
cacheDir: pipelineOptions.cache_dir,
|
|
48
|
+
logger
|
|
49
|
+
});
|
|
43
50
|
return pipeline(task, model, pipelineOptions);
|
|
44
51
|
}
|
|
45
52
|
|
|
@@ -94,6 +101,8 @@ export class LocalReranker {
|
|
|
94
101
|
this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
|
|
95
102
|
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
96
103
|
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
104
|
+
// issue #194:模型文件下载断点续传。只在显式传 true 时启用,与 embedder 同规。
|
|
105
|
+
this.resilientModelDownload = opts.resilientModelDownload === true;
|
|
97
106
|
this.logger = opts.logger ?? null;
|
|
98
107
|
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
99
108
|
// Injectable seam: async (query, passage) => number. When set, init()
|
|
@@ -129,7 +138,7 @@ export class LocalReranker {
|
|
|
129
138
|
}
|
|
130
139
|
|
|
131
140
|
_engineOptions() {
|
|
132
|
-
const options = { device: this.device, dtype: this.useDtype };
|
|
141
|
+
const options = { device: this.device, dtype: this.useDtype, resilientModelDownload: this.resilientModelDownload, logger: this.logger };
|
|
133
142
|
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
134
143
|
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
135
144
|
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
// 模型文件取件(issue #194)
|
|
2
|
+
//
|
|
3
|
+
// transformers.js 下载模型文件(如 model.onnx,约 1GB)走它自己的 env.fetch;undici
|
|
4
|
+
// socket 中途被重置(TypeError: terminated)时整份下载作废重来。4.2.0 的 env.fetch 是
|
|
5
|
+
// 官方文档化的可替换面(env.js: `fetch: DEFAULT_FETCH`),所以韧性做在这一层:对
|
|
6
|
+
// transformers.js 而言这仍然是一个普通的 fetch,成功路径的语义一点没变;变化只发生在
|
|
7
|
+
// 失败路径 —— 已收的字节落在磁盘 .part 里,重试(包括换一次进程)从断点继续。
|
|
8
|
+
//
|
|
9
|
+
// 硬规矩(与 runtime/download.js 的取件层同一套直觉):
|
|
10
|
+
// 1. 交给下游的响应永远是规范的 200 + 完整 content-length。hub.js 把非 200 一律当
|
|
11
|
+
// 失败处理,206 直接透传反而会破坏下游;断点拼接是我们内部的事,不能漏出去。
|
|
12
|
+
// 2. 续传前带 If-Range:断点对应的内容若在服务端被换过,服务器会回 200(全量)。
|
|
13
|
+
// 旧断点拼新内容是一份静默损坏的模型 —— 宁可整份重来,不可拼错。
|
|
14
|
+
// 3. 同一 URL 的 .part 同一时刻只允许一个写者。两个进程并发追加同一份字节是写损坏;
|
|
15
|
+
// 抢不到锁就按现状直接转发(裸 fetch),让持锁的那份去落盘。
|
|
16
|
+
//
|
|
17
|
+
// @module dsh-mneme/runtime/model-fetch
|
|
18
|
+
import { createHash } from "node:crypto";
|
|
19
|
+
import { closeSync, createReadStream, existsSync, futimesSync, mkdirSync, openSync, readFileSync, rmSync, statSync, writeFileSync, writeSync } from "node:fs";
|
|
20
|
+
import { Readable } from "node:stream";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
import { defaultModelCacheDir } from "./layout.js";
|
|
23
|
+
|
|
24
|
+
const DEFAULT_ATTEMPTS = 3;
|
|
25
|
+
const DEFAULT_RETRY_DELAY_MS = 1000;
|
|
26
|
+
const DEFAULT_MAX_RETRY_DELAY_MS = 15000;
|
|
27
|
+
// 看门狗是「连续无字节」上限而不是总时长上限:几百 MB~1GB 的文件在慢网下合法地要下
|
|
28
|
+
// 几十分钟,总时长上限会掐死健康但慢的下载;连续这么久一个字节都没有才是 socket 僵死。
|
|
29
|
+
const DEFAULT_IDLE_TIMEOUT_MS = 120000;
|
|
30
|
+
// 锁的 mtime 超过这么久没刷新 = 持锁方已死(活着的话每个进度节拍都会刷新),可以抢。
|
|
31
|
+
const STALE_LOCK_MS = 30 * 60 * 1000;
|
|
32
|
+
// 进度事件节流:1GB 按 64KB 一块是一万多个事件,逐个发出去日志没法看。
|
|
33
|
+
const PROGRESS_EMIT_INTERVAL_MS = 500;
|
|
34
|
+
|
|
35
|
+
/** URL → 稳定文件名。断点文件要跨进程、跨重启可复用,所以用内容寻址而不是 pid。 */
|
|
36
|
+
function partName(url) {
|
|
37
|
+
return createHash("sha256").update(url).digest("hex");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function readMeta(metaPath) {
|
|
41
|
+
try {
|
|
42
|
+
return readFileSync(metaPath, "utf8").trim();
|
|
43
|
+
} catch {
|
|
44
|
+
return "";
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function writeMeta(metaPath, etag) {
|
|
49
|
+
writeFileSync(metaPath, `${etag}\n`, "utf8");
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** 断点不可信(比源文件还大 / 内容对不上)时整体作废,而不是试着修剪。 */
|
|
53
|
+
function resetPart(partPath, metaPath) {
|
|
54
|
+
rmSync(partPath, { force: true });
|
|
55
|
+
rmSync(metaPath, { force: true });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* 拿 .part 的写锁。返回 fd(持有着,供流式期间刷新 mtime)或 null(别人在写)。
|
|
60
|
+
* 只尝试偷一次过期锁:再抢不到就放弃,调用方按现状裸转发。
|
|
61
|
+
*/
|
|
62
|
+
function acquireLock(lockPath, staleLockMs) {
|
|
63
|
+
for (let i = 0; i < 2; i++) {
|
|
64
|
+
try {
|
|
65
|
+
const fd = openSync(lockPath, "wx");
|
|
66
|
+
writeSync(fd, String(process.pid));
|
|
67
|
+
return fd;
|
|
68
|
+
} catch {
|
|
69
|
+
let age = Infinity;
|
|
70
|
+
try {
|
|
71
|
+
age = Date.now() - statSync(lockPath).mtimeMs;
|
|
72
|
+
} catch {}
|
|
73
|
+
if (age > staleLockMs) {
|
|
74
|
+
rmSync(lockPath, { force: true });
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function releaseLock(lockPath, lockFd) {
|
|
84
|
+
if (lockFd === null) return;
|
|
85
|
+
try {
|
|
86
|
+
closeSync(lockFd);
|
|
87
|
+
} catch {}
|
|
88
|
+
rmSync(lockPath, { force: true });
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** 进度事件的百分数。total 未知时给 null,让日志退化为「已收字节数」。 */
|
|
92
|
+
function percentOf(received, total) {
|
|
93
|
+
return Number.isFinite(total) && total > 0 ? Math.floor((received / total) * 100) : null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* 造一个「带断点续传的 fetch」。
|
|
98
|
+
* @param {object} opts - 选项。
|
|
99
|
+
* @param {Function} [opts.fetchImpl] - 底层 fetch,测试注入。
|
|
100
|
+
* @param {string} opts.partDir - 断点文件目录(应与模型缓存同盘)。
|
|
101
|
+
* @param {number} [opts.attempts] - 含首次在内的总尝试次数。
|
|
102
|
+
* @param {number} [opts.retryDelayMs] - 退避基数:1s → 2s → 4s,封顶 maxRetryDelayMs。
|
|
103
|
+
* @param {number} [opts.idleTimeoutMs] - 连续无字节的看门狗上限。
|
|
104
|
+
* @param {Function} [opts.onEvent] - resume / progress / restart / fail 事件回调。
|
|
105
|
+
* @param {Function} [opts.hash] - URL → 断点文件名,测试注入。
|
|
106
|
+
* @returns {Function} 与全局 fetch 同签名的函数。
|
|
107
|
+
*/
|
|
108
|
+
export function createResilientFetch({
|
|
109
|
+
fetchImpl = fetch,
|
|
110
|
+
partDir,
|
|
111
|
+
attempts = DEFAULT_ATTEMPTS,
|
|
112
|
+
retryDelayMs = DEFAULT_RETRY_DELAY_MS,
|
|
113
|
+
maxRetryDelayMs = DEFAULT_MAX_RETRY_DELAY_MS,
|
|
114
|
+
idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS,
|
|
115
|
+
onEvent = null,
|
|
116
|
+
hash = partName
|
|
117
|
+
} = {}) {
|
|
118
|
+
if (typeof partDir !== "string" || partDir === "") {
|
|
119
|
+
throw new TypeError("createResilientFetch:partDir 是必填的(.part 断点文件落在这里)");
|
|
120
|
+
}
|
|
121
|
+
mkdirSync(partDir, { recursive: true });
|
|
122
|
+
const emit = (event) => {
|
|
123
|
+
try {
|
|
124
|
+
onEvent?.(event);
|
|
125
|
+
} catch {
|
|
126
|
+
// 进度回调的异常不该打断下载本身。
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
return async function resilientFetch(input, init = undefined) {
|
|
131
|
+
const url = typeof input === "string" ? input : String(input?.url ?? input);
|
|
132
|
+
const method = String(init?.method ?? "GET").toUpperCase();
|
|
133
|
+
// 非 GET(HEAD 元数据预检等)与带 body 的请求不落盘:原样转发,语义与裸 fetch 一致。
|
|
134
|
+
if (method !== "GET" || init?.body != null) return fetchImpl(input, init);
|
|
135
|
+
|
|
136
|
+
const partPath = join(partDir, hash(url));
|
|
137
|
+
const metaPath = `${partPath}.meta`;
|
|
138
|
+
const lockPath = `${partPath}.lock`;
|
|
139
|
+
const lockFd = acquireLock(lockPath, STALE_LOCK_MS);
|
|
140
|
+
if (lockFd === null) {
|
|
141
|
+
// 另一个进程正在下同一个文件。并发追加同一份 .part 是写损坏,所以这边不参与落盘,
|
|
142
|
+
// 按改动前的行为裸转发一次 —— 持锁方会正常下完,之后的加载会命中缓存。
|
|
143
|
+
return fetchImpl(input, init);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
try {
|
|
147
|
+
return await downloadWithResume(url, init, {
|
|
148
|
+
fetchImpl,
|
|
149
|
+
partPath,
|
|
150
|
+
metaPath,
|
|
151
|
+
lockPath,
|
|
152
|
+
lockFd,
|
|
153
|
+
attempts,
|
|
154
|
+
retryDelayMs,
|
|
155
|
+
maxRetryDelayMs,
|
|
156
|
+
idleTimeoutMs,
|
|
157
|
+
emit
|
|
158
|
+
});
|
|
159
|
+
} catch (error) {
|
|
160
|
+
// 失败路径锁在这里释放;成功路径的锁随断点文件一起在响应体流关闭后清理 ——
|
|
161
|
+
// 若在返回响应时就放锁,另一个进程会在「.part 仍被下游流读着」的窗口里动它。
|
|
162
|
+
releaseLock(lockPath, lockFd);
|
|
163
|
+
throw error;
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async function downloadWithResume(url, init, ctx) {
|
|
169
|
+
const { fetchImpl, partPath, metaPath, lockPath, lockFd, attempts, retryDelayMs, maxRetryDelayMs, idleTimeoutMs, emit } = ctx;
|
|
170
|
+
let lastError = null;
|
|
171
|
+
let lastPercent = null;
|
|
172
|
+
let lastReceived = 0;
|
|
173
|
+
|
|
174
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
175
|
+
let offset = 0;
|
|
176
|
+
if (existsSync(partPath)) {
|
|
177
|
+
offset = statSync(partPath).size;
|
|
178
|
+
if (offset > 0) {
|
|
179
|
+
emit({ type: "resume", url, offset, attempt });
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const headers = new Headers(init?.headers);
|
|
184
|
+
if (offset > 0) {
|
|
185
|
+
headers.set("Range", `bytes=${offset}-`);
|
|
186
|
+
// If-Range:断点对应的内容在服务端被换过时,服务器回 200(全量)而不是 206 ——
|
|
187
|
+
// 那就整份重来。没有它,旧断点 + 新内容会拼出一份静默损坏的模型。
|
|
188
|
+
const storedEtag = readMeta(metaPath);
|
|
189
|
+
if (storedEtag) headers.set("If-Range", storedEtag);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const controller = new AbortController();
|
|
193
|
+
const onCallerAbort = () => controller.abort(init?.signal?.reason);
|
|
194
|
+
init?.signal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
195
|
+
let idleTimer = null;
|
|
196
|
+
const armIdle = () => {
|
|
197
|
+
if (!(idleTimeoutMs > 0)) return;
|
|
198
|
+
if (idleTimer !== null) clearTimeout(idleTimer);
|
|
199
|
+
idleTimer = setTimeout(
|
|
200
|
+
() => controller.abort(new Error(`连接空闲超过 ${Math.round(idleTimeoutMs / 1000)}s 无数据`)),
|
|
201
|
+
idleTimeoutMs
|
|
202
|
+
);
|
|
203
|
+
};
|
|
204
|
+
armIdle();
|
|
205
|
+
|
|
206
|
+
let fd = -1;
|
|
207
|
+
let received = 0;
|
|
208
|
+
let total = null;
|
|
209
|
+
let lastEmitAt = 0;
|
|
210
|
+
const reportProgress = (force = false) => {
|
|
211
|
+
const now = Date.now();
|
|
212
|
+
if (!force && now - lastEmitAt < PROGRESS_EMIT_INTERVAL_MS) return;
|
|
213
|
+
lastEmitAt = now;
|
|
214
|
+
lastReceived = offset + received;
|
|
215
|
+
lastPercent = percentOf(lastReceived, total);
|
|
216
|
+
futimesSync(lockFd, new Date(), new Date()); // 锁的活着证明与进度同节拍刷新。
|
|
217
|
+
emit({ type: "progress", url, received: lastReceived, total, percent: lastPercent, attempt });
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
const res = await fetchImpl(url, { ...init, headers, signal: controller.signal });
|
|
222
|
+
|
|
223
|
+
// Range 起点越过源文件末尾(断点比源还大,通常断点来自被替换前的旧文件):
|
|
224
|
+
// 断点不可信 —— 清掉,这次尝试作废,下一次从 0 开始。不处理的话会在 416 上
|
|
225
|
+
// 把重试次数全部烧完,报一条用户看不懂的错。
|
|
226
|
+
if (res.status === 416) {
|
|
227
|
+
emit({ type: "restart", url, offset, attempt, reason: "range-not-satisfiable" });
|
|
228
|
+
resetPart(partPath, metaPath);
|
|
229
|
+
lastError = new Error("HTTP 416(断点已重置)");
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
233
|
+
|
|
234
|
+
if (offset > 0 && res.status === 200) {
|
|
235
|
+
// 服务器无视了 Range,或 If-Range 判定内容已变:整份重来。
|
|
236
|
+
emit({ type: "restart", url, offset, attempt, reason: "range-rejected" });
|
|
237
|
+
resetPart(partPath, metaPath);
|
|
238
|
+
offset = 0;
|
|
239
|
+
}
|
|
240
|
+
if (res.status === 206) {
|
|
241
|
+
const cr = /bytes (\d+)-(\d+)\/(\d+)/.exec(res.headers.get("content-range") ?? "");
|
|
242
|
+
if (cr === null) throw new Error("206 响应缺少 Content-Range");
|
|
243
|
+
total = Number(cr[3]);
|
|
244
|
+
if (Number(cr[1]) !== offset) {
|
|
245
|
+
// 服务器续的位置和断点对不上:不猜(拼错的字节比下载失败更糟),清掉重来。
|
|
246
|
+
emit({ type: "restart", url, offset, attempt, reason: "offset-mismatch" });
|
|
247
|
+
resetPart(partPath, metaPath);
|
|
248
|
+
throw new Error(`断点偏移不一致(本地 ${offset},服务器 ${cr[1]}),断点已重置`);
|
|
249
|
+
}
|
|
250
|
+
} else {
|
|
251
|
+
total = Number(res.headers.get("content-length")) || null;
|
|
252
|
+
}
|
|
253
|
+
const etag = res.headers.get("etag");
|
|
254
|
+
if (etag) writeMeta(metaPath, etag);
|
|
255
|
+
|
|
256
|
+
fd = openSync(partPath, "a");
|
|
257
|
+
if (res.body !== null) {
|
|
258
|
+
const reader = res.body.getReader();
|
|
259
|
+
for (;;) {
|
|
260
|
+
const { done, value } = await reader.read();
|
|
261
|
+
if (done) break;
|
|
262
|
+
writeSync(fd, value);
|
|
263
|
+
received += value.length;
|
|
264
|
+
armIdle();
|
|
265
|
+
reportProgress();
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
// 服务端"干净地"提前关流(不抛错、但字节没给够)也是断流的一种,必须查出来。
|
|
269
|
+
if (total !== null && offset + received !== total) {
|
|
270
|
+
throw new Error(`下载不完整:收到 ${offset + received}/${total} 字节`);
|
|
271
|
+
}
|
|
272
|
+
clearTimeout(idleTimer);
|
|
273
|
+
idleTimer = null;
|
|
274
|
+
reportProgress(true);
|
|
275
|
+
return partResponse(partPath, metaPath, lockPath, lockFd, offset + received, etag);
|
|
276
|
+
} catch (error) {
|
|
277
|
+
lastError = error;
|
|
278
|
+
lastReceived = offset + received;
|
|
279
|
+
lastPercent = percentOf(lastReceived, total);
|
|
280
|
+
emit({ type: "fail", url, received: lastReceived, total, percent: lastPercent, attempt, error: String(error?.message ?? error) });
|
|
281
|
+
} finally {
|
|
282
|
+
if (fd >= 0) closeSync(fd);
|
|
283
|
+
if (idleTimer !== null) clearTimeout(idleTimer);
|
|
284
|
+
init?.signal?.removeEventListener("abort", onCallerAbort);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if (attempt < attempts) {
|
|
288
|
+
// 退避:1s → 2s → 4s,封顶 maxRetryDelayMs。
|
|
289
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(retryDelayMs * 2 ** (attempt - 1), maxRetryDelayMs)));
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// 最终失败的错误必须带中断位置(issue #194 的第 3 项):用户要能区分是网络断了
|
|
294
|
+
// 还是磁盘满了,而不是看到一句干巴巴的 TypeError: terminated。
|
|
295
|
+
const where = lastPercent !== null ? `中断于 ${lastPercent}%(${lastReceived} 字节)` : `已收 ${lastReceived} 字节`;
|
|
296
|
+
throw new Error(`模型文件下载失败(试了 ${attempts} 次,${where}):${lastError?.message ?? lastError}`);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* 把完成的断点文件包装成给下游的响应:规范的 200 + 完整 content-length,body 从磁盘
|
|
301
|
+
* 流式读出(1GB 不进内存)。流关闭即清理断点文件 —— 它给下游(FileCache.put /
|
|
302
|
+
* arrayBuffer)供字节的使命结束了,不清的话一个 1GB 的副本会一直躺在缓存旁边。
|
|
303
|
+
*/
|
|
304
|
+
function partResponse(partPath, metaPath, lockPath, lockFd, size, etag) {
|
|
305
|
+
const nodeStream = createReadStream(partPath);
|
|
306
|
+
const cleanup = () => {
|
|
307
|
+
try {
|
|
308
|
+
rmSync(partPath, { force: true });
|
|
309
|
+
rmSync(metaPath, { force: true });
|
|
310
|
+
releaseLock(lockPath, lockFd);
|
|
311
|
+
} catch {}
|
|
312
|
+
};
|
|
313
|
+
nodeStream.on("close", cleanup);
|
|
314
|
+
nodeStream.on("error", cleanup);
|
|
315
|
+
return new Response(Readable.toWeb(nodeStream), {
|
|
316
|
+
status: 200,
|
|
317
|
+
headers: {
|
|
318
|
+
"content-type": "application/octet-stream",
|
|
319
|
+
"content-length": String(size),
|
|
320
|
+
...(etag ? { etag } : {})
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** 把事件映射成日志。只发人该看的行:续传、重来、每次失败的位置 —— 不刷进度条。 */
|
|
326
|
+
export function logModelFetchEvent(logger, event) {
|
|
327
|
+
const kb = (n) => (n >= 1048576 ? `${(n / 1048576).toFixed(1)}MB` : `${Math.round(n / 1024)}KB`);
|
|
328
|
+
const where = (e) => (e.percent !== null && e.percent !== undefined ? `${e.percent}%(${kb(e.received)}${e.total ? `/${kb(e.total)}` : ""})` : kb(e.received ?? 0));
|
|
329
|
+
switch (event.type) {
|
|
330
|
+
case "resume":
|
|
331
|
+
logger?.info?.(`[dsh-mneme] 模型下载带断点续传:已有 ${kb(event.offset)},第 ${event.attempt} 次尝试`);
|
|
332
|
+
break;
|
|
333
|
+
case "restart":
|
|
334
|
+
logger?.warn?.(`[dsh-mneme] 断点不可用(${event.reason}),模型下载从头再来`);
|
|
335
|
+
break;
|
|
336
|
+
case "fail":
|
|
337
|
+
logger?.warn?.(`[dsh-mneme] 模型下载中断于 ${where(event)},第 ${event.attempt} 次尝试失败:${event.error}`);
|
|
338
|
+
break;
|
|
339
|
+
default:
|
|
340
|
+
break;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* 把弹性 fetch 装到 transformers.js 的 env 上(embedder / reranker 共用这一个入口)。
|
|
346
|
+
* @param {object} env - transformers.js 的 env(至少支持赋值 fetch)。
|
|
347
|
+
* @param {object} opts - 选项。
|
|
348
|
+
* @param {boolean} [opts.enabled] - 开关;关 = 完全不动 env(默认保持现状的出口)。
|
|
349
|
+
* @param {string} [opts.cacheDir] - 模型缓存目录,断点文件落在它的 .mneme-partial 下。
|
|
350
|
+
* @param {object} [opts.logger] - 接收下载进度日志的 logger。
|
|
351
|
+
* @param {object} [opts.rest] - 透传给 createResilientFetch 的其余选项(测试注入)。
|
|
352
|
+
* @returns {boolean} 是否实际安装。
|
|
353
|
+
*/
|
|
354
|
+
export function installResilientFetch(env, { enabled = true, cacheDir = "", logger = null, ...rest } = {}) {
|
|
355
|
+
if (!enabled || !env) return false;
|
|
356
|
+
const root = String(cacheDir ?? "").trim() || defaultModelCacheDir();
|
|
357
|
+
const partDir = join(root, ".mneme-partial");
|
|
358
|
+
mkdirSync(partDir, { recursive: true });
|
|
359
|
+
env.fetch = createResilientFetch({ partDir, onEvent: (event) => logModelFetchEvent(logger, event), ...rest });
|
|
360
|
+
return true;
|
|
361
|
+
}
|