@modusensus/dsh-mneme 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/lib/client.js +5 -1
- package/lib/config.js +10 -0
- package/lib/index.js +2 -0
- package/lib/inject.js +43 -1
- package/lib/local-embedder.js +13 -1
- package/lib/reranker.js +11 -2
- package/lib/runtime/model-fetch.js +361 -0
- package/lib/service.js +22 -6
- package/lib/settings.js +1 -0
- package/package.json +1 -1
- package/src/config.js +10 -0
- package/src/index.js +2 -0
- package/src/inject.js +43 -1
- package/src/local-embedder.js +13 -1
- package/src/reranker.js +11 -2
- package/src/runtime/model-fetch.js +361 -0
- package/src/service.js +22 -6
- package/src/settings.js +1 -0
- package/test/api.test.js +5 -4
- package/test/inject-rotation.test.js +161 -0
- package/test/local-embedder.test.js +20 -0
- package/test/runtime-model-fetch.test.js +348 -0
package/src/service.js
CHANGED
|
@@ -1235,7 +1235,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1235
1235
|
* fills + dedupes the remaining slots. Empty query / no cached recall /
|
|
1236
1236
|
* hybridInject off → pure legacy rule-based selection.
|
|
1237
1237
|
*/
|
|
1238
|
-
function injectCandidates({ query = "", maxItems = 5, threshold = 3, queryVector, scope = null } = {}) {
|
|
1238
|
+
function injectCandidates({ query = "", maxItems = 5, threshold = 3, queryVector, scope = null, rotate = null, rotateWindow = 0 } = {}) {
|
|
1239
1239
|
const q = String(query ?? "").trim();
|
|
1240
1240
|
// codingRetrospect 读取侧门控:编码记忆(rejected_solution / pitfall /
|
|
1241
1241
|
// constraint)只在编码任务时注入,防噪声污染其他业务;编码任务时按
|
|
@@ -1246,7 +1246,15 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1246
1246
|
// (quality_score null) count as 100 (weight 1), so legacy stores keep their
|
|
1247
1247
|
// exact summary>preference>importance ordering.
|
|
1248
1248
|
const qualityWeight = (m) => (m.quality_score != null ? m.quality_score / 100 : 1);
|
|
1249
|
-
|
|
1249
|
+
// Issue #205 补测:候选池必须比轮换窗口大——窗口 N 需要 maxItems×(N+1) 张
|
|
1250
|
+
// 不同的牌,池子只有 maxItems×2 时窗口一开就被吃光、只剩回填(重复率的
|
|
1251
|
+
// 结构性下限由池子大小决定)。无轮换时保持既有 maxItems×2 不变;规则路
|
|
1252
|
+
// 的 200 条上限同样给轮换让路(上限翻倍也只是多读一次 SQL LIMIT)。
|
|
1253
|
+
const rotateWindowN = Number.isInteger(rotateWindow) && rotateWindow > 0 ? rotateWindow : 0;
|
|
1254
|
+
const poolSize = rotateWindowN > 0
|
|
1255
|
+
? Math.max(maxItems * 2, maxItems * (rotateWindowN + 1))
|
|
1256
|
+
: maxItems * 2;
|
|
1257
|
+
const items = store.list({ limit: Math.max(200, poolSize), includeForgotten: false })
|
|
1250
1258
|
.filter((m) => !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
|
|
1251
1259
|
codingGate(m) &&
|
|
1252
1260
|
(m.type === "summary" || m.type === "preference" || m.importance >= threshold))
|
|
@@ -1277,7 +1285,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1277
1285
|
const semanticItems = [];
|
|
1278
1286
|
if (Array.isArray(queryVector) && queryVector.length && vectorIndex) {
|
|
1279
1287
|
try {
|
|
1280
|
-
const hits = vectorIndex.search(queryVector, { limit:
|
|
1288
|
+
const hits = vectorIndex.search(queryVector, { limit: poolSize, threshold: 0 });
|
|
1281
1289
|
for (const m of hits) {
|
|
1282
1290
|
if (m && !m.archived && INJECT_TYPES.has(m.type) && !m.forgotten &&
|
|
1283
1291
|
codingGate(m) &&
|
|
@@ -1300,7 +1308,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1300
1308
|
// 足以把当前话题顶到前排;门控与向量路径一致(含 importance 阈值)。
|
|
1301
1309
|
// 语义向量命中时本分支不参与,行为不变。
|
|
1302
1310
|
if (!semanticItems.length) {
|
|
1303
|
-
for (const hit of bm25Recall(q,
|
|
1311
|
+
for (const hit of bm25Recall(q, poolSize)) {
|
|
1304
1312
|
if (hit && !hit.archived && INJECT_TYPES.has(hit.type) && !hit.forgotten &&
|
|
1305
1313
|
codingGate(hit) &&
|
|
1306
1314
|
(hit.type === "summary" || hit.type === "preference" || hit.importance >= threshold)) {
|
|
@@ -1318,10 +1326,10 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1318
1326
|
};
|
|
1319
1327
|
for (const m of semanticItems) {
|
|
1320
1328
|
push(m);
|
|
1321
|
-
if (merged.length >=
|
|
1329
|
+
if (merged.length >= poolSize) break;
|
|
1322
1330
|
}
|
|
1323
1331
|
for (const m of items) {
|
|
1324
|
-
if (merged.length >=
|
|
1332
|
+
if (merged.length >= poolSize) break;
|
|
1325
1333
|
push(m);
|
|
1326
1334
|
}
|
|
1327
1335
|
candidates = merged;
|
|
@@ -1346,6 +1354,14 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1346
1354
|
if (config?.strictScope === true && scope) {
|
|
1347
1355
|
candidates = candidates.filter((m) => isVisibleInScope(m, scope));
|
|
1348
1356
|
}
|
|
1357
|
+
// Issue #205:注入位跨轮轮换。rotate = 最近 N 轮注入过的 id 集合(由注入层
|
|
1358
|
+
// 按会话维护并传入):这些条目本轮不再优先——新鲜者前置(各自内部相对次序
|
|
1359
|
+
// 保持),不足时按原序回填,槽位数与 touchRecall 语义均不变。rotate 为空时
|
|
1360
|
+
// 行为与既有排序完全一致。
|
|
1361
|
+
if (rotate && rotate.size > 0 && candidates.length > 0) {
|
|
1362
|
+
const fresh = candidates.filter((m) => !rotate.has(m.id));
|
|
1363
|
+
if (fresh.length > 0) candidates = [...fresh, ...candidates.filter((m) => rotate.has(m.id))];
|
|
1364
|
+
}
|
|
1349
1365
|
const selected = candidates.slice(0, maxItems);
|
|
1350
1366
|
touchRecalled(selected);
|
|
1351
1367
|
return selected;
|
package/src/settings.js
CHANGED
package/test/api.test.js
CHANGED
|
@@ -644,17 +644,18 @@ test("GET /api/dsh-mneme/features returns empty overrides and effective config d
|
|
|
644
644
|
assert.equal(res.statusCode, 200);
|
|
645
645
|
const data = JSON.parse(res.body);
|
|
646
646
|
assert.deepEqual(data.overrides, {});
|
|
647
|
-
// effective 覆盖全部
|
|
647
|
+
// effective 覆盖全部 52 个白名单键(含 v0.7.20 heatEnabled、Issue #89 新增
|
|
648
648
|
// dreamSkipInvalid/allowCrossTypeMerge/dreamMinIntervalMinutes、面板可调的
|
|
649
649
|
// dreamMaxTokens、睡眠路由 sleepProvider/sleepModel、PR1 新增的
|
|
650
650
|
// recallFusion/signalTransparency、issue #109 新增的实体抽取路由
|
|
651
651
|
// entityExtractionProvider/entityExtractionModel/entityExtractionReasoning、
|
|
652
652
|
// issue #127 新增的 summarize 五键、issue #126 新增的 sleepActionSet,
|
|
653
653
|
// 以及 issue #125 新增的候选集三键、issue #17(v0.8.0 A1/A3)新增的
|
|
654
|
-
// scopeEnabled/strictScope
|
|
654
|
+
// scopeEnabled/strictScope、issue #194 新增的 resilientModelDownload),
|
|
655
|
+
// 未覆盖时取 bundle 配置的解析默认值;
|
|
655
656
|
// dreamProvider/dreamModel 无 schema 默认值(Config({}) 解析为 undefined),
|
|
656
|
-
// 不编造给前端 →
|
|
657
|
-
assert.equal(Object.keys(data.effective).length,
|
|
657
|
+
// 不编造给前端 → 54 - 2 = 52
|
|
658
|
+
assert.equal(Object.keys(data.effective).length, 52);
|
|
658
659
|
assert.equal(data.effective.dreamSkipInvalid, true);
|
|
659
660
|
assert.equal(data.effective.allowCrossTypeMerge, false);
|
|
660
661
|
assert.equal(data.effective.dreamMinIntervalMinutes, 0);
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createStore } from "../src/store.js";
|
|
4
|
+
import { createService } from "../src/service.js";
|
|
5
|
+
import { createVectorIndex } from "../src/vector-index.js";
|
|
6
|
+
import { createInjector } from "../src/inject.js";
|
|
7
|
+
import { createSettings } from "../src/settings.js";
|
|
8
|
+
|
|
9
|
+
// 回归(issue #205):注入位跨轮轮换。长会话(几十上百轮、注入位有限)中
|
|
10
|
+
// 同一批条目在相邻轮次反复出现——报告者 15.5 小时实测 86 次注入 93% 重复,
|
|
11
|
+
// 库总览一条 43/43 块全勤。新配置 injectRotationTurns(默认 0 = 关闭):
|
|
12
|
+
// 同一条记忆在最近 N 个「不同用户查询」轮次注入过后,本轮不再优先——
|
|
13
|
+
// 新鲜者前置、不足按原序回填(槽位数不变);同一查询的重复渲染(工具轮)
|
|
14
|
+
// 不推进窗口;会话边界自动重置。
|
|
15
|
+
|
|
16
|
+
function setup(configOver = {}) {
|
|
17
|
+
const store = createStore(":memory:");
|
|
18
|
+
const service = createService({ store, mirror: null, config: {} });
|
|
19
|
+
const settings = createSettings(store.db);
|
|
20
|
+
const contexts = [];
|
|
21
|
+
const ctx = {
|
|
22
|
+
systemPrompt: {
|
|
23
|
+
context(def) {
|
|
24
|
+
contexts.push(def);
|
|
25
|
+
return () => {};
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const config = { maxInjectedItems: 2, importanceThreshold: 3, ...configOver };
|
|
30
|
+
createInjector(ctx, service, settings, config);
|
|
31
|
+
// 注入块是 contexts[0](memory),user-settings 是 contexts[1]
|
|
32
|
+
const render = (sessionId, query) => contexts[0].text({
|
|
33
|
+
agent: { session: { id: sessionId, snapshotEvents: () => query ? [
|
|
34
|
+
{ type: "user/message", data: { source: { kind: "user" }, content: [{ type: "text", text: query }] } }
|
|
35
|
+
] : [] } }
|
|
36
|
+
});
|
|
37
|
+
const titles = (text) => [...text.matchAll(/- \[([^\]]+)\] ([^((]*)/g)].map((m) => m[2].trim());
|
|
38
|
+
return { store, service, render, titles };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function seedMemories(service, n = 4) {
|
|
42
|
+
const saved = [];
|
|
43
|
+
for (let i = 0; i < n; i++) {
|
|
44
|
+
saved.push(service.saveWithDedupe({
|
|
45
|
+
type: "preference",
|
|
46
|
+
title: `偏好条目${i}`,
|
|
47
|
+
content: `第 ${i} 条独立的用户偏好内容,主题互不相同`,
|
|
48
|
+
importance: 5
|
|
49
|
+
}).memory);
|
|
50
|
+
}
|
|
51
|
+
return saved;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
test("#205: default (rotation off) repeats the same entries across turns", () => {
|
|
55
|
+
const { store, service, render, titles } = setup({});
|
|
56
|
+
seedMemories(service, 4);
|
|
57
|
+
const turn1 = titles(render("s1", "第一个问题"));
|
|
58
|
+
const turn2 = titles(render("s1", "第二个问题"));
|
|
59
|
+
assert.equal(turn1.length, 2);
|
|
60
|
+
assert.deepEqual(turn1, turn2, "rotation off → same entries repeat (current behavior)");
|
|
61
|
+
store.close();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("#205: rotation=1 swaps out previously injected entries, slots stay full", () => {
|
|
65
|
+
const { store, service, render, titles } = setup({ injectRotationTurns: 1 });
|
|
66
|
+
seedMemories(service, 4);
|
|
67
|
+
const turn1 = titles(render("s1", "第一个问题"));
|
|
68
|
+
const turn2 = titles(render("s1", "第二个问题"));
|
|
69
|
+
assert.equal(turn2.length, 2, "slots stay full");
|
|
70
|
+
for (const t of turn1) {
|
|
71
|
+
assert.ok(!turn2.includes(t), `previously injected "${t}" must rotate out`);
|
|
72
|
+
}
|
|
73
|
+
store.close();
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("#205: entry becomes eligible again after leaving the window", () => {
|
|
77
|
+
const { store, service, render, titles } = setup({ injectRotationTurns: 1 });
|
|
78
|
+
seedMemories(service, 4);
|
|
79
|
+
const turn1 = titles(render("s1", "q1"));
|
|
80
|
+
const turn2 = titles(render("s1", "q2")); // window = [q1] → turn1 条目被换下
|
|
81
|
+
const turn3 = titles(render("s1", "q3")); // window = [q2] → turn1 条目重新有资格
|
|
82
|
+
for (const t of turn2) assert.ok(!turn3.includes(t), `"${t}" (injected at turn2) rotates out at turn3`);
|
|
83
|
+
for (const t of turn1) assert.ok(turn3.includes(t), `"${t}" (turn1) is fresh again at turn3`);
|
|
84
|
+
store.close();
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("#205: repeated renders within the same query do not advance the window", () => {
|
|
88
|
+
const { store, service, render, titles } = setup({ injectRotationTurns: 1 });
|
|
89
|
+
seedMemories(service, 4);
|
|
90
|
+
const turn1a = titles(render("s1", "q1"));
|
|
91
|
+
const turn1b = titles(render("s1", "q1")); // 同一查询:工具调用轮,不推进
|
|
92
|
+
assert.deepEqual(turn1a, turn1b, "same query renders identically (window not advanced)");
|
|
93
|
+
const turn2 = titles(render("s1", "q2"));
|
|
94
|
+
for (const t of turn1a) assert.ok(!turn2.includes(t), "rotation happens on the next query only");
|
|
95
|
+
store.close();
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("#205: a new session starts fresh", () => {
|
|
99
|
+
const { store, service, render, titles } = setup({ injectRotationTurns: 1 });
|
|
100
|
+
seedMemories(service, 4);
|
|
101
|
+
const turn1 = titles(render("s1", "q1"));
|
|
102
|
+
const other = titles(render("s2", "q1")); // 新会话:历史为空
|
|
103
|
+
assert.deepEqual(other, turn1, "session boundary resets rotation history");
|
|
104
|
+
store.close();
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("#205: backfill keeps slots full when fresh candidates run out", () => {
|
|
108
|
+
const { store, service, render, titles } = setup({ injectRotationTurns: 1 });
|
|
109
|
+
seedMemories(service, 2); // 候选恰等于槽位
|
|
110
|
+
const turn1 = titles(render("s1", "q1"));
|
|
111
|
+
const turn2 = titles(render("s1", "q2"));
|
|
112
|
+
assert.equal(turn2.length, 2, "stale entries backfill when nothing fresh remains");
|
|
113
|
+
assert.deepEqual(turn1, turn2, "no fresh candidates → original selection returned");
|
|
114
|
+
store.close();
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// 回归(issue #205 补测,报告者读码发现):轮换窗口需要 maxItems×(N+1) 张
|
|
118
|
+
// 不同的牌,但候选池各有硬上限——语义路径截在 maxItems×2、合并层也是
|
|
119
|
+
// maxItems×2。池子只有十来张时,窗口一开就把池子吃光、只剩回填:重复率的
|
|
120
|
+
// 结构性下限由池子大小决定,轮换本身越不出箱子。修复:rotateWindow > 0 时
|
|
121
|
+
// 池子按 maxItems×(N+1) 扩容(语义检索、bm25、合并层三处同步),无轮换时
|
|
122
|
+
// 保持既有上限不变。旧测试样本(4 条记忆 + 注入位 2 = 恰好 maxItems×2)
|
|
123
|
+
// 正好测不到这个边界——本用例刻意用 8 条 + 窗口 2 压过旧上限。
|
|
124
|
+
test("#205 补测: semantic pool grows with the rotation window (no forced repeats)", () => {
|
|
125
|
+
const store = createStore(":memory:");
|
|
126
|
+
const service = createService({ store, mirror: null, config: {} });
|
|
127
|
+
const vectorIndex = createVectorIndex({ store });
|
|
128
|
+
service.setVectorIndex(vectorIndex);
|
|
129
|
+
const ids = [];
|
|
130
|
+
for (let i = 0; i < 8; i++) {
|
|
131
|
+
const theta = i * 0.1; // 角度递增 → 与 query [1,0,0] 的余弦严格递减
|
|
132
|
+
const m = service.saveWithDedupe({
|
|
133
|
+
type: "preference",
|
|
134
|
+
title: `条目${i}`,
|
|
135
|
+
content: `第 ${i} 条,主题互不相同`,
|
|
136
|
+
importance: 5
|
|
137
|
+
}).memory;
|
|
138
|
+
vectorIndex.saveEmbedding(m.id, [Math.cos(theta), Math.sin(theta), 0]);
|
|
139
|
+
ids.push(m.id);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const call = (rotate, rotateWindow) => service.injectCandidates({
|
|
143
|
+
query: "q",
|
|
144
|
+
queryVector: [1, 0, 0],
|
|
145
|
+
maxItems: 2,
|
|
146
|
+
threshold: 3,
|
|
147
|
+
rotate,
|
|
148
|
+
rotateWindow
|
|
149
|
+
}).map((m) => m.id);
|
|
150
|
+
|
|
151
|
+
const turn1 = call(null, 2);
|
|
152
|
+
assert.deepEqual(turn1, [ids[0], ids[1]], "baseline: top-2 by cosine");
|
|
153
|
+
const turn2 = call(new Set(turn1), 2);
|
|
154
|
+
assert.deepEqual(turn2, [ids[2], ids[3]], "window 1: next two fresh");
|
|
155
|
+
// 窗口 2 = 前两轮的并集(真实流程里注入层会把最近 N 轮的 id 合成一个集合)
|
|
156
|
+
const turn3 = call(new Set([...turn1, ...turn2]), 2);
|
|
157
|
+
// 旧上限(maxItems×2 = 4)下池子只有 m0..m3,第三轮 fresh 为空 → 只能回填
|
|
158
|
+
// m0/m1(强制重复)。池子扩到 6 后,m4/m5 仍在箱内可被轮换换上来。
|
|
159
|
+
assert.deepEqual(turn3, [ids[4], ids[5]], "window 2: pool grew with the window, no forced repeat");
|
|
160
|
+
store.close();
|
|
161
|
+
});
|
|
@@ -59,6 +59,26 @@ test("LocalEmbedder init loads via injected engineFactory with cache_dir", async
|
|
|
59
59
|
assert.equal(seen.options.device, "cpu");
|
|
60
60
|
});
|
|
61
61
|
|
|
62
|
+
test("LocalEmbedder init threads resilientModelDownload and logger to engineFactory", async () => {
|
|
63
|
+
// issue #194:开关与 logger 要能走到 defaultPipelineLoader(那里据此挂 env.fetch),
|
|
64
|
+
// 但不能混进 pipelineOptions 污染 transformers 的选项。
|
|
65
|
+
let seen = null;
|
|
66
|
+
const loader = async (_task, _model, options) => {
|
|
67
|
+
seen = options;
|
|
68
|
+
return makeFakeExtractor(512);
|
|
69
|
+
};
|
|
70
|
+
const logger = { info() {}, warn() {} };
|
|
71
|
+
const on = new LocalEmbedder({ cacheDir: "/tmp/model-cache", engineFactory: loader, resilientModelDownload: true, logger });
|
|
72
|
+
await on.init();
|
|
73
|
+
assert.equal(seen.resilientModelDownload, true);
|
|
74
|
+
assert.equal(seen.logger, logger);
|
|
75
|
+
|
|
76
|
+
seen = null;
|
|
77
|
+
const off = new LocalEmbedder({ engineFactory: loader });
|
|
78
|
+
await off.init();
|
|
79
|
+
assert.equal(seen.resilientModelDownload, false, "直连构造不显式给 true = 保持现状");
|
|
80
|
+
});
|
|
81
|
+
|
|
62
82
|
test("LocalEmbedder embed returns [n, dim] with mean pooling and chunking", async () => {
|
|
63
83
|
const calls = [];
|
|
64
84
|
const e = new LocalEmbedder({
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { tmpdir } from "node:os";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { createResilientFetch, installResilientFetch } from "../src/runtime/model-fetch.js";
|
|
9
|
+
|
|
10
|
+
// 模型文件取件层(issue #194)。transformers.js 下载几百 MB~1GB 的模型文件时,
|
|
11
|
+
// undici socket 中途被重置(TypeError: terminated)会让整份下载作废重来;报告者实测
|
|
12
|
+
// `curl -C -` 能稳定完成,说明服务端支持 Range。整条链都在本地 http 服务器上测:
|
|
13
|
+
// 断流、续传、退避重试、进度百分比 —— 不依赖外网。
|
|
14
|
+
|
|
15
|
+
/** 内容有变化的字节(0..250 循环):任何丢块 / 重复块都会在逐字节比对里现形。 */
|
|
16
|
+
function makeBody(size) {
|
|
17
|
+
const body = Buffer.alloc(size);
|
|
18
|
+
for (let i = 0; i < size; i++) body[i] = i % 251;
|
|
19
|
+
return body;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* 只服务内存里那份字节的服务器。
|
|
24
|
+
* - `cutFirst`:该 URL 的首次请求只发一半就掐断(等 30ms 再 destroy,保证客户端真的收到过字节);
|
|
25
|
+
* - `ignoreRange`:无视 Range 头,永远从 0 回全量 200 —— 逼出「续传被拒就从头再来」;
|
|
26
|
+
* - `seen` 记录收到的 Range 头与请求数。
|
|
27
|
+
*/
|
|
28
|
+
async function serve(files, { cutFirst = null, ignoreRange = false } = {}) {
|
|
29
|
+
const seen = { rangeHeader: null, requests: 0 };
|
|
30
|
+
let cutDone = false;
|
|
31
|
+
const server = createServer((req, res) => {
|
|
32
|
+
seen.requests++;
|
|
33
|
+
const body = files[req.url];
|
|
34
|
+
if (body === undefined) {
|
|
35
|
+
res.writeHead(404).end();
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const range = req.headers.range;
|
|
39
|
+
if (typeof range === "string") seen.rangeHeader = range;
|
|
40
|
+
const start = ignoreRange ? 0 : Number(/bytes=(\d+)-/.exec(range ?? "")?.[1] ?? 0);
|
|
41
|
+
|
|
42
|
+
if (cutFirst === req.url && !cutDone && start === 0) {
|
|
43
|
+
cutDone = true;
|
|
44
|
+
res.writeHead(200, {
|
|
45
|
+
"Content-Length": String(body.length),
|
|
46
|
+
ETag: '"immutable-etag"'
|
|
47
|
+
});
|
|
48
|
+
res.write(body.subarray(0, Math.floor(body.length / 2)));
|
|
49
|
+
// 立刻 destroy 会在客户端读到任何字节之前就断掉,那样就测不到「带着已收字节续传」。
|
|
50
|
+
setTimeout(() => res.destroy(), 30);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (start > 0) {
|
|
54
|
+
const rest = body.subarray(start);
|
|
55
|
+
res.writeHead(206, {
|
|
56
|
+
"Content-Range": `bytes ${start}-${body.length - 1}/${body.length}`,
|
|
57
|
+
ETag: '"immutable-etag"'
|
|
58
|
+
});
|
|
59
|
+
res.end(rest);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"immutable-etag"' });
|
|
63
|
+
res.end(body);
|
|
64
|
+
});
|
|
65
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
66
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
67
|
+
return { server, base, seen, close: () => new Promise((resolve) => server.close(resolve)) };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function partDirFor() {
|
|
71
|
+
return mkdtempSync(join(tmpdir(), "mneme-mfetch-"));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function testFetcher(partDir, opts = {}) {
|
|
75
|
+
return createResilientFetch({ partDir, retryDelayMs: 1, idleTimeoutMs: 5000, ...opts });
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
test("model-fetch:中途 terminated 后用 Range 续传,最终字节与源逐字节一致", async () => {
|
|
79
|
+
const body = makeBody(65536);
|
|
80
|
+
const srv = await serve({ "/model.onnx": body }, { cutFirst: "/model.onnx" });
|
|
81
|
+
const dir = partDirFor();
|
|
82
|
+
try {
|
|
83
|
+
const fetchImpl = testFetcher(dir);
|
|
84
|
+
const res = await fetchImpl(`${srv.base}/model.onnx`);
|
|
85
|
+
assert.equal(res.status, 200, "对 transformers.js 必须以 200 形态出现(hub 会把非 200 当失败)");
|
|
86
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
87
|
+
assert.ok(received.equals(body), "续传拼出的字节必须与源完全一致");
|
|
88
|
+
// 关键证据:第二次请求带了 Range —— 是续传,不是从头再来。
|
|
89
|
+
assert.match(String(srv.seen.rangeHeader), /^bytes=\d+-$/, `续传请求应当带 Range,实际=${srv.seen.rangeHeader}`);
|
|
90
|
+
} finally {
|
|
91
|
+
await srv.close();
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("model-fetch:每次失败都带中断百分比;重试耗尽时错误信息说清断在哪", async () => {
|
|
96
|
+
const body = makeBody(100000);
|
|
97
|
+
// 服务器掐在固定位置:每次请求都只发 30% 就断。
|
|
98
|
+
const server = createServer((_req, res) => {
|
|
99
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
100
|
+
res.write(body.subarray(0, 30000));
|
|
101
|
+
setTimeout(() => res.destroy(), 20);
|
|
102
|
+
});
|
|
103
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
104
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
105
|
+
const dir = partDirFor();
|
|
106
|
+
try {
|
|
107
|
+
const events = [];
|
|
108
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, onEvent: (e) => events.push(e) });
|
|
109
|
+
await assert.rejects(
|
|
110
|
+
() => fetchImpl(`${base}/big.bin`),
|
|
111
|
+
/中断于|30%/,
|
|
112
|
+
"最终错误必须带中断位置,让用户能区分网络问题与磁盘问题"
|
|
113
|
+
);
|
|
114
|
+
const fail = events.filter((e) => e.type === "fail");
|
|
115
|
+
assert.ok(fail.length >= 1, "每次失败都应发 fail 事件");
|
|
116
|
+
assert.equal(fail.at(-1).percent, 30, `断在 30%%(30000/100000),实际=${fail.at(-1).percent}`);
|
|
117
|
+
} finally {
|
|
118
|
+
await new Promise((resolve) => server.close(resolve));
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("model-fetch:If-Range 失配(服务端内容已换)时回 200,断点作废整份重来,不产出拼接损坏", async () => {
|
|
123
|
+
const v1 = makeBody(65536);
|
|
124
|
+
const v2 = Buffer.from(v1.map((b) => (b + 7) % 251));
|
|
125
|
+
let broken = false;
|
|
126
|
+
const server = createServer((req, res) => {
|
|
127
|
+
const range = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? 0);
|
|
128
|
+
// If-Range 与当前内容不符(或没带):真实服务器会回 200 全量 —— 内容换过了。
|
|
129
|
+
if (req.headers["if-range"] !== undefined && req.headers["if-range"] !== '"v2"') {
|
|
130
|
+
broken = true;
|
|
131
|
+
res.writeHead(200, { "Content-Length": String(v2.length), ETag: '"v2"' });
|
|
132
|
+
res.end(v2);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
if (range > 0 && !broken) {
|
|
136
|
+
res.writeHead(206, {
|
|
137
|
+
"Content-Range": `bytes ${range}-${v1.length - 1}/${v1.length}`,
|
|
138
|
+
ETag: '"v1"'
|
|
139
|
+
});
|
|
140
|
+
res.end(v1.subarray(range));
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (!broken) {
|
|
144
|
+
res.writeHead(200, { "Content-Length": String(v1.length), ETag: '"v1"' });
|
|
145
|
+
res.write(v1.subarray(0, Math.floor(v1.length / 2)));
|
|
146
|
+
setTimeout(() => res.destroy(), 30);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
151
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
152
|
+
const dir = partDirFor();
|
|
153
|
+
try {
|
|
154
|
+
const events = [];
|
|
155
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, onEvent: (e) => events.push(e) });
|
|
156
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
157
|
+
const body = Buffer.from(await res.arrayBuffer());
|
|
158
|
+
assert.ok(events.some((e) => e.type === "restart" && e.reason === "range-rejected"), "应发 range-rejected 重来事件");
|
|
159
|
+
assert.ok(body.equals(v2), "最终字节必须是新内容整份,而不是 v1 断点拼 v2 的损坏物");
|
|
160
|
+
} finally {
|
|
161
|
+
await new Promise((resolve) => server.close(resolve));
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test("model-fetch:服务器续传位置与断点对不上时清掉断点重来(拼错比失败更糟)", async () => {
|
|
166
|
+
const body = makeBody(65536);
|
|
167
|
+
let cutDone = false;
|
|
168
|
+
const server = createServer((req, res) => {
|
|
169
|
+
const range = req.headers.range;
|
|
170
|
+
if (typeof range === "string") {
|
|
171
|
+
// 故意回一个与断点无关的 206(从 0 开始):客户端必须识别出偏移不一致。
|
|
172
|
+
res.writeHead(206, { "Content-Range": `bytes 0-${body.length - 1}/${body.length}`, ETag: '"x"' });
|
|
173
|
+
res.end(body);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (!cutDone) {
|
|
177
|
+
cutDone = true;
|
|
178
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
179
|
+
res.write(body.subarray(0, 30000));
|
|
180
|
+
setTimeout(() => res.destroy(), 20);
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
184
|
+
res.end(body);
|
|
185
|
+
});
|
|
186
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
187
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
188
|
+
const dir = partDirFor();
|
|
189
|
+
try {
|
|
190
|
+
const events = [];
|
|
191
|
+
const fetchImpl = testFetcher(dir, { attempts: 3, onEvent: (e) => events.push(e) });
|
|
192
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
193
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
194
|
+
assert.ok(events.some((e) => e.type === "restart" && e.reason === "offset-mismatch"));
|
|
195
|
+
assert.ok(received.equals(body), "重置后重新下载的整份字节必须正确");
|
|
196
|
+
} finally {
|
|
197
|
+
await new Promise((resolve) => server.close(resolve));
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
test("model-fetch:416(断点越过文件末尾)时重置断点重试,而不是把重试次数烧完", async () => {
|
|
202
|
+
const body = makeBody(65536);
|
|
203
|
+
const server = createServer((req, res) => {
|
|
204
|
+
const start = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? -1);
|
|
205
|
+
if (start >= body.length) {
|
|
206
|
+
// 诚实的 416:Range 起点越过源文件末尾(断点来自被替换前的旧文件)。
|
|
207
|
+
res.writeHead(416, { "Content-Range": `bytes */${body.length}` });
|
|
208
|
+
res.end();
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
if (start > 0) {
|
|
212
|
+
res.writeHead(206, { "Content-Range": `bytes ${start}-${body.length - 1}/${body.length}`, ETag: '"x"' });
|
|
213
|
+
res.end(body.subarray(start));
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
217
|
+
res.end(body);
|
|
218
|
+
});
|
|
219
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
220
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
221
|
+
const url = `${base}/model.onnx`;
|
|
222
|
+
const dir = partDirFor();
|
|
223
|
+
try {
|
|
224
|
+
// 预置一份比源文件还大的断点(70000 > 65536)+ 它的 etag。
|
|
225
|
+
const partPath = join(dir, createHash("sha256").update(url).digest("hex"));
|
|
226
|
+
writeFileSync(partPath, Buffer.alloc(70000, 1));
|
|
227
|
+
writeFileSync(`${partPath}.meta`, '"old"\n');
|
|
228
|
+
const events = [];
|
|
229
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, onEvent: (e) => events.push(e) });
|
|
230
|
+
const res = await fetchImpl(url);
|
|
231
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
232
|
+
assert.ok(events.some((e) => e.type === "restart" && e.reason === "range-not-satisfiable"));
|
|
233
|
+
assert.ok(received.equals(body), "重置后重新下载的整份字节必须正确");
|
|
234
|
+
} finally {
|
|
235
|
+
await new Promise((resolve) => server.close(resolve));
|
|
236
|
+
}
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
test("model-fetch:连接僵死时被空闲看门狗掐断并重试,而不是永远挂着", async () => {
|
|
240
|
+
const body = makeBody(65536);
|
|
241
|
+
let stalled = false;
|
|
242
|
+
const server = createServer((_req, res) => {
|
|
243
|
+
if (!stalled) {
|
|
244
|
+
stalled = true;
|
|
245
|
+
res.writeHead(200, { "Content-Length": String(body.length) });
|
|
246
|
+
res.write(body.subarray(0, 10));
|
|
247
|
+
return; // 永远不 end:僵死的 socket。
|
|
248
|
+
}
|
|
249
|
+
res.writeHead(200, { "Content-Length": String(body.length) });
|
|
250
|
+
res.end(body);
|
|
251
|
+
});
|
|
252
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
253
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
254
|
+
const dir = partDirFor();
|
|
255
|
+
try {
|
|
256
|
+
const events = [];
|
|
257
|
+
const started = Date.now();
|
|
258
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, idleTimeoutMs: 150, retryDelayMs: 1, onEvent: (e) => events.push(e) });
|
|
259
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
260
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
261
|
+
assert.ok(Date.now() - started < 5000, "应当被看门狗及时掐断,而不是挂住");
|
|
262
|
+
assert.ok(events.some((e) => e.type === "fail" && /空闲/.test(e.error)), `失败事件应写明空闲超时,实际=${events.map((e) => e.error)}`);
|
|
263
|
+
assert.ok(received.equals(body));
|
|
264
|
+
} finally {
|
|
265
|
+
server.closeAllConnections();
|
|
266
|
+
await new Promise((resolve) => server.close(resolve));
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test("model-fetch:服务端干净地提前关流(字节没给够)被查出并续传补齐", async () => {
|
|
271
|
+
const body = makeBody(100000);
|
|
272
|
+
const server = createServer((req, res) => {
|
|
273
|
+
const range = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? 0);
|
|
274
|
+
if (range > 0) {
|
|
275
|
+
res.writeHead(206, { "Content-Range": `bytes ${range}-${body.length - 1}/${body.length}`, ETag: '"x"' });
|
|
276
|
+
res.end(body.subarray(range));
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
280
|
+
res.end(body.subarray(0, 30000)); // 声明 100000 只给 30000,然后"正常"结束。
|
|
281
|
+
});
|
|
282
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
283
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
284
|
+
const dir = partDirFor();
|
|
285
|
+
try {
|
|
286
|
+
const fetchImpl = testFetcher(dir, { attempts: 2 });
|
|
287
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
288
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
289
|
+
assert.ok(received.equals(body), "缺的字节要从断点补齐");
|
|
290
|
+
} finally {
|
|
291
|
+
await new Promise((resolve) => server.close(resolve));
|
|
292
|
+
}
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test("model-fetch:非 GET 与抢不到锁的请求原样直通,不落盘、不带 Range", async () => {
|
|
296
|
+
const dir = partDirFor();
|
|
297
|
+
const calls = [];
|
|
298
|
+
const stub = (input, init) => {
|
|
299
|
+
calls.push({ input: String(input), init: init ?? null });
|
|
300
|
+
return new Response("raw", { status: 200, headers: { "content-length": "3" } });
|
|
301
|
+
};
|
|
302
|
+
const fetchImpl = testFetcher(dir, { fetchImpl: stub });
|
|
303
|
+
await fetchImpl("https://example.com/meta", { method: "HEAD" });
|
|
304
|
+
// 锁名与被测实现一致:<sha256(url)>.lock;新鲜的锁 = 别的进程正在写。
|
|
305
|
+
const url = "https://example.com/model.onnx";
|
|
306
|
+
const locked = join(dir, `${createHash("sha256").update(url).digest("hex")}.lock`);
|
|
307
|
+
writeFileSync(locked, "9999");
|
|
308
|
+
await fetchImpl(url);
|
|
309
|
+
assert.equal(calls.length, 2, "两个请求都应直通底层 fetch");
|
|
310
|
+
assert.equal(calls[0].init.method, "HEAD");
|
|
311
|
+
assert.equal(calls[1].init, null, "直通不应改写调用参数");
|
|
312
|
+
assert.deepEqual(readdirSync(dir), [`${createHash("sha256").update(url).digest("hex")}.lock`], "直通不该产生断点文件");
|
|
313
|
+
rmSync(locked, { force: true });
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test("model-fetch:响应体消费完后断点文件、meta 与锁都清理干净", async () => {
|
|
317
|
+
const body = makeBody(65536);
|
|
318
|
+
const srv = await serve({ "/model.onnx": body }, { cutFirst: "/model.onnx" });
|
|
319
|
+
const dir = partDirFor();
|
|
320
|
+
try {
|
|
321
|
+
const fetchImpl = testFetcher(dir);
|
|
322
|
+
const res = await fetchImpl(`${srv.base}/model.onnx`);
|
|
323
|
+
await res.arrayBuffer(); // 消费完 = 下游(FileCache.put)拿完字节了。
|
|
324
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
325
|
+
assert.equal(readdirSync(dir).length, 0, `断点目录应已清空,实际=${readdirSync(dir)}`);
|
|
326
|
+
} finally {
|
|
327
|
+
await srv.close();
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test("installResilientFetch:开关关 = env 一字不动;开 = env.fetch 换成弹性实现", async () => {
|
|
332
|
+
const envOff = {};
|
|
333
|
+
assert.equal(installResilientFetch(envOff, { enabled: false }), false);
|
|
334
|
+
assert.equal(envOff.fetch, undefined, "关闭时不许碰 env(默认保持现状的出口)");
|
|
335
|
+
assert.equal(installResilientFetch(null, { enabled: true }), false, "env 缺席时不安装");
|
|
336
|
+
|
|
337
|
+
const cacheRoot = mkdtempSync(join(tmpdir(), "mneme-install-"));
|
|
338
|
+
const envOn = {};
|
|
339
|
+
assert.equal(installResilientFetch(envOn, { enabled: true, cacheDir: cacheRoot, fetchImpl: () => new Response("x") }), true);
|
|
340
|
+
assert.equal(typeof envOn.fetch, "function");
|
|
341
|
+
assert.ok(existsSync(join(cacheRoot, ".mneme-partial")), "断点目录建在模型缓存旁");
|
|
342
|
+
|
|
343
|
+
// 装上的 fetch 是能干活的:小文件直通下载 + 返回规范 200。
|
|
344
|
+
const res = await envOn.fetch("https://example.com/tiny.bin");
|
|
345
|
+
assert.equal(res.status, 200);
|
|
346
|
+
assert.equal(await res.text(), "x");
|
|
347
|
+
});
|
|
348
|
+
|