@modusensus/dsh-mneme 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -3
- package/lib/api-standalone.js +12 -0
- package/lib/api.js +12 -2
- package/lib/client.js +5 -1
- package/lib/config.js +18 -0
- package/lib/dream/decisions.js +55 -8
- package/lib/dream/sleep.js +36 -5
- package/lib/dream.js +18 -7
- package/lib/entities/extractor.js +2 -2
- package/lib/index.js +8 -0
- package/lib/inject.js +43 -1
- package/lib/lang.js +2 -2
- package/lib/local-embedder.js +21 -1
- package/lib/reranker.js +47 -8
- package/lib/runtime/model-fetch.js +361 -0
- package/lib/service.js +37 -5
- package/lib/settings.js +1 -0
- package/lib/store.js +85 -25
- package/lib/summarize.js +2 -2
- package/lib/tools.js +6 -19
- package/lib/vector-index.js +2 -8
- package/package.json +2 -1
- package/scripts/sync-test-badge.mjs +61 -0
- package/src/api-standalone.js +12 -0
- package/src/api.js +12 -2
- package/src/config.js +18 -0
- package/src/dream/decisions.js +55 -8
- package/src/dream/sleep.js +36 -5
- package/src/dream.js +18 -7
- package/src/entities/extractor.js +2 -2
- package/src/index.js +8 -0
- package/src/inject.js +43 -1
- package/src/lang.js +2 -2
- package/src/local-embedder.js +21 -1
- package/src/reranker.js +47 -8
- package/src/runtime/model-fetch.js +361 -0
- package/src/service.js +37 -5
- package/src/settings.js +1 -0
- package/src/store.js +85 -25
- package/src/summarize.js +2 -2
- package/src/tools.js +6 -19
- package/src/vector-index.js +2 -8
- package/test/api.test.js +10 -4
- package/test/dream-archive-guardrail.test.js +116 -0
- package/test/dream-coverage-degrade.test.js +86 -0
- package/test/dream.test.js +15 -11
- package/test/inject-first-round-bm25.test.js +89 -0
- package/test/inject-rotation.test.js +161 -0
- package/test/llm-message-source.test.js +177 -0
- package/test/local-embedder.test.js +20 -0
- package/test/reranker-single-logit.test.js +73 -0
- package/test/runtime-model-fetch.test.js +348 -0
- package/test/skip-invalid-restore.test.js +7 -5
- package/test/sleep-scheduler-rearm.test.js +89 -0
- package/test/sleep.test.js +2 -2
- package/test/standalone-api.test.js +34 -1
- package/test/store-vector-cache.test.js +98 -0
- package/test/tools-memory-get-schema.test.js +65 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createStore } from "../src/store.js";
|
|
4
|
+
import { createService } from "../src/service.js";
|
|
5
|
+
import { createDreamScheduler } from "../src/dream.js";
|
|
6
|
+
import { createSummarizer } from "../src/summarize.js";
|
|
7
|
+
import { runSleep } from "../src/dream/sleep.js";
|
|
8
|
+
import { extractEntities } from "../src/entities/extractor.js";
|
|
9
|
+
import { createVectorIndex } from "../src/vector-index.js";
|
|
10
|
+
|
|
11
|
+
// 回归:dsh-llm 的消息契约要求每条消息带 source。provider 适配器会直接读
|
|
12
|
+
// message.source.kind 来区分用户内容与工具结果(例如
|
|
13
|
+
// @mars-sea/dsh-commandcode-provider 的 messagesToCC),缺字段时请求在序列化
|
|
14
|
+
// 阶段就抛 "Cannot read properties of undefined (reading 'kind')",四条 LLM 管线
|
|
15
|
+
// (dream / sleep / summarize / entity extraction)全军覆没、耗时 0ms。
|
|
16
|
+
// 这里用同等严格的桩驱动真实管线,任何新加的消息漏了 source 都会失败。
|
|
17
|
+
function assertMessageSources(options) {
|
|
18
|
+
for (const message of options.messages ?? []) {
|
|
19
|
+
assert.equal(
|
|
20
|
+
message.source?.kind,
|
|
21
|
+
"plugin",
|
|
22
|
+
`LLM 消息缺 source(provider 会抛 TypeError): ${message.role} / ${String(message.content?.[0]?.text ?? "").slice(0, 40)}`
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const logger = { warn: () => {}, info: () => {} };
|
|
28
|
+
|
|
29
|
+
function makeService() {
|
|
30
|
+
const store = createStore(":memory:");
|
|
31
|
+
const service = createService({ store, mirror: null, config: {} });
|
|
32
|
+
return { store, service };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// dsh-llm chunk 形状与真实 provider 对齐:文本增量 + 终止 chunk。
|
|
36
|
+
function reply(text, finish = { reason: { kind: "stop" } }) {
|
|
37
|
+
return (async function* () {
|
|
38
|
+
yield { type: "text-delta", index: 0, text };
|
|
39
|
+
yield { type: "finish", ...finish };
|
|
40
|
+
})();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// ---------------------------------------------------------------- dream
|
|
44
|
+
|
|
45
|
+
test("dream: consolidation and summary messages carry source", async () => {
|
|
46
|
+
const { store, service } = makeService();
|
|
47
|
+
service.saveWithDedupe({ type: "project", title: "主题01", content: "规范内容", importance: 5 });
|
|
48
|
+
service.saveWithDedupe({ type: "project", title: "主题01·变体", content: "旧变体", importance: 3 });
|
|
49
|
+
|
|
50
|
+
const calls = [];
|
|
51
|
+
const ctx = {
|
|
52
|
+
logger,
|
|
53
|
+
agentDefaultModel: { currentSelection: () => ({ provider: "mock", model: "source-model" }) },
|
|
54
|
+
llm: {
|
|
55
|
+
async *stream(options) {
|
|
56
|
+
calls.push(options);
|
|
57
|
+
assertMessageSources(options);
|
|
58
|
+
const userText = options.messages.find((m) => m.role === "user")?.content?.[0]?.text ?? "";
|
|
59
|
+
yield* reply(userText.startsWith("id=") ? "[]" : "记忆库总览:已巩固。");
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const dream = createDreamScheduler({ onRun: () => Promise.resolve({ ok: true, skipped: true }) });
|
|
65
|
+
const run = await dream.runDream(ctx, service, {});
|
|
66
|
+
assert.equal(run.ok, true, "dream run succeeds against the strict provider stub");
|
|
67
|
+
assert.ok(calls.some((c) => c.purpose === "compaction"), "consolidation called the LLM");
|
|
68
|
+
assert.ok(calls.length >= 2, "consolidation + summary both called");
|
|
69
|
+
store.close();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// ---------------------------------------------------------------- summarize
|
|
73
|
+
|
|
74
|
+
test("summarize: distilled transcript and prompt messages carry source", async () => {
|
|
75
|
+
const { store, service } = makeService();
|
|
76
|
+
const events = [];
|
|
77
|
+
const calls = [];
|
|
78
|
+
const ctx = {
|
|
79
|
+
on(name, fn) {
|
|
80
|
+
events.push({ name, fn });
|
|
81
|
+
return () => {};
|
|
82
|
+
},
|
|
83
|
+
logger,
|
|
84
|
+
llm: {
|
|
85
|
+
stream(options) {
|
|
86
|
+
calls.push(options);
|
|
87
|
+
assertMessageSources(options);
|
|
88
|
+
return reply("[]", { kind: "ok" });
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
createSummarizer(ctx, service, { autoSummarize: true });
|
|
93
|
+
|
|
94
|
+
const handler = events.find((e) => e.name === "session/event").fn;
|
|
95
|
+
const session = {
|
|
96
|
+
id: "s-source",
|
|
97
|
+
requestHeader: () => ({ config: { provider: "mock", model: "source-model" } }),
|
|
98
|
+
events: [
|
|
99
|
+
{ type: "user/message", data: { source: { kind: "user" }, content: [{ type: "text", text: "帮我选型" }] } },
|
|
100
|
+
{ seq: 2, type: "turn/end" }
|
|
101
|
+
]
|
|
102
|
+
};
|
|
103
|
+
await handler(session, { seq: 2, type: "turn/end" });
|
|
104
|
+
|
|
105
|
+
assert.equal(calls.length, 1, "summarization called the LLM");
|
|
106
|
+
store.close();
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
// ---------------------------------------------------------------- sleep
|
|
110
|
+
|
|
111
|
+
test("sleep: conflict and pattern messages carry source", async () => {
|
|
112
|
+
const { store, service } = makeService();
|
|
113
|
+
const vectorIndex = createVectorIndex({ store });
|
|
114
|
+
const embedder = {
|
|
115
|
+
embedSingle: async () => [1, 0, 0],
|
|
116
|
+
embed: async () => [1, 0, 0],
|
|
117
|
+
schedule: () => {},
|
|
118
|
+
modelHash: "mock#1",
|
|
119
|
+
dimension: 3
|
|
120
|
+
};
|
|
121
|
+
service.setEmbedder(embedder);
|
|
122
|
+
service.setVectorIndex(vectorIndex);
|
|
123
|
+
// 冲突候选要在向量索引里真有向量(同 sleep.test.js 的 seedConflictPair);
|
|
124
|
+
// 同时给 pattern 阶段留下 2 条素材。
|
|
125
|
+
const a = service.saveWithDedupe({ type: "project", title: "主题X", content: "内容A 关于主题X", importance: 3 }).memory;
|
|
126
|
+
const b = service.saveWithDedupe({ type: "project", title: "主题X副本", content: "内容B 关于主题X", importance: 3 }).memory;
|
|
127
|
+
vectorIndex.saveEmbedding(a.id, [1, 0, 0]);
|
|
128
|
+
vectorIndex.saveEmbedding(b.id, [1, 0, 0]);
|
|
129
|
+
|
|
130
|
+
const calls = [];
|
|
131
|
+
const ctx = {
|
|
132
|
+
logger,
|
|
133
|
+
agentDefaultModel: { currentSelection: () => ({ provider: "mock", model: "sleep-model" }) },
|
|
134
|
+
llm: {
|
|
135
|
+
async *stream(options) {
|
|
136
|
+
calls.push(options);
|
|
137
|
+
assertMessageSources(options);
|
|
138
|
+
yield* reply("[]");
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
const config = {
|
|
143
|
+
sleepModeEnabled: true,
|
|
144
|
+
sleepIdleMinutes: 5,
|
|
145
|
+
sleepMinIntervalHours: 8,
|
|
146
|
+
sleepConflictStrictness: "normal",
|
|
147
|
+
sleepArchiveDays: 30,
|
|
148
|
+
sleepCompressDays: 90,
|
|
149
|
+
sleepPatternMinMemories: 2,
|
|
150
|
+
sleepMaxPatternPerRun: 3
|
|
151
|
+
};
|
|
152
|
+
const result = await runSleep(ctx, service, config, logger, { embedder, vectorIndex }, null);
|
|
153
|
+
|
|
154
|
+
assert.ok(calls.some((c) => c.purpose === "sleep-conflict"), "conflict phase called the LLM");
|
|
155
|
+
assert.ok(calls.some((c) => c.purpose === "sleep-pattern"), "pattern phase called the LLM");
|
|
156
|
+
assert.ok(result.runId, "sleep run produced a receipt");
|
|
157
|
+
store.close();
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
// ---------------------------------------------------------------- entity extraction
|
|
161
|
+
|
|
162
|
+
test("entity extraction: messages handed to the llm adapter carry source", async () => {
|
|
163
|
+
const { store, service } = makeService();
|
|
164
|
+
const memory = service.saveWithDedupe({ type: "project", title: "协作", content: "Alpha 与 Beta 一起干活", importance: 3 }).memory;
|
|
165
|
+
|
|
166
|
+
const calls = [];
|
|
167
|
+
const callLLM = async (messages) => {
|
|
168
|
+
calls.push(messages);
|
|
169
|
+
assertMessageSources({ messages });
|
|
170
|
+
return "[]";
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
await extractEntities(memory, { store, config: {}, callLLM, logger });
|
|
174
|
+
|
|
175
|
+
assert.equal(calls.length, 1, "extraction called the LLM adapter");
|
|
176
|
+
store.close();
|
|
177
|
+
});
|
|
@@ -59,6 +59,26 @@ test("LocalEmbedder init loads via injected engineFactory with cache_dir", async
|
|
|
59
59
|
assert.equal(seen.options.device, "cpu");
|
|
60
60
|
});
|
|
61
61
|
|
|
62
|
+
test("LocalEmbedder init threads resilientModelDownload and logger to engineFactory", async () => {
|
|
63
|
+
// issue #194:开关与 logger 要能走到 defaultPipelineLoader(那里据此挂 env.fetch),
|
|
64
|
+
// 但不能混进 pipelineOptions 污染 transformers 的选项。
|
|
65
|
+
let seen = null;
|
|
66
|
+
const loader = async (_task, _model, options) => {
|
|
67
|
+
seen = options;
|
|
68
|
+
return makeFakeExtractor(512);
|
|
69
|
+
};
|
|
70
|
+
const logger = { info() {}, warn() {} };
|
|
71
|
+
const on = new LocalEmbedder({ cacheDir: "/tmp/model-cache", engineFactory: loader, resilientModelDownload: true, logger });
|
|
72
|
+
await on.init();
|
|
73
|
+
assert.equal(seen.resilientModelDownload, true);
|
|
74
|
+
assert.equal(seen.logger, logger);
|
|
75
|
+
|
|
76
|
+
seen = null;
|
|
77
|
+
const off = new LocalEmbedder({ engineFactory: loader });
|
|
78
|
+
await off.init();
|
|
79
|
+
assert.equal(seen.resilientModelDownload, false, "直连构造不显式给 true = 保持现状");
|
|
80
|
+
});
|
|
81
|
+
|
|
62
82
|
test("LocalEmbedder embed returns [n, dim] with mean pooling and chunking", async () => {
|
|
63
83
|
const calls = [];
|
|
64
84
|
const e = new LocalEmbedder({
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { LocalReranker } from "../src/reranker.js";
|
|
4
|
+
|
|
5
|
+
// 回归(issue #188):Xenova/bge-reranker-base 是单 logit 交叉编码器——每个
|
|
6
|
+
// (query, passage) 对只输出 1 个 logit。tc 策略此前在 cols===1 时把 l1 退化成
|
|
7
|
+
// l0,sigmoid(l0 - l1) = sigmoid(0) = 0.5:所有候选分数恒为常量,重排静默失效
|
|
8
|
+
// (不抛错、日志照常 ready、语义状态照常 reranker="ready",只有分数是死的)。
|
|
9
|
+
// 单 logit 头的正类概率是 sigmoid(logit);sigmoid(l1 - l0) 只对双 logit 头
|
|
10
|
+
// (num_labels=2 的导出,如 bge-reranker-v2-m3)成立。修复后按 cols 分支,
|
|
11
|
+
// 两路各有回归;另加「分数全等 → warn 一次」的静默失效绊线。
|
|
12
|
+
|
|
13
|
+
function sigmoid(x) {
|
|
14
|
+
return 1 / (1 + Math.exp(-x));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function candidates(ids) {
|
|
18
|
+
return ids.map((id) => ({ id, title: `T-${id}`, content: `C-${id}` }));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// 只支持 text-classification 的假引擎:cascade 先试 "rerank"(transformers
|
|
22
|
+
// 4.2.0 无此任务,真机上也必然先失败),随后落到 tc。
|
|
23
|
+
function makeTcFactory(dims, data) {
|
|
24
|
+
return async (task) => {
|
|
25
|
+
if (task === "rerank") throw new Error("unsupported task: rerank");
|
|
26
|
+
assert.equal(task, "text-classification");
|
|
27
|
+
return {
|
|
28
|
+
tokenizer: (texts, _opts) => ({ texts }),
|
|
29
|
+
model: async (inputs) => ({ logits: { dims, data: Float64Array.from(data.slice(0, inputs.texts.length * dims[1])) } })
|
|
30
|
+
};
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
test("#188: single-logit head scores sigmoid(logit) instead of constant 0.5", async () => {
|
|
35
|
+
// 报告的实测 logits:相关 -0.556、无关 -10.193,区分度近 10——模型判断得
|
|
36
|
+
// 很好,只是旧公式把它们全抹成 0.5000。
|
|
37
|
+
const r = new LocalReranker({
|
|
38
|
+
engineFactory: makeTcFactory([3, 1], [-0.5561, -10.193, 8.7])
|
|
39
|
+
});
|
|
40
|
+
await r.init();
|
|
41
|
+
const out = await r.rerank("查询", candidates(["rel", "noise", "hit"]));
|
|
42
|
+
// sigmoid(-0.5561)≈0.3645 保留;sigmoid(-10.193)≈3.7e-5 被 0.1 阈值滤掉;
|
|
43
|
+
// sigmoid(8.7)≈0.9998 居首。修复前三者全 0.5、顺序不变。
|
|
44
|
+
assert.deepEqual(out.map((x) => x.id), ["hit", "rel"]);
|
|
45
|
+
assert.ok(Math.abs(out[0].score - sigmoid(8.7)) < 1e-6, "hit = sigmoid(logit)");
|
|
46
|
+
assert.ok(Math.abs(out[1].score - sigmoid(-0.5561)) < 1e-6, "rel = sigmoid(logit)");
|
|
47
|
+
r.dispose();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("#188: dual-logit head keeps softmax-positive-class semantics", async () => {
|
|
51
|
+
// l0=-2,l1=2 → sigmoid(4);l0=1,l1=1.5 → sigmoid(0.5)——与原实现逐字节等价。
|
|
52
|
+
const r = new LocalReranker({
|
|
53
|
+
engineFactory: makeTcFactory([2, 2], [-2, 2, 1, 1.5])
|
|
54
|
+
});
|
|
55
|
+
await r.init();
|
|
56
|
+
const out = await r.rerank("查询", candidates(["a", "b"]));
|
|
57
|
+
assert.deepEqual(out.map((x) => x.id), ["a", "b"]);
|
|
58
|
+
assert.ok(Math.abs(out[0].score - sigmoid(4)) < 1e-6);
|
|
59
|
+
assert.ok(Math.abs(out[1].score - sigmoid(0.5)) < 1e-6);
|
|
60
|
+
r.dispose();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("#188: constant-score batch warns once (silent-failure tripwire)", async () => {
|
|
64
|
+
const warns = [];
|
|
65
|
+
const r = new LocalReranker({
|
|
66
|
+
scorePair: async () => 0.5,
|
|
67
|
+
logger: { warn: (m) => warns.push(m), info: () => {} }
|
|
68
|
+
});
|
|
69
|
+
await r.rerank("查询", candidates(["a", "b", "c"]));
|
|
70
|
+
await r.rerank("查询", candidates(["a", "b", "c"]));
|
|
71
|
+
assert.equal(warns.length, 1, "constant-score batch warns exactly once across calls");
|
|
72
|
+
r.dispose();
|
|
73
|
+
});
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { tmpdir } from "node:os";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { createResilientFetch, installResilientFetch } from "../src/runtime/model-fetch.js";
|
|
9
|
+
|
|
10
|
+
// 模型文件取件层(issue #194)。transformers.js 下载几百 MB~1GB 的模型文件时,
|
|
11
|
+
// undici socket 中途被重置(TypeError: terminated)会让整份下载作废重来;报告者实测
|
|
12
|
+
// `curl -C -` 能稳定完成,说明服务端支持 Range。整条链都在本地 http 服务器上测:
|
|
13
|
+
// 断流、续传、退避重试、进度百分比 —— 不依赖外网。
|
|
14
|
+
|
|
15
|
+
/** 内容有变化的字节(0..250 循环):任何丢块 / 重复块都会在逐字节比对里现形。 */
|
|
16
|
+
function makeBody(size) {
|
|
17
|
+
const body = Buffer.alloc(size);
|
|
18
|
+
for (let i = 0; i < size; i++) body[i] = i % 251;
|
|
19
|
+
return body;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* 只服务内存里那份字节的服务器。
|
|
24
|
+
* - `cutFirst`:该 URL 的首次请求只发一半就掐断(等 30ms 再 destroy,保证客户端真的收到过字节);
|
|
25
|
+
* - `ignoreRange`:无视 Range 头,永远从 0 回全量 200 —— 逼出「续传被拒就从头再来」;
|
|
26
|
+
* - `seen` 记录收到的 Range 头与请求数。
|
|
27
|
+
*/
|
|
28
|
+
async function serve(files, { cutFirst = null, ignoreRange = false } = {}) {
|
|
29
|
+
const seen = { rangeHeader: null, requests: 0 };
|
|
30
|
+
let cutDone = false;
|
|
31
|
+
const server = createServer((req, res) => {
|
|
32
|
+
seen.requests++;
|
|
33
|
+
const body = files[req.url];
|
|
34
|
+
if (body === undefined) {
|
|
35
|
+
res.writeHead(404).end();
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const range = req.headers.range;
|
|
39
|
+
if (typeof range === "string") seen.rangeHeader = range;
|
|
40
|
+
const start = ignoreRange ? 0 : Number(/bytes=(\d+)-/.exec(range ?? "")?.[1] ?? 0);
|
|
41
|
+
|
|
42
|
+
if (cutFirst === req.url && !cutDone && start === 0) {
|
|
43
|
+
cutDone = true;
|
|
44
|
+
res.writeHead(200, {
|
|
45
|
+
"Content-Length": String(body.length),
|
|
46
|
+
ETag: '"immutable-etag"'
|
|
47
|
+
});
|
|
48
|
+
res.write(body.subarray(0, Math.floor(body.length / 2)));
|
|
49
|
+
// 立刻 destroy 会在客户端读到任何字节之前就断掉,那样就测不到「带着已收字节续传」。
|
|
50
|
+
setTimeout(() => res.destroy(), 30);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (start > 0) {
|
|
54
|
+
const rest = body.subarray(start);
|
|
55
|
+
res.writeHead(206, {
|
|
56
|
+
"Content-Range": `bytes ${start}-${body.length - 1}/${body.length}`,
|
|
57
|
+
ETag: '"immutable-etag"'
|
|
58
|
+
});
|
|
59
|
+
res.end(rest);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"immutable-etag"' });
|
|
63
|
+
res.end(body);
|
|
64
|
+
});
|
|
65
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
66
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
67
|
+
return { server, base, seen, close: () => new Promise((resolve) => server.close(resolve)) };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function partDirFor() {
|
|
71
|
+
return mkdtempSync(join(tmpdir(), "mneme-mfetch-"));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function testFetcher(partDir, opts = {}) {
|
|
75
|
+
return createResilientFetch({ partDir, retryDelayMs: 1, idleTimeoutMs: 5000, ...opts });
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
test("model-fetch:中途 terminated 后用 Range 续传,最终字节与源逐字节一致", async () => {
|
|
79
|
+
const body = makeBody(65536);
|
|
80
|
+
const srv = await serve({ "/model.onnx": body }, { cutFirst: "/model.onnx" });
|
|
81
|
+
const dir = partDirFor();
|
|
82
|
+
try {
|
|
83
|
+
const fetchImpl = testFetcher(dir);
|
|
84
|
+
const res = await fetchImpl(`${srv.base}/model.onnx`);
|
|
85
|
+
assert.equal(res.status, 200, "对 transformers.js 必须以 200 形态出现(hub 会把非 200 当失败)");
|
|
86
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
87
|
+
assert.ok(received.equals(body), "续传拼出的字节必须与源完全一致");
|
|
88
|
+
// 关键证据:第二次请求带了 Range —— 是续传,不是从头再来。
|
|
89
|
+
assert.match(String(srv.seen.rangeHeader), /^bytes=\d+-$/, `续传请求应当带 Range,实际=${srv.seen.rangeHeader}`);
|
|
90
|
+
} finally {
|
|
91
|
+
await srv.close();
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("model-fetch:每次失败都带中断百分比;重试耗尽时错误信息说清断在哪", async () => {
|
|
96
|
+
const body = makeBody(100000);
|
|
97
|
+
// 服务器掐在固定位置:每次请求都只发 30% 就断。
|
|
98
|
+
const server = createServer((_req, res) => {
|
|
99
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
100
|
+
res.write(body.subarray(0, 30000));
|
|
101
|
+
setTimeout(() => res.destroy(), 20);
|
|
102
|
+
});
|
|
103
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
104
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
105
|
+
const dir = partDirFor();
|
|
106
|
+
try {
|
|
107
|
+
const events = [];
|
|
108
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, onEvent: (e) => events.push(e) });
|
|
109
|
+
await assert.rejects(
|
|
110
|
+
() => fetchImpl(`${base}/big.bin`),
|
|
111
|
+
/中断于|30%/,
|
|
112
|
+
"最终错误必须带中断位置,让用户能区分网络问题与磁盘问题"
|
|
113
|
+
);
|
|
114
|
+
const fail = events.filter((e) => e.type === "fail");
|
|
115
|
+
assert.ok(fail.length >= 1, "每次失败都应发 fail 事件");
|
|
116
|
+
assert.equal(fail.at(-1).percent, 30, `断在 30%%(30000/100000),实际=${fail.at(-1).percent}`);
|
|
117
|
+
} finally {
|
|
118
|
+
await new Promise((resolve) => server.close(resolve));
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("model-fetch:If-Range 失配(服务端内容已换)时回 200,断点作废整份重来,不产出拼接损坏", async () => {
|
|
123
|
+
const v1 = makeBody(65536);
|
|
124
|
+
const v2 = Buffer.from(v1.map((b) => (b + 7) % 251));
|
|
125
|
+
let broken = false;
|
|
126
|
+
const server = createServer((req, res) => {
|
|
127
|
+
const range = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? 0);
|
|
128
|
+
// If-Range 与当前内容不符(或没带):真实服务器会回 200 全量 —— 内容换过了。
|
|
129
|
+
if (req.headers["if-range"] !== undefined && req.headers["if-range"] !== '"v2"') {
|
|
130
|
+
broken = true;
|
|
131
|
+
res.writeHead(200, { "Content-Length": String(v2.length), ETag: '"v2"' });
|
|
132
|
+
res.end(v2);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
if (range > 0 && !broken) {
|
|
136
|
+
res.writeHead(206, {
|
|
137
|
+
"Content-Range": `bytes ${range}-${v1.length - 1}/${v1.length}`,
|
|
138
|
+
ETag: '"v1"'
|
|
139
|
+
});
|
|
140
|
+
res.end(v1.subarray(range));
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (!broken) {
|
|
144
|
+
res.writeHead(200, { "Content-Length": String(v1.length), ETag: '"v1"' });
|
|
145
|
+
res.write(v1.subarray(0, Math.floor(v1.length / 2)));
|
|
146
|
+
setTimeout(() => res.destroy(), 30);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
151
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
152
|
+
const dir = partDirFor();
|
|
153
|
+
try {
|
|
154
|
+
const events = [];
|
|
155
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, onEvent: (e) => events.push(e) });
|
|
156
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
157
|
+
const body = Buffer.from(await res.arrayBuffer());
|
|
158
|
+
assert.ok(events.some((e) => e.type === "restart" && e.reason === "range-rejected"), "应发 range-rejected 重来事件");
|
|
159
|
+
assert.ok(body.equals(v2), "最终字节必须是新内容整份,而不是 v1 断点拼 v2 的损坏物");
|
|
160
|
+
} finally {
|
|
161
|
+
await new Promise((resolve) => server.close(resolve));
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test("model-fetch:服务器续传位置与断点对不上时清掉断点重来(拼错比失败更糟)", async () => {
|
|
166
|
+
const body = makeBody(65536);
|
|
167
|
+
let cutDone = false;
|
|
168
|
+
const server = createServer((req, res) => {
|
|
169
|
+
const range = req.headers.range;
|
|
170
|
+
if (typeof range === "string") {
|
|
171
|
+
// 故意回一个与断点无关的 206(从 0 开始):客户端必须识别出偏移不一致。
|
|
172
|
+
res.writeHead(206, { "Content-Range": `bytes 0-${body.length - 1}/${body.length}`, ETag: '"x"' });
|
|
173
|
+
res.end(body);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (!cutDone) {
|
|
177
|
+
cutDone = true;
|
|
178
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
179
|
+
res.write(body.subarray(0, 30000));
|
|
180
|
+
setTimeout(() => res.destroy(), 20);
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
184
|
+
res.end(body);
|
|
185
|
+
});
|
|
186
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
187
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
188
|
+
const dir = partDirFor();
|
|
189
|
+
try {
|
|
190
|
+
const events = [];
|
|
191
|
+
const fetchImpl = testFetcher(dir, { attempts: 3, onEvent: (e) => events.push(e) });
|
|
192
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
193
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
194
|
+
assert.ok(events.some((e) => e.type === "restart" && e.reason === "offset-mismatch"));
|
|
195
|
+
assert.ok(received.equals(body), "重置后重新下载的整份字节必须正确");
|
|
196
|
+
} finally {
|
|
197
|
+
await new Promise((resolve) => server.close(resolve));
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
test("model-fetch:416(断点越过文件末尾)时重置断点重试,而不是把重试次数烧完", async () => {
|
|
202
|
+
const body = makeBody(65536);
|
|
203
|
+
const server = createServer((req, res) => {
|
|
204
|
+
const start = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? -1);
|
|
205
|
+
if (start >= body.length) {
|
|
206
|
+
// 诚实的 416:Range 起点越过源文件末尾(断点来自被替换前的旧文件)。
|
|
207
|
+
res.writeHead(416, { "Content-Range": `bytes */${body.length}` });
|
|
208
|
+
res.end();
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
if (start > 0) {
|
|
212
|
+
res.writeHead(206, { "Content-Range": `bytes ${start}-${body.length - 1}/${body.length}`, ETag: '"x"' });
|
|
213
|
+
res.end(body.subarray(start));
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
217
|
+
res.end(body);
|
|
218
|
+
});
|
|
219
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
220
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
221
|
+
const url = `${base}/model.onnx`;
|
|
222
|
+
const dir = partDirFor();
|
|
223
|
+
try {
|
|
224
|
+
// 预置一份比源文件还大的断点(70000 > 65536)+ 它的 etag。
|
|
225
|
+
const partPath = join(dir, createHash("sha256").update(url).digest("hex"));
|
|
226
|
+
writeFileSync(partPath, Buffer.alloc(70000, 1));
|
|
227
|
+
writeFileSync(`${partPath}.meta`, '"old"\n');
|
|
228
|
+
const events = [];
|
|
229
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, onEvent: (e) => events.push(e) });
|
|
230
|
+
const res = await fetchImpl(url);
|
|
231
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
232
|
+
assert.ok(events.some((e) => e.type === "restart" && e.reason === "range-not-satisfiable"));
|
|
233
|
+
assert.ok(received.equals(body), "重置后重新下载的整份字节必须正确");
|
|
234
|
+
} finally {
|
|
235
|
+
await new Promise((resolve) => server.close(resolve));
|
|
236
|
+
}
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
test("model-fetch:连接僵死时被空闲看门狗掐断并重试,而不是永远挂着", async () => {
|
|
240
|
+
const body = makeBody(65536);
|
|
241
|
+
let stalled = false;
|
|
242
|
+
const server = createServer((_req, res) => {
|
|
243
|
+
if (!stalled) {
|
|
244
|
+
stalled = true;
|
|
245
|
+
res.writeHead(200, { "Content-Length": String(body.length) });
|
|
246
|
+
res.write(body.subarray(0, 10));
|
|
247
|
+
return; // 永远不 end:僵死的 socket。
|
|
248
|
+
}
|
|
249
|
+
res.writeHead(200, { "Content-Length": String(body.length) });
|
|
250
|
+
res.end(body);
|
|
251
|
+
});
|
|
252
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
253
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
254
|
+
const dir = partDirFor();
|
|
255
|
+
try {
|
|
256
|
+
const events = [];
|
|
257
|
+
const started = Date.now();
|
|
258
|
+
const fetchImpl = testFetcher(dir, { attempts: 2, idleTimeoutMs: 150, retryDelayMs: 1, onEvent: (e) => events.push(e) });
|
|
259
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
260
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
261
|
+
assert.ok(Date.now() - started < 5000, "应当被看门狗及时掐断,而不是挂住");
|
|
262
|
+
assert.ok(events.some((e) => e.type === "fail" && /空闲/.test(e.error)), `失败事件应写明空闲超时,实际=${events.map((e) => e.error)}`);
|
|
263
|
+
assert.ok(received.equals(body));
|
|
264
|
+
} finally {
|
|
265
|
+
server.closeAllConnections();
|
|
266
|
+
await new Promise((resolve) => server.close(resolve));
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test("model-fetch:服务端干净地提前关流(字节没给够)被查出并续传补齐", async () => {
|
|
271
|
+
const body = makeBody(100000);
|
|
272
|
+
const server = createServer((req, res) => {
|
|
273
|
+
const range = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? 0);
|
|
274
|
+
if (range > 0) {
|
|
275
|
+
res.writeHead(206, { "Content-Range": `bytes ${range}-${body.length - 1}/${body.length}`, ETag: '"x"' });
|
|
276
|
+
res.end(body.subarray(range));
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
res.writeHead(200, { "Content-Length": String(body.length), ETag: '"x"' });
|
|
280
|
+
res.end(body.subarray(0, 30000)); // 声明 100000 只给 30000,然后"正常"结束。
|
|
281
|
+
});
|
|
282
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
283
|
+
const base = `http://127.0.0.1:${server.address().port}`;
|
|
284
|
+
const dir = partDirFor();
|
|
285
|
+
try {
|
|
286
|
+
const fetchImpl = testFetcher(dir, { attempts: 2 });
|
|
287
|
+
const res = await fetchImpl(`${base}/model.onnx`);
|
|
288
|
+
const received = Buffer.from(await res.arrayBuffer());
|
|
289
|
+
assert.ok(received.equals(body), "缺的字节要从断点补齐");
|
|
290
|
+
} finally {
|
|
291
|
+
await new Promise((resolve) => server.close(resolve));
|
|
292
|
+
}
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test("model-fetch:非 GET 与抢不到锁的请求原样直通,不落盘、不带 Range", async () => {
|
|
296
|
+
const dir = partDirFor();
|
|
297
|
+
const calls = [];
|
|
298
|
+
const stub = (input, init) => {
|
|
299
|
+
calls.push({ input: String(input), init: init ?? null });
|
|
300
|
+
return new Response("raw", { status: 200, headers: { "content-length": "3" } });
|
|
301
|
+
};
|
|
302
|
+
const fetchImpl = testFetcher(dir, { fetchImpl: stub });
|
|
303
|
+
await fetchImpl("https://example.com/meta", { method: "HEAD" });
|
|
304
|
+
// 锁名与被测实现一致:<sha256(url)>.lock;新鲜的锁 = 别的进程正在写。
|
|
305
|
+
const url = "https://example.com/model.onnx";
|
|
306
|
+
const locked = join(dir, `${createHash("sha256").update(url).digest("hex")}.lock`);
|
|
307
|
+
writeFileSync(locked, "9999");
|
|
308
|
+
await fetchImpl(url);
|
|
309
|
+
assert.equal(calls.length, 2, "两个请求都应直通底层 fetch");
|
|
310
|
+
assert.equal(calls[0].init.method, "HEAD");
|
|
311
|
+
assert.equal(calls[1].init, null, "直通不应改写调用参数");
|
|
312
|
+
assert.deepEqual(readdirSync(dir), [`${createHash("sha256").update(url).digest("hex")}.lock`], "直通不该产生断点文件");
|
|
313
|
+
rmSync(locked, { force: true });
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test("model-fetch:响应体消费完后断点文件、meta 与锁都清理干净", async () => {
|
|
317
|
+
const body = makeBody(65536);
|
|
318
|
+
const srv = await serve({ "/model.onnx": body }, { cutFirst: "/model.onnx" });
|
|
319
|
+
const dir = partDirFor();
|
|
320
|
+
try {
|
|
321
|
+
const fetchImpl = testFetcher(dir);
|
|
322
|
+
const res = await fetchImpl(`${srv.base}/model.onnx`);
|
|
323
|
+
await res.arrayBuffer(); // 消费完 = 下游(FileCache.put)拿完字节了。
|
|
324
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
325
|
+
assert.equal(readdirSync(dir).length, 0, `断点目录应已清空,实际=${readdirSync(dir)}`);
|
|
326
|
+
} finally {
|
|
327
|
+
await srv.close();
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test("installResilientFetch:开关关 = env 一字不动;开 = env.fetch 换成弹性实现", async () => {
|
|
332
|
+
const envOff = {};
|
|
333
|
+
assert.equal(installResilientFetch(envOff, { enabled: false }), false);
|
|
334
|
+
assert.equal(envOff.fetch, undefined, "关闭时不许碰 env(默认保持现状的出口)");
|
|
335
|
+
assert.equal(installResilientFetch(null, { enabled: true }), false, "env 缺席时不安装");
|
|
336
|
+
|
|
337
|
+
const cacheRoot = mkdtempSync(join(tmpdir(), "mneme-install-"));
|
|
338
|
+
const envOn = {};
|
|
339
|
+
assert.equal(installResilientFetch(envOn, { enabled: true, cacheDir: cacheRoot, fetchImpl: () => new Response("x") }), true);
|
|
340
|
+
assert.equal(typeof envOn.fetch, "function");
|
|
341
|
+
assert.ok(existsSync(join(cacheRoot, ".mneme-partial")), "断点目录建在模型缓存旁");
|
|
342
|
+
|
|
343
|
+
// 装上的 fetch 是能干活的:小文件直通下载 + 返回规范 200。
|
|
344
|
+
const res = await envOn.fetch("https://example.com/tiny.bin");
|
|
345
|
+
assert.equal(res.status, 200);
|
|
346
|
+
assert.equal(await res.text(), "x");
|
|
347
|
+
});
|
|
348
|
+
|
|
@@ -151,17 +151,19 @@ test("validateDecisions skipInvalid: conflict without a distinct winner/loser is
|
|
|
151
151
|
|
|
152
152
|
// ------------------------------------------------------- skipInvalid 与全局闸门
|
|
153
153
|
|
|
154
|
-
test("validateDecisions skipInvalid: valid subset below the coverage floor
|
|
154
|
+
test("validateDecisions skipInvalid: valid subset below the coverage floor degrades (Issue #104 方向 1)", () => {
|
|
155
155
|
const snap = makeSnap(["p", "j", "x"]);
|
|
156
156
|
const decisions = [
|
|
157
157
|
{ action: "archive", ids: ["p"], reason: "stale" },
|
|
158
158
|
{ action: "merge", ids: ["p", "j"], keepSource: "p", title: "跨类型", content: "m", importance: 4 }
|
|
159
159
|
];
|
|
160
|
-
const { ok, errors, skipped } = validateDecisions(decisions, snap, { skipInvalid: true });
|
|
161
|
-
assert.equal(ok,
|
|
160
|
+
const { ok, errors, skipped, coverageShortfall } = validateDecisions(decisions, snap, { skipInvalid: true });
|
|
161
|
+
assert.equal(ok, true, "valid subset applies — shortfall degrades instead of rejecting");
|
|
162
|
+
assert.deepEqual(errors, []);
|
|
162
163
|
assert.equal(skipped.length, 1);
|
|
163
|
-
assert.
|
|
164
|
-
|
|
164
|
+
assert.match(String(coverageShortfall), /coverage/, "shortfall reason surfaced for audit");
|
|
165
|
+
// claimed {p} = 1/3 = 33% < 50%;j/x 隐式 keep 补齐
|
|
166
|
+
assert.deepEqual(decisions.map((d) => d.action), ["archive", "keep", "keep"]);
|
|
165
167
|
});
|
|
166
168
|
|
|
167
169
|
test("validateDecisions skipInvalid: updates skipped for other reasons do not count toward the update cap", () => {
|