@kira905/dsh-memory-toolkit 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -1
- package/README.en.md +225 -2
- package/README.md +208 -2
- package/cordis.patch.yml +32 -0
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +11 -1
|
@@ -1,78 +1,117 @@
|
|
|
1
|
-
// 记忆引擎
|
|
1
|
+
// 记忆引擎 · 离线层 —— 编码桥联调(编码通道契约)
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
3
|
+
// 一个进程管全流程:起桥 → 轮询 /health 直到模型就绪 → 打两个 /encode →
|
|
4
|
+
// 确认第二个实例顺延到下一个端口 → 确认非回环绑定被拒 → 关桥。
|
|
5
|
+
// 为什么不起后台进程:SSH 会话一结束会把 detached 进程一起带走(跨机踩过的坑),
|
|
6
|
+
// 「起 + 测 + 关」必须落在同一条命令里 —— 本脚本就是那条命令。
|
|
6
7
|
import { startBridge } from '../serve.mjs';
|
|
7
8
|
import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
10
|
+
const FALLBACK_PORT = 8201;
|
|
11
|
+
const POLL_MS = 400;
|
|
12
|
+
const LOAD_BUDGET_MS = 60000;
|
|
13
|
+
const LOOPBACK = '127.0.0.1';
|
|
14
|
+
|
|
15
|
+
const delay = (ms) => new Promise((done) => setTimeout(done, ms));
|
|
16
|
+
|
|
17
|
+
/** 按要打印的键投影一份快照,保持声明顺序。 */
|
|
18
|
+
const pick = (snapshot, keys) => Object.fromEntries(keys.map((key) => [key, snapshot[key]]));
|
|
19
|
+
|
|
20
|
+
const meanSquare = (vector) => vector.reduce((sum, value) => sum + value * value, 0);
|
|
21
|
+
|
|
22
|
+
async function callJson(port, route, init) {
|
|
23
|
+
const response = await fetch(`http://${LOOPBACK}:${port}${route}`, init);
|
|
24
|
+
return response.json();
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const healthOf = (port) => callJson(port, '/health');
|
|
28
|
+
|
|
29
|
+
/** 正常请求的 /encode;只有传输层或 JSON 坏掉才会抛。 */
|
|
30
|
+
const encodeTexts = (port, texts) =>
|
|
31
|
+
callJson(port, '/encode', {
|
|
32
|
+
method: 'POST',
|
|
33
|
+
headers: { 'content-type': 'application/json' },
|
|
34
|
+
body: JSON.stringify({ texts }),
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
/** 故意打坏请求的 /encode;状态码先取下来再解析正文。 */
|
|
38
|
+
async function encodeBroken(port, payload) {
|
|
39
|
+
const response = await fetch(`http://${LOOPBACK}:${port}/encode`, {
|
|
40
|
+
method: 'POST',
|
|
41
|
+
headers: { 'content-type': 'application/json' },
|
|
42
|
+
body: JSON.stringify(payload),
|
|
43
|
+
});
|
|
44
|
+
return { status: response.status, body: await response.json() };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* 轮询 /health 直到模型就绪。
|
|
49
|
+
* 桥报了加载错误也只等到预算耗尽才抛 —— 与桥的懒加载行为对齐,期间的加载失败
|
|
50
|
+
* 不算终局(模型可能重试后起来)。
|
|
51
|
+
*/
|
|
52
|
+
async function awaitReady(port, budget = LOAD_BUDGET_MS) {
|
|
53
|
+
const startedAt = Date.now();
|
|
54
|
+
const overdue = () => Date.now() - startedAt > budget;
|
|
11
55
|
for (;;) {
|
|
12
56
|
try {
|
|
13
|
-
const
|
|
14
|
-
if (
|
|
15
|
-
if (
|
|
16
|
-
} catch (
|
|
17
|
-
if (
|
|
57
|
+
const snapshot = await healthOf(port);
|
|
58
|
+
if (snapshot.ready) return snapshot;
|
|
59
|
+
if (snapshot.lastError) throw new Error(`模型加载失败:${snapshot.lastError}`);
|
|
60
|
+
} catch (err) {
|
|
61
|
+
if (overdue()) throw err;
|
|
18
62
|
}
|
|
19
|
-
if (
|
|
20
|
-
await
|
|
63
|
+
if (overdue()) throw new Error('等待模型就绪超时');
|
|
64
|
+
await delay(POLL_MS);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function reportPortShift(first, second) {
|
|
69
|
+
const verdict = second.port === first.port + 1 ? ' ✅' : ' ❌';
|
|
70
|
+
console.log(`端口顺延检查:首个=${first.port},第二个=${second.port}${verdict}`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** 除了回环,桥不许绑任何地址。 */
|
|
74
|
+
async function probeLoopbackGuard(modelDir) {
|
|
75
|
+
try {
|
|
76
|
+
await startBridge({ flags: { 'model-dir': modelDir, bind: '0.0.0.0' }, log: () => {} });
|
|
77
|
+
return '未拦截 ❌';
|
|
78
|
+
} catch (err) {
|
|
79
|
+
return `已拦截 ✅(${String(err.message).slice(0, 40)}…)`;
|
|
21
80
|
}
|
|
22
81
|
}
|
|
23
82
|
|
|
24
83
|
async function main() {
|
|
25
84
|
const { flags } = parseArgs();
|
|
26
85
|
const modelDir = flags['model-dir'];
|
|
27
|
-
const wantPort = Number(flags.port ||
|
|
28
|
-
const
|
|
29
|
-
const log = (m) => {
|
|
30
|
-
logs.push(m);
|
|
31
|
-
console.log(m);
|
|
32
|
-
};
|
|
86
|
+
const wantPort = Number(flags.port || FALLBACK_PORT);
|
|
87
|
+
const log = (message) => console.log(message);
|
|
33
88
|
|
|
34
89
|
console.log('=== 编码桥实测(仅回环) ===');
|
|
35
|
-
const
|
|
36
|
-
const
|
|
37
|
-
|
|
90
|
+
const primary = await startBridge({ flags: { 'model-dir': modelDir, port: String(wantPort) }, log });
|
|
91
|
+
const secondary = await startBridge({ flags: { 'model-dir': modelDir, port: String(wantPort), warm: false }, log });
|
|
92
|
+
reportPortShift(primary, secondary);
|
|
38
93
|
|
|
39
|
-
const
|
|
40
|
-
console.log(`/health(加载中):${JSON.stringify(
|
|
94
|
+
const whileLoading = await healthOf(primary.port);
|
|
95
|
+
console.log(`/health(加载中):${JSON.stringify(pick(whileLoading, ['ready', 'modelLoading', 'model', 'dim']))}`);
|
|
41
96
|
|
|
42
|
-
const ready = await
|
|
43
|
-
console.log(`/health(就绪):${JSON.stringify(
|
|
97
|
+
const ready = await awaitReady(primary.port);
|
|
98
|
+
console.log(`/health(就绪):${JSON.stringify(pick(ready, ['ready', 'model', 'dim', 'port']))}`);
|
|
44
99
|
|
|
45
|
-
const
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
console.log(`/encode 2 条:${ms.toFixed(1)}ms · model=${enc.model} dim=${enc.dim} · 首向量前 5 维 ${enc.vectors[0].slice(0, 5).map((x) => x.toFixed(4)).join(', ')}`);
|
|
53
|
-
const norm = Math.sqrt(enc.vectors[0].reduce((s, x) => s + x * x, 0));
|
|
54
|
-
console.log(`首向量 L2 范数:${norm.toFixed(6)}(契约要求归一化 ⇒ 应 ≈1)`);
|
|
55
|
-
|
|
56
|
-
const bad = await fetch(`http://127.0.0.1:${b1.port}/encode`, {
|
|
57
|
-
method: 'POST',
|
|
58
|
-
headers: { 'content-type': 'application/json' },
|
|
59
|
-
body: JSON.stringify({ nope: 1 }),
|
|
60
|
-
}).then((r) => ({ status: r.status, body: r.json() }));
|
|
61
|
-
const badRes = await bad.body;
|
|
62
|
-
console.log(`坏请求处理:HTTP ${bad.status} ${JSON.stringify(badRes)}`);
|
|
100
|
+
const startedAt = performance.now();
|
|
101
|
+
const encoded = await encodeTexts(primary.port, ['跨机同步的挡推裁决', '整夜干活']);
|
|
102
|
+
const elapsed = performance.now() - startedAt;
|
|
103
|
+
const firstVector = encoded.vectors[0];
|
|
104
|
+
const headDims = firstVector.slice(0, 5).map((value) => value.toFixed(4)).join(', ');
|
|
105
|
+
console.log(`/encode 2 条:${elapsed.toFixed(1)}ms · model=${encoded.model} dim=${encoded.dim} · 首向量前 5 维 ${headDims}`);
|
|
106
|
+
console.log(`首向量 L2 范数:${Math.sqrt(meanSquare(firstVector)).toFixed(6)}(契约要求归一化 ⇒ 应 ≈1)`);
|
|
63
107
|
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
} catch (e) {
|
|
69
|
-
return `已拦截 ✅(${String(e.message).slice(0, 40)}…)`;
|
|
70
|
-
}
|
|
71
|
-
})();
|
|
72
|
-
console.log(`非回环绑定拦截:${nonLoop}`);
|
|
108
|
+
const broken = await encodeBroken(primary.port, { nope: 1 });
|
|
109
|
+
console.log(`坏请求处理:HTTP ${broken.status} ${JSON.stringify(broken.body)}`);
|
|
110
|
+
|
|
111
|
+
console.log(`非回环绑定拦截:${await probeLoopbackGuard(modelDir)}`);
|
|
73
112
|
|
|
74
|
-
await
|
|
75
|
-
await
|
|
113
|
+
await secondary.close();
|
|
114
|
+
await primary.close();
|
|
76
115
|
console.log('桥已关闭');
|
|
77
116
|
process.exitCode = 0;
|
|
78
117
|
}
|
|
@@ -1,92 +1,179 @@
|
|
|
1
|
-
// 记忆引擎
|
|
1
|
+
// 记忆引擎 · 离线层 —— 模型权重获取器(纯 Node,不依赖 curl)
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// node 自带 TLS 栈不受 schannel 影响 ⇒ 用本脚本取权重。
|
|
3
|
+
// 为什么不用 curl:Windows 自带的 curl.exe 走 schannel 做 TLS,而 Hub 上的大文件
|
|
4
|
+
// 会 302 到 Xet CAS 桥(cas-bridge.xethub.hf.co);部分 Windows 主机对这个域名的
|
|
5
|
+
// 握手直接失败(curl: (35) SEC_E_ILLEGAL_MESSAGE),反倒是直连下发的小文件没事。
|
|
6
|
+
// Node 自带独立 TLS 栈,能绕开 schannel 这条路。
|
|
8
7
|
//
|
|
9
8
|
// 用法:
|
|
10
9
|
// node tools/fetch-model.mjs --out <模型目录> [--base https://hf-mirror.com]
|
|
11
|
-
// [--repo Xenova/bge-small-zh-v1.5] [--variant fp32|int8] [--force]
|
|
12
|
-
//
|
|
13
|
-
import
|
|
14
|
-
import
|
|
15
|
-
import
|
|
16
|
-
import {
|
|
17
|
-
import { pipeline } from 'node:stream/promises';
|
|
10
|
+
// [--repo Xenova/bge-small-zh-v1.5] [--variant fp32|int8|fp16] [--force]
|
|
11
|
+
// 幂等:本地文件的字节数与服务端 content-length 相同就不动它;--force 要求重下。
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, statSync } from 'node:fs';
|
|
14
|
+
import { open } from 'node:fs/promises';
|
|
15
|
+
import { basename, dirname, join, resolve, sep } from 'node:path';
|
|
18
16
|
import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
|
|
19
17
|
|
|
20
|
-
const
|
|
18
|
+
const MIB = 1024 * 1024;
|
|
19
|
+
|
|
20
|
+
/** 所有变体都要的配套文件,按这个顺序取。 */
|
|
21
|
+
const SHARED_FILES = [
|
|
21
22
|
'config.json',
|
|
22
23
|
'tokenizer.json',
|
|
23
24
|
'tokenizer_config.json',
|
|
24
25
|
'special_tokens_map.json',
|
|
25
26
|
'vocab.txt',
|
|
26
27
|
];
|
|
27
|
-
|
|
28
|
+
|
|
29
|
+
/** 变体名 → 仓内的权重文件(键顺序会出现在报错文案里,别乱动)。 */
|
|
30
|
+
const WEIGHTS = {
|
|
28
31
|
fp32: 'onnx/model.onnx',
|
|
29
32
|
int8: 'onnx/model_quantized.onnx',
|
|
30
33
|
fp16: 'onnx/model_fp16.onnx',
|
|
31
34
|
};
|
|
32
35
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
/** 命令行没给时的兜底值。 */
|
|
37
|
+
const BAREFALLS = {
|
|
38
|
+
base: 'https://hf-mirror.com',
|
|
39
|
+
repo: 'Xenova/bge-small-zh-v1.5',
|
|
40
|
+
variant: 'fp32',
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const USAGE =
|
|
44
|
+
'用法: node tools/fetch-model.mjs --out <dir> [--base <url>] [--repo <owner/name>] [--variant fp32|int8|fp16] [--force]';
|
|
45
|
+
|
|
46
|
+
const pause = (ms) => new Promise((done) => setTimeout(done, ms));
|
|
47
|
+
|
|
48
|
+
/** sha256 前 32 位 hex:够人对一眼,又不至于刷出 64 位。 */
|
|
49
|
+
const shortDigest = (filePath) => createHash('sha256').update(readFileSync(filePath)).digest('hex').slice(0, 32);
|
|
50
|
+
|
|
51
|
+
const asMegabytes = (bytes) => (bytes / MIB).toFixed(1);
|
|
52
|
+
|
|
53
|
+
/** 每跨过 10 % 才打一行进度,免得刷屏。 */
|
|
54
|
+
function announce(name, received, total, cursor, say) {
|
|
55
|
+
if (!total) return;
|
|
56
|
+
const percent = Math.floor((received / total) * 100);
|
|
57
|
+
if (percent < cursor.last + 10) return;
|
|
58
|
+
cursor.last = percent;
|
|
59
|
+
say(` ${name}: ${percent}%(${asMegabytes(received)}/${asMegabytes(total)} MB)`);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** 把一块 Uint8Array 全写进去;短写就接着写剩下的。 */
|
|
63
|
+
async function writeAll(handle, chunk) {
|
|
64
|
+
let written = 0;
|
|
65
|
+
while (written < chunk.byteLength) {
|
|
66
|
+
const { bytesWritten } = await handle.write(chunk, written, chunk.byteLength - written);
|
|
67
|
+
if (!bytesWritten) throw new Error('写入中断(0 字节)');
|
|
68
|
+
written += bytesWritten;
|
|
41
69
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* 把响应体逐块落到临时文件里,每块都 await ⇒ 套接字永远不会跑到磁盘前面。
|
|
74
|
+
* 落完怎么处理(改名 / 丢弃)由调用方决定。
|
|
75
|
+
*/
|
|
76
|
+
async function spoolTo(partialPath, body, expected, name, say) {
|
|
77
|
+
if (!body) throw new Error(`${name}:响应没有正文`);
|
|
78
|
+
const reader = body.getReader();
|
|
79
|
+
const handle = await open(partialPath, 'w');
|
|
80
|
+
const cursor = { last: -1 };
|
|
81
|
+
let received = 0;
|
|
82
|
+
try {
|
|
83
|
+
for (;;) {
|
|
84
|
+
const { done, value } = await reader.read();
|
|
85
|
+
if (done) break;
|
|
86
|
+
if (!value) continue;
|
|
87
|
+
await writeAll(handle, value);
|
|
88
|
+
received += value.byteLength;
|
|
89
|
+
announce(name, received, expected, cursor, say);
|
|
55
90
|
}
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
if (total && fs.statSync(part).size !== total) {
|
|
59
|
-
throw new Error(`${name} 大小不符:期望 ${total},实际 ${fs.statSync(part).size}`);
|
|
91
|
+
} finally {
|
|
92
|
+
await handle.close();
|
|
60
93
|
}
|
|
61
|
-
fs.renameSync(part, dest);
|
|
62
|
-
return { file: name, status: 'fetched', size: fs.statSync(dest).size, sha: shortSha(dest) };
|
|
63
94
|
}
|
|
64
95
|
|
|
65
|
-
|
|
66
|
-
|
|
96
|
+
/**
|
|
97
|
+
* 取一个远端路径到 destination。
|
|
98
|
+
* 返回 {file,status:'skip'|'fetched',size,sha};传输层出问题就抛。
|
|
99
|
+
*/
|
|
100
|
+
async function pullFile({ url, destination, overwrite, say }) {
|
|
101
|
+
const response = await fetch(url, { method: 'GET', redirect: 'follow' });
|
|
102
|
+
if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} (${url})`);
|
|
103
|
+
|
|
104
|
+
const expected = Number(response.headers.get('content-length') || 0);
|
|
105
|
+
const name = basename(destination);
|
|
106
|
+
const localSize = existsSync(destination) ? statSync(destination).size : -1;
|
|
107
|
+
|
|
108
|
+
if (!overwrite && expected > 0 && localSize === expected) {
|
|
109
|
+
await response.body?.cancel?.();
|
|
110
|
+
return { file: name, status: 'skip', size: expected, sha: shortDigest(destination) };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
mkdirSync(dirname(destination), { recursive: true });
|
|
114
|
+
const partialPath = `${destination}.part`;
|
|
115
|
+
await spoolTo(partialPath, response.body, expected, name, say);
|
|
116
|
+
|
|
117
|
+
const landed = statSync(partialPath).size;
|
|
118
|
+
if (expected && landed !== expected) throw new Error(`${name} 大小不符:期望 ${expected},实际 ${landed}`);
|
|
119
|
+
renameSync(partialPath, destination);
|
|
120
|
+
return { file: name, status: 'fetched', size: statSync(destination).size, sha: shortDigest(destination) };
|
|
67
121
|
}
|
|
68
122
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
log(` ⚠ ${String(e && e.message ? e.message : e)}`);
|
|
84
|
-
await new Promise((r) => setTimeout(r, 1500 * attempt));
|
|
85
|
-
}
|
|
123
|
+
/**
|
|
124
|
+
* 一个远端路径 + 重试预算。
|
|
125
|
+
* 调用方把 retries 设成 0 时返回 null(一条记录都不写,保持原语义)。
|
|
126
|
+
*/
|
|
127
|
+
async function attemptFile({ url, destination, force, log, retries, name }) {
|
|
128
|
+
let failure = null;
|
|
129
|
+
for (let round = 1; round <= retries; round += 1) {
|
|
130
|
+
log(` → ${name}${round > 1 ? `(第 ${round} 次尝试)` : ''}`);
|
|
131
|
+
try {
|
|
132
|
+
return await pullFile({ url, destination, overwrite: force, say: log });
|
|
133
|
+
} catch (err) {
|
|
134
|
+
failure = err;
|
|
135
|
+
log(` ⚠ ${String(err && err.message ? err.message : err)}`);
|
|
136
|
+
await pause(1500 * round);
|
|
86
137
|
}
|
|
87
|
-
|
|
88
|
-
|
|
138
|
+
}
|
|
139
|
+
if (!failure) return null;
|
|
140
|
+
return { file: name, status: 'FAILED', error: String(failure.message || failure) };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** 逐条打结果,并定进程退出码。 */
|
|
144
|
+
function summarize(results) {
|
|
145
|
+
console.log('\n=== 结果 ===');
|
|
146
|
+
let failed = 0;
|
|
147
|
+
for (const item of results) {
|
|
148
|
+
if (item.status === 'FAILED') {
|
|
149
|
+
failed += 1;
|
|
150
|
+
console.log(`❌ ${item.file}:${item.error}`);
|
|
151
|
+
continue;
|
|
89
152
|
}
|
|
153
|
+
const mark = item.status === 'skip' ? '⏭️ ' : '✅';
|
|
154
|
+
console.log(`${mark} ${item.file.padEnd(28)} ${(item.size / MIB).toFixed(2)} MB sha256=${item.sha}`);
|
|
155
|
+
}
|
|
156
|
+
console.log(failed ? `\n❌ ${failed} 个文件失败` : '\n✅ 全部就绪');
|
|
157
|
+
process.exitCode = failed ? 1 : 0;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* 把某个变体需要的文件全取到 out 里。
|
|
162
|
+
* 普通下载失败不会让本函数 reject —— 那些都变成 FAILED 记录。
|
|
163
|
+
*/
|
|
164
|
+
export async function fetchModel({ out, base, repo, variant, force, log, retries = 3 }) {
|
|
165
|
+
const remotePaths = [...SHARED_FILES, WEIGHTS[variant]];
|
|
166
|
+
const results = [];
|
|
167
|
+
for (const remotePath of remotePaths) {
|
|
168
|
+
const settled = await attemptFile({
|
|
169
|
+
url: `${base}/${repo}/resolve/main/${remotePath}`,
|
|
170
|
+
destination: join(out, remotePath.replace(/\//g, sep)),
|
|
171
|
+
force,
|
|
172
|
+
log,
|
|
173
|
+
retries,
|
|
174
|
+
name: remotePath,
|
|
175
|
+
});
|
|
176
|
+
if (settled) results.push(settled);
|
|
90
177
|
}
|
|
91
178
|
return results;
|
|
92
179
|
}
|
|
@@ -94,35 +181,28 @@ export async function fetchModel({ out, base, repo, variant, force, log, retries
|
|
|
94
181
|
async function main() {
|
|
95
182
|
const { flags } = parseArgs();
|
|
96
183
|
if (flags.help) {
|
|
97
|
-
console.log(
|
|
184
|
+
console.log(USAGE);
|
|
98
185
|
return;
|
|
99
186
|
}
|
|
100
|
-
const
|
|
101
|
-
if (!
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
187
|
+
const requestedOut = flags.out || process.env.MEMORY_ENGINE_MODEL_DIR;
|
|
188
|
+
if (!requestedOut) throw new Error('必须指定 --out(模型落盘目录)');
|
|
189
|
+
|
|
190
|
+
const settings = {
|
|
191
|
+
out: resolve(requestedOut),
|
|
192
|
+
base: flags.base || BAREFALLS.base,
|
|
193
|
+
repo: flags.repo || BAREFALLS.repo,
|
|
194
|
+
variant: flags.variant || BAREFALLS.variant,
|
|
107
195
|
force: Boolean(flags.force),
|
|
108
196
|
};
|
|
109
|
-
console.log(`模型获取器:${
|
|
110
|
-
console.log(`镜像 :${
|
|
111
|
-
console.log(`目标目录 :${
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
let failed = 0;
|
|
116
|
-
for (const r of results) {
|
|
117
|
-
if (r.status === 'FAILED') {
|
|
118
|
-
failed += 1;
|
|
119
|
-
console.log(`❌ ${r.file}:${r.error}`);
|
|
120
|
-
} else {
|
|
121
|
-
console.log(`${r.status === 'skip' ? '⏭️ ' : '✅'} ${r.file.padEnd(28)} ${(r.size / 1048576).toFixed(2)} MB sha256=${r.sha}`);
|
|
122
|
-
}
|
|
197
|
+
console.log(`模型获取器:${settings.repo}(${settings.variant})`);
|
|
198
|
+
console.log(`镜像 :${settings.base}`);
|
|
199
|
+
console.log(`目标目录 :${settings.out}`);
|
|
200
|
+
|
|
201
|
+
if (!WEIGHTS[settings.variant]) {
|
|
202
|
+
throw new Error(`未知 variant=${settings.variant}(可选 ${Object.keys(WEIGHTS).join('/')})`);
|
|
123
203
|
}
|
|
124
|
-
|
|
125
|
-
|
|
204
|
+
|
|
205
|
+
summarize(await fetchModel({ ...settings, log: (message) => console.log(message) }));
|
|
126
206
|
}
|
|
127
207
|
|
|
128
208
|
if (isMainEntry(import.meta.url)) await main();
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
// 记忆引擎
|
|
1
|
+
// 记忆引擎 · 离线层 —— 真实模型召回冒烟(给人读的证据)
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
3
|
+
// 单独成一个脚本的理由:中文查询经 cmd/ssh 传过去会变乱码,所以把探针查询集写死在
|
|
4
|
+
// 这个文件里当数据用;在目标机上跑一条命令,就能拿到「真模型 + 真语料」的可读召回
|
|
5
|
+
// 证据(含短暗号)。
|
|
6
|
+
import { existsSync } from 'node:fs';
|
|
6
7
|
import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
|
|
7
8
|
import { resolveConfig } from '../lib/config.mjs';
|
|
8
9
|
import { openVectorsStore, countVectors, getMetaAll } from '../lib/vectors-store.mjs';
|
|
@@ -10,7 +11,8 @@ import { openMemorySource } from '../lib/memory-source.mjs';
|
|
|
10
11
|
import { createLocalEmbedder } from '../lib/embedder.mjs';
|
|
11
12
|
import { recall, buildVectorIndex, shouldTrigger } from '../lib/recall.mjs';
|
|
12
13
|
|
|
13
|
-
|
|
14
|
+
/** 固定探针集:日常暗号、工程问题,外加一个裸端口号。 */
|
|
15
|
+
const PROBES = [
|
|
14
16
|
'退下吧',
|
|
15
17
|
'整夜干活',
|
|
16
18
|
'去深水区',
|
|
@@ -23,16 +25,84 @@ const QUERIES = [
|
|
|
23
25
|
'3080 端口',
|
|
24
26
|
];
|
|
25
27
|
|
|
28
|
+
/** CLI 里能改配置路径的几个开关。 */
|
|
29
|
+
const PATH_FLAGS = [
|
|
30
|
+
['memoryDb', 'memory-db'],
|
|
31
|
+
['vectorsDb', 'vectors-db'],
|
|
32
|
+
['modelDir', 'model-dir'],
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
const LABEL_COLUMNS = 9;
|
|
36
|
+
const HIT_INDENT = ' ';
|
|
37
|
+
const MAX_BRIEF_CHARS = 58;
|
|
38
|
+
const TOP_K = 5;
|
|
39
|
+
|
|
40
|
+
/** 字符串的终端宽度:CJK 码点按两列算(首列才能对齐)。 */
|
|
41
|
+
const widthOf = (text) => [...text].reduce((cols, ch) => cols + (ch.codePointAt(0) > 0x2e7f ? 2 : 1), 0);
|
|
42
|
+
|
|
43
|
+
const padLabel = (text) => text + ' '.repeat(Math.max(0, LABEL_COLUMNS - widthOf(text)));
|
|
44
|
+
|
|
45
|
+
const collapse = (text) => String(text).replace(/\s+/g, ' ');
|
|
46
|
+
|
|
47
|
+
/** 一条记忆的单行、截断版摘要。 */
|
|
48
|
+
const excerpt = (text) => collapse(text).slice(0, MAX_BRIEF_CHARS);
|
|
49
|
+
|
|
50
|
+
const gateVerdict = (gate) => (gate.trigger ? '是' : `否(${gate.reason})`);
|
|
51
|
+
|
|
52
|
+
/** 把命令行开关翻成 resolveConfig 的覆盖项;没给的键一律不出现。 */
|
|
53
|
+
function overridesFrom(flags) {
|
|
54
|
+
const overrides = {};
|
|
55
|
+
for (const [key, flag] of PATH_FLAGS) {
|
|
56
|
+
if (flags[flag]) overrides[key] = flags[flag];
|
|
57
|
+
}
|
|
58
|
+
if (flags.model) overrides.modelId = flags.model;
|
|
59
|
+
if (flags.dim) overrides.dim = Number(flags.dim);
|
|
60
|
+
return overrides;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** 表头段:查之前先把语料形状交代清楚。 */
|
|
64
|
+
function printCorpora({ config, source, store, embedder }) {
|
|
65
|
+
const rows = [
|
|
66
|
+
['主机', process.env.COMPUTERNAME || '?'],
|
|
67
|
+
['记忆库', `${config.memoryDb}(${source.count()} 条 / pinned ${source.pinnedIds().length} 条)`],
|
|
68
|
+
['向量库', `${config.vectorsDb}(${countVectors(store.db, 'memory')} 条向量)`],
|
|
69
|
+
['meta', JSON.stringify(getMetaAll(store.db))],
|
|
70
|
+
['模型', `${config.modelId} dim=${embedder.dim}`],
|
|
71
|
+
];
|
|
72
|
+
console.log('=== 真实模型召回冒烟 ===');
|
|
73
|
+
for (const [label, value] of rows) console.log(`${padLabel(label)}: ${value}`);
|
|
74
|
+
console.log('');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function printHit(hit, row) {
|
|
78
|
+
const stored = row ? row.pinned : '?';
|
|
79
|
+
const text = excerpt(row ? row.text : '(已删除)');
|
|
80
|
+
console.log(`${HIT_INDENT}#${hit.refId} score=${hit.score.toFixed(5)} [${hit.sources.join('+')}] pinned=${stored} · ${text}…`);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** 跑一条探针;返回这条命中里漏进了几条 pinned。 */
|
|
84
|
+
async function runProbe({ query, store, source, embedder, index, pinnedIds }) {
|
|
85
|
+
const gate = shouldTrigger(query);
|
|
86
|
+
const startedAt = performance.now();
|
|
87
|
+
const outcome = await recall({ query, store, source, embedder, index, resultTopK: TOP_K });
|
|
88
|
+
const elapsed = performance.now() - startedAt;
|
|
89
|
+
|
|
90
|
+
const ftsMatched = outcome.stats.ftsMatched ?? 0;
|
|
91
|
+
console.log(`【${query}】触发=${gateVerdict(gate)} 命中 ${outcome.hits.length} 条 · ${elapsed.toFixed(1)}ms · FTS命中=${ftsMatched}`);
|
|
92
|
+
|
|
93
|
+
let leaked = 0;
|
|
94
|
+
for (const hit of outcome.hits) {
|
|
95
|
+
if (pinnedIds.has(String(hit.refId))) leaked += 1;
|
|
96
|
+
printHit(hit, source.getById(hit.id));
|
|
97
|
+
}
|
|
98
|
+
console.log('');
|
|
99
|
+
return leaked;
|
|
100
|
+
}
|
|
101
|
+
|
|
26
102
|
async function main() {
|
|
27
103
|
const { flags } = parseArgs();
|
|
28
|
-
const config = resolveConfig(
|
|
29
|
-
|
|
30
|
-
vectorsDb: flags['vectors-db'] || undefined,
|
|
31
|
-
modelDir: flags['model-dir'] || undefined,
|
|
32
|
-
modelId: flags.model || undefined,
|
|
33
|
-
dim: flags.dim ? Number(flags.dim) : undefined,
|
|
34
|
-
});
|
|
35
|
-
if (!fs.existsSync(config.memoryDb)) throw new Error(`记忆库不存在:${config.memoryDb}`);
|
|
104
|
+
const config = resolveConfig(overridesFrom(flags));
|
|
105
|
+
if (!existsSync(config.memoryDb)) throw new Error(`记忆库不存在:${config.memoryDb}`);
|
|
36
106
|
|
|
37
107
|
const embedder = await createLocalEmbedder({
|
|
38
108
|
modelDir: config.modelDir,
|
|
@@ -43,34 +113,20 @@ async function main() {
|
|
|
43
113
|
const source = openMemorySource(config.memoryDb);
|
|
44
114
|
const index = buildVectorIndex(store.db, { source: 'memory' });
|
|
45
115
|
|
|
46
|
-
|
|
47
|
-
console.log(`主机 : ${process.env.COMPUTERNAME || '?'}`);
|
|
48
|
-
console.log(`记忆库 : ${config.memoryDb}(${source.count()} 条 / pinned ${source.pinnedIds().length} 条)`);
|
|
49
|
-
console.log(`向量库 : ${config.vectorsDb}(${countVectors(store.db, 'memory')} 条向量)`);
|
|
50
|
-
console.log(`meta : ${JSON.stringify(getMetaAll(store.db))}`);
|
|
51
|
-
console.log(`模型 : ${config.modelId} dim=${embedder.dim}`);
|
|
52
|
-
console.log('');
|
|
116
|
+
printCorpora({ config, source, store, embedder });
|
|
53
117
|
|
|
54
|
-
const
|
|
55
|
-
let
|
|
56
|
-
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
const r = await recall({ query: q, store, source, embedder, index, resultTopK: 5 });
|
|
60
|
-
const ms = performance.now() - t0;
|
|
61
|
-
console.log(`【${q}】触发=${gate.trigger ? '是' : `否(${gate.reason})`} 命中 ${r.hits.length} 条 · ${ms.toFixed(1)}ms · FTS命中=${r.stats.ftsMatched ?? 0}`);
|
|
62
|
-
for (const h of r.hits) {
|
|
63
|
-
if (pinned.has(String(h.refId))) pinnedLeak += 1;
|
|
64
|
-
const row = source.getById(h.id);
|
|
65
|
-
const brief = (row ? row.text : '(已删除)').replace(/\s+/g, ' ').slice(0, 58);
|
|
66
|
-
console.log(` #${h.refId} score=${h.score.toFixed(5)} [${h.sources.join('+')}] pinned=${row ? row.pinned : '?'} · ${brief}…`);
|
|
118
|
+
const pinnedIds = new Set(source.pinnedIds().map(String));
|
|
119
|
+
let leakedTotal = 0;
|
|
120
|
+
try {
|
|
121
|
+
for (const query of PROBES) {
|
|
122
|
+
leakedTotal += await runProbe({ query, store, source, embedder, index, pinnedIds });
|
|
67
123
|
}
|
|
68
|
-
console.log('');
|
|
124
|
+
console.log(`pinned 泄漏检查:${leakedTotal === 0 ? '✅ 0 条(排除生效)' : `❌ ${leakedTotal} 条`}`);
|
|
125
|
+
} finally {
|
|
126
|
+
source.close();
|
|
127
|
+
store.db.close();
|
|
69
128
|
}
|
|
70
|
-
|
|
71
|
-
source.close();
|
|
72
|
-
store.db.close();
|
|
73
|
-
process.exitCode = pinnedLeak === 0 ? 0 : 4;
|
|
129
|
+
process.exitCode = leakedTotal === 0 ? 0 : 4;
|
|
74
130
|
}
|
|
75
131
|
|
|
76
132
|
if (isMainEntry(import.meta.url)) await main();
|