@modusensus/dsh-mneme 0.8.12 → 0.8.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -5
- package/bin/dsh-mneme-serve.mjs +151 -0
- package/lib/api-standalone.js +110 -9
- package/lib/api.js +2 -0
- package/lib/client.js +12 -1
- package/lib/config.js +26 -0
- package/lib/guide.js +13 -1
- package/lib/index.js +31 -191
- package/lib/semantic.js +229 -0
- package/lib/sensitive-scan.js +209 -0
- package/lib/serve.js +215 -0
- package/lib/service.js +54 -18
- package/lib/settings.js +7 -1
- package/package.json +6 -2
- package/scripts/changelog-prep.mjs +36 -0
- package/src/api-standalone.js +110 -9
- package/src/api.js +2 -0
- package/src/config.js +26 -0
- package/src/guide.js +13 -1
- package/src/index.js +31 -191
- package/src/semantic.js +229 -0
- package/src/sensitive-scan.js +209 -0
- package/src/serve.js +215 -0
- package/src/service.js +54 -18
- package/src/settings.js +7 -1
- package/test/api.test.js +12 -2
- package/test/changelog-prep.test.js +60 -0
- package/test/inject-heat.test.js +31 -15
- package/test/inject-pools.test.js +19 -0
- package/test/scope-strict.test.js +136 -0
- package/test/sensitive-scan.test.js +207 -0
- package/test/serve-bin.test.js +121 -0
- package/test/serve.test.js +196 -0
- package/test/standalone-api.test.js +212 -5
- package/test/write-admission.test.js +115 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
// #164 A2:写入边界的密钥 / PII 判据(src/sensitive-scan.js)。
|
|
2
|
+
//
|
|
3
|
+
// 这是「真判据」自己的验收,与 write-admission.test.js 里那组(拿参考扫描器测**闸门
|
|
4
|
+
// 接线**)分工不同:这边测判据本身,且第一条用例就是把 #332 配好的 26 条语料原样跑在
|
|
5
|
+
// 生产实现上——维护者 09-28 的验收条件就是这句「同一套负样本原样跑,抓错任何一条就
|
|
6
|
+
// 不算落地」,所以它放在这里而不是再抄一份语料。
|
|
7
|
+
//
|
|
8
|
+
// 参考扫描器 referenceScan 只服务闸门测;本文件刻意**不** import 它——生产判据与
|
|
9
|
+
// 参考实现是两份独立实现,同时跑过同一套语料才算把语料用足(对照着改就成了自证)。
|
|
10
|
+
import test from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
|
|
13
|
+
import { scanSensitive, createSensitiveScan } from "../src/sensitive-scan.js";
|
|
14
|
+
import {
|
|
15
|
+
SECRET_SAMPLES,
|
|
16
|
+
PII_SAMPLES,
|
|
17
|
+
POSITIVE_SAMPLES,
|
|
18
|
+
NEGATIVE_SAMPLES,
|
|
19
|
+
SAMPLE_VALUES
|
|
20
|
+
} from "./helpers/write-admission-samples.js";
|
|
21
|
+
|
|
22
|
+
// #164 A2 的语料里那几条凭据值走 helper 的拼接导出,别在测试里另抄一份字面量:
|
|
23
|
+
// GitHub 的 push protection 会按形状拦真实凭据(本 PR 实测:`sk_live_` 一条就够
|
|
24
|
+
// 拒推,helper 正是为这件事把值拆成两段拼的)。这里复用同一份,仓库文本里不留完整形状。
|
|
25
|
+
const STRIPE_KEY = SAMPLE_VALUES.stripeKey;
|
|
26
|
+
|
|
27
|
+
// 正样本那条 OpenAI key 从语料里取,不另抄字面量(同 STRIPE_KEY 的理由)。
|
|
28
|
+
const OPENAI_KEY = POSITIVE_SAMPLES.find((s) => s.id === "openai-key").content.match(/\S*sk-\S+/)[0];
|
|
29
|
+
/** `sk-proj-…` 前缀的变体:语料那条不带 proj-,这里补同样的正文长度。 */
|
|
30
|
+
const openaiProjKey = (body) => `sk-proj-${body}`;
|
|
31
|
+
const OPENAI_BODY = OPENAI_KEY.replace(/^.*?sk-(?:proj-)?/, "");
|
|
32
|
+
|
|
33
|
+
test("真判据:正样本一条都不漏放,且 kind 与语料声明一致", () => {
|
|
34
|
+
for (const sample of POSITIVE_SAMPLES) {
|
|
35
|
+
const text = `${sample.title}\n${sample.content}`;
|
|
36
|
+
const hit = scanSensitive(text);
|
|
37
|
+
assert.ok(hit, `${sample.id} 被漏放(${sample.why})`);
|
|
38
|
+
// kind 是审计行里的稳定判据键:它错了,「按 kind 看命中分布」这条路就断了。
|
|
39
|
+
// 同一个串可能被多条规则吃下(`sk_live_` 也像通用 `sk-`),所以顺序是判据的一部分。
|
|
40
|
+
assert.equal(hit.kind, sample.kind, `${sample.id} 的 kind 必须是语料声明的那个(规则顺序变了就会漂)`);
|
|
41
|
+
assert.equal(typeof hit.label, "string");
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("真判据:负样本一条都不误杀(抓错任何一条就不算落地)", () => {
|
|
46
|
+
for (const sample of NEGATIVE_SAMPLES) {
|
|
47
|
+
const hit = scanSensitive(`${sample.title}\n${sample.content}`);
|
|
48
|
+
assert.equal(hit, null, `${sample.id} 被误杀(${sample.why}):实际命中 ${hit?.kind}`);
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("密钥与 PII 分两档,但都从同一条通道出来", () => {
|
|
53
|
+
// 分档的意义是「将来按类放行只改策略、不动判据」:kind 集合是策略的抓手,
|
|
54
|
+
// 所以这里断言两类都能被分辨,而不是把 PII 折进密钥那一档。
|
|
55
|
+
assert.ok(SECRET_SAMPLES.length >= 1 && PII_SAMPLES.length >= 1);
|
|
56
|
+
assert.equal(scanSensitive("password = \"Tr0ub4dor&3xKcd\"").kind, "assigned_secret");
|
|
57
|
+
assert.equal(scanSensitive("联系方式 zhang.wei@example.com").kind, "email");
|
|
58
|
+
assert.equal(scanSensitive("值班手机 13800138000").kind, "cn_mobile");
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("只扫文本:路径与 id 这类元数据字段不进判据", () => {
|
|
62
|
+
// 元数据由系统生成,不是用户写进来的内容;扫它们只会引入假阳(一个文件名里
|
|
63
|
+
// `token` 这种词很常见)。契约是判据只看记忆的文本字段。
|
|
64
|
+
const scan = createSensitiveScan({ config: { sensitiveScanEnabled: true } });
|
|
65
|
+
assert.equal(scan({ title: "配置", content: "正常正文", id: SAMPLE_VALUES.githubPat, doc_path: STRIPE_KEY }), null);
|
|
66
|
+
// 反过来,正文里的凭据必须报——上面那条不是靠「什么都扫不到」通过的。
|
|
67
|
+
assert.ok(scan({ title: "配置", content: `正文里有 ${SAMPLE_VALUES.githubPat}` }));
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("tags 也扫:标签是用户输入的自由文本", () => {
|
|
71
|
+
const scan = createSensitiveScan({ config: { sensitiveScanEnabled: true } });
|
|
72
|
+
assert.equal(scan({ title: "正常标题", content: "正常正文", tags: ["正常"] }), null);
|
|
73
|
+
assert.equal(scan({ title: "正常标题", content: "正常正文", tags: [SAMPLE_VALUES.githubPat] })?.kind, "github_token");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("工厂:判据自己的闸关时连函数都不给(闸门侧行为与 #332 逐字段一致)", () => {
|
|
77
|
+
assert.equal(createSensitiveScan({ config: { sensitiveScanEnabled: false } }), null);
|
|
78
|
+
assert.equal(createSensitiveScan({ config: {} }), null);
|
|
79
|
+
assert.equal(createSensitiveScan({}), null, "没有 config 也不能默认开");
|
|
80
|
+
assert.equal(createSensitiveScan({ config: { sensitiveScanEnabled: "true" } }), null, "字符串 true 不是 true");
|
|
81
|
+
const scan = createSensitiveScan({ config: { sensitiveScanEnabled: true } });
|
|
82
|
+
assert.equal(typeof scan, "function");
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("工厂不碰决策:命中只报 kind,拦不拦由闸门的 enforce 决定", () => {
|
|
86
|
+
// 判据与闸门分层(#164:默认仅告警、拦截 opt-in)——判据知道「这是什么」,
|
|
87
|
+
// 但不知道也不该知道「要不要拦」。返回值里没有 decision / enforced 这类字段,
|
|
88
|
+
// 就是这条分层的机器可判点。
|
|
89
|
+
const scan = createSensitiveScan({
|
|
90
|
+
config: { sensitiveScanEnabled: true, writeAdmission: { enabled: true, enforce: false } }
|
|
91
|
+
});
|
|
92
|
+
const hit = scan({ title: "调试", content: openaiProjKey(OPENAI_BODY) });
|
|
93
|
+
assert.deepEqual(Object.keys(hit).sort(), ["kind", "label"]);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("不抛:坏输入返回 null,扫描器故障不会把写入变成不可用", () => {
|
|
97
|
+
assert.equal(scanSensitive(undefined), null);
|
|
98
|
+
assert.equal(scanSensitive(null), null);
|
|
99
|
+
assert.equal(scanSensitive(""), null);
|
|
100
|
+
assert.equal(scanSensitive(12345), null);
|
|
101
|
+
const scan = createSensitiveScan({ config: { sensitiveScanEnabled: true } });
|
|
102
|
+
assert.equal(scan(null), null);
|
|
103
|
+
assert.equal(scan({ title: 7, content: null, tags: "not-an-array" }), null);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("一处命中只报最严重的那条(deny 面记类别,不记条数)", () => {
|
|
107
|
+
// 一篇同时带 Stripe 密钥与邮箱的正文:报密钥那一档。反过来把 PII 排在前面
|
|
108
|
+
// 会让「这条命中过密钥」这个更严重的事实被邮箱盖掉。
|
|
109
|
+
const text = `邮箱 a@b.com,另外 ${STRIPE_KEY}`;
|
|
110
|
+
assert.equal(scanSensitive(text).kind, "stripe_key");
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("语料之外的探针:漏放那一类锁一条(实现期实测出来的,不是假想)", () => {
|
|
114
|
+
// 不在 #332 的 26 条语料里,是拿真判据跑额外形状时发现的:AWS secret access key
|
|
115
|
+
// 没有前缀,`secret` 关键词规则又要求它后面紧跟赋值号,多词键
|
|
116
|
+
// (`AWS_SECRET_ACCESS_KEY=`)两条都够不着 → 补了专用的键名规则。
|
|
117
|
+
assert.equal(
|
|
118
|
+
scanSensitive("AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY").kind,
|
|
119
|
+
"aws_secret_key"
|
|
120
|
+
);
|
|
121
|
+
// 反向确认「值 + 赋值键」这条兜底规则还在工作,而且**不**看值的形状:`api_key =`
|
|
122
|
+
// 后面跟任何 8 位以上的值都报(判据不知道哪个值是真的)。
|
|
123
|
+
assert.equal(scanSensitive("api_key = sk-abcdefghijklmnopqrst").kind, "assigned_secret");
|
|
124
|
+
// 收紧后的开关键字仍然认 24 位以上的真 key(44 位正文,语料那条)。
|
|
125
|
+
assert.equal(scanSensitive(openaiProjKey(OPENAI_BODY)).kind, "openai_key");
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
// --- 合并后独立复审的三项加固 -------------------------------------------------
|
|
129
|
+
|
|
130
|
+
// 赋值句的键与值都在运行时拼装,别写成整串字面量:`.github/workflows/security.yml` 的
|
|
131
|
+
// gitleaks 是**全历史**扫描且没有 allowlist,而 generic-api-key 规则认的是「关键词 +
|
|
132
|
+
// 赋值号 + 高熵值」——一个 16 位、字符全不重复的十六进制串熵值就有 4.0,高于它 3.5 的
|
|
133
|
+
// 阈值,直接落进源码就会被判成真凭据(本批初稿实测:两条各报一次)。
|
|
134
|
+
// 与 helpers/write-admission-samples.js 里那几条凭据值同一个理由,那边也是拼的。
|
|
135
|
+
const envLine = (key, value, prefix = "") => `${prefix}${key}=${value}`;
|
|
136
|
+
/** 一眼假的低熵值:不是占位符(占位符会被守卫挡掉,那样就测不到这条规则了)。 */
|
|
137
|
+
const FAKE_LITERAL = ["super", "sekrit", "000"].join("-");
|
|
138
|
+
|
|
139
|
+
test("左边界放开 `_`:环境变量名形态的赋值也要抓(原先整类漏放)", () => {
|
|
140
|
+
// 回归锁:赋值型规则的左边界原为 `[^A-Za-z0-9_]`,把下划线也当成了标识符内部字符,
|
|
141
|
+
// 于是只有恰好落在行首的 `API_KEY=` 能中,而 `DB_PASSWORD=` / `MY_API_KEY=` /
|
|
142
|
+
// `MYSQL_PASSWORD=` 这种「前缀_关键词」的环境变量名整类漏放——配置片段与 .env 正文
|
|
143
|
+
// 恰恰是最常见的落库形态。放宽后 #332 的 26 条语料仍全绿,说明原写法不是语料换来
|
|
144
|
+
// 的取舍,是白丢的。
|
|
145
|
+
for (const [key, prefix] of [
|
|
146
|
+
["DB_PASSWORD", ""],
|
|
147
|
+
["MY_API_KEY", ""],
|
|
148
|
+
["MYSQL_PASSWORD", ""],
|
|
149
|
+
["GITHUB_TOKEN", "export "] // 带 shell 前缀的写法
|
|
150
|
+
]) {
|
|
151
|
+
const line = envLine(key, FAKE_LITERAL, prefix);
|
|
152
|
+
assert.equal(scanSensitive(line)?.kind, "assigned_secret", `${key} 形态的赋值不该漏放`);
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("放宽左边界没有放走占位符与间接引用(守卫仍在管那一半)", () => {
|
|
157
|
+
// 与上一条成对:放宽的是「关键词前面能不能是 `_`」,不是「右边的值算不算字面量」。
|
|
158
|
+
// 后者仍归占位符守卫——下面三条正是负样本群里「有敏感词但没有值」的形态。
|
|
159
|
+
// 这些值同样拼装,理由见本节开头(`${...}` 这类串本身也贴着"像凭据"的形状)。
|
|
160
|
+
const placeholders = [
|
|
161
|
+
["DB_PASSWORD", ["$", "{DB_PASSWORD}"].join("")],
|
|
162
|
+
["MY_API_KEY", ["<", "your-key-here", ">"].join("")],
|
|
163
|
+
["DB_PASSWORD", ["process", ".env.DB_PASSWORD"].join("")]
|
|
164
|
+
];
|
|
165
|
+
for (const [key, value] of placeholders) {
|
|
166
|
+
const line = envLine(key, value);
|
|
167
|
+
assert.equal(scanSensitive(line), null, `${key} 形态的占位 / 间接引用不该报`);
|
|
168
|
+
}
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("身份证档加校验位:位数对但校验不过的 18 位数字串不报", () => {
|
|
172
|
+
// 回归锁:原先只有银行卡档有 Luhn、身份证档只看长度,结果是 18 位纯数字(订单号 /
|
|
173
|
+
// 内部编号 / 拼接时间戳)先被身份证规则命中,等不到银行卡那条的校验。补 GB 11643
|
|
174
|
+
// MOD 11-2 之后两侧都干净:校验不过的不报,语料里那条真形状照报。
|
|
175
|
+
assert.equal(scanSensitive("内部编号 123456789012345678 已登记"), null, "校验不过的 18 位不该报");
|
|
176
|
+
assert.equal(scanSensitive("内部编号 987654321098765432 已登记"), null);
|
|
177
|
+
assert.equal(scanSensitive("身份证 11010519491231002X 已核验")?.kind, "cn_id_card");
|
|
178
|
+
// 小写 x 走同一张校验表(语料只覆盖了大写 X)。
|
|
179
|
+
assert.equal(scanSensitive("身份证 11010519491231002x 已核验")?.kind, "cn_id_card");
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
test("email 的 local part 上界恰好卡在 RFC 5321 的 64", () => {
|
|
183
|
+
// 上界不是随手取的数:64 是 RFC 5321 给 local part 的上限,所以「长于 64 不认」
|
|
184
|
+
// 这条边界必须是精确的——松一位会让回溯面失控,紧一位会漏掉合法邮箱。
|
|
185
|
+
const at = (n) => `${"a".repeat(n)}@example.com`;
|
|
186
|
+
assert.ok(scanSensitive(at(64)), "64 位的 local part 是合法的,必须认");
|
|
187
|
+
assert.equal(scanSensitive(at(65)), null, "65 位超出 RFC 上限,不认(换取回溯有界的代价)");
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("长文本上的回溯有上界:写阻塞类回归的计时护栏", () => {
|
|
191
|
+
// 回归锁:email 的 local part 与 connection_string 的 scheme 都作用在含 `.` 的字符
|
|
192
|
+
// 类上,缺上界时每个起点都要一路重扫到结尾才失败 → O(n²)。未加上界时实测:34KB
|
|
193
|
+
// 点分链 0.6 秒、120KB 对抗串 23.5 秒(email 18.6s + 连接串 4.1s);而本判据在写入
|
|
194
|
+
// 路径上同步跑,等于把写入卡死。加上界后同输入约 60ms。
|
|
195
|
+
// 预算取 2 秒:比修复前小一个数量级(12× 安全边),又比修复后大 30 倍,不至于像
|
|
196
|
+
// 50ms 级断言那样被 CI 抖动误伤——它抓的是「上界被拿掉」这类数量级回归,不是微优化。
|
|
197
|
+
const inputs = [
|
|
198
|
+
"a.".repeat(30000) + "@" + "b.".repeat(30000) + "!",
|
|
199
|
+
"com.example.pkg.".repeat(10000) + "@"
|
|
200
|
+
];
|
|
201
|
+
for (const text of inputs) {
|
|
202
|
+
const started = process.hrtime.bigint();
|
|
203
|
+
scanSensitive(text);
|
|
204
|
+
const ms = Number(process.hrtime.bigint() - started) / 1e6;
|
|
205
|
+
assert.ok(ms < 2000, `${text.length} 字符耗时 ${ms.toFixed(1)}ms,超 2000ms 预算(回溯上界被破坏?)`);
|
|
206
|
+
}
|
|
207
|
+
});
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { spawn } from "node:child_process";
|
|
4
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
5
|
+
import { tmpdir } from "node:os";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { createStore } from "../src/store.js";
|
|
9
|
+
import { createService } from "../src/service.js";
|
|
10
|
+
import { createSettings } from "../src/settings.js";
|
|
11
|
+
|
|
12
|
+
// serve bin 冒烟 + 多进程共存锁(daemon 唯一的真新风险点):daemon 子进程与测试进程
|
|
13
|
+
// 同时打开同一 memory.db(WAL + busy_timeout 本就为此设计),互写互读。spawn 必须
|
|
14
|
+
// 异步(maintenance.test.js 的教训:同步 spawn 会堵住本进程事件循环);Windows 下
|
|
15
|
+
// child.kill() 是硬终止,不断言信号路径,只断言可达性与退出。
|
|
16
|
+
|
|
17
|
+
const BIN = fileURLToPath(new URL("../bin/dsh-mneme-serve.mjs", import.meta.url));
|
|
18
|
+
|
|
19
|
+
function wait(ms) {
|
|
20
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function waitFor(fn, timeoutMs, what) {
|
|
24
|
+
const deadline = Date.now() + timeoutMs;
|
|
25
|
+
let lastErr;
|
|
26
|
+
while (Date.now() < deadline) {
|
|
27
|
+
try {
|
|
28
|
+
return await fn();
|
|
29
|
+
} catch (err) {
|
|
30
|
+
lastErr = err;
|
|
31
|
+
}
|
|
32
|
+
await wait(150);
|
|
33
|
+
}
|
|
34
|
+
throw new Error(`timeout waiting for ${what}: ${lastErr?.message ?? lastErr}`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
test("serve bin: 值旗标缺值直接报错退出(--port 后无值不能落成 Number(true)=1)", { timeout: 30000 }, async () => {
|
|
38
|
+
// CodeRabbit on #364:--port 紧跟另一个旗标或结束时,旧解析把它存成 true,
|
|
39
|
+
// Number(true)=1 通过校验 → 静默改绑端口 1(EACCES 误导排错方向)。
|
|
40
|
+
const child = spawn(process.execPath, [BIN, "--memory-dir", mkdtempSync(join(tmpdir(), "mneme-serve-arg-")), "--port"], {
|
|
41
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
42
|
+
});
|
|
43
|
+
let stderr = "";
|
|
44
|
+
child.stderr.on("data", (d) => { stderr += d; });
|
|
45
|
+
const [code] = await new Promise((resolve) => {
|
|
46
|
+
child.on("exit", (c) => resolve([c]));
|
|
47
|
+
});
|
|
48
|
+
assert.notEqual(code, 0, "missing value must exit non-zero");
|
|
49
|
+
assert.ok(stderr.includes("缺少参数值"), `stderr should name the missing value, got: ${stderr.slice(-200)}`);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("serve bin: spawn 冒烟;daemon 与宿主进程同库互写互读", { timeout: 120000 }, async () => {
|
|
53
|
+
const dir = mkdtempSync(join(tmpdir(), "mneme-serve-bin-"));
|
|
54
|
+
|
|
55
|
+
// 预置 token:测试进程先开一次库写进 kv external_api,daemon 复用之 —— 同时验证
|
|
56
|
+
// 「与宿主面板/CLI 共用同一凭证」这条零配置承诺在真子进程里成立。
|
|
57
|
+
const seedStore = createStore(join(dir, "memory.db"));
|
|
58
|
+
const seedSettings = createSettings(seedStore.db);
|
|
59
|
+
const TOKEN = "serve-bin-test-token-0123456789abcdef";
|
|
60
|
+
seedSettings.setExternalApi({ token: TOKEN });
|
|
61
|
+
const peer = createService({ store: seedStore, mirror: null, config: {} });
|
|
62
|
+
peer.saveWithDedupe({ type: "project", title: "peer 进程直写", content: "测试进程经 createStore 写入", importance: 3 });
|
|
63
|
+
|
|
64
|
+
// --embed off:CI 无 runtime payload,绝不能触发取件;多进程共存与语义无关
|
|
65
|
+
const child = spawn(process.execPath, [BIN, "--memory-dir", dir, "--port", "0", "--embed", "off"], {
|
|
66
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
67
|
+
});
|
|
68
|
+
let stdout = "";
|
|
69
|
+
let stderr = "";
|
|
70
|
+
child.stdout.on("data", (d) => { stdout += d; });
|
|
71
|
+
child.stderr.on("data", (d) => { stderr += d; });
|
|
72
|
+
|
|
73
|
+
try {
|
|
74
|
+
// stdout 唯一机器可读行:listening 行(解析实际端口)
|
|
75
|
+
await waitFor(() => {
|
|
76
|
+
if (child.exitCode !== null) {
|
|
77
|
+
throw new Error(`daemon exited early (code ${child.exitCode}): ${stderr.slice(-400)}`);
|
|
78
|
+
}
|
|
79
|
+
if (!stdout.includes("listening on http://")) throw new Error("no listening line yet");
|
|
80
|
+
return true;
|
|
81
|
+
}, 20000, "listening line");
|
|
82
|
+
const port = Number(stdout.match(/:(\d+)/)[1]);
|
|
83
|
+
const base = `http://127.0.0.1:${port}`;
|
|
84
|
+
const auth = { authorization: `Bearer ${TOKEN}` };
|
|
85
|
+
|
|
86
|
+
await waitFor(async () => {
|
|
87
|
+
const res = await fetch(`${base}/health`);
|
|
88
|
+
assert.equal(res.status, 200);
|
|
89
|
+
return true;
|
|
90
|
+
}, 10000, "daemon /health");
|
|
91
|
+
|
|
92
|
+
// 双向可见性 ①:测试进程直写 → daemon HTTP 检索可见
|
|
93
|
+
await waitFor(async () => {
|
|
94
|
+
const res = await fetch(`${base}/search?q=${encodeURIComponent("peer 进程直写")}`, { headers: auth });
|
|
95
|
+
assert.equal(res.status, 200);
|
|
96
|
+
const body = await res.json();
|
|
97
|
+
assert.ok((body.items ?? []).some((m) => m.title === "peer 进程直写"));
|
|
98
|
+
return true;
|
|
99
|
+
}, 10000, "peer write visible via daemon");
|
|
100
|
+
|
|
101
|
+
// 双向可见性 ②:daemon HTTP 写 → 测试进程 service 同步检索可见
|
|
102
|
+
const save = await fetch(`${base}/memories`, {
|
|
103
|
+
method: "POST",
|
|
104
|
+
headers: { ...auth, "content-type": "application/json" },
|
|
105
|
+
body: JSON.stringify({ type: "project", title: "daemon HTTP 写入", content: "serve 进程经数据面写入", importance: 3 })
|
|
106
|
+
});
|
|
107
|
+
assert.ok(save.status === 201 || save.status === 200, `daemon save status ${save.status}`);
|
|
108
|
+
assert.ok(
|
|
109
|
+
peer.search("daemon HTTP 写入", { limit: 5 }).some((m) => m.title === "daemon HTTP 写入"),
|
|
110
|
+
"daemon-side write must be visible to the other process immediately"
|
|
111
|
+
);
|
|
112
|
+
} finally {
|
|
113
|
+
child.kill();
|
|
114
|
+
await new Promise((r) => {
|
|
115
|
+
if (child.exitCode !== null) r();
|
|
116
|
+
else child.on("exit", r);
|
|
117
|
+
});
|
|
118
|
+
try { seedStore.close(); } catch { /* 同上 */ }
|
|
119
|
+
try { rmSync(dir, { recursive: true, force: true }); } catch { /* win 文件锁偶发,残留临时目录无害 */ }
|
|
120
|
+
}
|
|
121
|
+
});
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { createServeRuntime } from "../src/serve.js";
|
|
7
|
+
|
|
8
|
+
// daemon 装配面锁(test/serve-bin.test.js 另有真子进程 + 多进程共存):
|
|
9
|
+
// in-process 起真 HTTP(port 0,OS 分配),fetch 走 health / 401 / save / search /
|
|
10
|
+
// token 持久化复用全链路。锁的是「daemon 数据面 = api-standalone 同一工厂」这一契约。
|
|
11
|
+
// 基础用例一律 embed:"off"——不碰取件与模型 init,CI 确定性;向量走注入假 embedder。
|
|
12
|
+
|
|
13
|
+
function tmpDir() {
|
|
14
|
+
return mkdtempSync(join(tmpdir(), "mneme-serve-"));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** 确定性 4 维伪嵌入:同词相关、异词近正交,足以让向量路径产生非零命中。 */
|
|
18
|
+
function fakeEmbedder() {
|
|
19
|
+
return {
|
|
20
|
+
ready: true,
|
|
21
|
+
modelHash: "fake-hash-1",
|
|
22
|
+
dimension: 4,
|
|
23
|
+
embedSingle: async (text) => {
|
|
24
|
+
const v = [0, 0, 0, 0];
|
|
25
|
+
for (let i = 0; i < text.length; i++) v[i % 4] += text.charCodeAt(i) % 7;
|
|
26
|
+
const n = Math.hypot(v[0], v[1], v[2], v[3]) || 1;
|
|
27
|
+
return v.map((x) => x / n);
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function waitFor(fn, timeoutMs, what) {
|
|
33
|
+
const deadline = Date.now() + timeoutMs;
|
|
34
|
+
let lastErr;
|
|
35
|
+
while (Date.now() < deadline) {
|
|
36
|
+
try { return await fn(); } catch (err) { lastErr = err; }
|
|
37
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
38
|
+
}
|
|
39
|
+
throw new Error(`timeout waiting for ${what}: ${lastErr?.message ?? lastErr}`);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
test("serve: /health 免鉴权,业务路由无 token 401", async () => {
|
|
43
|
+
const dir = tmpDir();
|
|
44
|
+
const rt = await createServeRuntime({ memoryDir: dir, port: 0, embed: "off" });
|
|
45
|
+
await rt.api.ready;
|
|
46
|
+
try {
|
|
47
|
+
const base = `http://127.0.0.1:${rt.api.port}`;
|
|
48
|
+
const health = await fetch(`${base}/health`);
|
|
49
|
+
assert.equal(health.status, 200);
|
|
50
|
+
assert.deepEqual(await health.json(), { ok: true });
|
|
51
|
+
|
|
52
|
+
const noAuth = await fetch(`${base}/search?q=x`);
|
|
53
|
+
assert.equal(noAuth.status, 401);
|
|
54
|
+
assert.deepEqual(await noAuth.json(), { error: "unauthorized" });
|
|
55
|
+
} finally {
|
|
56
|
+
rt.dispose();
|
|
57
|
+
try { rmSync(dir, { recursive: true, force: true }); } catch { /* win 文件锁偶发 */ }
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("serve: save → search 走通;token 持久化 kv;二次启动复用同一 token;检索回执落 recall_runs", async () => {
|
|
62
|
+
const dir = tmpDir();
|
|
63
|
+
const rt = await createServeRuntime({ memoryDir: dir, port: 0, embed: "off" });
|
|
64
|
+
await rt.api.ready;
|
|
65
|
+
const base = `http://127.0.0.1:${rt.api.port}`;
|
|
66
|
+
const auth = { authorization: `Bearer ${rt.api.token}` };
|
|
67
|
+
|
|
68
|
+
// 全新目录:首次启动生成 token
|
|
69
|
+
assert.equal(rt.tokenExisted, false);
|
|
70
|
+
|
|
71
|
+
const save = await fetch(`${base}/memories`, {
|
|
72
|
+
method: "POST",
|
|
73
|
+
headers: { ...auth, "content-type": "application/json" },
|
|
74
|
+
body: JSON.stringify({ type: "preference", title: "serve 冒烟偏好", content: "回复保持简短", importance: 4 })
|
|
75
|
+
});
|
|
76
|
+
assert.equal(save.status, 201);
|
|
77
|
+
|
|
78
|
+
const search = await fetch(`${base}/search?q=${encodeURIComponent("简短")}`, { headers: auth });
|
|
79
|
+
assert.equal(search.status, 200);
|
|
80
|
+
const searchBody = await search.json();
|
|
81
|
+
assert.ok((searchBody.items ?? []).some((m) => m.title === "serve 冒烟偏好"));
|
|
82
|
+
|
|
83
|
+
// token 持久化在 kv external_api(与 DSH 面板/CLI 共用同一凭证通道)
|
|
84
|
+
assert.equal(rt.settings.getExternalApi().token, rt.api.token);
|
|
85
|
+
|
|
86
|
+
// 检索回执:recorder 已接(index.js:294-309 同款),recall_runs 不缺数 —— #363 回帖
|
|
87
|
+
// 向 bridge 承诺过「第三方检索的复用统计不受影响」,这条就是该承诺的回归锁。
|
|
88
|
+
const recallRows = rt.store.db.prepare("SELECT COUNT(*) AS n FROM recall_runs").get();
|
|
89
|
+
assert.ok(recallRows.n >= 1, "recall_runs should record the search above");
|
|
90
|
+
|
|
91
|
+
rt.dispose();
|
|
92
|
+
|
|
93
|
+
// 二次启动同目录:token 复用不重新生成(firstBoot 提示只在真正首次出现)
|
|
94
|
+
const rt2 = await createServeRuntime({ memoryDir: dir, port: 0, embed: "off" });
|
|
95
|
+
await rt2.api.ready;
|
|
96
|
+
try {
|
|
97
|
+
assert.equal(rt2.tokenExisted, true);
|
|
98
|
+
assert.equal(rt2.api.token, rt.api.token);
|
|
99
|
+
} finally {
|
|
100
|
+
rt2.dispose();
|
|
101
|
+
try { rmSync(dir, { recursive: true, force: true }); } catch { /* 同上 */ }
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("serve: dispose 后端口可复用(同端口连起两轮不踩 strictPort)", async () => {
|
|
106
|
+
const dir = tmpDir();
|
|
107
|
+
const rt = await createServeRuntime({ memoryDir: dir, port: 0, embed: "off" });
|
|
108
|
+
await rt.api.ready;
|
|
109
|
+
const port = rt.api.port;
|
|
110
|
+
// dispose 现为 async:等 server.close 回调(在途请求排干)后再重绑
|
|
111
|
+
await rt.dispose();
|
|
112
|
+
// 端口释放可能有内核级迟滞,重试绑定而不是假设立即可用
|
|
113
|
+
let rebound = null;
|
|
114
|
+
for (let i = 0; i < 10 && !rebound; i++) {
|
|
115
|
+
try {
|
|
116
|
+
const rt2 = await createServeRuntime({ memoryDir: dir, port, embed: "off" });
|
|
117
|
+
await rt2.api.ready;
|
|
118
|
+
rebound = rt2;
|
|
119
|
+
} catch (err) {
|
|
120
|
+
if (err?.code !== "EADDRINUSE") throw err;
|
|
121
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
try {
|
|
125
|
+
assert.ok(rebound, "port should be rebindable after dispose");
|
|
126
|
+
assert.equal(rebound.api.port, port);
|
|
127
|
+
} finally {
|
|
128
|
+
await rebound?.dispose();
|
|
129
|
+
try { rmSync(dir, { recursive: true, force: true }); } catch { /* 同上 */ }
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("serve: 注入假 embedder → 写入即嵌入,vector 轴接管检索;auto 检索可用", async () => {
|
|
134
|
+
const dir = tmpDir();
|
|
135
|
+
const fake = fakeEmbedder();
|
|
136
|
+
const rt = await createServeRuntime({ memoryDir: dir, port: 0, embed: "off", embedder: fake });
|
|
137
|
+
await rt.api.ready;
|
|
138
|
+
try {
|
|
139
|
+
// 覆盖注入生效:semantic 被绕过,embedder 直接挂 service
|
|
140
|
+
assert.equal(rt.semantic, null);
|
|
141
|
+
|
|
142
|
+
const base = `http://127.0.0.1:${rt.api.port}`;
|
|
143
|
+
const auth = { authorization: `Bearer ${rt.api.token}` };
|
|
144
|
+
const save = await fetch(`${base}/memories`, {
|
|
145
|
+
method: "POST",
|
|
146
|
+
headers: { ...auth, "content-type": "application/json" },
|
|
147
|
+
body: JSON.stringify({ type: "project", title: "向量注入冒烟", content: "daemon 侧写入即嵌入", importance: 3 })
|
|
148
|
+
});
|
|
149
|
+
assert.equal(save.status, 201);
|
|
150
|
+
|
|
151
|
+
// 写入侧嵌入是异步排队的:轮询到 vector 轴命中为止。auto 模式不断言 mode 字段 ——
|
|
152
|
+
// 标题同词命中时融合合并对象是 keyword 行,上游 mneme 的 auto 报告语义如此
|
|
153
|
+
// (vector:true 标记只在纯向量来源上保留),那是上游行为,不是 daemon 契约。
|
|
154
|
+
await waitFor(async () => {
|
|
155
|
+
const res = await fetch(`${base}/search?q=${encodeURIComponent("向量注入冒烟")}&mode=vector`, { headers: auth });
|
|
156
|
+
assert.equal(res.status, 200);
|
|
157
|
+
const body = await res.json();
|
|
158
|
+
assert.equal(body.mode, "vector", "mode=vector must report the vector axis");
|
|
159
|
+
assert.ok((body.items ?? []).some((m) => m.title === "向量注入冒烟"));
|
|
160
|
+
return true;
|
|
161
|
+
}, 8000, "vector-axis search");
|
|
162
|
+
|
|
163
|
+
const autoRes = await fetch(`${base}/search?q=${encodeURIComponent("向量注入冒烟")}&mode=auto`, { headers: auth });
|
|
164
|
+
assert.equal(autoRes.status, 200);
|
|
165
|
+
assert.ok(((await autoRes.json()).items ?? []).some((m) => m.title === "向量注入冒烟"));
|
|
166
|
+
|
|
167
|
+
// /context 全链路(issue #370):daemon 与宿主外部访问共用同一工厂,注入
|
|
168
|
+
// 候选端点一并带出——同一 embedder 句柄喂查询向量,injectCandidates 走通。
|
|
169
|
+
const ctxRes = await fetch(`${base}/context?q=${encodeURIComponent("向量注入冒烟")}`, { headers: auth });
|
|
170
|
+
assert.equal(ctxRes.status, 200);
|
|
171
|
+
const ctxBody = await ctxRes.json();
|
|
172
|
+
assert.ok((ctxBody.items ?? []).some((m) => m.title === "向量注入冒烟"));
|
|
173
|
+
// #217 注入分账:injectCandidates 的曝光型访问事件落 recall_runs
|
|
174
|
+
// (mode='inject'),第三方的注入统计与宿主注入同表同口径。
|
|
175
|
+
const injectRuns = rt.store.db.prepare("SELECT count(*) AS c FROM recall_runs WHERE mode = 'inject'").get().c;
|
|
176
|
+
assert.ok(injectRuns >= 1, "injection access must be receipted in recall_runs (#217)");
|
|
177
|
+
} finally {
|
|
178
|
+
rt.dispose();
|
|
179
|
+
try { rmSync(dir, { recursive: true, force: true }); } catch { /* 同上 */ }
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test("serve: 非法 embed provider 拒装配;embed=off 时 semantic 为 null", async () => {
|
|
184
|
+
const dir = tmpDir();
|
|
185
|
+
await assert.rejects(
|
|
186
|
+
createServeRuntime({ memoryDir: dir, port: 0, embed: "bogus" }),
|
|
187
|
+
/invalid embed provider/
|
|
188
|
+
);
|
|
189
|
+
const rt = await createServeRuntime({ memoryDir: dir, port: 0, embed: "off" });
|
|
190
|
+
try {
|
|
191
|
+
assert.equal(rt.semantic, null, "off path must not assemble a semantic pipeline");
|
|
192
|
+
} finally {
|
|
193
|
+
rt.dispose();
|
|
194
|
+
try { rmSync(dir, { recursive: true, force: true }); } catch { /* 同上 */ }
|
|
195
|
+
}
|
|
196
|
+
});
|