@modusensus/dsh-mneme 0.8.9 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/assets/icon.svg +3 -0
- package/lib/api-standalone.js +11 -1
- package/lib/api.js +6 -1
- package/lib/client.js +208 -59
- package/lib/config.js +26 -0
- package/lib/index.js +10 -4
- package/lib/inject.js +68 -9
- package/lib/service.js +28 -3
- package/lib/settings.js +9 -1
- package/lib/tools.js +17 -8
- package/lib/write-admission.js +165 -22
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +6 -2
- package/scripts/benchmark-recall.js +57 -7
- package/src/api-standalone.js +11 -1
- package/src/api.js +6 -1
- package/src/config.js +26 -0
- package/src/index.js +10 -4
- package/src/inject.js +68 -9
- package/src/service.js +28 -3
- package/src/settings.js +9 -1
- package/src/tools.js +17 -8
- package/src/write-admission.js +165 -22
- package/test/api.test.js +17 -2
- package/test/client.test.js +245 -45
- package/test/epistemic.test.js +27 -0
- package/test/helpers/write-admission-samples.js +308 -0
- package/test/inject.test.js +137 -0
- package/test/recall-anti-update.test.js +82 -0
- package/test/write-admission.test.js +208 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
// #254 第 1 级「密钥 / PII」判据的回归样本集。
|
|
2
|
+
//
|
|
3
|
+
// 这份语料是给**判据**用的,不是给闸门用的:闸门(src/write-admission.js)只消费
|
|
4
|
+
// `sensitiveScan` 返回的 {kind,label},判据本身是 #164 A2 的落地范围(记在维护者
|
|
5
|
+
// 排期里)。语料放在这里有两个用途——判据落地时有一份现成的正负样本可跑;现在就能
|
|
6
|
+
// 把「接线通不通」钉住(下面导出的 referenceScan 是只给测试用的参考实现)。
|
|
7
|
+
//
|
|
8
|
+
// 三条口径:
|
|
9
|
+
//
|
|
10
|
+
// 1. 正样本分两类。SECRET_SAMPLES(密钥凭据)与 PII_SAMPLES(个人信息)都收,但
|
|
11
|
+
// 两者的误杀面差一个量级:密钥串在正常项目记忆里几乎不出现(出现了就是事故),
|
|
12
|
+
// 而邮箱 / 电话 / 订单号在项目记忆里完全可能是正当内容。所以 `kind` 必须落进
|
|
13
|
+
// 审计行(见 write-admission.js 的 record)——enforce 打开前先按 kind 看命中
|
|
14
|
+
// 分布,将来要按类放行也只改策略、不动判据。
|
|
15
|
+
//
|
|
16
|
+
// 2. 负样本专挑两类最容易误杀的文本:**含敏感词但没有值**("把 API key 放进环境
|
|
17
|
+
// 变量")、**高熵但不是凭据**(sha256 校验和、UUID、Luhn 合法的示例卡号之外的
|
|
18
|
+
// 长数字)。一个只按关键词命中的扫描器会在这组里全军覆没,这正是这组样本的
|
|
19
|
+
// 用处——它量的不是「抓得住多少」,而是「抓错多少」。
|
|
20
|
+
//
|
|
21
|
+
// 3. 样本里的凭据都是公开占位值(AWS 官方文档的 AKIAIOSFODNN7EXAMPLE、GitHub /
|
|
22
|
+
// OpenAI / Stripe 的文档格式、Stripe 与银行的官方测试卡号),不是真凭据。
|
|
23
|
+
//
|
|
24
|
+
// 用法(测试侧):
|
|
25
|
+
// import { allSamples, NEGATIVE_SAMPLES, referenceScan } from "./helpers/write-admission-samples.js";
|
|
26
|
+
|
|
27
|
+
// 这几条样本的**值**在密钥扫描器眼里就是真凭据的形状。字面量直接写进仓库有两重代价:
|
|
28
|
+
// GitHub 侧 push protection 会直接拒推(GH013),CI 的 gitleaks 全历史扫描会红掉
|
|
29
|
+
// (.github/workflows/security.yml)。拆成拼接后仓库文本里不再含完整形状,而运行时拼
|
|
30
|
+
// 出来的值仍然是判据要面对的那个形状——语料测的东西一点没变。
|
|
31
|
+
//
|
|
32
|
+
// 两套扫描器的规则**并不重合**:push protection 拦的是 Slack / Stripe,gitleaks 拦的是
|
|
33
|
+
// GitHub PAT / JWT。所以这几条统一走拼接,别按「上一次哪条红了」逐条打补丁——扫描器
|
|
34
|
+
// 升级一次规则就又得回来补。
|
|
35
|
+
//
|
|
36
|
+
// 别为了绕开扫描器把它们简化成一眼假的占位串:正样本的价值恰恰在于压在检测器的形状
|
|
37
|
+
// 边界上,改成 `xoxb-xxx` 就等于把这条样本删了。AWS 那条用的是官方文档自己的示例值
|
|
38
|
+
// AKIAIOSFODNN7EXAMPLE,两套扫描器本来就认它是样例,保持字面量。
|
|
39
|
+
const assemble = (...parts) => parts.join("");
|
|
40
|
+
|
|
41
|
+
/** 语料里那几条被扫描器盯着形状的凭据值。测试侧要拼句子时从这里取,别另抄一份。 */
|
|
42
|
+
export const SAMPLE_VALUES = {
|
|
43
|
+
githubPat: assemble("ghp_", "16C7e42F292c6912E7710c838347Ae178B4a"),
|
|
44
|
+
slackToken: assemble("xoxb-", "123456789012-1234567890123-AbCdEfGhIjKlMnOpQrStUvWx"),
|
|
45
|
+
jwt: assemble(
|
|
46
|
+
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
|
|
47
|
+
".eyJzdWIiOiIxMjM0NTY3ODkwIn0",
|
|
48
|
+
".dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"
|
|
49
|
+
),
|
|
50
|
+
stripeKey: assemble("sk_", "live_", "51H8Qw2KZvN3mPqRsTuVwXyZ0123456789abcdef")
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
/** 密钥凭据类正样本。kind 与 label 是判据该报回来的东西。 */
|
|
54
|
+
export const SECRET_SAMPLES = [
|
|
55
|
+
{
|
|
56
|
+
id: "aws-access-key",
|
|
57
|
+
kind: "aws_access_key",
|
|
58
|
+
label: "AWS access key id",
|
|
59
|
+
title: "部署脚本",
|
|
60
|
+
content: "部署脚本里写死了 AKIAIOSFODNN7EXAMPLE,回头要换掉",
|
|
61
|
+
why: "AKIA + 16 位大写字母数字,长度与字符集都固定"
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
id: "github-pat",
|
|
65
|
+
kind: "github_token",
|
|
66
|
+
label: "GitHub token",
|
|
67
|
+
title: "CI 配置",
|
|
68
|
+
content: `CI 的 token 是 ${SAMPLE_VALUES.githubPat},别再贴出来`,
|
|
69
|
+
why: "ghp_ 前缀 + 36 位,GitHub PAT 的固定形状"
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
id: "openai-key",
|
|
73
|
+
kind: "openai_key",
|
|
74
|
+
label: "OpenAI API key",
|
|
75
|
+
title: "调试记录",
|
|
76
|
+
content: "临时用 sk-proj-abcdefghijklmnopqrstuvwxyz0123456789ABCD 调了一次",
|
|
77
|
+
why: "sk-/sk-proj- 前缀 + 长串"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
id: "slack-token",
|
|
81
|
+
kind: "slack_token",
|
|
82
|
+
label: "Slack token",
|
|
83
|
+
title: "webhook",
|
|
84
|
+
content: `webhook 配的是 ${SAMPLE_VALUES.slackToken}`,
|
|
85
|
+
why: "xoxb- 前缀 + 分段数字 + 混合串"
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
id: "pem-private-key",
|
|
89
|
+
kind: "private_key",
|
|
90
|
+
label: "PEM private key",
|
|
91
|
+
title: "issue 正文",
|
|
92
|
+
content: "把 -----BEGIN RSA PRIVATE KEY----- 整段贴到了 issue 里",
|
|
93
|
+
why: "PEM 头是明文私钥的确定标志,有没有正文都不影响判定"
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
id: "assigned-literal",
|
|
97
|
+
kind: "assigned_secret",
|
|
98
|
+
label: "assigned credential literal",
|
|
99
|
+
title: "本地配置",
|
|
100
|
+
content: 'config 里 password = "Tr0ub4dor&3xKcd" 这一行得删掉',
|
|
101
|
+
why: "赋值号右边是字面量凭据——这是最常见的一类,也是最需要防误杀的一类"
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
id: "jwt",
|
|
105
|
+
kind: "jwt",
|
|
106
|
+
label: "JSON Web Token",
|
|
107
|
+
title: "抓包",
|
|
108
|
+
content: `抓到的 Authorization 是 ${SAMPLE_VALUES.jwt}`,
|
|
109
|
+
why: "三段 base64url,header 恒以 eyJ 开头"
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
id: "connection-string",
|
|
113
|
+
kind: "connection_string",
|
|
114
|
+
label: "credentials in URL",
|
|
115
|
+
title: "连不上",
|
|
116
|
+
content: "postgres://admin:S3cr3tPa55w0rd@db.internal:5432/prod 连不上",
|
|
117
|
+
why: "URL 里带 user:password@host 就是明文凭据"
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
id: "stripe-key",
|
|
121
|
+
kind: "stripe_key",
|
|
122
|
+
label: "Stripe secret key",
|
|
123
|
+
title: "支付联调",
|
|
124
|
+
content: `联调用的 ${SAMPLE_VALUES.stripeKey} 忘了轮换`,
|
|
125
|
+
why: "sk_live_ 前缀是生产密钥(测试密钥是 sk_test_,不该报)"
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
id: "npm-auth-token",
|
|
129
|
+
kind: "npm_token",
|
|
130
|
+
label: "npm auth token",
|
|
131
|
+
title: ".npmrc",
|
|
132
|
+
content: "//registry.npmjs.org/:_authToken=npm_abcdefghijklmnopqrstuvwxyz0123456789",
|
|
133
|
+
why: "按 _authToken= 这个固定键定位,不必猜 token 的形状"
|
|
134
|
+
}
|
|
135
|
+
];
|
|
136
|
+
|
|
137
|
+
/** PII 类正样本。误杀面比密钥大一档,见文件头第 1 条。 */
|
|
138
|
+
export const PII_SAMPLES = [
|
|
139
|
+
{
|
|
140
|
+
id: "email-phone",
|
|
141
|
+
kind: "email",
|
|
142
|
+
label: "email address",
|
|
143
|
+
title: "客户联系方式",
|
|
144
|
+
content: "客户的联系方式是 zhang.wei@example.com,别外传",
|
|
145
|
+
why: "RFC 形状的邮箱;项目记忆里出现它是可能的,所以更适合先告警"
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
id: "cn-mobile",
|
|
149
|
+
kind: "cn_mobile",
|
|
150
|
+
label: "mainland mobile number",
|
|
151
|
+
title: "值班表",
|
|
152
|
+
content: "值班手机 13800138000,紧急情况打这个",
|
|
153
|
+
why: "1[3-9] + 9 位,中国手机号的固定形状"
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
id: "cn-id-card",
|
|
157
|
+
kind: "cn_id_card",
|
|
158
|
+
label: "mainland ID number",
|
|
159
|
+
title: "实名核验",
|
|
160
|
+
content: "身份证 11010519491231002X 已经核验过",
|
|
161
|
+
why: "17 位数字 + 校验位(数字或 X)"
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
id: "bank-card",
|
|
165
|
+
kind: "bank_card",
|
|
166
|
+
label: "payment card number",
|
|
167
|
+
title: "对公账户",
|
|
168
|
+
content: "银行卡 4111111111111111 是对公账户",
|
|
169
|
+
why: "4 开头的 16 位;这里用 Visa 的公开测试号,不用真卡号"
|
|
170
|
+
}
|
|
171
|
+
];
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* 负样本:必须**一条都不命中**。每条都写清它为什么容易误杀。
|
|
175
|
+
* 这是本文件里最值钱的部分——正样本只证明「抓得住」,负样本才约束「抓错多少」。
|
|
176
|
+
*/
|
|
177
|
+
export const NEGATIVE_SAMPLES = [
|
|
178
|
+
{
|
|
179
|
+
id: "keyword-no-value",
|
|
180
|
+
title: "密钥管理",
|
|
181
|
+
content: "把 API key 放进环境变量,别写进代码——轮换流程见 docs/handbook/12-secrets.md",
|
|
182
|
+
why: "整句在讨论凭据管理,没有一个字面量值"
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
id: "token-as-noun",
|
|
186
|
+
title: "重试策略",
|
|
187
|
+
content: "token 刷新失败时退避重试三次,第三次仍失败就报错",
|
|
188
|
+
why: "token 是普通名词,后面没有赋值号"
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
id: "redaction-placeholder",
|
|
192
|
+
title: "脱敏示例",
|
|
193
|
+
content: "password: <redacted>",
|
|
194
|
+
why: "占位符不是值——纯关键词扫描器在这里必错"
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
id: "env-var-indirection",
|
|
198
|
+
title: "配置读取",
|
|
199
|
+
content: "secret = process.env.MNEME_SECRET",
|
|
200
|
+
why: "赋值右边是环境变量读取,不是字面量"
|
|
201
|
+
},
|
|
202
|
+
{
|
|
203
|
+
id: "template-var",
|
|
204
|
+
title: "模板",
|
|
205
|
+
// 这条样本**故意**就是一个模板占位符——它量的正是「看到 = 或 : 就报」的误杀。
|
|
206
|
+
// biome-ignore lint/suspicious/noTemplateCurlyInString: 样本内容本身就是待测文本,不是要插值的模板
|
|
207
|
+
content: "token: ${TOKEN}",
|
|
208
|
+
why: "模板变量同理——看到 = 或 : 就报会误杀这一整类"
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
id: "hash-not-secret",
|
|
212
|
+
title: "校验和",
|
|
213
|
+
content: "sha256 摘要 3f786850e387550fdab836ed7e6dc881de23001b 不是密钥,只是一个校验和",
|
|
214
|
+
why: "高熵十六进制串,但不是凭据;按熵阈值判会误杀全部哈希"
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
id: "uuid",
|
|
218
|
+
title: "订单号",
|
|
219
|
+
content: "订单号 550e8400-e29b-41d4-a716-446655440000 是 UUID v4",
|
|
220
|
+
why: "同样是高熵串;UUID 在项目记忆里到处都是"
|
|
221
|
+
},
|
|
222
|
+
{
|
|
223
|
+
id: "prefix-only",
|
|
224
|
+
title: "格式说明",
|
|
225
|
+
content: "GitHub PAT 的前缀是 ghp_,新格式是 github_pat_",
|
|
226
|
+
why: "只提到前缀,后面没有 token 正文"
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
id: "sk-prefix-only",
|
|
230
|
+
title: "key 前缀",
|
|
231
|
+
content: "sk- 开头的是 OpenAI 的 key 前缀",
|
|
232
|
+
why: "同上——前缀本身不是凭据"
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
id: "discuss-rotation",
|
|
236
|
+
title: "轮换周期",
|
|
237
|
+
content: "把密钥轮换周期定成 90 天,到期前一周提醒",
|
|
238
|
+
why: "在讨论密钥的生命周期"
|
|
239
|
+
},
|
|
240
|
+
{
|
|
241
|
+
id: "discuss-hashing",
|
|
242
|
+
title: "密码处理",
|
|
243
|
+
content: "用户在设置页输入密码后调用 bcrypt.hash 存哈希,明文不落盘",
|
|
244
|
+
why: "讨论密码处理方式,不含任何密码"
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
id: "discuss-pii",
|
|
248
|
+
title: "脱敏要求",
|
|
249
|
+
content: "手机号必须脱敏后再入库,日志里也不能出现",
|
|
250
|
+
why: "在讨论 PII 策略,不含具体号码"
|
|
251
|
+
}
|
|
252
|
+
];
|
|
253
|
+
|
|
254
|
+
/** 全部正样本(密钥 + PII),每条都带期望的 kind。 */
|
|
255
|
+
export const POSITIVE_SAMPLES = [...SECRET_SAMPLES, ...PII_SAMPLES];
|
|
256
|
+
|
|
257
|
+
/** 正负样本一起,便于整组遍历。 */
|
|
258
|
+
export const allSamples = [...POSITIVE_SAMPLES, ...NEGATIVE_SAMPLES];
|
|
259
|
+
|
|
260
|
+
// --- 参考扫描器(仅供测试) ---------------------------------------------------
|
|
261
|
+
// 这不是要发货的实现,也**不是** #164 A2 的实现——A2 归维护者排期。它的作用只有
|
|
262
|
+
// 两个:证明闸门侧 `sensitiveScan` 的接线真的能把命中变成拒绝;给上面的语料一个
|
|
263
|
+
// 可执行的对照。A2 落地时把同一个语料原样跑在真扫描器上,这个函数就可以删掉。
|
|
264
|
+
//
|
|
265
|
+
// 写法上刻意体现一件事:**先看形状,再看关键词**。纯关键词匹配在负样本组里会
|
|
266
|
+
// 全军覆没(见 NEGATIVE_SAMPLES 的 why),所以这里每条模式都带长度或字符集约束,
|
|
267
|
+
// 赋值型那条还要多一道占位符守卫。
|
|
268
|
+
const SECRET_PATTERNS = [
|
|
269
|
+
{ kind: "aws_access_key", label: "AWS access key id", re: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
|
270
|
+
{ kind: "github_token", label: "GitHub token", re: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36}\b/ },
|
|
271
|
+
{ kind: "slack_token", label: "Slack token", re: /\bxox[abprs]-[A-Za-z0-9-]{10,}\b/ },
|
|
272
|
+
{ kind: "private_key", label: "PEM private key", re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
|
|
273
|
+
{ kind: "jwt", label: "JSON Web Token", re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/ },
|
|
274
|
+
{ kind: "connection_string", label: "credentials in URL", re: /\b[a-z][a-z0-9+.-]*:\/\/[^\s/:@]+:[^\s/:@]{6,}@/ },
|
|
275
|
+
{ kind: "stripe_key", label: "Stripe secret key", re: /\bsk_live_[A-Za-z0-9]{16,}\b/ },
|
|
276
|
+
{ kind: "npm_token", label: "npm auth token", re: /_authToken\s*=\s*[A-Za-z0-9_-]{20,}/ },
|
|
277
|
+
{ kind: "openai_key", label: "OpenAI API key", re: /\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/ },
|
|
278
|
+
{
|
|
279
|
+
kind: "assigned_secret",
|
|
280
|
+
label: "assigned credential literal",
|
|
281
|
+
re: /\b(?:password|passwd|pwd|secret|api[_-]?key|token)\b\s*[:=]\s*["']?([^\s"']{8,})/i,
|
|
282
|
+
// 占位符 / 间接引用守卫:命中关键词但没有字面量值时不算命中。
|
|
283
|
+
guard: (value) => !/^(?:<[^>]*>|\$\{|\$[A-Z_]+$|process\.env|redacted|xx+|\*+|\u2026)/i.test(value)
|
|
284
|
+
}
|
|
285
|
+
];
|
|
286
|
+
|
|
287
|
+
const PII_PATTERNS = [
|
|
288
|
+
{ kind: "email", label: "email address", re: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/ },
|
|
289
|
+
{ kind: "cn_mobile", label: "mainland mobile number", re: /\b1[3-9]\d{9}\b/ },
|
|
290
|
+
{ kind: "cn_id_card", label: "mainland ID number", re: /\b\d{17}[\dXx]\b/ },
|
|
291
|
+
{ kind: "bank_card", label: "payment card number", re: /\b(?:4\d{15}|5[1-5]\d{14}|62\d{14,17})\b/ }
|
|
292
|
+
];
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* 参考扫描器:命中返回 `{kind, label}`,未命中返回 null——与
|
|
296
|
+
* `createWriteAdmission` 的 `sensitiveScan` 契约一致(不抛)。
|
|
297
|
+
* 先密钥后 PII:一条文本同时像两类时(如带凭据的 URL 也像邮箱),报更严重的那类。
|
|
298
|
+
*/
|
|
299
|
+
export function referenceScan(memory) {
|
|
300
|
+
const text = [memory?.title, memory?.content].filter((s) => typeof s === "string").join("\n");
|
|
301
|
+
for (const p of [...SECRET_PATTERNS, ...PII_PATTERNS]) {
|
|
302
|
+
const m = text.match(p.re);
|
|
303
|
+
if (!m) continue;
|
|
304
|
+
if (p.guard && !p.guard(m[1] ?? "")) continue;
|
|
305
|
+
return { kind: p.kind, label: p.label };
|
|
306
|
+
}
|
|
307
|
+
return null;
|
|
308
|
+
}
|
package/test/inject.test.js
CHANGED
|
@@ -281,3 +281,140 @@ test("injection snapshot: records adaptive budget and rotation state (issue #179
|
|
|
281
281
|
assert.equal(snap.rotated, 0, "no suppression on first render of a fresh session");
|
|
282
282
|
assert.equal(snap.scoped, null, "no scope when strict scope is off");
|
|
283
283
|
});
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
// ---- issue #334: emoji 代理对截断——注入路径的 4 处截断点绝不留孤立代理项 ----
|
|
287
|
+
//
|
|
288
|
+
// 回归类(报告 #334):UTF-16 slice 切在代理对中间留下孤立低位代理项,序列化成
|
|
289
|
+
// 非法 UTF-8,DeepSeek API 对每个请求回 400;畸形文本随注入消息被永久写进会话
|
|
290
|
+
// 历史,之后每一轮都带着它——会话级不可逆损坏。锁的是「少一个字符都行,绝不留
|
|
291
|
+
// 半个」这个语义本身,不是某个具体上限值。
|
|
292
|
+
|
|
293
|
+
// 孤立代理项检测:合法文本不含未配对的 UTF-16 代理项。
|
|
294
|
+
function hasLoneSurrogate(text) {
|
|
295
|
+
for (let i = 0; i < text.length; i++) {
|
|
296
|
+
const c = text.charCodeAt(i);
|
|
297
|
+
if (c >= 0xd800 && c <= 0xdbff) {
|
|
298
|
+
const next = text.charCodeAt(i + 1);
|
|
299
|
+
if (!(next >= 0xdc00 && next <= 0xdfff)) return true;
|
|
300
|
+
i++;
|
|
301
|
+
} else if (c >= 0xdc00 && c <= 0xdfff) return true;
|
|
302
|
+
}
|
|
303
|
+
return false;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// 复现 #334 的现场形状:`## Review 18/40 — 🔴` 恰好在 800 字符上限处切到 🔴。
|
|
307
|
+
const EMOJI_PAD = "# Review 18/40 — ".padEnd(799, "x");
|
|
308
|
+
test("issue #334: hot round response truncated mid-emoji leaves no lone surrogate", () => {
|
|
309
|
+
const { contexts } = setup();
|
|
310
|
+
// 799 个 ASCII + 🔴(2 个 UTF-16 码元)= 801:slice(0, 800) 恰好切在代理对中间。
|
|
311
|
+
const body = EMOJI_PAD + "🔴";
|
|
312
|
+
assert.equal(body.length, 801, "fixture: cut point lands inside the surrogate pair");
|
|
313
|
+
const text = contexts[0].text({
|
|
314
|
+
agent: {
|
|
315
|
+
session: {
|
|
316
|
+
id: "s-emoji",
|
|
317
|
+
events: [
|
|
318
|
+
{ type: "user/message", data: { source: { kind: "user" }, content: ["看下评审进度"] } },
|
|
319
|
+
{ type: "assistant/message", data: { message: { content: [{ type: "text", text: body }] } } }
|
|
320
|
+
]
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
assert.ok(text.includes("[短期上下文]"), "hot block rendered");
|
|
325
|
+
assert.ok(!hasLoneSurrogate(text), "no lone surrogate survives the hot-round truncation");
|
|
326
|
+
assert.ok(text.includes("Review 18/40"), "truncated body still present up to the cut");
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
test("issue #334: memory content truncated mid-emoji carries no lone surrogate", () => {
|
|
330
|
+
const { contexts, service } = setup();
|
|
331
|
+
// injectContentMaxChars 最小 60:凑一条「恰好把 🔴 切成两半」的记忆。
|
|
332
|
+
const content = "r".repeat(59) + "🔴";
|
|
333
|
+
service.saveWithDedupe({ type: "preference", title: "评审", content, importance: 5 });
|
|
334
|
+
const text = contexts[0].text({});
|
|
335
|
+
assert.ok(!hasLoneSurrogate(text), "no lone surrogate in the truncated memory entry");
|
|
336
|
+
assert.ok(text.includes("评审"), "entry still rendered by title");
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
test("issue #334: lastUserQuery truncation never produces a lone surrogate", () => {
|
|
340
|
+
const { contexts } = setup();
|
|
341
|
+
const text = contexts[0].text({
|
|
342
|
+
agent: {
|
|
343
|
+
session: {
|
|
344
|
+
id: "s-q",
|
|
345
|
+
snapshotEvents: () => [
|
|
346
|
+
// 499 个 ASCII + 🔴:lastUserQuery 的 slice(0, 500) 切在代理对中间。
|
|
347
|
+
{ type: "user/message", data: { source: { kind: "user" }, content: [{ type: "text", text: "q".repeat(499) + "🔴" }] } },
|
|
348
|
+
{ type: "assistant/message", data: { message: { content: [{ type: "text", text: "答" }] } } }
|
|
349
|
+
]
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
});
|
|
353
|
+
assert.ok(text.includes("[短期上下文]"), "round rendered from snapshot events");
|
|
354
|
+
assert.ok(!hasLoneSurrogate(text), "no lone surrogate from the query truncation path");
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
// ---- issue #34 恢复(#333):对话开始注入当前时间,per-session 闩锁 ----
|
|
358
|
+
|
|
359
|
+
test("issue #34/#333: injectTimePrefix off by default (byte-identical behavior)", () => {
|
|
360
|
+
const { contexts, service } = setup();
|
|
361
|
+
service.saveWithDedupe({ type: "preference", title: "语言", content: "用户用中文交流", importance: 5 });
|
|
362
|
+
const text = contexts[0].text({ agent: { session: { id: "s-t0" } } });
|
|
363
|
+
assert.ok(!text.includes("当前时间"), "no time prefix when the flag is off (default)");
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
test("issue #34/#333: enabled prefix appears once per session and carries the date format", () => {
|
|
367
|
+
const { contexts, service } = setup({ injectTimePrefix: true });
|
|
368
|
+
service.saveWithDedupe({ type: "preference", title: "语言", content: "用户用中文交流", importance: 5 });
|
|
369
|
+
const first = contexts[0].text({ agent: { session: { id: "s-t1" } } });
|
|
370
|
+
assert.match(first, /\[当前时间: \d{4}-\d{2}-\d{2} 周. \d{2}:\d{2}\]/, "prefix format matches v0.7.2 shape");
|
|
371
|
+
assert.ok(first.startsWith("[当前时间: "), "prefix leads the block");
|
|
372
|
+
// 同一会话第二次渲染:闩锁生效,不再出现。
|
|
373
|
+
const second = contexts[0].text({ agent: { session: { id: "s-t1" } } });
|
|
374
|
+
assert.ok(!second.includes("当前时间"), "latched: same session never re-injects");
|
|
375
|
+
// 新会话:重新注入。
|
|
376
|
+
const third = contexts[0].text({ agent: { session: { id: "s-t2" } } });
|
|
377
|
+
assert.match(third, /\[当前时间: /, "new session re-injects");
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
test("issue #34/#333: time prefix coexists with hot context (time first, then hot, then memory)", () => {
|
|
381
|
+
const { contexts } = setup({ injectTimePrefix: true, hotMemoryEnabled: true });
|
|
382
|
+
const text = contexts[0].text({
|
|
383
|
+
agent: {
|
|
384
|
+
session: {
|
|
385
|
+
id: "s-t3",
|
|
386
|
+
snapshotEvents: () => [
|
|
387
|
+
{ type: "user/message", data: { source: { kind: "user" }, content: ["问题"] } },
|
|
388
|
+
{ type: "assistant/message", data: { message: { content: [{ type: "text", text: "回答" }] } } }
|
|
389
|
+
]
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
const ti = text.indexOf("当前时间");
|
|
394
|
+
const hi = text.indexOf("[短期上下文]");
|
|
395
|
+
assert.ok(ti >= 0 && hi > ti, "time prefix precedes the hot block");
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
test("issue #34/#333: latch survives interleaved sessions (A-B-A never re-injects A)", () => {
|
|
399
|
+
const { contexts, service } = setup({ injectTimePrefix: true });
|
|
400
|
+
service.saveWithDedupe({ type: "preference", title: "语言", content: "用户用中文交流", importance: 5 });
|
|
401
|
+
contexts[0].text({ agent: { session: { id: "sA" } } });
|
|
402
|
+
contexts[0].text({ agent: { session: { id: "sB" } } });
|
|
403
|
+
// 回到 sA:单值闩锁会在这里重复注入(#335 review);集合闩锁不会。
|
|
404
|
+
const back = contexts[0].text({ agent: { session: { id: "sA" } } });
|
|
405
|
+
assert.ok(!back.includes("当前时间"), "interleaved return to session A must not re-inject");
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
test("issue #334: snapshot query carries no lone surrogate (query path cut at exact limit)", () => {
|
|
409
|
+
// #335 review:先 slice 再 safeSlice 时,恰为上限的串会绕过检查。现在
|
|
410
|
+
// lastUserQuery 直接 safeSlice 全文,快照里的 query 必须干净。
|
|
411
|
+
const { contexts } = setup();
|
|
412
|
+
const events = [
|
|
413
|
+
{ type: "user/message", data: { source: { kind: "user" }, content: [{ type: "text", text: "q".repeat(499) + "\u{1F534}" }] } },
|
|
414
|
+
{ type: "assistant/message", data: { message: { content: [{ type: "text", text: "ans" }] } } }
|
|
415
|
+
];
|
|
416
|
+
contexts[0].text({ agent: { session: { id: "s-q2", snapshotEvents: () => events } } });
|
|
417
|
+
const snap = getInjectionSnapshot();
|
|
418
|
+
assert.ok(snap.query.length > 0, "query recorded");
|
|
419
|
+
assert.ok(!hasLoneSurrogate(snap.query), "snapshot query is surrogate-clean");
|
|
420
|
+
});
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Anti-update 回归集(E6,#280 链;StatemenBench anti-update probes)。
|
|
2
|
+
//
|
|
3
|
+
// 锁的回归类:**supersede 后旧值复现**。取代关系没有专属列——落库状态完全
|
|
4
|
+
// 由 applySupersede(dream/decisions.js)的产物表达:loser 的 archived=1 +
|
|
5
|
+
// 正文 (superseded by: …) 注记。四条默认检索路径各自过滤 archived(keyword
|
|
6
|
+
// store.js:1528、vector store.js:1564、BM25 池 service.js bm25Recall、entity
|
|
7
|
+
// service.js:470),本文件把它们钉成「排除,而非降权」:
|
|
8
|
+
// - 任何一路让归档旧条回到候选集 → 红;
|
|
9
|
+
// - 归档行被误删/误遗忘(取代是归档不是删除)→ 红;
|
|
10
|
+
// - includeArchived 显式口子仍在(排除发生在检索层,不是行消失)→ 红。
|
|
11
|
+
// seedService 已种好取代对(winner = mem_rust_switch,loser = mem_rust_switch_old)。
|
|
12
|
+
import test from "node:test";
|
|
13
|
+
import assert from "node:assert/strict";
|
|
14
|
+
import { runBenchmark, runFusionBenchmark, seedService } from "../scripts/benchmark-recall.js";
|
|
15
|
+
|
|
16
|
+
const WINNER = "mem_rust_switch";
|
|
17
|
+
const LOSER = "mem_rust_switch_old";
|
|
18
|
+
|
|
19
|
+
test("benchmark 里带 forbidden 的 anti-update 用例在 legacy/fused 两配置下都零泄露", async () => {
|
|
20
|
+
const report = await runBenchmark({ topK: 5 });
|
|
21
|
+
for (const run of report.runs) {
|
|
22
|
+
const probes = run.rows.filter((r) => r.forbidden?.length);
|
|
23
|
+
assert.ok(probes.length >= 2, `${run.config}: anti-update probes present`);
|
|
24
|
+
for (const r of probes) {
|
|
25
|
+
assert.equal(r.forbiddenHit, 0, `${run.config} "${r.query}": superseded old row leaked`);
|
|
26
|
+
assert.equal(r.recall, 1, `${run.config} "${r.query}": winner must still answer the topic`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("fusion 三个配方同样零泄露(排除发生在融合上游,与配方无关)", async () => {
|
|
32
|
+
const report = await runFusionBenchmark({ topK: 5 });
|
|
33
|
+
for (const run of report.runs) {
|
|
34
|
+
for (const r of run.rows) {
|
|
35
|
+
if (!r.forbidden?.length) continue;
|
|
36
|
+
assert.equal(r.forbiddenHit, 0, `${run.config} "${r.query}": leaked via fusion recipe`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("mode=keyword / vector / auto 三条路径都不再返回被取代旧值", async () => {
|
|
42
|
+
const svc = seedService();
|
|
43
|
+
// 查询用 winner 标题「语言迁移决策」:toy 哈希嵌入按连续 CJK 段整段分词,
|
|
44
|
+
// 只有这个串在 keyword(LIKE 子串)与 vector(token 精确重叠)两条路上都
|
|
45
|
+
// 能命中 winner。注意 loser 的 superseded-by 注记里恰好也含这个串——
|
|
46
|
+
// 归档排除一旦失效,注记自己就会把旧值钓回来,本用例立即红。
|
|
47
|
+
for (const mode of ["keyword", "vector", "auto"]) {
|
|
48
|
+
const rows = await svc.searchMemories("语言迁移决策", { mode, topK: 10, useRerank: false });
|
|
49
|
+
const ids = rows.map((r) => r.id);
|
|
50
|
+
assert.ok(!ids.includes(LOSER), `mode=${mode}: archived loser resurfaced`);
|
|
51
|
+
assert.ok(ids.includes(WINNER), `mode=${mode}: winner missing from default recall`);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("BM25 的候选池(store.list)不含归档行——降权化改动在这里最先红", async () => {
|
|
56
|
+
const svc = seedService();
|
|
57
|
+
// bm25Recall 的取数源就是 store.list({limit, includeForgotten:false}) 再
|
|
58
|
+
// filter !m.archived;list 层面归档行本就不回来,这里锁「池子是干净的」。
|
|
59
|
+
const pool = svc.list({ limit: 500, includeForgotten: false });
|
|
60
|
+
assert.ok(pool.length > 0, "pool non-empty");
|
|
61
|
+
assert.ok(!pool.some((m) => m.id === LOSER), "archived loser must not enter the bm25 pool");
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("取代是归档不是删除:旧行还在、未遗忘、带 superseded-by 注记", async () => {
|
|
65
|
+
const svc = seedService();
|
|
66
|
+
const loser = svc.getById(LOSER);
|
|
67
|
+
assert.ok(loser, "loser row must still exist");
|
|
68
|
+
assert.equal(loser.archived, true, "loser must be archived (supersede semantics)");
|
|
69
|
+
assert.equal(loser.forgotten, false, "supersede must not forget the row (heat/forget 是另一条线)");
|
|
70
|
+
assert.ok(loser.content.includes("(superseded by: 语言迁移决策)"), "content carries the superseded-by note");
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("includeArchived 显式口子仍在:排除发生在检索层,不是行消失", async () => {
|
|
74
|
+
const svc = seedService();
|
|
75
|
+
// store.search 是整串 LIKE 子串扫描(不切词),所以这里用 loser 正文的
|
|
76
|
+
// 连续子串作探针;被取代旧条不在默认结果里、但在 includeArchived 下回来。
|
|
77
|
+
const probe = "选型理由是部署简单";
|
|
78
|
+
const without = svc.search(probe, { limit: 50 });
|
|
79
|
+
const withArchived = svc.search(probe, { limit: 50, includeArchived: true });
|
|
80
|
+
assert.ok(!without.some((m) => m.id === LOSER), "default search excludes the archived loser");
|
|
81
|
+
assert.ok(withArchived.some((m) => m.id === LOSER), "includeArchived must still surface it (row exists)");
|
|
82
|
+
});
|