@nebulaaa/auditor 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auditor.cjs +430 -0
- package/dist/auditor.d.ts +160 -0
- package/dist/auditor.min.cjs +1 -0
- package/dist/auditor.min.mjs +1 -0
- package/dist/auditor.mjs +428 -0
- package/dist/package.json +34 -0
- package/package.json +70 -6
- package/README.md +0 -3
package/dist/auditor.cjs
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var dict = require('@nebulaaa/dict');
|
|
4
|
+
var lru = require('@nebulaaa/lru');
|
|
5
|
+
var plugin = require('@nebulaaa/plugin');
|
|
6
|
+
|
|
7
|
+
/** AC 自动机 */
|
|
8
|
+
class ACAutomaton {
|
|
9
|
+
constructor(ignoreCase = false) {
|
|
10
|
+
this._root = {
|
|
11
|
+
Children: new Map(),
|
|
12
|
+
Fail: null,
|
|
13
|
+
Words: [],
|
|
14
|
+
Output: [],
|
|
15
|
+
};
|
|
16
|
+
this._built = false;
|
|
17
|
+
this._interferenceWords = new Set();
|
|
18
|
+
this._ignoreCase = ignoreCase;
|
|
19
|
+
this._wordChars = new Map();
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* 大小写归一(仅 ASCII 字母,逐码点等长替换,保证匹配位置与原始文本索引一致)
|
|
23
|
+
*/
|
|
24
|
+
normalize(text) {
|
|
25
|
+
if (!this._ignoreCase) {
|
|
26
|
+
return text;
|
|
27
|
+
}
|
|
28
|
+
let out = '';
|
|
29
|
+
for (const ch of text) {
|
|
30
|
+
const code = ch.codePointAt(0);
|
|
31
|
+
out += code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : ch;
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* 添加敏感词
|
|
37
|
+
* @param word 敏感词
|
|
38
|
+
*/
|
|
39
|
+
AddWord(word) {
|
|
40
|
+
if (!word || word.length === 0) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
let current = this._root;
|
|
44
|
+
const chars = Array.from(this.normalize(word)); // 归一后分词(IgnoreCase 关闭时原样返回)
|
|
45
|
+
for (const char of chars) {
|
|
46
|
+
let child = current.Children.get(char);
|
|
47
|
+
if (!child) {
|
|
48
|
+
child = {
|
|
49
|
+
Children: new Map(),
|
|
50
|
+
Fail: null,
|
|
51
|
+
Words: [],
|
|
52
|
+
Output: [],
|
|
53
|
+
};
|
|
54
|
+
current.Children.set(char, child);
|
|
55
|
+
}
|
|
56
|
+
current = child;
|
|
57
|
+
}
|
|
58
|
+
// 归一后同一路径只保留首词:IgnoreCase 下 "abc"/"ABC" 共享路径,
|
|
59
|
+
// 变体重复入表会让 Match 对同一区间输出多条重复结果(下游命中计数虚高)
|
|
60
|
+
if (current.Words.length === 0) {
|
|
61
|
+
current.Words.push(word);
|
|
62
|
+
this._wordChars.set(word, chars);
|
|
63
|
+
this._built = false; // 标记需要重新构建 fail 指针
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* 构建 fail 指针(使用 BFS)
|
|
68
|
+
*/
|
|
69
|
+
Build() {
|
|
70
|
+
if (this._built) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const queue = [];
|
|
74
|
+
// 第一层节点的 fail 指针指向 root
|
|
75
|
+
for (const child of this._root.Children.values()) {
|
|
76
|
+
child.Fail = this._root;
|
|
77
|
+
child.Output = [...child.Words]; // 由自身词重新生成(fail 为 root,无合并项)
|
|
78
|
+
queue.push(child);
|
|
79
|
+
}
|
|
80
|
+
// BFS 构建 fail 指针(头指针索引出队:array.shift 为 O(n),万级节点的大词库构建退化为 O(n²))
|
|
81
|
+
let head = 0;
|
|
82
|
+
while (head < queue.length) {
|
|
83
|
+
const current = queue[head++];
|
|
84
|
+
for (const [char, child] of current.Children) {
|
|
85
|
+
queue.push(child);
|
|
86
|
+
// 查找 fail 指针
|
|
87
|
+
let fail = current.Fail;
|
|
88
|
+
while (fail && !fail.Children.has(char)) {
|
|
89
|
+
fail = fail.Fail;
|
|
90
|
+
}
|
|
91
|
+
child.Fail = fail ? fail.Children.get(char) || this._root : this._root;
|
|
92
|
+
// 合并 output:由自身词重新生成(幂等,重复构建不再累积;fail 节点层级更浅、已先构建完成)
|
|
93
|
+
child.Output = [...child.Words, ...child.Fail.Output];
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
this._built = true;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* 匹配文本,返回所有匹配结果
|
|
100
|
+
* @param text 待匹配的文本
|
|
101
|
+
* @returns 匹配结果列表
|
|
102
|
+
*/
|
|
103
|
+
Match(text) {
|
|
104
|
+
if (!this._built) {
|
|
105
|
+
this.Build();
|
|
106
|
+
}
|
|
107
|
+
const results = [];
|
|
108
|
+
let current = this._root;
|
|
109
|
+
const chars = Array.from(this.normalize(text)); // 归一后分词(逐码点等长,位置索引不变)
|
|
110
|
+
// 记录匹配时的原始位置
|
|
111
|
+
for (let i = 0; i < chars.length; i++) {
|
|
112
|
+
const char = chars[i];
|
|
113
|
+
// 如果是干扰字符,跳过
|
|
114
|
+
if (this._interferenceWords.has(char)) {
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
// 沿着 fail 指针查找匹配的子节点
|
|
118
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
119
|
+
current = current.Fail;
|
|
120
|
+
}
|
|
121
|
+
if (current.Children.has(char)) {
|
|
122
|
+
current = current.Children.get(char);
|
|
123
|
+
// 输出所有匹配的敏感词
|
|
124
|
+
for (const word of current.Output) {
|
|
125
|
+
// 计算在原始文本中的起始位置(跳过干扰字符)
|
|
126
|
+
let j = i;
|
|
127
|
+
// 从当前位置向前查找匹配的起始位置(AddWord 已缓存归一后的码点数组)
|
|
128
|
+
const wordChars = this._wordChars.get(word);
|
|
129
|
+
let wordIndex = wordChars.length - 1;
|
|
130
|
+
// 从后向前匹配,找到原始文本中的位置
|
|
131
|
+
while (j >= 0 && wordIndex >= 0) {
|
|
132
|
+
if (this._interferenceWords.has(chars[j])) {
|
|
133
|
+
j--;
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
if (chars[j] === wordChars[wordIndex]) {
|
|
137
|
+
wordIndex--;
|
|
138
|
+
}
|
|
139
|
+
j--;
|
|
140
|
+
}
|
|
141
|
+
// 回扫必须完整命中:前向 AC 已保证该词是当前位置的后缀(两侧对称跳过干扰字符),
|
|
142
|
+
// wordIndex >= 0 说明不变量被破坏(理论不可达),防御性跳过而非产出错误区间
|
|
143
|
+
if (wordIndex >= 0) {
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
results.push({
|
|
147
|
+
Word: word,
|
|
148
|
+
Start: j + 1,
|
|
149
|
+
End: i + 1,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return results;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* 检测文本是否包含敏感词
|
|
158
|
+
* @param text 待检测的文本
|
|
159
|
+
* @returns 是否包含敏感词
|
|
160
|
+
*/
|
|
161
|
+
Test(text) {
|
|
162
|
+
if (!this._built) {
|
|
163
|
+
this.Build();
|
|
164
|
+
}
|
|
165
|
+
let current = this._root;
|
|
166
|
+
const chars = Array.from(this.normalize(text));
|
|
167
|
+
for (let i = 0; i < chars.length; i++) {
|
|
168
|
+
const char = chars[i];
|
|
169
|
+
// 如果是干扰字符,跳过
|
|
170
|
+
if (this._interferenceWords.has(char)) {
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
174
|
+
current = current.Fail;
|
|
175
|
+
}
|
|
176
|
+
if (current.Children.has(char)) {
|
|
177
|
+
current = current.Children.get(char);
|
|
178
|
+
if (current.Output.length > 0) {
|
|
179
|
+
return true;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* 清空所有节点
|
|
187
|
+
*/
|
|
188
|
+
Clear() {
|
|
189
|
+
this._root = {
|
|
190
|
+
Children: new Map(),
|
|
191
|
+
Fail: null,
|
|
192
|
+
Words: [],
|
|
193
|
+
Output: [],
|
|
194
|
+
};
|
|
195
|
+
this._wordChars.clear();
|
|
196
|
+
this._built = false;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* 注入干扰词(仅单字符生效:匹配按单字符查表)
|
|
200
|
+
* @param words 干扰词列表
|
|
201
|
+
* @returns 被忽略的多字符词(由调用方决定告警方式,保持算法类无日志副作用)
|
|
202
|
+
*/
|
|
203
|
+
AddInterferenceWords(words) {
|
|
204
|
+
const ignored = [];
|
|
205
|
+
for (const word of words) {
|
|
206
|
+
if (!word || word.length === 0) {
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
const chars = Array.from(this.normalize(word));
|
|
210
|
+
if (chars.length !== 1) {
|
|
211
|
+
// 多字符干扰词在按字符匹配的查表中永不命中——返回给调用方显式告警而非静默失效
|
|
212
|
+
ignored.push(word);
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
this._interferenceWords.add(chars[0]);
|
|
216
|
+
}
|
|
217
|
+
return ignored;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* 清空干扰词
|
|
221
|
+
*/
|
|
222
|
+
ClearInterferenceWords() {
|
|
223
|
+
this._interferenceWords.clear();
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* 获取干扰词数量
|
|
227
|
+
*/
|
|
228
|
+
GetInterferenceWordCount() {
|
|
229
|
+
return this._interferenceWords.size;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* 敏感词审计员插件
|
|
234
|
+
*/
|
|
235
|
+
class AuditorPlugin extends plugin.Plugin {
|
|
236
|
+
constructor(deps, cfg) {
|
|
237
|
+
// 默认值填充必须在 super 之前:基类构造会冻结 cfg,冻结后 defineProperty 会抛错
|
|
238
|
+
dict.Override(cfg, {
|
|
239
|
+
Replacement: '█',
|
|
240
|
+
IgnoreCase: false,
|
|
241
|
+
CacheSize: 1000,
|
|
242
|
+
});
|
|
243
|
+
super(deps, cfg);
|
|
244
|
+
// 缓存容量须为正整数:无论是否启用缓存都在构造期校验,配置错误不留到运行期
|
|
245
|
+
if (!Number.isInteger(cfg.CacheSize) || cfg.CacheSize <= 0) {
|
|
246
|
+
throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${cfg.CacheSize}`);
|
|
247
|
+
}
|
|
248
|
+
this._acAutomaton = new ACAutomaton(cfg.IgnoreCase);
|
|
249
|
+
this._sensitiveWords = new Set();
|
|
250
|
+
this._acAutomaton.AddInterferenceWords(' \t\r\n'.split(''));
|
|
251
|
+
if (cfg.EnableCache) {
|
|
252
|
+
this._testCache = new lru.LRU(cfg.CacheSize);
|
|
253
|
+
this._findCache = new lru.LRU(cfg.CacheSize);
|
|
254
|
+
this._replaceCache = new lru.LRU(cfg.CacheSize);
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
this._testCache = null;
|
|
258
|
+
this._findCache = null;
|
|
259
|
+
this._replaceCache = null;
|
|
260
|
+
}
|
|
261
|
+
// 构建敏感词自动机
|
|
262
|
+
this.buildAutomaton();
|
|
263
|
+
}
|
|
264
|
+
Test(text) {
|
|
265
|
+
if (!text || text.length === 0) {
|
|
266
|
+
return false;
|
|
267
|
+
}
|
|
268
|
+
// 检查缓存
|
|
269
|
+
if (this._testCache) {
|
|
270
|
+
const cached = this._testCache.Get(text);
|
|
271
|
+
if (cached !== null) {
|
|
272
|
+
return cached;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
const result = this._acAutomaton.Test(text);
|
|
276
|
+
// 更新缓存
|
|
277
|
+
if (this._testCache) {
|
|
278
|
+
this._testCache.Set(text, result);
|
|
279
|
+
}
|
|
280
|
+
return result;
|
|
281
|
+
}
|
|
282
|
+
Find(text) {
|
|
283
|
+
if (!text || text.length === 0) {
|
|
284
|
+
return { Matched: false, Words: [], Matches: [] };
|
|
285
|
+
}
|
|
286
|
+
// 检查缓存
|
|
287
|
+
if (this._findCache) {
|
|
288
|
+
const cached = this._findCache.Get(text);
|
|
289
|
+
if (cached !== null) {
|
|
290
|
+
// 返回浅拷贝,防止外部修改影响缓存
|
|
291
|
+
return {
|
|
292
|
+
Matched: cached.Matched,
|
|
293
|
+
Words: [...cached.Words],
|
|
294
|
+
Matches: cached.Matches.map((m) => (Object.assign({}, m))),
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
// 匹配文本,自动跳过干扰词
|
|
299
|
+
const matches = this._acAutomaton.Match(text);
|
|
300
|
+
// 去重敏感词
|
|
301
|
+
const wordSet = new Set();
|
|
302
|
+
for (const match of matches) {
|
|
303
|
+
wordSet.add(match.Word);
|
|
304
|
+
}
|
|
305
|
+
const result = {
|
|
306
|
+
Matched: matches.length > 0,
|
|
307
|
+
Words: Array.from(wordSet),
|
|
308
|
+
Matches: matches,
|
|
309
|
+
};
|
|
310
|
+
// 更新缓存(存储副本,防止后续修改影响缓存)
|
|
311
|
+
if (this._findCache) {
|
|
312
|
+
this._findCache.Set(text, {
|
|
313
|
+
Matched: result.Matched,
|
|
314
|
+
Words: [...result.Words],
|
|
315
|
+
Matches: result.Matches.map((m) => (Object.assign({}, m))),
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
return result;
|
|
319
|
+
}
|
|
320
|
+
Replace(text, replacement) {
|
|
321
|
+
var _a;
|
|
322
|
+
if (!text || text.length === 0) {
|
|
323
|
+
return text;
|
|
324
|
+
}
|
|
325
|
+
// 检查缓存(JSON 编码组合键:字符串拼接的 "_" 分隔符可出现在 text/replacement 中造成碰撞)
|
|
326
|
+
const cacheKey = JSON.stringify([text, replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement]);
|
|
327
|
+
if (this._replaceCache) {
|
|
328
|
+
const cached = this._replaceCache.Get(cacheKey);
|
|
329
|
+
if (cached !== null) {
|
|
330
|
+
return cached;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const result = this.Find(text);
|
|
334
|
+
if (!result.Matched) {
|
|
335
|
+
// 干净文本是最高频流量,同样入缓存(组合键已防碰撞),避免反复全量匹配
|
|
336
|
+
(_a = this._replaceCache) === null || _a === void 0 ? void 0 : _a.Set(cacheKey, text);
|
|
337
|
+
return text;
|
|
338
|
+
}
|
|
339
|
+
// 在码点数组上操作:matches 的位置基于 Array.from 分词(码点索引),
|
|
340
|
+
// 直接对原字符串 slice 会因代理对(emoji 等)长度不一致而切错位置
|
|
341
|
+
const chars = Array.from(text);
|
|
342
|
+
// 合并重叠区间:重叠匹配若逐个替换,从后向前切片会互相错位(吞字或残留敏感片段)
|
|
343
|
+
const sorted = [...result.Matches].sort((a, b) => a.Start - b.Start);
|
|
344
|
+
const ranges = [];
|
|
345
|
+
for (const match of sorted) {
|
|
346
|
+
const last = ranges[ranges.length - 1];
|
|
347
|
+
if (last && match.Start < last.end) {
|
|
348
|
+
last.end = Math.max(last.end, match.End); // 重叠:并入上一区间
|
|
349
|
+
}
|
|
350
|
+
else {
|
|
351
|
+
ranges.push({ start: match.Start, end: match.End });
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
// 每个合并区间整体替换为一个 replacement 串(非逐字符屏蔽);空串为合法替换值(删除敏感词)
|
|
355
|
+
const replaceChar = replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement;
|
|
356
|
+
for (let i = ranges.length - 1; i >= 0; i--) {
|
|
357
|
+
const { start, end } = ranges[i];
|
|
358
|
+
chars.splice(start, end - start, replaceChar); // 从后向前,前面的索引不受影响
|
|
359
|
+
}
|
|
360
|
+
const replacedText = chars.join('');
|
|
361
|
+
// 更新缓存
|
|
362
|
+
if (this._replaceCache) {
|
|
363
|
+
this._replaceCache.Set(cacheKey, replacedText);
|
|
364
|
+
}
|
|
365
|
+
return replacedText;
|
|
366
|
+
}
|
|
367
|
+
InjectSensitiveWords(words) {
|
|
368
|
+
let added = false;
|
|
369
|
+
for (const word of words) {
|
|
370
|
+
if (word && word.length > 0 && !this._sensitiveWords.has(word)) {
|
|
371
|
+
this._sensitiveWords.add(word);
|
|
372
|
+
added = true;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
// 无新增词时词库未变:跳过自动机重建与缓存清理,重复注入不打断既有缓存
|
|
376
|
+
if (!added) {
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
this.buildAutomaton();
|
|
380
|
+
this.clearCache();
|
|
381
|
+
}
|
|
382
|
+
InjectInterferenceWords(words) {
|
|
383
|
+
const ignored = this._acAutomaton.AddInterferenceWords(words);
|
|
384
|
+
// 多字符干扰词永不命中:经插件日志显式告警(词库内容不落 console)
|
|
385
|
+
if (ignored.length > 0) {
|
|
386
|
+
const logger = this.Logger;
|
|
387
|
+
for (const word of ignored) {
|
|
388
|
+
logger.Warn(`干扰词须为单个字符,已忽略 >>> ${word}`);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
this.clearCache();
|
|
392
|
+
}
|
|
393
|
+
GetSensitiveWordCount() {
|
|
394
|
+
return this._sensitiveWords.size;
|
|
395
|
+
}
|
|
396
|
+
GetInterferenceWordCount() {
|
|
397
|
+
return this._acAutomaton.GetInterferenceWordCount();
|
|
398
|
+
}
|
|
399
|
+
ClearSensitiveWords() {
|
|
400
|
+
this._sensitiveWords.clear();
|
|
401
|
+
this.buildAutomaton();
|
|
402
|
+
this.clearCache();
|
|
403
|
+
}
|
|
404
|
+
ClearInterferenceWords() {
|
|
405
|
+
this._acAutomaton.ClearInterferenceWords();
|
|
406
|
+
this.clearCache();
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* 构建 AC 自动机
|
|
410
|
+
*/
|
|
411
|
+
buildAutomaton() {
|
|
412
|
+
this._acAutomaton.Clear();
|
|
413
|
+
const words = this._sensitiveWords;
|
|
414
|
+
for (const word of words) {
|
|
415
|
+
this._acAutomaton.AddWord(word);
|
|
416
|
+
}
|
|
417
|
+
this._acAutomaton.Build();
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* 清空缓存
|
|
421
|
+
*/
|
|
422
|
+
clearCache() {
|
|
423
|
+
var _a, _b, _c;
|
|
424
|
+
(_a = this._testCache) === null || _a === void 0 ? void 0 : _a.Clear();
|
|
425
|
+
(_b = this._findCache) === null || _b === void 0 ? void 0 : _b.Clear();
|
|
426
|
+
(_c = this._replaceCache) === null || _c === void 0 ? void 0 : _c.Clear();
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
exports.AuditorPlugin = AuditorPlugin;
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { PluginCfg, PluginDeps, IPlugin, Plugin } from '@nebulaaa/plugin';
|
|
2
|
+
|
|
3
|
+
/** 敏感词审计员插件依赖 */
|
|
4
|
+
interface AuditorDeps extends PluginDeps {
|
|
5
|
+
}
|
|
6
|
+
/** 敏感词审计员插件配置 */
|
|
7
|
+
interface AuditorCfg extends PluginCfg {
|
|
8
|
+
/**
|
|
9
|
+
* 敏感词替换字符
|
|
10
|
+
* @default '█'
|
|
11
|
+
*/
|
|
12
|
+
readonly Replacement?: string;
|
|
13
|
+
/**
|
|
14
|
+
* 是否启用缓存
|
|
15
|
+
* @default true
|
|
16
|
+
*/
|
|
17
|
+
readonly EnableCache?: boolean;
|
|
18
|
+
/**
|
|
19
|
+
* 缓存大小
|
|
20
|
+
* @default 1000
|
|
21
|
+
* @tip 须为正整数,非法值构造期显式报错(无论是否启用缓存均校验)
|
|
22
|
+
*/
|
|
23
|
+
readonly CacheSize?: number;
|
|
24
|
+
/**
|
|
25
|
+
* 是否忽略大小写
|
|
26
|
+
* @default false
|
|
27
|
+
* @tip 仅归一 ASCII 字母(A-Z↔a-z),逐码点等长替换不影响位置索引;
|
|
28
|
+
* 全角字母/数字等全半角变体不归一
|
|
29
|
+
*/
|
|
30
|
+
readonly IgnoreCase?: boolean;
|
|
31
|
+
}
|
|
32
|
+
/** 匹配位置信息 */
|
|
33
|
+
interface IAuditorMatchPosition {
|
|
34
|
+
/** 匹配到的敏感词 */
|
|
35
|
+
Word: string;
|
|
36
|
+
/**
|
|
37
|
+
* 起始位置(**码点索引**,`Array.from` 语义)
|
|
38
|
+
* @tip 非 UTF-16 下标:文本含 emoji 等代理对时,直接 `str.slice(Start, End)` 会切错位置
|
|
39
|
+
*/
|
|
40
|
+
Start: number;
|
|
41
|
+
/**
|
|
42
|
+
* 结束位置(码点索引,不包含)
|
|
43
|
+
* @tip 同上,切分须按码点进行(如 `Array.from(str).slice(Start, End)`)
|
|
44
|
+
*/
|
|
45
|
+
End: number;
|
|
46
|
+
}
|
|
47
|
+
/** 匹配结果 */
|
|
48
|
+
interface IAuditorMatchResult {
|
|
49
|
+
/** 是否匹配到敏感词 */
|
|
50
|
+
Matched: boolean;
|
|
51
|
+
/** 匹配到的敏感词列表(去重,按首次出现顺序) */
|
|
52
|
+
Words: string[];
|
|
53
|
+
/** 所有匹配位置信息 */
|
|
54
|
+
Matches: IAuditorMatchPosition[];
|
|
55
|
+
}
|
|
56
|
+
/** AC 自动机节点接口 */
|
|
57
|
+
interface IACNode {
|
|
58
|
+
/** 子节点映射 */
|
|
59
|
+
Children: Map<string, IACNode>;
|
|
60
|
+
/** 失败指针 */
|
|
61
|
+
Fail: IACNode | null;
|
|
62
|
+
/** 该节点自身的完整敏感词(词尾节点非空) */
|
|
63
|
+
Words: string[];
|
|
64
|
+
/** 匹配输出(自身词 + fail 链词,构建时由 words 重新生成) */
|
|
65
|
+
Output: string[];
|
|
66
|
+
}
|
|
67
|
+
/** AC 自动机匹配结果(与外部匹配位置结构一致,复用同一形状避免重复定义) */
|
|
68
|
+
type IACMatchResult = IAuditorMatchPosition;
|
|
69
|
+
/** 敏感词审计员插件接口 */
|
|
70
|
+
interface IAuditorPlugin extends IPlugin<AuditorDeps, AuditorCfg> {
|
|
71
|
+
/**
|
|
72
|
+
* 检测文本是否包含敏感词
|
|
73
|
+
* @param text 待检测的文本
|
|
74
|
+
* @returns 是否包含敏感词
|
|
75
|
+
*/
|
|
76
|
+
Test(text: string): boolean;
|
|
77
|
+
/**
|
|
78
|
+
* 查找文本中的所有敏感词
|
|
79
|
+
* @param text 待查找的文本
|
|
80
|
+
* @returns 匹配结果(匹配位置为码点索引)
|
|
81
|
+
*/
|
|
82
|
+
Find(text: string): IAuditorMatchResult;
|
|
83
|
+
/**
|
|
84
|
+
* 替换文本中的敏感词
|
|
85
|
+
*
|
|
86
|
+
* - 每个匹配区间(重叠匹配先合并为不重叠区间)整体替换为一个
|
|
87
|
+
* replacement 串(非逐字符屏蔽)
|
|
88
|
+
* - 区间内的干扰字符一并被替换(变体表达整体屏蔽)
|
|
89
|
+
* @param text 待替换的文本
|
|
90
|
+
* @param replacement 替换串,缺省使用配置的替换字符;传空串为删除敏感词语义
|
|
91
|
+
* @returns 替换后的文本
|
|
92
|
+
*/
|
|
93
|
+
Replace(text: string, replacement?: string): string;
|
|
94
|
+
/**
|
|
95
|
+
* 注入敏感词
|
|
96
|
+
* @tip 重复注入已存在的词为幂等操作,不触发自动机重建与缓存清理
|
|
97
|
+
* @param words 敏感词列表
|
|
98
|
+
*/
|
|
99
|
+
InjectSensitiveWords(words: string[]): void;
|
|
100
|
+
/**
|
|
101
|
+
* 注入干扰词
|
|
102
|
+
*
|
|
103
|
+
* - 干扰词在匹配时被跳过(如 "敏 感 词" 仍命中),仅单字符
|
|
104
|
+
* 生效,多字符词将被忽略并告警
|
|
105
|
+
* - 构造期默认注入空白字符(空格/制表符/回车/换行),故全新实例的
|
|
106
|
+
* GetInterferenceWordCount 返回 4
|
|
107
|
+
* - 敏感词本身包含干扰字符时该词将无法命中,注入时须避开词库字符
|
|
108
|
+
* @param words 干扰词列表
|
|
109
|
+
*/
|
|
110
|
+
InjectInterferenceWords(words: string[]): void;
|
|
111
|
+
/**
|
|
112
|
+
* 获取当前敏感词数量
|
|
113
|
+
* @returns 敏感词数量
|
|
114
|
+
*/
|
|
115
|
+
GetSensitiveWordCount(): number;
|
|
116
|
+
/**
|
|
117
|
+
* 获取当前干扰词数量
|
|
118
|
+
* @returns 干扰词数量(含构造期默认注入的空白字符)
|
|
119
|
+
*/
|
|
120
|
+
GetInterferenceWordCount(): number;
|
|
121
|
+
/** 清空所有敏感词 */
|
|
122
|
+
ClearSensitiveWords(): void;
|
|
123
|
+
/**
|
|
124
|
+
* 清空所有干扰词(含构造期默认注入的空白字符)
|
|
125
|
+
* @tip 清空后 "敏 感 词" 这类夹空白的变体不再命中
|
|
126
|
+
*/
|
|
127
|
+
ClearInterferenceWords(): void;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* 敏感词审计员插件
|
|
132
|
+
*/
|
|
133
|
+
declare class AuditorPlugin extends Plugin<AuditorDeps, AuditorCfg> implements IAuditorPlugin {
|
|
134
|
+
private _acAutomaton;
|
|
135
|
+
private _testCache;
|
|
136
|
+
private _findCache;
|
|
137
|
+
private _replaceCache;
|
|
138
|
+
private _sensitiveWords;
|
|
139
|
+
constructor(deps: AuditorDeps, cfg: AuditorCfg);
|
|
140
|
+
Test(text: string): boolean;
|
|
141
|
+
Find(text: string): IAuditorMatchResult;
|
|
142
|
+
Replace(text: string, replacement?: string): string;
|
|
143
|
+
InjectSensitiveWords(words: string[]): void;
|
|
144
|
+
InjectInterferenceWords(words: string[]): void;
|
|
145
|
+
GetSensitiveWordCount(): number;
|
|
146
|
+
GetInterferenceWordCount(): number;
|
|
147
|
+
ClearSensitiveWords(): void;
|
|
148
|
+
ClearInterferenceWords(): void;
|
|
149
|
+
/**
|
|
150
|
+
* 构建 AC 自动机
|
|
151
|
+
*/
|
|
152
|
+
private buildAutomaton;
|
|
153
|
+
/**
|
|
154
|
+
* 清空缓存
|
|
155
|
+
*/
|
|
156
|
+
private clearCache;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export { AuditorPlugin };
|
|
160
|
+
export type { AuditorCfg, AuditorDeps, IACMatchResult, IACNode, IAuditorMatchPosition, IAuditorMatchResult, IAuditorPlugin };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var t=require("@nebulaaa/dict"),e=require("@nebulaaa/lru"),r=require("@nebulaaa/plugin");class s{constructor(t=!1){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._built=!1,this._interferenceWords=new Set,this._ignoreCase=t,this._wordChars=new Map}normalize(t){if(!this._ignoreCase)return t;let e="";for(const r of t){const t=r.codePointAt(0);e+=t>=65&&t<=90?String.fromCharCode(t+32):r}return e}AddWord(t){if(!t||0===t.length)return;let e=this._root;const r=Array.from(this.normalize(t));for(const t of r){let r=e.Children.get(t);r||(r={Children:new Map,Fail:null,Words:[],Output:[]},e.Children.set(t,r)),e=r}0===e.Words.length&&(e.Words.push(t),this._wordChars.set(t,r),this._built=!1)}Build(){if(this._built)return;const t=[];for(const e of this._root.Children.values())e.Fail=this._root,e.Output=[...e.Words],t.push(e);let e=0;for(;e<t.length;){const r=t[e++];for(const[e,s]of r.Children){t.push(s);let n=r.Fail;for(;n&&!n.Children.has(e);)n=n.Fail;s.Fail=n&&n.Children.get(e)||this._root,s.Output=[...s.Words,...s.Fail.Output]}}this._built=!0}Match(t){this._built||this.Build();const e=[];let r=this._root;const s=Array.from(this.normalize(t));for(let t=0;t<s.length;t++){const n=s[t];if(!this._interferenceWords.has(n)){for(;r!==this._root&&!r.Children.has(n);)r=r.Fail;if(r.Children.has(n)){r=r.Children.get(n);for(const n of r.Output){let r=t;const i=this._wordChars.get(n);let o=i.length-1;for(;r>=0&&o>=0;)this._interferenceWords.has(s[r])||s[r]===i[o]&&o--,r--;o>=0||e.push({Word:n,Start:r+1,End:t+1})}}}}return e}Test(t){this._built||this.Build();let e=this._root;const r=Array.from(this.normalize(t));for(let t=0;t<r.length;t++){const s=r[t];if(!this._interferenceWords.has(s)){for(;e!==this._root&&!e.Children.has(s);)e=e.Fail;if(e.Children.has(s)&&(e=e.Children.get(s),e.Output.length>0))return!0}}return!1}Clear(){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._wordChars.clear(),this._built=!1}AddInterferenceWords(t){const e=[];for(const r of t){if(!r||0===r.length)continue;const t=Array.from(this.normalize(r));1===t.length?this._interferenceWords.add(t[0]):e.push(r)}return e}ClearInterferenceWords(){this._interferenceWords.clear()}GetInterferenceWordCount(){return this._interferenceWords.size}}class n extends r.Plugin{constructor(r,n){if(t.Override(n,{Replacement:"█",IgnoreCase:!1,CacheSize:1e3}),super(r,n),!Number.isInteger(n.CacheSize)||n.CacheSize<=0)throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${n.CacheSize}`);this._acAutomaton=new s(n.IgnoreCase),this._sensitiveWords=new Set,this._acAutomaton.AddInterferenceWords(" \t\r\n".split("")),n.EnableCache?(this._testCache=new e.LRU(n.CacheSize),this._findCache=new e.LRU(n.CacheSize),this._replaceCache=new e.LRU(n.CacheSize)):(this._testCache=null,this._findCache=null,this._replaceCache=null),this.buildAutomaton()}Test(t){if(!t||0===t.length)return!1;if(this._testCache){const e=this._testCache.Get(t);if(null!==e)return e}const e=this._acAutomaton.Test(t);return this._testCache&&this._testCache.Set(t,e),e}Find(t){if(!t||0===t.length)return{Matched:!1,Words:[],Matches:[]};if(this._findCache){const e=this._findCache.Get(t);if(null!==e)return{Matched:e.Matched,Words:[...e.Words],Matches:e.Matches.map(t=>Object.assign({},t))}}const e=this._acAutomaton.Match(t),r=new Set;for(const t of e)r.add(t.Word);const s={Matched:e.length>0,Words:Array.from(r),Matches:e};return this._findCache&&this._findCache.Set(t,{Matched:s.Matched,Words:[...s.Words],Matches:s.Matches.map(t=>Object.assign({},t))}),s}Replace(t,e){var r;if(!t||0===t.length)return t;const s=JSON.stringify([t,null!=e?e:this.Cfg.Replacement]);if(this._replaceCache){const t=this._replaceCache.Get(s);if(null!==t)return t}const n=this.Find(t);if(!n.Matched)return null===(r=this._replaceCache)||void 0===r||r.Set(s,t),t;const i=Array.from(t),o=[...n.Matches].sort((t,e)=>t.Start-e.Start),h=[];for(const t of o){const e=h[h.length-1];e&&t.Start<e.end?e.end=Math.max(e.end,t.End):h.push({start:t.Start,end:t.End})}const a=null!=e?e:this.Cfg.Replacement;for(let t=h.length-1;t>=0;t--){const{start:e,end:r}=h[t];i.splice(e,r-e,a)}const c=i.join("");return this._replaceCache&&this._replaceCache.Set(s,c),c}InjectSensitiveWords(t){let e=!1;for(const r of t)r&&r.length>0&&!this._sensitiveWords.has(r)&&(this._sensitiveWords.add(r),e=!0);e&&(this.buildAutomaton(),this.clearCache())}InjectInterferenceWords(t){const e=this._acAutomaton.AddInterferenceWords(t);if(e.length>0){const t=this.Logger;for(const r of e)t.Warn(`干扰词须为单个字符,已忽略 >>> ${r}`)}this.clearCache()}GetSensitiveWordCount(){return this._sensitiveWords.size}GetInterferenceWordCount(){return this._acAutomaton.GetInterferenceWordCount()}ClearSensitiveWords(){this._sensitiveWords.clear(),this.buildAutomaton(),this.clearCache()}ClearInterferenceWords(){this._acAutomaton.ClearInterferenceWords(),this.clearCache()}buildAutomaton(){this._acAutomaton.Clear();const t=this._sensitiveWords;for(const e of t)this._acAutomaton.AddWord(e);this._acAutomaton.Build()}clearCache(){var t,e,r;null===(t=this._testCache)||void 0===t||t.Clear(),null===(e=this._findCache)||void 0===e||e.Clear(),null===(r=this._replaceCache)||void 0===r||r.Clear()}}exports.AuditorPlugin=n;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{Override as t}from"@nebulaaa/dict";import{LRU as e}from"@nebulaaa/lru";import{Plugin as r}from"@nebulaaa/plugin";class s{constructor(t=!1){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._built=!1,this._interferenceWords=new Set,this._ignoreCase=t,this._wordChars=new Map}normalize(t){if(!this._ignoreCase)return t;let e="";for(const r of t){const t=r.codePointAt(0);e+=t>=65&&t<=90?String.fromCharCode(t+32):r}return e}AddWord(t){if(!t||0===t.length)return;let e=this._root;const r=Array.from(this.normalize(t));for(const t of r){let r=e.Children.get(t);r||(r={Children:new Map,Fail:null,Words:[],Output:[]},e.Children.set(t,r)),e=r}0===e.Words.length&&(e.Words.push(t),this._wordChars.set(t,r),this._built=!1)}Build(){if(this._built)return;const t=[];for(const e of this._root.Children.values())e.Fail=this._root,e.Output=[...e.Words],t.push(e);let e=0;for(;e<t.length;){const r=t[e++];for(const[e,s]of r.Children){t.push(s);let n=r.Fail;for(;n&&!n.Children.has(e);)n=n.Fail;s.Fail=n&&n.Children.get(e)||this._root,s.Output=[...s.Words,...s.Fail.Output]}}this._built=!0}Match(t){this._built||this.Build();const e=[];let r=this._root;const s=Array.from(this.normalize(t));for(let t=0;t<s.length;t++){const n=s[t];if(!this._interferenceWords.has(n)){for(;r!==this._root&&!r.Children.has(n);)r=r.Fail;if(r.Children.has(n)){r=r.Children.get(n);for(const n of r.Output){let r=t;const i=this._wordChars.get(n);let o=i.length-1;for(;r>=0&&o>=0;)this._interferenceWords.has(s[r])||s[r]===i[o]&&o--,r--;o>=0||e.push({Word:n,Start:r+1,End:t+1})}}}}return e}Test(t){this._built||this.Build();let e=this._root;const r=Array.from(this.normalize(t));for(let t=0;t<r.length;t++){const s=r[t];if(!this._interferenceWords.has(s)){for(;e!==this._root&&!e.Children.has(s);)e=e.Fail;if(e.Children.has(s)&&(e=e.Children.get(s),e.Output.length>0))return!0}}return!1}Clear(){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._wordChars.clear(),this._built=!1}AddInterferenceWords(t){const e=[];for(const r of t){if(!r||0===r.length)continue;const t=Array.from(this.normalize(r));1===t.length?this._interferenceWords.add(t[0]):e.push(r)}return e}ClearInterferenceWords(){this._interferenceWords.clear()}GetInterferenceWordCount(){return this._interferenceWords.size}}class n extends r{constructor(r,n){if(t(n,{Replacement:"█",IgnoreCase:!1,CacheSize:1e3}),super(r,n),!Number.isInteger(n.CacheSize)||n.CacheSize<=0)throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${n.CacheSize}`);this._acAutomaton=new s(n.IgnoreCase),this._sensitiveWords=new Set,this._acAutomaton.AddInterferenceWords(" \t\r\n".split("")),n.EnableCache?(this._testCache=new e(n.CacheSize),this._findCache=new e(n.CacheSize),this._replaceCache=new e(n.CacheSize)):(this._testCache=null,this._findCache=null,this._replaceCache=null),this.buildAutomaton()}Test(t){if(!t||0===t.length)return!1;if(this._testCache){const e=this._testCache.Get(t);if(null!==e)return e}const e=this._acAutomaton.Test(t);return this._testCache&&this._testCache.Set(t,e),e}Find(t){if(!t||0===t.length)return{Matched:!1,Words:[],Matches:[]};if(this._findCache){const e=this._findCache.Get(t);if(null!==e)return{Matched:e.Matched,Words:[...e.Words],Matches:e.Matches.map(t=>Object.assign({},t))}}const e=this._acAutomaton.Match(t),r=new Set;for(const t of e)r.add(t.Word);const s={Matched:e.length>0,Words:Array.from(r),Matches:e};return this._findCache&&this._findCache.Set(t,{Matched:s.Matched,Words:[...s.Words],Matches:s.Matches.map(t=>Object.assign({},t))}),s}Replace(t,e){var r;if(!t||0===t.length)return t;const s=JSON.stringify([t,null!=e?e:this.Cfg.Replacement]);if(this._replaceCache){const t=this._replaceCache.Get(s);if(null!==t)return t}const n=this.Find(t);if(!n.Matched)return null===(r=this._replaceCache)||void 0===r||r.Set(s,t),t;const i=Array.from(t),o=[...n.Matches].sort((t,e)=>t.Start-e.Start),h=[];for(const t of o){const e=h[h.length-1];e&&t.Start<e.end?e.end=Math.max(e.end,t.End):h.push({start:t.Start,end:t.End})}const a=null!=e?e:this.Cfg.Replacement;for(let t=h.length-1;t>=0;t--){const{start:e,end:r}=h[t];i.splice(e,r-e,a)}const c=i.join("");return this._replaceCache&&this._replaceCache.Set(s,c),c}InjectSensitiveWords(t){let e=!1;for(const r of t)r&&r.length>0&&!this._sensitiveWords.has(r)&&(this._sensitiveWords.add(r),e=!0);e&&(this.buildAutomaton(),this.clearCache())}InjectInterferenceWords(t){const e=this._acAutomaton.AddInterferenceWords(t);if(e.length>0){const t=this.Logger;for(const r of e)t.Warn(`干扰词须为单个字符,已忽略 >>> ${r}`)}this.clearCache()}GetSensitiveWordCount(){return this._sensitiveWords.size}GetInterferenceWordCount(){return this._acAutomaton.GetInterferenceWordCount()}ClearSensitiveWords(){this._sensitiveWords.clear(),this.buildAutomaton(),this.clearCache()}ClearInterferenceWords(){this._acAutomaton.ClearInterferenceWords(),this.clearCache()}buildAutomaton(){this._acAutomaton.Clear();const t=this._sensitiveWords;for(const e of t)this._acAutomaton.AddWord(e);this._acAutomaton.Build()}clearCache(){var t,e,r;null===(t=this._testCache)||void 0===t||t.Clear(),null===(e=this._findCache)||void 0===e||e.Clear(),null===(r=this._replaceCache)||void 0===r||r.Clear()}}export{n as AuditorPlugin};
|
package/dist/auditor.mjs
ADDED
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
import { Override } from '@nebulaaa/dict';
|
|
2
|
+
import { LRU } from '@nebulaaa/lru';
|
|
3
|
+
import { Plugin } from '@nebulaaa/plugin';
|
|
4
|
+
|
|
5
|
+
/** AC 自动机 */
|
|
6
|
+
class ACAutomaton {
|
|
7
|
+
constructor(ignoreCase = false) {
|
|
8
|
+
this._root = {
|
|
9
|
+
Children: new Map(),
|
|
10
|
+
Fail: null,
|
|
11
|
+
Words: [],
|
|
12
|
+
Output: [],
|
|
13
|
+
};
|
|
14
|
+
this._built = false;
|
|
15
|
+
this._interferenceWords = new Set();
|
|
16
|
+
this._ignoreCase = ignoreCase;
|
|
17
|
+
this._wordChars = new Map();
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* 大小写归一(仅 ASCII 字母,逐码点等长替换,保证匹配位置与原始文本索引一致)
|
|
21
|
+
*/
|
|
22
|
+
normalize(text) {
|
|
23
|
+
if (!this._ignoreCase) {
|
|
24
|
+
return text;
|
|
25
|
+
}
|
|
26
|
+
let out = '';
|
|
27
|
+
for (const ch of text) {
|
|
28
|
+
const code = ch.codePointAt(0);
|
|
29
|
+
out += code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : ch;
|
|
30
|
+
}
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* 添加敏感词
|
|
35
|
+
* @param word 敏感词
|
|
36
|
+
*/
|
|
37
|
+
AddWord(word) {
|
|
38
|
+
if (!word || word.length === 0) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
let current = this._root;
|
|
42
|
+
const chars = Array.from(this.normalize(word)); // 归一后分词(IgnoreCase 关闭时原样返回)
|
|
43
|
+
for (const char of chars) {
|
|
44
|
+
let child = current.Children.get(char);
|
|
45
|
+
if (!child) {
|
|
46
|
+
child = {
|
|
47
|
+
Children: new Map(),
|
|
48
|
+
Fail: null,
|
|
49
|
+
Words: [],
|
|
50
|
+
Output: [],
|
|
51
|
+
};
|
|
52
|
+
current.Children.set(char, child);
|
|
53
|
+
}
|
|
54
|
+
current = child;
|
|
55
|
+
}
|
|
56
|
+
// 归一后同一路径只保留首词:IgnoreCase 下 "abc"/"ABC" 共享路径,
|
|
57
|
+
// 变体重复入表会让 Match 对同一区间输出多条重复结果(下游命中计数虚高)
|
|
58
|
+
if (current.Words.length === 0) {
|
|
59
|
+
current.Words.push(word);
|
|
60
|
+
this._wordChars.set(word, chars);
|
|
61
|
+
this._built = false; // 标记需要重新构建 fail 指针
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* 构建 fail 指针(使用 BFS)
|
|
66
|
+
*/
|
|
67
|
+
Build() {
|
|
68
|
+
if (this._built) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const queue = [];
|
|
72
|
+
// 第一层节点的 fail 指针指向 root
|
|
73
|
+
for (const child of this._root.Children.values()) {
|
|
74
|
+
child.Fail = this._root;
|
|
75
|
+
child.Output = [...child.Words]; // 由自身词重新生成(fail 为 root,无合并项)
|
|
76
|
+
queue.push(child);
|
|
77
|
+
}
|
|
78
|
+
// BFS 构建 fail 指针(头指针索引出队:array.shift 为 O(n),万级节点的大词库构建退化为 O(n²))
|
|
79
|
+
let head = 0;
|
|
80
|
+
while (head < queue.length) {
|
|
81
|
+
const current = queue[head++];
|
|
82
|
+
for (const [char, child] of current.Children) {
|
|
83
|
+
queue.push(child);
|
|
84
|
+
// 查找 fail 指针
|
|
85
|
+
let fail = current.Fail;
|
|
86
|
+
while (fail && !fail.Children.has(char)) {
|
|
87
|
+
fail = fail.Fail;
|
|
88
|
+
}
|
|
89
|
+
child.Fail = fail ? fail.Children.get(char) || this._root : this._root;
|
|
90
|
+
// 合并 output:由自身词重新生成(幂等,重复构建不再累积;fail 节点层级更浅、已先构建完成)
|
|
91
|
+
child.Output = [...child.Words, ...child.Fail.Output];
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
this._built = true;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* 匹配文本,返回所有匹配结果
|
|
98
|
+
* @param text 待匹配的文本
|
|
99
|
+
* @returns 匹配结果列表
|
|
100
|
+
*/
|
|
101
|
+
Match(text) {
|
|
102
|
+
if (!this._built) {
|
|
103
|
+
this.Build();
|
|
104
|
+
}
|
|
105
|
+
const results = [];
|
|
106
|
+
let current = this._root;
|
|
107
|
+
const chars = Array.from(this.normalize(text)); // 归一后分词(逐码点等长,位置索引不变)
|
|
108
|
+
// 记录匹配时的原始位置
|
|
109
|
+
for (let i = 0; i < chars.length; i++) {
|
|
110
|
+
const char = chars[i];
|
|
111
|
+
// 如果是干扰字符,跳过
|
|
112
|
+
if (this._interferenceWords.has(char)) {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
// 沿着 fail 指针查找匹配的子节点
|
|
116
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
117
|
+
current = current.Fail;
|
|
118
|
+
}
|
|
119
|
+
if (current.Children.has(char)) {
|
|
120
|
+
current = current.Children.get(char);
|
|
121
|
+
// 输出所有匹配的敏感词
|
|
122
|
+
for (const word of current.Output) {
|
|
123
|
+
// 计算在原始文本中的起始位置(跳过干扰字符)
|
|
124
|
+
let j = i;
|
|
125
|
+
// 从当前位置向前查找匹配的起始位置(AddWord 已缓存归一后的码点数组)
|
|
126
|
+
const wordChars = this._wordChars.get(word);
|
|
127
|
+
let wordIndex = wordChars.length - 1;
|
|
128
|
+
// 从后向前匹配,找到原始文本中的位置
|
|
129
|
+
while (j >= 0 && wordIndex >= 0) {
|
|
130
|
+
if (this._interferenceWords.has(chars[j])) {
|
|
131
|
+
j--;
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
if (chars[j] === wordChars[wordIndex]) {
|
|
135
|
+
wordIndex--;
|
|
136
|
+
}
|
|
137
|
+
j--;
|
|
138
|
+
}
|
|
139
|
+
// 回扫必须完整命中:前向 AC 已保证该词是当前位置的后缀(两侧对称跳过干扰字符),
|
|
140
|
+
// wordIndex >= 0 说明不变量被破坏(理论不可达),防御性跳过而非产出错误区间
|
|
141
|
+
if (wordIndex >= 0) {
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
results.push({
|
|
145
|
+
Word: word,
|
|
146
|
+
Start: j + 1,
|
|
147
|
+
End: i + 1,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return results;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* 检测文本是否包含敏感词
|
|
156
|
+
* @param text 待检测的文本
|
|
157
|
+
* @returns 是否包含敏感词
|
|
158
|
+
*/
|
|
159
|
+
Test(text) {
|
|
160
|
+
if (!this._built) {
|
|
161
|
+
this.Build();
|
|
162
|
+
}
|
|
163
|
+
let current = this._root;
|
|
164
|
+
const chars = Array.from(this.normalize(text));
|
|
165
|
+
for (let i = 0; i < chars.length; i++) {
|
|
166
|
+
const char = chars[i];
|
|
167
|
+
// 如果是干扰字符,跳过
|
|
168
|
+
if (this._interferenceWords.has(char)) {
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
172
|
+
current = current.Fail;
|
|
173
|
+
}
|
|
174
|
+
if (current.Children.has(char)) {
|
|
175
|
+
current = current.Children.get(char);
|
|
176
|
+
if (current.Output.length > 0) {
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return false;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* 清空所有节点
|
|
185
|
+
*/
|
|
186
|
+
Clear() {
|
|
187
|
+
this._root = {
|
|
188
|
+
Children: new Map(),
|
|
189
|
+
Fail: null,
|
|
190
|
+
Words: [],
|
|
191
|
+
Output: [],
|
|
192
|
+
};
|
|
193
|
+
this._wordChars.clear();
|
|
194
|
+
this._built = false;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* 注入干扰词(仅单字符生效:匹配按单字符查表)
|
|
198
|
+
* @param words 干扰词列表
|
|
199
|
+
* @returns 被忽略的多字符词(由调用方决定告警方式,保持算法类无日志副作用)
|
|
200
|
+
*/
|
|
201
|
+
AddInterferenceWords(words) {
|
|
202
|
+
const ignored = [];
|
|
203
|
+
for (const word of words) {
|
|
204
|
+
if (!word || word.length === 0) {
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
const chars = Array.from(this.normalize(word));
|
|
208
|
+
if (chars.length !== 1) {
|
|
209
|
+
// 多字符干扰词在按字符匹配的查表中永不命中——返回给调用方显式告警而非静默失效
|
|
210
|
+
ignored.push(word);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
this._interferenceWords.add(chars[0]);
|
|
214
|
+
}
|
|
215
|
+
return ignored;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* 清空干扰词
|
|
219
|
+
*/
|
|
220
|
+
ClearInterferenceWords() {
|
|
221
|
+
this._interferenceWords.clear();
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* 获取干扰词数量
|
|
225
|
+
*/
|
|
226
|
+
GetInterferenceWordCount() {
|
|
227
|
+
return this._interferenceWords.size;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* 敏感词审计员插件
|
|
232
|
+
*/
|
|
233
|
+
class AuditorPlugin extends Plugin {
|
|
234
|
+
constructor(deps, cfg) {
|
|
235
|
+
// 默认值填充必须在 super 之前:基类构造会冻结 cfg,冻结后 defineProperty 会抛错
|
|
236
|
+
Override(cfg, {
|
|
237
|
+
Replacement: '█',
|
|
238
|
+
IgnoreCase: false,
|
|
239
|
+
CacheSize: 1000,
|
|
240
|
+
});
|
|
241
|
+
super(deps, cfg);
|
|
242
|
+
// 缓存容量须为正整数:无论是否启用缓存都在构造期校验,配置错误不留到运行期
|
|
243
|
+
if (!Number.isInteger(cfg.CacheSize) || cfg.CacheSize <= 0) {
|
|
244
|
+
throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${cfg.CacheSize}`);
|
|
245
|
+
}
|
|
246
|
+
this._acAutomaton = new ACAutomaton(cfg.IgnoreCase);
|
|
247
|
+
this._sensitiveWords = new Set();
|
|
248
|
+
this._acAutomaton.AddInterferenceWords(' \t\r\n'.split(''));
|
|
249
|
+
if (cfg.EnableCache) {
|
|
250
|
+
this._testCache = new LRU(cfg.CacheSize);
|
|
251
|
+
this._findCache = new LRU(cfg.CacheSize);
|
|
252
|
+
this._replaceCache = new LRU(cfg.CacheSize);
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
this._testCache = null;
|
|
256
|
+
this._findCache = null;
|
|
257
|
+
this._replaceCache = null;
|
|
258
|
+
}
|
|
259
|
+
// 构建敏感词自动机
|
|
260
|
+
this.buildAutomaton();
|
|
261
|
+
}
|
|
262
|
+
Test(text) {
|
|
263
|
+
if (!text || text.length === 0) {
|
|
264
|
+
return false;
|
|
265
|
+
}
|
|
266
|
+
// 检查缓存
|
|
267
|
+
if (this._testCache) {
|
|
268
|
+
const cached = this._testCache.Get(text);
|
|
269
|
+
if (cached !== null) {
|
|
270
|
+
return cached;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
const result = this._acAutomaton.Test(text);
|
|
274
|
+
// 更新缓存
|
|
275
|
+
if (this._testCache) {
|
|
276
|
+
this._testCache.Set(text, result);
|
|
277
|
+
}
|
|
278
|
+
return result;
|
|
279
|
+
}
|
|
280
|
+
Find(text) {
|
|
281
|
+
if (!text || text.length === 0) {
|
|
282
|
+
return { Matched: false, Words: [], Matches: [] };
|
|
283
|
+
}
|
|
284
|
+
// 检查缓存
|
|
285
|
+
if (this._findCache) {
|
|
286
|
+
const cached = this._findCache.Get(text);
|
|
287
|
+
if (cached !== null) {
|
|
288
|
+
// 返回浅拷贝,防止外部修改影响缓存
|
|
289
|
+
return {
|
|
290
|
+
Matched: cached.Matched,
|
|
291
|
+
Words: [...cached.Words],
|
|
292
|
+
Matches: cached.Matches.map((m) => (Object.assign({}, m))),
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
// 匹配文本,自动跳过干扰词
|
|
297
|
+
const matches = this._acAutomaton.Match(text);
|
|
298
|
+
// 去重敏感词
|
|
299
|
+
const wordSet = new Set();
|
|
300
|
+
for (const match of matches) {
|
|
301
|
+
wordSet.add(match.Word);
|
|
302
|
+
}
|
|
303
|
+
const result = {
|
|
304
|
+
Matched: matches.length > 0,
|
|
305
|
+
Words: Array.from(wordSet),
|
|
306
|
+
Matches: matches,
|
|
307
|
+
};
|
|
308
|
+
// 更新缓存(存储副本,防止后续修改影响缓存)
|
|
309
|
+
if (this._findCache) {
|
|
310
|
+
this._findCache.Set(text, {
|
|
311
|
+
Matched: result.Matched,
|
|
312
|
+
Words: [...result.Words],
|
|
313
|
+
Matches: result.Matches.map((m) => (Object.assign({}, m))),
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
return result;
|
|
317
|
+
}
|
|
318
|
+
Replace(text, replacement) {
|
|
319
|
+
var _a;
|
|
320
|
+
if (!text || text.length === 0) {
|
|
321
|
+
return text;
|
|
322
|
+
}
|
|
323
|
+
// 检查缓存(JSON 编码组合键:字符串拼接的 "_" 分隔符可出现在 text/replacement 中造成碰撞)
|
|
324
|
+
const cacheKey = JSON.stringify([text, replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement]);
|
|
325
|
+
if (this._replaceCache) {
|
|
326
|
+
const cached = this._replaceCache.Get(cacheKey);
|
|
327
|
+
if (cached !== null) {
|
|
328
|
+
return cached;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
const result = this.Find(text);
|
|
332
|
+
if (!result.Matched) {
|
|
333
|
+
// 干净文本是最高频流量,同样入缓存(组合键已防碰撞),避免反复全量匹配
|
|
334
|
+
(_a = this._replaceCache) === null || _a === void 0 ? void 0 : _a.Set(cacheKey, text);
|
|
335
|
+
return text;
|
|
336
|
+
}
|
|
337
|
+
// 在码点数组上操作:matches 的位置基于 Array.from 分词(码点索引),
|
|
338
|
+
// 直接对原字符串 slice 会因代理对(emoji 等)长度不一致而切错位置
|
|
339
|
+
const chars = Array.from(text);
|
|
340
|
+
// 合并重叠区间:重叠匹配若逐个替换,从后向前切片会互相错位(吞字或残留敏感片段)
|
|
341
|
+
const sorted = [...result.Matches].sort((a, b) => a.Start - b.Start);
|
|
342
|
+
const ranges = [];
|
|
343
|
+
for (const match of sorted) {
|
|
344
|
+
const last = ranges[ranges.length - 1];
|
|
345
|
+
if (last && match.Start < last.end) {
|
|
346
|
+
last.end = Math.max(last.end, match.End); // 重叠:并入上一区间
|
|
347
|
+
}
|
|
348
|
+
else {
|
|
349
|
+
ranges.push({ start: match.Start, end: match.End });
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
// 每个合并区间整体替换为一个 replacement 串(非逐字符屏蔽);空串为合法替换值(删除敏感词)
|
|
353
|
+
const replaceChar = replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement;
|
|
354
|
+
for (let i = ranges.length - 1; i >= 0; i--) {
|
|
355
|
+
const { start, end } = ranges[i];
|
|
356
|
+
chars.splice(start, end - start, replaceChar); // 从后向前,前面的索引不受影响
|
|
357
|
+
}
|
|
358
|
+
const replacedText = chars.join('');
|
|
359
|
+
// 更新缓存
|
|
360
|
+
if (this._replaceCache) {
|
|
361
|
+
this._replaceCache.Set(cacheKey, replacedText);
|
|
362
|
+
}
|
|
363
|
+
return replacedText;
|
|
364
|
+
}
|
|
365
|
+
InjectSensitiveWords(words) {
|
|
366
|
+
let added = false;
|
|
367
|
+
for (const word of words) {
|
|
368
|
+
if (word && word.length > 0 && !this._sensitiveWords.has(word)) {
|
|
369
|
+
this._sensitiveWords.add(word);
|
|
370
|
+
added = true;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
// 无新增词时词库未变:跳过自动机重建与缓存清理,重复注入不打断既有缓存
|
|
374
|
+
if (!added) {
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
this.buildAutomaton();
|
|
378
|
+
this.clearCache();
|
|
379
|
+
}
|
|
380
|
+
InjectInterferenceWords(words) {
|
|
381
|
+
const ignored = this._acAutomaton.AddInterferenceWords(words);
|
|
382
|
+
// 多字符干扰词永不命中:经插件日志显式告警(词库内容不落 console)
|
|
383
|
+
if (ignored.length > 0) {
|
|
384
|
+
const logger = this.Logger;
|
|
385
|
+
for (const word of ignored) {
|
|
386
|
+
logger.Warn(`干扰词须为单个字符,已忽略 >>> ${word}`);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
this.clearCache();
|
|
390
|
+
}
|
|
391
|
+
GetSensitiveWordCount() {
|
|
392
|
+
return this._sensitiveWords.size;
|
|
393
|
+
}
|
|
394
|
+
GetInterferenceWordCount() {
|
|
395
|
+
return this._acAutomaton.GetInterferenceWordCount();
|
|
396
|
+
}
|
|
397
|
+
ClearSensitiveWords() {
|
|
398
|
+
this._sensitiveWords.clear();
|
|
399
|
+
this.buildAutomaton();
|
|
400
|
+
this.clearCache();
|
|
401
|
+
}
|
|
402
|
+
ClearInterferenceWords() {
|
|
403
|
+
this._acAutomaton.ClearInterferenceWords();
|
|
404
|
+
this.clearCache();
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* 构建 AC 自动机
|
|
408
|
+
*/
|
|
409
|
+
buildAutomaton() {
|
|
410
|
+
this._acAutomaton.Clear();
|
|
411
|
+
const words = this._sensitiveWords;
|
|
412
|
+
for (const word of words) {
|
|
413
|
+
this._acAutomaton.AddWord(word);
|
|
414
|
+
}
|
|
415
|
+
this._acAutomaton.Build();
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* 清空缓存
|
|
419
|
+
*/
|
|
420
|
+
clearCache() {
|
|
421
|
+
var _a, _b, _c;
|
|
422
|
+
(_a = this._testCache) === null || _a === void 0 ? void 0 : _a.Clear();
|
|
423
|
+
(_b = this._findCache) === null || _b === void 0 ? void 0 : _b.Clear();
|
|
424
|
+
(_c = this._replaceCache) === null || _c === void 0 ? void 0 : _c.Clear();
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
export { AuditorPlugin };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@nebulaaa/auditor",
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"description": "敏感词审计员插件",
|
|
5
|
+
"author": "doooreyn",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"keywords": [
|
|
8
|
+
"auditor",
|
|
9
|
+
"nebulaaa"
|
|
10
|
+
],
|
|
11
|
+
"engines": {
|
|
12
|
+
"node": ">=16.0.0",
|
|
13
|
+
"npm": ">=10.0.0"
|
|
14
|
+
},
|
|
15
|
+
"main": "./auditor.cjs",
|
|
16
|
+
"module": "./auditor.mjs",
|
|
17
|
+
"types": "./auditor.d.ts",
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"types": "./auditor.d.ts",
|
|
21
|
+
"require": "./auditor.cjs",
|
|
22
|
+
"import": "./auditor.mjs",
|
|
23
|
+
"default": "./auditor.cjs"
|
|
24
|
+
},
|
|
25
|
+
"./min": {
|
|
26
|
+
"require": "./auditor.min.cjs",
|
|
27
|
+
"import": "./auditor.min.mjs"
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"publishConfig": {
|
|
31
|
+
"registry": "https://registry.npmjs.org/",
|
|
32
|
+
"access": "public"
|
|
33
|
+
}
|
|
34
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,70 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@nebulaaa/auditor",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "@nebulaaa/auditor",
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"description": "敏感词审计员插件",
|
|
5
|
+
"main": "./dist/auditor.cjs",
|
|
6
|
+
"module": "./dist/auditor.mjs",
|
|
7
|
+
"types": "./dist/auditor.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"require": "./dist/auditor.cjs",
|
|
11
|
+
"import": "./dist/auditor.mjs",
|
|
12
|
+
"types": "./dist/auditor.d.ts",
|
|
13
|
+
"default": "./dist/auditor.cjs"
|
|
14
|
+
},
|
|
15
|
+
"./min": {
|
|
16
|
+
"require": "./dist/auditor.min.cjs",
|
|
17
|
+
"import": "./dist/auditor.min.mjs"
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"files": [
|
|
21
|
+
"dist"
|
|
22
|
+
],
|
|
23
|
+
"scripts": {
|
|
24
|
+
"clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"",
|
|
25
|
+
"build": "npm run clean && rollup -c rollup.config.mjs && node ../scripts/write-dist-package.mjs auditor",
|
|
26
|
+
"test": "jest"
|
|
27
|
+
},
|
|
28
|
+
"keywords": [
|
|
29
|
+
"auditor",
|
|
30
|
+
"nebulaaa"
|
|
31
|
+
],
|
|
32
|
+
"author": "doooreyn",
|
|
33
|
+
"license": "MIT",
|
|
34
|
+
"devEngines": {
|
|
35
|
+
"packageManager": {
|
|
36
|
+
"name": "npm",
|
|
37
|
+
"version": ">=10.0.0",
|
|
38
|
+
"onFail": "warn"
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"engines": {
|
|
42
|
+
"node": ">=16.0.0",
|
|
43
|
+
"npm": ">=10.0.0"
|
|
44
|
+
},
|
|
45
|
+
"publishConfig": {
|
|
46
|
+
"registry": "https://registry.npmjs.org/",
|
|
47
|
+
"access": "public"
|
|
48
|
+
},
|
|
49
|
+
"allowScripts": {
|
|
50
|
+
"unrs-resolver@1.12.2": true,
|
|
51
|
+
"@parcel/watcher@2.6.0": true
|
|
52
|
+
},
|
|
53
|
+
"devDependencies": {
|
|
54
|
+
"@rollup/plugin-terser": "^1.0.2",
|
|
55
|
+
"@rollup/plugin-typescript": "^12.1.2",
|
|
56
|
+
"@types/jest": "^30.0.0",
|
|
57
|
+
"@types/node": "^22.10.2",
|
|
58
|
+
"jest": "^30.5.2",
|
|
59
|
+
"rollup": "^4.28.1",
|
|
60
|
+
"rollup-plugin-dts": "^6.1.1",
|
|
61
|
+
"ts-jest": "^29.1.2",
|
|
62
|
+
"ts-node": "^10.9.2",
|
|
63
|
+
"typescript": "5.7.3"
|
|
64
|
+
},
|
|
65
|
+
"dependencies": {
|
|
66
|
+
"@nebulaaa/dict": "^1.1.0",
|
|
67
|
+
"@nebulaaa/lru": "^1.1.0",
|
|
68
|
+
"@nebulaaa/plugin": "^1.0.0"
|
|
69
|
+
}
|
|
70
|
+
}
|
package/README.md
DELETED