@nebulaaa/auditor 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auditor.cjs +405 -0
- package/dist/auditor.d.ts +155 -0
- package/dist/auditor.min.cjs +1 -0
- package/dist/auditor.min.mjs +1 -0
- package/dist/auditor.mjs +403 -0
- package/dist/package.json +34 -0
- package/package.json +70 -6
- package/README.md +0 -3
package/dist/auditor.cjs
ADDED
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var dict = require('@nebulaaa/dict');
|
|
4
|
+
var lru = require('@nebulaaa/lru');
|
|
5
|
+
var plugin = require('@nebulaaa/plugin');
|
|
6
|
+
|
|
7
|
+
/** AC 自动机 */
|
|
8
|
+
class ACAutomaton {
|
|
9
|
+
constructor(ignoreCase = false) {
|
|
10
|
+
this._root = {
|
|
11
|
+
Children: new Map(),
|
|
12
|
+
Fail: null,
|
|
13
|
+
Words: [],
|
|
14
|
+
Output: [],
|
|
15
|
+
};
|
|
16
|
+
this._built = false;
|
|
17
|
+
this._interferenceWords = new Set();
|
|
18
|
+
this._ignoreCase = ignoreCase;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* 大小写归一(仅 ASCII 字母,逐码点等长替换,保证匹配位置与原始文本索引一致)
|
|
22
|
+
*/
|
|
23
|
+
normalize(text) {
|
|
24
|
+
if (!this._ignoreCase) {
|
|
25
|
+
return text;
|
|
26
|
+
}
|
|
27
|
+
let out = '';
|
|
28
|
+
for (const ch of text) {
|
|
29
|
+
const code = ch.codePointAt(0);
|
|
30
|
+
out += code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : ch;
|
|
31
|
+
}
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* 添加敏感词
|
|
36
|
+
* @param word 敏感词
|
|
37
|
+
*/
|
|
38
|
+
AddWord(word) {
|
|
39
|
+
if (!word || word.length === 0) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
let current = this._root;
|
|
43
|
+
const chars = Array.from(this.normalize(word)); // 归一后分词(IgnoreCase 关闭时原样返回)
|
|
44
|
+
for (const char of chars) {
|
|
45
|
+
let child = current.Children.get(char);
|
|
46
|
+
if (!child) {
|
|
47
|
+
child = {
|
|
48
|
+
Children: new Map(),
|
|
49
|
+
Fail: null,
|
|
50
|
+
Words: [],
|
|
51
|
+
Output: [],
|
|
52
|
+
};
|
|
53
|
+
current.Children.set(char, child);
|
|
54
|
+
}
|
|
55
|
+
current = child;
|
|
56
|
+
}
|
|
57
|
+
if (!current.Words.includes(word)) {
|
|
58
|
+
// 同词重复添加幂等:自身词独立存放,output 由 build 重新生成
|
|
59
|
+
current.Words.push(word);
|
|
60
|
+
}
|
|
61
|
+
this._built = false; // 标记需要重新构建 fail 指针
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* 构建 fail 指针(使用 BFS)
|
|
65
|
+
*/
|
|
66
|
+
Build() {
|
|
67
|
+
if (this._built) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const queue = [];
|
|
71
|
+
// 第一层节点的 fail 指针指向 root
|
|
72
|
+
for (const child of this._root.Children.values()) {
|
|
73
|
+
child.Fail = this._root;
|
|
74
|
+
child.Output = [...child.Words]; // 由自身词重新生成(fail 为 root,无合并项)
|
|
75
|
+
queue.push(child);
|
|
76
|
+
}
|
|
77
|
+
// BFS 构建 fail 指针(头指针索引出队:array.shift 为 O(n),万级节点的大词库构建退化为 O(n²))
|
|
78
|
+
let head = 0;
|
|
79
|
+
while (head < queue.length) {
|
|
80
|
+
const current = queue[head++];
|
|
81
|
+
for (const [char, child] of current.Children) {
|
|
82
|
+
queue.push(child);
|
|
83
|
+
// 查找 fail 指针
|
|
84
|
+
let fail = current.Fail;
|
|
85
|
+
while (fail && !fail.Children.has(char)) {
|
|
86
|
+
fail = fail.Fail;
|
|
87
|
+
}
|
|
88
|
+
child.Fail = fail ? fail.Children.get(char) || this._root : this._root;
|
|
89
|
+
// 合并 output:由自身词重新生成(幂等,重复构建不再累积;fail 节点层级更浅、已先构建完成)
|
|
90
|
+
child.Output = [...child.Words, ...child.Fail.Output];
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
this._built = true;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* 匹配文本,返回所有匹配结果
|
|
97
|
+
* @param text 待匹配的文本
|
|
98
|
+
* @returns 匹配结果列表
|
|
99
|
+
*/
|
|
100
|
+
Match(text) {
|
|
101
|
+
if (!this._built) {
|
|
102
|
+
this.Build();
|
|
103
|
+
}
|
|
104
|
+
const results = [];
|
|
105
|
+
let current = this._root;
|
|
106
|
+
const chars = Array.from(this.normalize(text)); // 归一后分词(逐码点等长,位置索引不变)
|
|
107
|
+
// 记录匹配时的原始位置
|
|
108
|
+
for (let i = 0; i < chars.length; i++) {
|
|
109
|
+
const char = chars[i];
|
|
110
|
+
// 如果是干扰字符,跳过
|
|
111
|
+
if (this._interferenceWords.has(char)) {
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
// 沿着 fail 指针查找匹配的子节点
|
|
115
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
116
|
+
current = current.Fail;
|
|
117
|
+
}
|
|
118
|
+
if (current.Children.has(char)) {
|
|
119
|
+
current = current.Children.get(char);
|
|
120
|
+
// 输出所有匹配的敏感词
|
|
121
|
+
for (const word of current.Output) {
|
|
122
|
+
// 计算在原始文本中的起始位置(跳过干扰字符)
|
|
123
|
+
let j = i;
|
|
124
|
+
// 从当前位置向前查找匹配的起始位置
|
|
125
|
+
const wordChars = Array.from(this.normalize(word));
|
|
126
|
+
let wordIndex = wordChars.length - 1;
|
|
127
|
+
// 从后向前匹配,找到原始文本中的位置
|
|
128
|
+
while (j >= 0 && wordIndex >= 0) {
|
|
129
|
+
if (this._interferenceWords.has(chars[j])) {
|
|
130
|
+
j--;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
if (chars[j] === wordChars[wordIndex]) {
|
|
134
|
+
wordIndex--;
|
|
135
|
+
}
|
|
136
|
+
j--;
|
|
137
|
+
}
|
|
138
|
+
results.push({
|
|
139
|
+
Word: word,
|
|
140
|
+
Start: j + 1,
|
|
141
|
+
End: i + 1,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return results;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* 检测文本是否包含敏感词
|
|
150
|
+
* @param text 待检测的文本
|
|
151
|
+
* @returns 是否包含敏感词
|
|
152
|
+
*/
|
|
153
|
+
Test(text) {
|
|
154
|
+
if (!this._built) {
|
|
155
|
+
this.Build();
|
|
156
|
+
}
|
|
157
|
+
let current = this._root;
|
|
158
|
+
const chars = Array.from(this.normalize(text));
|
|
159
|
+
for (let i = 0; i < chars.length; i++) {
|
|
160
|
+
const char = chars[i];
|
|
161
|
+
// 如果是干扰字符,跳过
|
|
162
|
+
if (this._interferenceWords.has(char)) {
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
166
|
+
current = current.Fail;
|
|
167
|
+
}
|
|
168
|
+
if (current.Children.has(char)) {
|
|
169
|
+
current = current.Children.get(char);
|
|
170
|
+
if (current.Output.length > 0) {
|
|
171
|
+
return true;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* 清空所有节点
|
|
179
|
+
*/
|
|
180
|
+
Clear() {
|
|
181
|
+
this._root = {
|
|
182
|
+
Children: new Map(),
|
|
183
|
+
Fail: null,
|
|
184
|
+
Words: [],
|
|
185
|
+
Output: [],
|
|
186
|
+
};
|
|
187
|
+
this._built = false;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* 注入干扰词(仅单字符生效:匹配按单字符查表)
|
|
191
|
+
* @param words 干扰词列表
|
|
192
|
+
*/
|
|
193
|
+
AddInterferenceWords(words) {
|
|
194
|
+
for (const word of words) {
|
|
195
|
+
if (!word || word.length === 0) {
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
const chars = Array.from(this.normalize(word));
|
|
199
|
+
if (chars.length !== 1) {
|
|
200
|
+
// 多字符干扰词在按字符匹配的查表中永不命中——显式告警而非静默失效
|
|
201
|
+
console.warn(`addInterferenceWords: 干扰词须为单个字符,已忽略 "${word}"`);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
this._interferenceWords.add(chars[0]);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* 清空干扰词
|
|
209
|
+
*/
|
|
210
|
+
ClearInterferenceWords() {
|
|
211
|
+
this._interferenceWords.clear();
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* 获取干扰词数量
|
|
215
|
+
*/
|
|
216
|
+
GetInterferenceWordCount() {
|
|
217
|
+
return this._interferenceWords.size;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* 敏感词审计员插件
|
|
222
|
+
*/
|
|
223
|
+
class AuditorPlugin extends plugin.Plugin {
|
|
224
|
+
constructor(deps, cfg) {
|
|
225
|
+
// 默认值填充必须在 super 之前:基类构造会冻结 cfg,冻结后 defineProperty 会抛错
|
|
226
|
+
dict.Override(cfg, {
|
|
227
|
+
Replacement: '█',
|
|
228
|
+
IgnoreCase: false,
|
|
229
|
+
CacheSize: 1000,
|
|
230
|
+
});
|
|
231
|
+
super(deps, cfg);
|
|
232
|
+
this._acAutomaton = new ACAutomaton(cfg.IgnoreCase);
|
|
233
|
+
this._sensitiveWords = new Set();
|
|
234
|
+
this._acAutomaton.AddInterferenceWords(' \t\r\n'.split(''));
|
|
235
|
+
if (cfg.EnableCache) {
|
|
236
|
+
// 缓存容量须为正整数:非法配置构造期显式失败,错误信息带插件上下文便于定位
|
|
237
|
+
if (!Number.isInteger(cfg.CacheSize) || cfg.CacheSize <= 0) {
|
|
238
|
+
throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${cfg.CacheSize}`);
|
|
239
|
+
}
|
|
240
|
+
this._testCache = new lru.LRU(cfg.CacheSize);
|
|
241
|
+
this._findCache = new lru.LRU(cfg.CacheSize);
|
|
242
|
+
this._replaceCache = new lru.LRU(cfg.CacheSize);
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
this._testCache = null;
|
|
246
|
+
this._findCache = null;
|
|
247
|
+
this._replaceCache = null;
|
|
248
|
+
}
|
|
249
|
+
// 构建敏感词自动机
|
|
250
|
+
this.buildAutomaton();
|
|
251
|
+
}
|
|
252
|
+
Test(text) {
|
|
253
|
+
if (!text || text.length === 0) {
|
|
254
|
+
return false;
|
|
255
|
+
}
|
|
256
|
+
// 检查缓存
|
|
257
|
+
if (this._testCache) {
|
|
258
|
+
const cached = this._testCache.Get(text);
|
|
259
|
+
if (cached !== null) {
|
|
260
|
+
return cached;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
const result = this._acAutomaton.Test(text);
|
|
264
|
+
// 更新缓存
|
|
265
|
+
if (this._testCache) {
|
|
266
|
+
this._testCache.Set(text, result);
|
|
267
|
+
}
|
|
268
|
+
return result;
|
|
269
|
+
}
|
|
270
|
+
Find(text) {
|
|
271
|
+
if (!text || text.length === 0) {
|
|
272
|
+
return { Matched: false, Words: [], Matches: [] };
|
|
273
|
+
}
|
|
274
|
+
// 检查缓存
|
|
275
|
+
if (this._findCache) {
|
|
276
|
+
const cached = this._findCache.Get(text);
|
|
277
|
+
if (cached !== null) {
|
|
278
|
+
// 返回浅拷贝,防止外部修改影响缓存
|
|
279
|
+
return {
|
|
280
|
+
Matched: cached.Matched,
|
|
281
|
+
Words: [...cached.Words],
|
|
282
|
+
Matches: cached.Matches.map((m) => (Object.assign({}, m))),
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
// 匹配文本,自动跳过干扰词
|
|
287
|
+
const matches = this._acAutomaton.Match(text);
|
|
288
|
+
// 去重敏感词
|
|
289
|
+
const wordSet = new Set();
|
|
290
|
+
for (const match of matches) {
|
|
291
|
+
wordSet.add(match.Word);
|
|
292
|
+
}
|
|
293
|
+
const result = {
|
|
294
|
+
Matched: matches.length > 0,
|
|
295
|
+
Words: Array.from(wordSet),
|
|
296
|
+
Matches: matches,
|
|
297
|
+
};
|
|
298
|
+
// 更新缓存(存储副本,防止后续修改影响缓存)
|
|
299
|
+
if (this._findCache) {
|
|
300
|
+
this._findCache.Set(text, {
|
|
301
|
+
Matched: result.Matched,
|
|
302
|
+
Words: [...result.Words],
|
|
303
|
+
Matches: result.Matches.map((m) => (Object.assign({}, m))),
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
return result;
|
|
307
|
+
}
|
|
308
|
+
Replace(text, replacement) {
|
|
309
|
+
var _a;
|
|
310
|
+
if (!text || text.length === 0) {
|
|
311
|
+
return text;
|
|
312
|
+
}
|
|
313
|
+
// 检查缓存(JSON 编码组合键:字符串拼接的 "_" 分隔符可出现在 text/replacement 中造成碰撞)
|
|
314
|
+
const cacheKey = JSON.stringify([text, replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement]);
|
|
315
|
+
if (this._replaceCache) {
|
|
316
|
+
const cached = this._replaceCache.Get(cacheKey);
|
|
317
|
+
if (cached !== null) {
|
|
318
|
+
return cached;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
const result = this.Find(text);
|
|
322
|
+
if (!result.Matched) {
|
|
323
|
+
// 干净文本是最高频流量,同样入缓存(组合键已防碰撞),避免反复全量匹配
|
|
324
|
+
(_a = this._replaceCache) === null || _a === void 0 ? void 0 : _a.Set(cacheKey, text);
|
|
325
|
+
return text;
|
|
326
|
+
}
|
|
327
|
+
// 在码点数组上操作:matches 的位置基于 Array.from 分词(码点索引),
|
|
328
|
+
// 直接对原字符串 slice 会因代理对(emoji 等)长度不一致而切错位置
|
|
329
|
+
const chars = Array.from(text);
|
|
330
|
+
// 合并重叠区间:重叠匹配若逐个替换,从后向前切片会互相错位(吞字或残留敏感片段)
|
|
331
|
+
const sorted = [...result.Matches].sort((a, b) => a.Start - b.Start);
|
|
332
|
+
const ranges = [];
|
|
333
|
+
for (const match of sorted) {
|
|
334
|
+
const last = ranges[ranges.length - 1];
|
|
335
|
+
if (last && match.Start < last.end) {
|
|
336
|
+
last.end = Math.max(last.end, match.End); // 重叠:并入上一区间
|
|
337
|
+
}
|
|
338
|
+
else {
|
|
339
|
+
ranges.push({ start: match.Start, end: match.End });
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
// 每个合并区间整体替换为一个 replacement 串(非逐字符屏蔽);空串为合法替换值(删除敏感词)
|
|
343
|
+
const replaceChar = replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement;
|
|
344
|
+
for (let i = ranges.length - 1; i >= 0; i--) {
|
|
345
|
+
const { start, end } = ranges[i];
|
|
346
|
+
chars.splice(start, end - start, replaceChar); // 从后向前,前面的索引不受影响
|
|
347
|
+
}
|
|
348
|
+
const replacedText = chars.join('');
|
|
349
|
+
// 更新缓存
|
|
350
|
+
if (this._replaceCache) {
|
|
351
|
+
this._replaceCache.Set(cacheKey, replacedText);
|
|
352
|
+
}
|
|
353
|
+
return replacedText;
|
|
354
|
+
}
|
|
355
|
+
InjectSensitiveWords(words) {
|
|
356
|
+
for (const word of words) {
|
|
357
|
+
if (word && word.length > 0) {
|
|
358
|
+
this._sensitiveWords.add(word);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
this.buildAutomaton();
|
|
362
|
+
this.clearCache();
|
|
363
|
+
}
|
|
364
|
+
InjectInterferenceWords(words) {
|
|
365
|
+
this._acAutomaton.AddInterferenceWords(words);
|
|
366
|
+
this.clearCache();
|
|
367
|
+
}
|
|
368
|
+
GetSensitiveWordCount() {
|
|
369
|
+
return this._sensitiveWords.size;
|
|
370
|
+
}
|
|
371
|
+
GetInterferenceWordCount() {
|
|
372
|
+
return this._acAutomaton.GetInterferenceWordCount();
|
|
373
|
+
}
|
|
374
|
+
ClearSensitiveWords() {
|
|
375
|
+
this._sensitiveWords.clear();
|
|
376
|
+
this.buildAutomaton();
|
|
377
|
+
this.clearCache();
|
|
378
|
+
}
|
|
379
|
+
ClearInterferenceWords() {
|
|
380
|
+
this._acAutomaton.ClearInterferenceWords();
|
|
381
|
+
this.clearCache();
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* 构建 AC 自动机
|
|
385
|
+
*/
|
|
386
|
+
buildAutomaton() {
|
|
387
|
+
this._acAutomaton.Clear();
|
|
388
|
+
const words = this._sensitiveWords;
|
|
389
|
+
for (const word of words) {
|
|
390
|
+
this._acAutomaton.AddWord(word);
|
|
391
|
+
}
|
|
392
|
+
this._acAutomaton.Build();
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* 清空缓存
|
|
396
|
+
*/
|
|
397
|
+
clearCache() {
|
|
398
|
+
var _a, _b, _c;
|
|
399
|
+
(_a = this._testCache) === null || _a === void 0 ? void 0 : _a.Clear();
|
|
400
|
+
(_b = this._findCache) === null || _b === void 0 ? void 0 : _b.Clear();
|
|
401
|
+
(_c = this._replaceCache) === null || _c === void 0 ? void 0 : _c.Clear();
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
exports.AuditorPlugin = AuditorPlugin;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { PluginCfg, PluginDeps, IPlugin, Plugin } from '@nebulaaa/plugin';
|
|
2
|
+
|
|
3
|
+
/** 敏感词审计员插件依赖 */
|
|
4
|
+
interface AuditorDeps extends PluginDeps {
|
|
5
|
+
}
|
|
6
|
+
/** 敏感词审计员插件配置 */
|
|
7
|
+
interface AuditorCfg extends PluginCfg {
|
|
8
|
+
/**
|
|
9
|
+
* 敏感词替换字符
|
|
10
|
+
* @default '█'
|
|
11
|
+
*/
|
|
12
|
+
readonly Replacement?: string;
|
|
13
|
+
/**
|
|
14
|
+
* 是否启用缓存
|
|
15
|
+
* @default true
|
|
16
|
+
*/
|
|
17
|
+
readonly EnableCache?: boolean;
|
|
18
|
+
/**
|
|
19
|
+
* 缓存大小
|
|
20
|
+
* @default 1000
|
|
21
|
+
* @tip 须为正整数,非法值构造期显式报错
|
|
22
|
+
*/
|
|
23
|
+
readonly CacheSize?: number;
|
|
24
|
+
/**
|
|
25
|
+
* 是否忽略大小写
|
|
26
|
+
* @default false
|
|
27
|
+
* @tip 仅归一 ASCII 字母(A-Z↔a-z),逐码点等长替换不影响位置索引;
|
|
28
|
+
* 全角字母/数字等全半角变体不归一
|
|
29
|
+
*/
|
|
30
|
+
readonly IgnoreCase?: boolean;
|
|
31
|
+
}
|
|
32
|
+
/** 匹配位置信息 */
|
|
33
|
+
interface IAuditorMatchPosition {
|
|
34
|
+
/** 匹配到的敏感词 */
|
|
35
|
+
Word: string;
|
|
36
|
+
/** 起始位置(字符索引) */
|
|
37
|
+
Start: number;
|
|
38
|
+
/** 结束位置(字符索引,不包含) */
|
|
39
|
+
End: number;
|
|
40
|
+
}
|
|
41
|
+
/** 匹配结果 */
|
|
42
|
+
interface IAuditorMatchResult {
|
|
43
|
+
/** 是否匹配到敏感词 */
|
|
44
|
+
Matched: boolean;
|
|
45
|
+
/** 匹配到的敏感词列表(去重,按首次出现顺序) */
|
|
46
|
+
Words: string[];
|
|
47
|
+
/** 所有匹配位置信息 */
|
|
48
|
+
Matches: IAuditorMatchPosition[];
|
|
49
|
+
}
|
|
50
|
+
/** AC 自动机节点接口 */
|
|
51
|
+
interface IACNode {
|
|
52
|
+
/** 子节点映射 */
|
|
53
|
+
Children: Map<string, IACNode>;
|
|
54
|
+
/** 失败指针 */
|
|
55
|
+
Fail: IACNode | null;
|
|
56
|
+
/** 该节点自身的完整敏感词(词尾节点非空) */
|
|
57
|
+
Words: string[];
|
|
58
|
+
/** 匹配输出(自身词 + fail 链词,构建时由 words 重新生成) */
|
|
59
|
+
Output: string[];
|
|
60
|
+
}
|
|
61
|
+
/** AC 自动机匹配结果 */
|
|
62
|
+
interface IACMatchResult {
|
|
63
|
+
/** 匹配到的敏感词 */
|
|
64
|
+
Word: string;
|
|
65
|
+
/** 起始位置 */
|
|
66
|
+
Start: number;
|
|
67
|
+
/** 结束位置 */
|
|
68
|
+
End: number;
|
|
69
|
+
}
|
|
70
|
+
/** 敏感词审计员插件接口 */
|
|
71
|
+
interface IAuditorPlugin extends IPlugin<AuditorDeps, AuditorCfg> {
|
|
72
|
+
/**
|
|
73
|
+
* 检测文本是否包含敏感词
|
|
74
|
+
* @param text 待检测的文本
|
|
75
|
+
* @returns 是否包含敏感词
|
|
76
|
+
*/
|
|
77
|
+
Test(text: string): boolean;
|
|
78
|
+
/**
|
|
79
|
+
* 查找文本中的所有敏感词
|
|
80
|
+
* @param text 待查找的文本
|
|
81
|
+
* @returns 匹配结果
|
|
82
|
+
*/
|
|
83
|
+
Find(text: string): IAuditorMatchResult;
|
|
84
|
+
/**
|
|
85
|
+
* 替换文本中的敏感词
|
|
86
|
+
*
|
|
87
|
+
* - 每个匹配区间(重叠匹配先合并为不重叠区间)整体替换为一个
|
|
88
|
+
* replacement 串(非逐字符屏蔽)
|
|
89
|
+
* - 区间内的干扰字符一并被替换(变体表达整体屏蔽)
|
|
90
|
+
* @param text 待替换的文本
|
|
91
|
+
* @param replacement 替换串,缺省使用配置的替换字符;传空串为删除敏感词语义
|
|
92
|
+
* @returns 替换后的文本
|
|
93
|
+
*/
|
|
94
|
+
Replace(text: string, replacement?: string): string;
|
|
95
|
+
/**
|
|
96
|
+
* 注入敏感词
|
|
97
|
+
* @param words 敏感词列表
|
|
98
|
+
*/
|
|
99
|
+
InjectSensitiveWords(words: string[]): void;
|
|
100
|
+
/**
|
|
101
|
+
* 注入干扰词
|
|
102
|
+
*
|
|
103
|
+
* - 干扰词在匹配时被跳过(如 "敏 感 词" 仍命中),仅单字符
|
|
104
|
+
* 生效,多字符词将被忽略并告警
|
|
105
|
+
* - 敏感词本身包含干扰字符时该词将无法命中,注入时须避开词库字符
|
|
106
|
+
* @param words 干扰词列表
|
|
107
|
+
*/
|
|
108
|
+
InjectInterferenceWords(words: string[]): void;
|
|
109
|
+
/**
|
|
110
|
+
* 获取当前敏感词数量
|
|
111
|
+
* @returns 敏感词数量
|
|
112
|
+
*/
|
|
113
|
+
GetSensitiveWordCount(): number;
|
|
114
|
+
/**
|
|
115
|
+
* 获取当前干扰词数量
|
|
116
|
+
* @returns 干扰词数量
|
|
117
|
+
*/
|
|
118
|
+
GetInterferenceWordCount(): number;
|
|
119
|
+
/** 清空所有敏感词 */
|
|
120
|
+
ClearSensitiveWords(): void;
|
|
121
|
+
/** 清空所有干扰词 */
|
|
122
|
+
ClearInterferenceWords(): void;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* 敏感词审计员插件
|
|
127
|
+
*/
|
|
128
|
+
declare class AuditorPlugin extends Plugin<AuditorDeps, AuditorCfg> implements IAuditorPlugin {
|
|
129
|
+
private _acAutomaton;
|
|
130
|
+
private _testCache;
|
|
131
|
+
private _findCache;
|
|
132
|
+
private _replaceCache;
|
|
133
|
+
private _sensitiveWords;
|
|
134
|
+
constructor(deps: AuditorDeps, cfg: AuditorCfg);
|
|
135
|
+
Test(text: string): boolean;
|
|
136
|
+
Find(text: string): IAuditorMatchResult;
|
|
137
|
+
Replace(text: string, replacement?: string): string;
|
|
138
|
+
InjectSensitiveWords(words: string[]): void;
|
|
139
|
+
InjectInterferenceWords(words: string[]): void;
|
|
140
|
+
GetSensitiveWordCount(): number;
|
|
141
|
+
GetInterferenceWordCount(): number;
|
|
142
|
+
ClearSensitiveWords(): void;
|
|
143
|
+
ClearInterferenceWords(): void;
|
|
144
|
+
/**
|
|
145
|
+
* 构建 AC 自动机
|
|
146
|
+
*/
|
|
147
|
+
private buildAutomaton;
|
|
148
|
+
/**
|
|
149
|
+
* 清空缓存
|
|
150
|
+
*/
|
|
151
|
+
private clearCache;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export { AuditorPlugin };
|
|
155
|
+
export type { AuditorCfg, AuditorDeps, IACMatchResult, IACNode, IAuditorMatchPosition, IAuditorMatchResult, IAuditorPlugin };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var t=require("@nebulaaa/dict"),e=require("@nebulaaa/lru"),r=require("@nebulaaa/plugin");class n{constructor(t=!1){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._built=!1,this._interferenceWords=new Set,this._ignoreCase=t}normalize(t){if(!this._ignoreCase)return t;let e="";for(const r of t){const t=r.codePointAt(0);e+=t>=65&&t<=90?String.fromCharCode(t+32):r}return e}AddWord(t){if(!t||0===t.length)return;let e=this._root;const r=Array.from(this.normalize(t));for(const t of r){let r=e.Children.get(t);r||(r={Children:new Map,Fail:null,Words:[],Output:[]},e.Children.set(t,r)),e=r}e.Words.includes(t)||e.Words.push(t),this._built=!1}Build(){if(this._built)return;const t=[];for(const e of this._root.Children.values())e.Fail=this._root,e.Output=[...e.Words],t.push(e);let e=0;for(;e<t.length;){const r=t[e++];for(const[e,n]of r.Children){t.push(n);let s=r.Fail;for(;s&&!s.Children.has(e);)s=s.Fail;n.Fail=s&&s.Children.get(e)||this._root,n.Output=[...n.Words,...n.Fail.Output]}}this._built=!0}Match(t){this._built||this.Build();const e=[];let r=this._root;const n=Array.from(this.normalize(t));for(let t=0;t<n.length;t++){const s=n[t];if(!this._interferenceWords.has(s)){for(;r!==this._root&&!r.Children.has(s);)r=r.Fail;if(r.Children.has(s)){r=r.Children.get(s);for(const s of r.Output){let r=t;const i=Array.from(this.normalize(s));let o=i.length-1;for(;r>=0&&o>=0;)this._interferenceWords.has(n[r])||n[r]===i[o]&&o--,r--;e.push({Word:s,Start:r+1,End:t+1})}}}}return e}Test(t){this._built||this.Build();let e=this._root;const r=Array.from(this.normalize(t));for(let t=0;t<r.length;t++){const n=r[t];if(!this._interferenceWords.has(n)){for(;e!==this._root&&!e.Children.has(n);)e=e.Fail;if(e.Children.has(n)&&(e=e.Children.get(n),e.Output.length>0))return!0}}return!1}Clear(){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._built=!1}AddInterferenceWords(t){for(const e of t){if(!e||0===e.length)continue;const t=Array.from(this.normalize(e));1===t.length?this._interferenceWords.add(t[0]):console.warn(`addInterferenceWords: 干扰词须为单个字符,已忽略 "${e}"`)}}ClearInterferenceWords(){this._interferenceWords.clear()}GetInterferenceWordCount(){return this._interferenceWords.size}}class s extends r.Plugin{constructor(r,s){if(t.Override(s,{Replacement:"█",IgnoreCase:!1,CacheSize:1e3}),super(r,s),this._acAutomaton=new n(s.IgnoreCase),this._sensitiveWords=new Set,this._acAutomaton.AddInterferenceWords(" \t\r\n".split("")),s.EnableCache){if(!Number.isInteger(s.CacheSize)||s.CacheSize<=0)throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${s.CacheSize}`);this._testCache=new e.LRU(s.CacheSize),this._findCache=new e.LRU(s.CacheSize),this._replaceCache=new e.LRU(s.CacheSize)}else this._testCache=null,this._findCache=null,this._replaceCache=null;this.buildAutomaton()}Test(t){if(!t||0===t.length)return!1;if(this._testCache){const e=this._testCache.Get(t);if(null!==e)return e}const e=this._acAutomaton.Test(t);return this._testCache&&this._testCache.Set(t,e),e}Find(t){if(!t||0===t.length)return{Matched:!1,Words:[],Matches:[]};if(this._findCache){const e=this._findCache.Get(t);if(null!==e)return{Matched:e.Matched,Words:[...e.Words],Matches:e.Matches.map(t=>Object.assign({},t))}}const e=this._acAutomaton.Match(t),r=new Set;for(const t of e)r.add(t.Word);const n={Matched:e.length>0,Words:Array.from(r),Matches:e};return this._findCache&&this._findCache.Set(t,{Matched:n.Matched,Words:[...n.Words],Matches:n.Matches.map(t=>Object.assign({},t))}),n}Replace(t,e){var r;if(!t||0===t.length)return t;const n=JSON.stringify([t,null!=e?e:this.Cfg.Replacement]);if(this._replaceCache){const t=this._replaceCache.Get(n);if(null!==t)return t}const s=this.Find(t);if(!s.Matched)return null===(r=this._replaceCache)||void 0===r||r.Set(n,t),t;const i=Array.from(t),o=[...s.Matches].sort((t,e)=>t.Start-e.Start),h=[];for(const t of o){const e=h[h.length-1];e&&t.Start<e.end?e.end=Math.max(e.end,t.End):h.push({start:t.Start,end:t.End})}const a=null!=e?e:this.Cfg.Replacement;for(let t=h.length-1;t>=0;t--){const{start:e,end:r}=h[t];i.splice(e,r-e,a)}const c=i.join("");return this._replaceCache&&this._replaceCache.Set(n,c),c}InjectSensitiveWords(t){for(const e of t)e&&e.length>0&&this._sensitiveWords.add(e);this.buildAutomaton(),this.clearCache()}InjectInterferenceWords(t){this._acAutomaton.AddInterferenceWords(t),this.clearCache()}GetSensitiveWordCount(){return this._sensitiveWords.size}GetInterferenceWordCount(){return this._acAutomaton.GetInterferenceWordCount()}ClearSensitiveWords(){this._sensitiveWords.clear(),this.buildAutomaton(),this.clearCache()}ClearInterferenceWords(){this._acAutomaton.ClearInterferenceWords(),this.clearCache()}buildAutomaton(){this._acAutomaton.Clear();const t=this._sensitiveWords;for(const e of t)this._acAutomaton.AddWord(e);this._acAutomaton.Build()}clearCache(){var t,e,r;null===(t=this._testCache)||void 0===t||t.Clear(),null===(e=this._findCache)||void 0===e||e.Clear(),null===(r=this._replaceCache)||void 0===r||r.Clear()}}exports.AuditorPlugin=s;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{Override as t}from"@nebulaaa/dict";import{LRU as e}from"@nebulaaa/lru";import{Plugin as r}from"@nebulaaa/plugin";class n{constructor(t=!1){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._built=!1,this._interferenceWords=new Set,this._ignoreCase=t}normalize(t){if(!this._ignoreCase)return t;let e="";for(const r of t){const t=r.codePointAt(0);e+=t>=65&&t<=90?String.fromCharCode(t+32):r}return e}AddWord(t){if(!t||0===t.length)return;let e=this._root;const r=Array.from(this.normalize(t));for(const t of r){let r=e.Children.get(t);r||(r={Children:new Map,Fail:null,Words:[],Output:[]},e.Children.set(t,r)),e=r}e.Words.includes(t)||e.Words.push(t),this._built=!1}Build(){if(this._built)return;const t=[];for(const e of this._root.Children.values())e.Fail=this._root,e.Output=[...e.Words],t.push(e);let e=0;for(;e<t.length;){const r=t[e++];for(const[e,n]of r.Children){t.push(n);let s=r.Fail;for(;s&&!s.Children.has(e);)s=s.Fail;n.Fail=s&&s.Children.get(e)||this._root,n.Output=[...n.Words,...n.Fail.Output]}}this._built=!0}Match(t){this._built||this.Build();const e=[];let r=this._root;const n=Array.from(this.normalize(t));for(let t=0;t<n.length;t++){const s=n[t];if(!this._interferenceWords.has(s)){for(;r!==this._root&&!r.Children.has(s);)r=r.Fail;if(r.Children.has(s)){r=r.Children.get(s);for(const s of r.Output){let r=t;const i=Array.from(this.normalize(s));let o=i.length-1;for(;r>=0&&o>=0;)this._interferenceWords.has(n[r])||n[r]===i[o]&&o--,r--;e.push({Word:s,Start:r+1,End:t+1})}}}}return e}Test(t){this._built||this.Build();let e=this._root;const r=Array.from(this.normalize(t));for(let t=0;t<r.length;t++){const n=r[t];if(!this._interferenceWords.has(n)){for(;e!==this._root&&!e.Children.has(n);)e=e.Fail;if(e.Children.has(n)&&(e=e.Children.get(n),e.Output.length>0))return!0}}return!1}Clear(){this._root={Children:new Map,Fail:null,Words:[],Output:[]},this._built=!1}AddInterferenceWords(t){for(const e of t){if(!e||0===e.length)continue;const t=Array.from(this.normalize(e));1===t.length?this._interferenceWords.add(t[0]):console.warn(`addInterferenceWords: 干扰词须为单个字符,已忽略 "${e}"`)}}ClearInterferenceWords(){this._interferenceWords.clear()}GetInterferenceWordCount(){return this._interferenceWords.size}}class s extends r{constructor(r,s){if(t(s,{Replacement:"█",IgnoreCase:!1,CacheSize:1e3}),super(r,s),this._acAutomaton=new n(s.IgnoreCase),this._sensitiveWords=new Set,this._acAutomaton.AddInterferenceWords(" \t\r\n".split("")),s.EnableCache){if(!Number.isInteger(s.CacheSize)||s.CacheSize<=0)throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${s.CacheSize}`);this._testCache=new e(s.CacheSize),this._findCache=new e(s.CacheSize),this._replaceCache=new e(s.CacheSize)}else this._testCache=null,this._findCache=null,this._replaceCache=null;this.buildAutomaton()}Test(t){if(!t||0===t.length)return!1;if(this._testCache){const e=this._testCache.Get(t);if(null!==e)return e}const e=this._acAutomaton.Test(t);return this._testCache&&this._testCache.Set(t,e),e}Find(t){if(!t||0===t.length)return{Matched:!1,Words:[],Matches:[]};if(this._findCache){const e=this._findCache.Get(t);if(null!==e)return{Matched:e.Matched,Words:[...e.Words],Matches:e.Matches.map(t=>Object.assign({},t))}}const e=this._acAutomaton.Match(t),r=new Set;for(const t of e)r.add(t.Word);const n={Matched:e.length>0,Words:Array.from(r),Matches:e};return this._findCache&&this._findCache.Set(t,{Matched:n.Matched,Words:[...n.Words],Matches:n.Matches.map(t=>Object.assign({},t))}),n}Replace(t,e){var r;if(!t||0===t.length)return t;const n=JSON.stringify([t,null!=e?e:this.Cfg.Replacement]);if(this._replaceCache){const t=this._replaceCache.Get(n);if(null!==t)return t}const s=this.Find(t);if(!s.Matched)return null===(r=this._replaceCache)||void 0===r||r.Set(n,t),t;const i=Array.from(t),o=[...s.Matches].sort((t,e)=>t.Start-e.Start),h=[];for(const t of o){const e=h[h.length-1];e&&t.Start<e.end?e.end=Math.max(e.end,t.End):h.push({start:t.Start,end:t.End})}const a=null!=e?e:this.Cfg.Replacement;for(let t=h.length-1;t>=0;t--){const{start:e,end:r}=h[t];i.splice(e,r-e,a)}const c=i.join("");return this._replaceCache&&this._replaceCache.Set(n,c),c}InjectSensitiveWords(t){for(const e of t)e&&e.length>0&&this._sensitiveWords.add(e);this.buildAutomaton(),this.clearCache()}InjectInterferenceWords(t){this._acAutomaton.AddInterferenceWords(t),this.clearCache()}GetSensitiveWordCount(){return this._sensitiveWords.size}GetInterferenceWordCount(){return this._acAutomaton.GetInterferenceWordCount()}ClearSensitiveWords(){this._sensitiveWords.clear(),this.buildAutomaton(),this.clearCache()}ClearInterferenceWords(){this._acAutomaton.ClearInterferenceWords(),this.clearCache()}buildAutomaton(){this._acAutomaton.Clear();const t=this._sensitiveWords;for(const e of t)this._acAutomaton.AddWord(e);this._acAutomaton.Build()}clearCache(){var t,e,r;null===(t=this._testCache)||void 0===t||t.Clear(),null===(e=this._findCache)||void 0===e||e.Clear(),null===(r=this._replaceCache)||void 0===r||r.Clear()}}export{s as AuditorPlugin};
|
package/dist/auditor.mjs
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
import { Override } from '@nebulaaa/dict';
|
|
2
|
+
import { LRU } from '@nebulaaa/lru';
|
|
3
|
+
import { Plugin } from '@nebulaaa/plugin';
|
|
4
|
+
|
|
5
|
+
/** AC 自动机 */
|
|
6
|
+
class ACAutomaton {
|
|
7
|
+
constructor(ignoreCase = false) {
|
|
8
|
+
this._root = {
|
|
9
|
+
Children: new Map(),
|
|
10
|
+
Fail: null,
|
|
11
|
+
Words: [],
|
|
12
|
+
Output: [],
|
|
13
|
+
};
|
|
14
|
+
this._built = false;
|
|
15
|
+
this._interferenceWords = new Set();
|
|
16
|
+
this._ignoreCase = ignoreCase;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* 大小写归一(仅 ASCII 字母,逐码点等长替换,保证匹配位置与原始文本索引一致)
|
|
20
|
+
*/
|
|
21
|
+
normalize(text) {
|
|
22
|
+
if (!this._ignoreCase) {
|
|
23
|
+
return text;
|
|
24
|
+
}
|
|
25
|
+
let out = '';
|
|
26
|
+
for (const ch of text) {
|
|
27
|
+
const code = ch.codePointAt(0);
|
|
28
|
+
out += code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : ch;
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* 添加敏感词
|
|
34
|
+
* @param word 敏感词
|
|
35
|
+
*/
|
|
36
|
+
AddWord(word) {
|
|
37
|
+
if (!word || word.length === 0) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
let current = this._root;
|
|
41
|
+
const chars = Array.from(this.normalize(word)); // 归一后分词(IgnoreCase 关闭时原样返回)
|
|
42
|
+
for (const char of chars) {
|
|
43
|
+
let child = current.Children.get(char);
|
|
44
|
+
if (!child) {
|
|
45
|
+
child = {
|
|
46
|
+
Children: new Map(),
|
|
47
|
+
Fail: null,
|
|
48
|
+
Words: [],
|
|
49
|
+
Output: [],
|
|
50
|
+
};
|
|
51
|
+
current.Children.set(char, child);
|
|
52
|
+
}
|
|
53
|
+
current = child;
|
|
54
|
+
}
|
|
55
|
+
if (!current.Words.includes(word)) {
|
|
56
|
+
// 同词重复添加幂等:自身词独立存放,output 由 build 重新生成
|
|
57
|
+
current.Words.push(word);
|
|
58
|
+
}
|
|
59
|
+
this._built = false; // 标记需要重新构建 fail 指针
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* 构建 fail 指针(使用 BFS)
|
|
63
|
+
*/
|
|
64
|
+
Build() {
|
|
65
|
+
if (this._built) {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
const queue = [];
|
|
69
|
+
// 第一层节点的 fail 指针指向 root
|
|
70
|
+
for (const child of this._root.Children.values()) {
|
|
71
|
+
child.Fail = this._root;
|
|
72
|
+
child.Output = [...child.Words]; // 由自身词重新生成(fail 为 root,无合并项)
|
|
73
|
+
queue.push(child);
|
|
74
|
+
}
|
|
75
|
+
// BFS 构建 fail 指针(头指针索引出队:array.shift 为 O(n),万级节点的大词库构建退化为 O(n²))
|
|
76
|
+
let head = 0;
|
|
77
|
+
while (head < queue.length) {
|
|
78
|
+
const current = queue[head++];
|
|
79
|
+
for (const [char, child] of current.Children) {
|
|
80
|
+
queue.push(child);
|
|
81
|
+
// 查找 fail 指针
|
|
82
|
+
let fail = current.Fail;
|
|
83
|
+
while (fail && !fail.Children.has(char)) {
|
|
84
|
+
fail = fail.Fail;
|
|
85
|
+
}
|
|
86
|
+
child.Fail = fail ? fail.Children.get(char) || this._root : this._root;
|
|
87
|
+
// 合并 output:由自身词重新生成(幂等,重复构建不再累积;fail 节点层级更浅、已先构建完成)
|
|
88
|
+
child.Output = [...child.Words, ...child.Fail.Output];
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
this._built = true;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* 匹配文本,返回所有匹配结果
|
|
95
|
+
* @param text 待匹配的文本
|
|
96
|
+
* @returns 匹配结果列表
|
|
97
|
+
*/
|
|
98
|
+
Match(text) {
|
|
99
|
+
if (!this._built) {
|
|
100
|
+
this.Build();
|
|
101
|
+
}
|
|
102
|
+
const results = [];
|
|
103
|
+
let current = this._root;
|
|
104
|
+
const chars = Array.from(this.normalize(text)); // 归一后分词(逐码点等长,位置索引不变)
|
|
105
|
+
// 记录匹配时的原始位置
|
|
106
|
+
for (let i = 0; i < chars.length; i++) {
|
|
107
|
+
const char = chars[i];
|
|
108
|
+
// 如果是干扰字符,跳过
|
|
109
|
+
if (this._interferenceWords.has(char)) {
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
// 沿着 fail 指针查找匹配的子节点
|
|
113
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
114
|
+
current = current.Fail;
|
|
115
|
+
}
|
|
116
|
+
if (current.Children.has(char)) {
|
|
117
|
+
current = current.Children.get(char);
|
|
118
|
+
// 输出所有匹配的敏感词
|
|
119
|
+
for (const word of current.Output) {
|
|
120
|
+
// 计算在原始文本中的起始位置(跳过干扰字符)
|
|
121
|
+
let j = i;
|
|
122
|
+
// 从当前位置向前查找匹配的起始位置
|
|
123
|
+
const wordChars = Array.from(this.normalize(word));
|
|
124
|
+
let wordIndex = wordChars.length - 1;
|
|
125
|
+
// 从后向前匹配,找到原始文本中的位置
|
|
126
|
+
while (j >= 0 && wordIndex >= 0) {
|
|
127
|
+
if (this._interferenceWords.has(chars[j])) {
|
|
128
|
+
j--;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (chars[j] === wordChars[wordIndex]) {
|
|
132
|
+
wordIndex--;
|
|
133
|
+
}
|
|
134
|
+
j--;
|
|
135
|
+
}
|
|
136
|
+
results.push({
|
|
137
|
+
Word: word,
|
|
138
|
+
Start: j + 1,
|
|
139
|
+
End: i + 1,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return results;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* 检测文本是否包含敏感词
|
|
148
|
+
* @param text 待检测的文本
|
|
149
|
+
* @returns 是否包含敏感词
|
|
150
|
+
*/
|
|
151
|
+
Test(text) {
|
|
152
|
+
if (!this._built) {
|
|
153
|
+
this.Build();
|
|
154
|
+
}
|
|
155
|
+
let current = this._root;
|
|
156
|
+
const chars = Array.from(this.normalize(text));
|
|
157
|
+
for (let i = 0; i < chars.length; i++) {
|
|
158
|
+
const char = chars[i];
|
|
159
|
+
// 如果是干扰字符,跳过
|
|
160
|
+
if (this._interferenceWords.has(char)) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
while (current !== this._root && !current.Children.has(char)) {
|
|
164
|
+
current = current.Fail;
|
|
165
|
+
}
|
|
166
|
+
if (current.Children.has(char)) {
|
|
167
|
+
current = current.Children.get(char);
|
|
168
|
+
if (current.Output.length > 0) {
|
|
169
|
+
return true;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* 清空所有节点
|
|
177
|
+
*/
|
|
178
|
+
Clear() {
|
|
179
|
+
this._root = {
|
|
180
|
+
Children: new Map(),
|
|
181
|
+
Fail: null,
|
|
182
|
+
Words: [],
|
|
183
|
+
Output: [],
|
|
184
|
+
};
|
|
185
|
+
this._built = false;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* 注入干扰词(仅单字符生效:匹配按单字符查表)
|
|
189
|
+
* @param words 干扰词列表
|
|
190
|
+
*/
|
|
191
|
+
AddInterferenceWords(words) {
|
|
192
|
+
for (const word of words) {
|
|
193
|
+
if (!word || word.length === 0) {
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const chars = Array.from(this.normalize(word));
|
|
197
|
+
if (chars.length !== 1) {
|
|
198
|
+
// 多字符干扰词在按字符匹配的查表中永不命中——显式告警而非静默失效
|
|
199
|
+
console.warn(`addInterferenceWords: 干扰词须为单个字符,已忽略 "${word}"`);
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
this._interferenceWords.add(chars[0]);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* 清空干扰词
|
|
207
|
+
*/
|
|
208
|
+
ClearInterferenceWords() {
|
|
209
|
+
this._interferenceWords.clear();
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* 获取干扰词数量
|
|
213
|
+
*/
|
|
214
|
+
GetInterferenceWordCount() {
|
|
215
|
+
return this._interferenceWords.size;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* 敏感词审计员插件
|
|
220
|
+
*/
|
|
221
|
+
class AuditorPlugin extends Plugin {
|
|
222
|
+
constructor(deps, cfg) {
|
|
223
|
+
// 默认值填充必须在 super 之前:基类构造会冻结 cfg,冻结后 defineProperty 会抛错
|
|
224
|
+
Override(cfg, {
|
|
225
|
+
Replacement: '█',
|
|
226
|
+
IgnoreCase: false,
|
|
227
|
+
CacheSize: 1000,
|
|
228
|
+
});
|
|
229
|
+
super(deps, cfg);
|
|
230
|
+
this._acAutomaton = new ACAutomaton(cfg.IgnoreCase);
|
|
231
|
+
this._sensitiveWords = new Set();
|
|
232
|
+
this._acAutomaton.AddInterferenceWords(' \t\r\n'.split(''));
|
|
233
|
+
if (cfg.EnableCache) {
|
|
234
|
+
// 缓存容量须为正整数:非法配置构造期显式失败,错误信息带插件上下文便于定位
|
|
235
|
+
if (!Number.isInteger(cfg.CacheSize) || cfg.CacheSize <= 0) {
|
|
236
|
+
throw new Error(`AuditorPlugin CacheSize 须为正整数 >>> ${cfg.CacheSize}`);
|
|
237
|
+
}
|
|
238
|
+
this._testCache = new LRU(cfg.CacheSize);
|
|
239
|
+
this._findCache = new LRU(cfg.CacheSize);
|
|
240
|
+
this._replaceCache = new LRU(cfg.CacheSize);
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
this._testCache = null;
|
|
244
|
+
this._findCache = null;
|
|
245
|
+
this._replaceCache = null;
|
|
246
|
+
}
|
|
247
|
+
// 构建敏感词自动机
|
|
248
|
+
this.buildAutomaton();
|
|
249
|
+
}
|
|
250
|
+
Test(text) {
|
|
251
|
+
if (!text || text.length === 0) {
|
|
252
|
+
return false;
|
|
253
|
+
}
|
|
254
|
+
// 检查缓存
|
|
255
|
+
if (this._testCache) {
|
|
256
|
+
const cached = this._testCache.Get(text);
|
|
257
|
+
if (cached !== null) {
|
|
258
|
+
return cached;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
const result = this._acAutomaton.Test(text);
|
|
262
|
+
// 更新缓存
|
|
263
|
+
if (this._testCache) {
|
|
264
|
+
this._testCache.Set(text, result);
|
|
265
|
+
}
|
|
266
|
+
return result;
|
|
267
|
+
}
|
|
268
|
+
Find(text) {
|
|
269
|
+
if (!text || text.length === 0) {
|
|
270
|
+
return { Matched: false, Words: [], Matches: [] };
|
|
271
|
+
}
|
|
272
|
+
// 检查缓存
|
|
273
|
+
if (this._findCache) {
|
|
274
|
+
const cached = this._findCache.Get(text);
|
|
275
|
+
if (cached !== null) {
|
|
276
|
+
// 返回浅拷贝,防止外部修改影响缓存
|
|
277
|
+
return {
|
|
278
|
+
Matched: cached.Matched,
|
|
279
|
+
Words: [...cached.Words],
|
|
280
|
+
Matches: cached.Matches.map((m) => (Object.assign({}, m))),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// 匹配文本,自动跳过干扰词
|
|
285
|
+
const matches = this._acAutomaton.Match(text);
|
|
286
|
+
// 去重敏感词
|
|
287
|
+
const wordSet = new Set();
|
|
288
|
+
for (const match of matches) {
|
|
289
|
+
wordSet.add(match.Word);
|
|
290
|
+
}
|
|
291
|
+
const result = {
|
|
292
|
+
Matched: matches.length > 0,
|
|
293
|
+
Words: Array.from(wordSet),
|
|
294
|
+
Matches: matches,
|
|
295
|
+
};
|
|
296
|
+
// 更新缓存(存储副本,防止后续修改影响缓存)
|
|
297
|
+
if (this._findCache) {
|
|
298
|
+
this._findCache.Set(text, {
|
|
299
|
+
Matched: result.Matched,
|
|
300
|
+
Words: [...result.Words],
|
|
301
|
+
Matches: result.Matches.map((m) => (Object.assign({}, m))),
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
return result;
|
|
305
|
+
}
|
|
306
|
+
Replace(text, replacement) {
|
|
307
|
+
var _a;
|
|
308
|
+
if (!text || text.length === 0) {
|
|
309
|
+
return text;
|
|
310
|
+
}
|
|
311
|
+
// 检查缓存(JSON 编码组合键:字符串拼接的 "_" 分隔符可出现在 text/replacement 中造成碰撞)
|
|
312
|
+
const cacheKey = JSON.stringify([text, replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement]);
|
|
313
|
+
if (this._replaceCache) {
|
|
314
|
+
const cached = this._replaceCache.Get(cacheKey);
|
|
315
|
+
if (cached !== null) {
|
|
316
|
+
return cached;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
const result = this.Find(text);
|
|
320
|
+
if (!result.Matched) {
|
|
321
|
+
// 干净文本是最高频流量,同样入缓存(组合键已防碰撞),避免反复全量匹配
|
|
322
|
+
(_a = this._replaceCache) === null || _a === void 0 ? void 0 : _a.Set(cacheKey, text);
|
|
323
|
+
return text;
|
|
324
|
+
}
|
|
325
|
+
// 在码点数组上操作:matches 的位置基于 Array.from 分词(码点索引),
|
|
326
|
+
// 直接对原字符串 slice 会因代理对(emoji 等)长度不一致而切错位置
|
|
327
|
+
const chars = Array.from(text);
|
|
328
|
+
// 合并重叠区间:重叠匹配若逐个替换,从后向前切片会互相错位(吞字或残留敏感片段)
|
|
329
|
+
const sorted = [...result.Matches].sort((a, b) => a.Start - b.Start);
|
|
330
|
+
const ranges = [];
|
|
331
|
+
for (const match of sorted) {
|
|
332
|
+
const last = ranges[ranges.length - 1];
|
|
333
|
+
if (last && match.Start < last.end) {
|
|
334
|
+
last.end = Math.max(last.end, match.End); // 重叠:并入上一区间
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
ranges.push({ start: match.Start, end: match.End });
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
// 每个合并区间整体替换为一个 replacement 串(非逐字符屏蔽);空串为合法替换值(删除敏感词)
|
|
341
|
+
const replaceChar = replacement !== null && replacement !== void 0 ? replacement : this.Cfg.Replacement;
|
|
342
|
+
for (let i = ranges.length - 1; i >= 0; i--) {
|
|
343
|
+
const { start, end } = ranges[i];
|
|
344
|
+
chars.splice(start, end - start, replaceChar); // 从后向前,前面的索引不受影响
|
|
345
|
+
}
|
|
346
|
+
const replacedText = chars.join('');
|
|
347
|
+
// 更新缓存
|
|
348
|
+
if (this._replaceCache) {
|
|
349
|
+
this._replaceCache.Set(cacheKey, replacedText);
|
|
350
|
+
}
|
|
351
|
+
return replacedText;
|
|
352
|
+
}
|
|
353
|
+
InjectSensitiveWords(words) {
|
|
354
|
+
for (const word of words) {
|
|
355
|
+
if (word && word.length > 0) {
|
|
356
|
+
this._sensitiveWords.add(word);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
this.buildAutomaton();
|
|
360
|
+
this.clearCache();
|
|
361
|
+
}
|
|
362
|
+
InjectInterferenceWords(words) {
|
|
363
|
+
this._acAutomaton.AddInterferenceWords(words);
|
|
364
|
+
this.clearCache();
|
|
365
|
+
}
|
|
366
|
+
GetSensitiveWordCount() {
|
|
367
|
+
return this._sensitiveWords.size;
|
|
368
|
+
}
|
|
369
|
+
GetInterferenceWordCount() {
|
|
370
|
+
return this._acAutomaton.GetInterferenceWordCount();
|
|
371
|
+
}
|
|
372
|
+
ClearSensitiveWords() {
|
|
373
|
+
this._sensitiveWords.clear();
|
|
374
|
+
this.buildAutomaton();
|
|
375
|
+
this.clearCache();
|
|
376
|
+
}
|
|
377
|
+
ClearInterferenceWords() {
|
|
378
|
+
this._acAutomaton.ClearInterferenceWords();
|
|
379
|
+
this.clearCache();
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* 构建 AC 自动机
|
|
383
|
+
*/
|
|
384
|
+
buildAutomaton() {
|
|
385
|
+
this._acAutomaton.Clear();
|
|
386
|
+
const words = this._sensitiveWords;
|
|
387
|
+
for (const word of words) {
|
|
388
|
+
this._acAutomaton.AddWord(word);
|
|
389
|
+
}
|
|
390
|
+
this._acAutomaton.Build();
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* 清空缓存
|
|
394
|
+
*/
|
|
395
|
+
clearCache() {
|
|
396
|
+
var _a, _b, _c;
|
|
397
|
+
(_a = this._testCache) === null || _a === void 0 ? void 0 : _a.Clear();
|
|
398
|
+
(_b = this._findCache) === null || _b === void 0 ? void 0 : _b.Clear();
|
|
399
|
+
(_c = this._replaceCache) === null || _c === void 0 ? void 0 : _c.Clear();
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
export { AuditorPlugin };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@nebulaaa/auditor",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "敏感词审计员插件",
|
|
5
|
+
"author": "doooreyn",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"keywords": [
|
|
8
|
+
"auditor",
|
|
9
|
+
"nebulaaa"
|
|
10
|
+
],
|
|
11
|
+
"engines": {
|
|
12
|
+
"node": ">=16.0.0",
|
|
13
|
+
"npm": ">=10.0.0"
|
|
14
|
+
},
|
|
15
|
+
"main": "./auditor.cjs",
|
|
16
|
+
"module": "./auditor.mjs",
|
|
17
|
+
"types": "./auditor.d.ts",
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"types": "./auditor.d.ts",
|
|
21
|
+
"require": "./auditor.cjs",
|
|
22
|
+
"import": "./auditor.mjs",
|
|
23
|
+
"default": "./auditor.cjs"
|
|
24
|
+
},
|
|
25
|
+
"./min": {
|
|
26
|
+
"require": "./auditor.min.cjs",
|
|
27
|
+
"import": "./auditor.min.mjs"
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"publishConfig": {
|
|
31
|
+
"registry": "https://registry.npmjs.org/",
|
|
32
|
+
"access": "public"
|
|
33
|
+
}
|
|
34
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,70 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@nebulaaa/auditor",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "@nebulaaa/auditor",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "敏感词审计员插件",
|
|
5
|
+
"main": "./dist/auditor.cjs",
|
|
6
|
+
"module": "./dist/auditor.mjs",
|
|
7
|
+
"types": "./dist/auditor.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"require": "./dist/auditor.cjs",
|
|
11
|
+
"import": "./dist/auditor.mjs",
|
|
12
|
+
"types": "./dist/auditor.d.ts",
|
|
13
|
+
"default": "./dist/auditor.cjs"
|
|
14
|
+
},
|
|
15
|
+
"./min": {
|
|
16
|
+
"require": "./dist/auditor.min.cjs",
|
|
17
|
+
"import": "./dist/auditor.min.mjs"
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"files": [
|
|
21
|
+
"dist"
|
|
22
|
+
],
|
|
23
|
+
"scripts": {
|
|
24
|
+
"clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"",
|
|
25
|
+
"build": "npm run clean && rollup -c rollup.config.mjs && node ../scripts/write-dist-package.mjs auditor",
|
|
26
|
+
"test": "jest"
|
|
27
|
+
},
|
|
28
|
+
"keywords": [
|
|
29
|
+
"auditor",
|
|
30
|
+
"nebulaaa"
|
|
31
|
+
],
|
|
32
|
+
"author": "doooreyn",
|
|
33
|
+
"license": "MIT",
|
|
34
|
+
"devEngines": {
|
|
35
|
+
"packageManager": {
|
|
36
|
+
"name": "npm",
|
|
37
|
+
"version": ">=10.0.0",
|
|
38
|
+
"onFail": "warn"
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"engines": {
|
|
42
|
+
"node": ">=16.0.0",
|
|
43
|
+
"npm": ">=10.0.0"
|
|
44
|
+
},
|
|
45
|
+
"publishConfig": {
|
|
46
|
+
"registry": "https://registry.npmjs.org/",
|
|
47
|
+
"access": "public"
|
|
48
|
+
},
|
|
49
|
+
"allowScripts": {
|
|
50
|
+
"unrs-resolver@1.12.2": true,
|
|
51
|
+
"@parcel/watcher@2.6.0": true
|
|
52
|
+
},
|
|
53
|
+
"devDependencies": {
|
|
54
|
+
"@rollup/plugin-terser": "^1.0.2",
|
|
55
|
+
"@rollup/plugin-typescript": "^12.1.2",
|
|
56
|
+
"@types/jest": "^30.0.0",
|
|
57
|
+
"@types/node": "^22.10.2",
|
|
58
|
+
"jest": "^30.5.2",
|
|
59
|
+
"rollup": "^4.28.1",
|
|
60
|
+
"rollup-plugin-dts": "^6.1.1",
|
|
61
|
+
"ts-jest": "^29.1.2",
|
|
62
|
+
"ts-node": "^10.9.2",
|
|
63
|
+
"typescript": "5.7.3"
|
|
64
|
+
},
|
|
65
|
+
"dependencies": {
|
|
66
|
+
"@nebulaaa/dict": "^1.1.0",
|
|
67
|
+
"@nebulaaa/lru": "^1.1.0",
|
|
68
|
+
"@nebulaaa/plugin": "^1.0.0"
|
|
69
|
+
}
|
|
70
|
+
}
|
package/README.md
DELETED