dsh-mask 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +70 -0
- package/CHANGELOG.md +23 -0
- package/LICENSE +195 -0
- package/README.es.md +170 -0
- package/README.hi.md +170 -0
- package/README.md +188 -0
- package/README.pt.md +170 -0
- package/README.zh.md +188 -0
- package/SECURITY.md +45 -0
- package/THIRD_PARTY_NOTICES.md +41 -0
- package/cordis.patch.yml +54 -0
- package/index.mjs +379 -0
- package/lib/constants.mjs +90 -0
- package/lib/domain.mjs +26 -0
- package/lib/errors.mjs +87 -0
- package/lib/gate.mjs +41 -0
- package/lib/mask.mjs +45 -0
- package/lib/sanitize.mjs +45 -0
- package/lib/store.mjs +125 -0
- package/lib/strip.mjs +255 -0
- package/package.json +141 -0
- package/types.d.ts +47 -0
package/lib/mask.mjs
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// lib/mask.mjs — 消息遮罩助手(零依赖;把 UserMessage 的文本内容块替换为占位符)。
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* 遮罩单条 UserMessage 的文本内容块(保留消息结构与来源,仅替换 text 块)。
|
|
5
|
+
* 非文本块(图片、工具调用等)原样保留,向前兼容。
|
|
6
|
+
* @param {object} message - UserMessage(含 content 数组)。
|
|
7
|
+
* @param {import('./strip.mjs').Stripper} stripper - 累计脱敏器。
|
|
8
|
+
* @returns {{message: object, replaced: number}} 遮罩后消息与本次替换数。
|
|
9
|
+
*/
|
|
10
|
+
export function maskMessage(message, stripper) {
|
|
11
|
+
if (message === null || typeof message !== 'object' || !Array.isArray(message.content)) {
|
|
12
|
+
return { message, replaced: 0 }
|
|
13
|
+
}
|
|
14
|
+
let replaced = 0
|
|
15
|
+
let changed = false
|
|
16
|
+
const content = message.content.map((block) => {
|
|
17
|
+
if (block === null || typeof block !== 'object' || block.type !== 'text' || typeof block.text !== 'string') {
|
|
18
|
+
return block
|
|
19
|
+
}
|
|
20
|
+
const result = stripper.stripInto(block.text)
|
|
21
|
+
if (result.replaced > 0) {
|
|
22
|
+
replaced += result.replaced
|
|
23
|
+
changed = true
|
|
24
|
+
}
|
|
25
|
+
return { ...block, text: result.text }
|
|
26
|
+
})
|
|
27
|
+
if (!changed) return { message, replaced: 0 }
|
|
28
|
+
return { message: { ...message, content }, replaced }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* 遮罩消息数组,返回遮罩后数组与总替换数。
|
|
33
|
+
* @param {object[]} messages - UserMessage 数组。
|
|
34
|
+
* @param {import('./strip.mjs').Stripper} stripper - 累计脱敏器。
|
|
35
|
+
* @returns {{messages: object[], replaced: number}} 遮罩后数组与总替换数。
|
|
36
|
+
*/
|
|
37
|
+
export function maskMessages(messages, stripper) {
|
|
38
|
+
let replaced = 0
|
|
39
|
+
const masked = (Array.isArray(messages) ? messages : []).map((message) => {
|
|
40
|
+
const result = maskMessage(message, stripper)
|
|
41
|
+
replaced += result.replaced
|
|
42
|
+
return result.message
|
|
43
|
+
})
|
|
44
|
+
return { messages: masked, replaced }
|
|
45
|
+
}
|
package/lib/sanitize.mjs
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// lib/sanitize.mjs — 展示/日志脱敏纯函数(零依赖)。
|
|
2
|
+
//
|
|
3
|
+
// 规则:任何可能携带 PII 原文的文本在进入日志、命令结果或工具结果前都必须
|
|
4
|
+
// 过这里的纯函数。函数绝不抛错、绝不访问 I/O;输入不合法时返回保守的脱敏文本。
|
|
5
|
+
// 复用 lib/strip.mjs 的检测器词汇,保证"脱敏"与"遮罩"覆盖同一套 PII 类型。
|
|
6
|
+
|
|
7
|
+
import { BUILTIN_PATTERNS } from './strip.mjs'
|
|
8
|
+
|
|
9
|
+
/** 合并全部内置检测器为单条脱敏正则(按 PII 原文整体打码)。 */
|
|
10
|
+
const REDACT_RE = new RegExp(BUILTIN_PATTERNS.map(pattern => pattern.source.source).join('|'), 'gu')
|
|
11
|
+
|
|
12
|
+
/** key=value 形态的凭据(值整体打码;键名保留供定位)。 */
|
|
13
|
+
const CREDENTIAL_ASSIGNMENT = /\b((?:api[_-]?key|access[_-]?token|auth[_-]?token|refresh[_-]?token|password|passwd|client[_-]?secret|private[_-]?key)\s*=\s*)([^\s&;,]+)/giu
|
|
14
|
+
|
|
15
|
+
/** URL userinfo(user:password@)——错误消息可能回显远端地址。 */
|
|
16
|
+
const URL_USERINFO = /\/\/([^/\s:@]+):([^/\s@]+)@/gu
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* 文本脱敏:PII 实体、密钥赋值、URL 凭据整体打码。
|
|
20
|
+
* 非字符串输入先 String() 强转(绝不抛错——测试锁定此契约)。
|
|
21
|
+
* @param {unknown} text - 任意输出文本(错误消息、命令/工具结果等)。
|
|
22
|
+
* @returns {string} 脱敏文本。
|
|
23
|
+
*/
|
|
24
|
+
export function redactText(text) {
|
|
25
|
+
if (typeof text !== 'string') return String(text)
|
|
26
|
+
return text
|
|
27
|
+
.replace(URL_USERINFO, '//***@')
|
|
28
|
+
.replace(CREDENTIAL_ASSIGNMENT, '$1***')
|
|
29
|
+
.replace(REDACT_RE, '***')
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* 映射表脱敏:把 {占位符: 原文} 归约为 {占位符: '***'} 的安全摘要。
|
|
34
|
+
* 用于任何"必须展示映射却绝不含原文"的边界;正常路径根本不应序列化映射。
|
|
35
|
+
* @param {unknown} mapping - 占位符到原文的映射(或任意输入)。
|
|
36
|
+
* @returns {Record<string, string>} 仅含占位符键的安全摘要。
|
|
37
|
+
*/
|
|
38
|
+
export function redactMapping(mapping) {
|
|
39
|
+
if (mapping === null || typeof mapping !== 'object') return /** @type {Record<string, string>} */ ({})
|
|
40
|
+
const out = /** @type {Record<string, string>} */ ({})
|
|
41
|
+
for (const [placeholder] of Object.entries(mapping)) {
|
|
42
|
+
out[placeholder] = '***'
|
|
43
|
+
}
|
|
44
|
+
return out
|
|
45
|
+
}
|
package/lib/store.mjs
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// lib/store.mjs — 恢复表存储(内存 Stripper + 受控 storageDomain 持久化,零 I/O 除领域读写)。
|
|
2
|
+
|
|
3
|
+
import { createStripper } from './strip.mjs'
|
|
4
|
+
import { DEFAULTS, RESTORE_TABLE } from './constants.mjs'
|
|
5
|
+
import { registryUnavailable } from './errors.mjs'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* 恢复表:每会话一个累计 Stripper(内存),可选持久化映射到 storageDomain。
|
|
9
|
+
*
|
|
10
|
+
* 不变量:PII 原文只出现在这里的内存 Map 与受控 storageDomain(persist 开启时),
|
|
11
|
+
* 绝不进会话日志。mask/stats 走内存;restore 内存未命中时若 persist 开启则从
|
|
12
|
+
* 领域回载(跨重启还原)。
|
|
13
|
+
*/
|
|
14
|
+
export class RestoreStore {
|
|
15
|
+
/**
|
|
16
|
+
* @param {object} options - {entities, maxEntries, maxSessions, persist, domainPromise, onError}。
|
|
17
|
+
* domainPromise 为 Promise<{table(name): {get, put}}> | null。
|
|
18
|
+
*/
|
|
19
|
+
constructor(options) {
|
|
20
|
+
this.entities = options.entities ?? DEFAULTS.ENTITIES
|
|
21
|
+
this.maxEntries = options.maxEntries ?? DEFAULTS.MAX_RESTORE_ENTRIES_PER_SESSION
|
|
22
|
+
this.maxSessions = options.maxSessions ?? DEFAULTS.MAX_SESSIONS
|
|
23
|
+
this.persistEnabled = options.persist ?? DEFAULTS.PERSIST_RESTORE_TABLE
|
|
24
|
+
this.domainPromise = options.domainPromise ?? null
|
|
25
|
+
this.onError = options.onError ?? (() => {})
|
|
26
|
+
/** @type {Map<string, import('./strip.mjs').Stripper>} 会话 id -> 累计脱敏器(LRU)。 */
|
|
27
|
+
this.strippers = new Map()
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* 取得(或创建)会话的累计脱敏器。
|
|
32
|
+
* @param {string} sessionId - 会话 id。
|
|
33
|
+
* @returns {import('./strip.mjs').Stripper} 脱敏器。
|
|
34
|
+
*/
|
|
35
|
+
stripperFor(sessionId) {
|
|
36
|
+
let stripper = this.strippers.get(sessionId)
|
|
37
|
+
if (stripper === undefined) {
|
|
38
|
+
stripper = createStripper({ entities: this.entities, maxEntries: this.maxEntries })
|
|
39
|
+
this.strippers.set(sessionId, stripper)
|
|
40
|
+
this._pruneSessions()
|
|
41
|
+
}
|
|
42
|
+
return stripper
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* 累计遮罩一段文本(本会话占位符不重置)。
|
|
47
|
+
* @param {string} sessionId - 会话 id。
|
|
48
|
+
* @param {string} text - 待遮罩文本。
|
|
49
|
+
* @returns {{text: string, replaced: number, distribution: Record<string, number>}} 遮罩结果 + 本次增量。
|
|
50
|
+
*/
|
|
51
|
+
mask(sessionId, text) {
|
|
52
|
+
return this.stripperFor(sessionId).stripInto(text)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* 会话累计统计(审计用,绝不含原文)。
|
|
57
|
+
* @param {string} sessionId - 会话 id。
|
|
58
|
+
* @returns {{replaced: number, distribution: Record<string, number>}} 统计。
|
|
59
|
+
*/
|
|
60
|
+
stats(sessionId) {
|
|
61
|
+
return this.stripperFor(sessionId).stats()
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* 还原一段文本中的占位符;内存未命中且 persist 开启时从领域回载。
|
|
66
|
+
* @param {string} sessionId - 会话 id。
|
|
67
|
+
* @param {string} text - 含占位符文本。
|
|
68
|
+
* @returns {Promise<string>} 还原后的文本。
|
|
69
|
+
*/
|
|
70
|
+
async restore(sessionId, text) {
|
|
71
|
+
let stripper = this.strippers.get(sessionId)
|
|
72
|
+
if (stripper === undefined && this.persistEnabled && this.domainPromise !== null) {
|
|
73
|
+
stripper = await this._load(sessionId)
|
|
74
|
+
}
|
|
75
|
+
if (stripper === undefined) return text
|
|
76
|
+
return stripper.restore(text)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* 持久化会话映射到 storageDomain(尽力而为;失败仅上报,绝不阻断遮罩)。
|
|
81
|
+
* @param {string} sessionId - 会话 id。
|
|
82
|
+
* @returns {Promise<void>}
|
|
83
|
+
*/
|
|
84
|
+
async persist(sessionId) {
|
|
85
|
+
if (!this.persistEnabled || this.domainPromise === null) return
|
|
86
|
+
const stripper = this.strippers.get(sessionId)
|
|
87
|
+
if (stripper === undefined) return
|
|
88
|
+
try {
|
|
89
|
+
const domain = await this.domainPromise
|
|
90
|
+
const table = domain.table(RESTORE_TABLE)
|
|
91
|
+
await table.put(sessionId, { sessionId, entries: stripper.mapping(), updatedAt: Date.now() })
|
|
92
|
+
} catch (error) {
|
|
93
|
+
this.onError(registryUnavailable(error instanceof Error ? error.message : String(error)))
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* 从领域回载会话映射。
|
|
99
|
+
* @param {string} sessionId - 会话 id。
|
|
100
|
+
* @returns {Promise<import('./strip.mjs').Stripper|undefined>} 脱敏器或 undefined。
|
|
101
|
+
*/
|
|
102
|
+
async _load(sessionId) {
|
|
103
|
+
try {
|
|
104
|
+
const domain = await this.domainPromise
|
|
105
|
+
const record = await domain.table(RESTORE_TABLE).get(sessionId)
|
|
106
|
+
if (record === undefined || record.entries === undefined) return undefined
|
|
107
|
+
const stripper = createStripper({ entities: this.entities, maxEntries: this.maxEntries })
|
|
108
|
+
stripper.loadMapping(record.entries)
|
|
109
|
+
this.strippers.set(sessionId, stripper)
|
|
110
|
+
this._pruneSessions()
|
|
111
|
+
return stripper
|
|
112
|
+
} catch (error) {
|
|
113
|
+
this.onError(registryUnavailable(error instanceof Error ? error.message : String(error)))
|
|
114
|
+
return undefined
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** 会话数超限时逐出最久未用(映射仍在领域,restore 按需回载)。 */
|
|
119
|
+
_pruneSessions() {
|
|
120
|
+
while (this.strippers.size > this.maxSessions) {
|
|
121
|
+
const oldest = this.strippers.keys().next().value
|
|
122
|
+
this.strippers.delete(oldest)
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
package/lib/strip.mjs
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
// lib/strip.mjs — PII 脱敏/还原纯函数(零依赖)。
|
|
2
|
+
//
|
|
3
|
+
// 移植自上游 Pii-Stripper-Middleware 的 core.py(发送前匿名化 + 返回后恢复),
|
|
4
|
+
// 逐条对齐:正则检测、重叠消解(同起点取高分、区间不重叠)、相同原文复用同一
|
|
5
|
+
// 占位符、还原按占位符长度降序。新增:KEY(密钥)检测器(规格要求的"密钥")、
|
|
6
|
+
// 每会话累计计数与类型分布(审计用,绝不含明文)、有界逐出(maxEntries)。
|
|
7
|
+
// 函数绝不访问 I/O;输入非字符串时保守转字符串。
|
|
8
|
+
|
|
9
|
+
import { ENTITY_LABELS, LIMITS } from './constants.mjs'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* 内置正则检测器(实体 -> 多条 pattern,各自带置信度)。
|
|
13
|
+
* 与上游 REGEX_PATTERNS + 中文自定义识别器对齐,新增 key(密钥)检测器。
|
|
14
|
+
* 每条 pattern 都以 g 标志编译(供 matchAll 遍历)。
|
|
15
|
+
*/
|
|
16
|
+
export const BUILTIN_PATTERNS = Object.freeze([
|
|
17
|
+
{ entity: 'phone', source: /(?<!\d)1[3-9]\d{9}(?!\d)/gu, score: 0.95 },
|
|
18
|
+
{ entity: 'id-card', source: /(?<!\d)\d{17}[\dXx](?!\d)/gu, score: 0.92 },
|
|
19
|
+
{ entity: 'email', source: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/gu, score: 0.9 },
|
|
20
|
+
{ entity: 'bank-card', source: /(?<!\d)\d{16,19}(?!\d)/gu, score: 0.65 },
|
|
21
|
+
{ entity: 'ip', source: /\b(?:\d{1,3}\.){3}\d{1,3}\b/gu, score: 0.85 },
|
|
22
|
+
{ entity: 'key', source: /sk-[A-Za-z0-9_-]{16,}/gu, score: 0.95 },
|
|
23
|
+
{ entity: 'key', source: /gh[pousr]_[A-Za-z0-9]{16,}/gu, score: 0.95 },
|
|
24
|
+
{ entity: 'key', source: /xox[baprs]-[A-Za-z0-9-]{10,}/gu, score: 0.9 },
|
|
25
|
+
{ entity: 'key', source: /AKIA[0-9A-Z]{16}/gu, score: 0.9 },
|
|
26
|
+
{ entity: 'key', source: /Bearer[ \t]+[A-Za-z0-9._~+/=-]{8,}/gu, score: 0.9 },
|
|
27
|
+
])
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* 检测到的一个 PII 实体片段。
|
|
31
|
+
* @typedef {{text: string, entity: string, label: string, start: number, end: number, score: number}} PIIEntity
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* PII 脱敏器:把 PII 替换为 `<LABEL_N>` 占位符,并能按映射还原。
|
|
36
|
+
*
|
|
37
|
+
* 同一会话内跨请求累计(stripInto 不重置),保证同一原文始终复用同一占位符,
|
|
38
|
+
* 使模型在整段历史里看到的占位符语义一致;strip() 是单次模式(每次重置),
|
|
39
|
+
* 供 mask_test 这类"试跑一段文本"的独立场景使用。
|
|
40
|
+
*/
|
|
41
|
+
export class Stripper {
|
|
42
|
+
/**
|
|
43
|
+
* @param {object} options - {patterns: Array<{entity: string, source: RegExp, score: number}>, maxEntries?: number}。
|
|
44
|
+
*/
|
|
45
|
+
constructor(options) {
|
|
46
|
+
this.patterns = options.patterns
|
|
47
|
+
this.maxEntries = options.maxEntries ?? LIMITS.MAX_RESTORE_ENTRIES
|
|
48
|
+
this.reset()
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** 清空占位符计数、映射与统计(每次单次 strip 前调用)。 */
|
|
52
|
+
reset() {
|
|
53
|
+
/** @type {Map<string, number>} 标签 -> 已生成占位符数(单调递增,保证跨请求唯一)。 */
|
|
54
|
+
this.counter = new Map()
|
|
55
|
+
/** @type {Map<string, string>} 占位符 -> 原文(还原用)。 */
|
|
56
|
+
this.placeholderMap = new Map()
|
|
57
|
+
/** @type {Map<string, string>} 原文 -> 占位符(相同原文复用)。 */
|
|
58
|
+
this.valueMap = new Map()
|
|
59
|
+
this.totalReplaced = 0
|
|
60
|
+
/** @type {Map<string, number>} 标签 -> 累计替换次数(审计分布)。 */
|
|
61
|
+
this.distribution = new Map()
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* 单次脱敏:先重置再替换,返回脱敏文本(上游 strip 语义)。
|
|
66
|
+
* @param {unknown} text - 待脱敏文本(非字符串保守转字符串)。
|
|
67
|
+
* @returns {string} 脱敏后的文本。
|
|
68
|
+
*/
|
|
69
|
+
strip(text) {
|
|
70
|
+
this.reset()
|
|
71
|
+
return this._apply(text)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* 累计脱敏:不重置,返回本次替换的增量统计。
|
|
76
|
+
* @param {unknown} text - 待脱敏文本(非字符串保守转字符串)。
|
|
77
|
+
* @returns {{text: string, replaced: number, distribution: Record<string, number>}} 脱敏文本 + 本次增量。
|
|
78
|
+
*/
|
|
79
|
+
stripInto(text) {
|
|
80
|
+
const startReplaced = this.totalReplaced
|
|
81
|
+
const startDist = new Map(this.distribution)
|
|
82
|
+
const out = this._apply(text)
|
|
83
|
+
const replaced = this.totalReplaced - startReplaced
|
|
84
|
+
const distribution = /** @type {Record<string, number>} */ ({})
|
|
85
|
+
for (const [label, count] of this.distribution) {
|
|
86
|
+
const before = startDist.get(label) ?? 0
|
|
87
|
+
if (count > before) distribution[label] = count - before
|
|
88
|
+
}
|
|
89
|
+
return { text: out, replaced, distribution }
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* 把脱敏文本中的占位符还原为原文(按占位符长度降序,避免短占位符提前匹配)。
|
|
94
|
+
* @param {string} text - 含占位符的文本。
|
|
95
|
+
* @returns {string} 还原后的文本。
|
|
96
|
+
*/
|
|
97
|
+
restore(text) {
|
|
98
|
+
if (typeof text !== 'string') return String(text)
|
|
99
|
+
let result = text
|
|
100
|
+
const entries = [...this.placeholderMap.entries()].sort((a, b) => b[0].length - a[0].length)
|
|
101
|
+
for (const [placeholder, original] of entries) {
|
|
102
|
+
result = result.split(placeholder).join(original)
|
|
103
|
+
}
|
|
104
|
+
return result
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** @returns {Record<string, string>} {占位符: 原文} 只读副本。 */
|
|
108
|
+
mapping() {
|
|
109
|
+
return Object.fromEntries(this.placeholderMap)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* 从持久化的 {占位符: 原文} 映射回载(恢复表跨重启还原)。
|
|
114
|
+
* 回载后重建各标签计数,保证后续新占位符编号继续单调不冲突。
|
|
115
|
+
* 累计统计(totalReplaced/distribution)属于进程内遥测,不回载。
|
|
116
|
+
* @param {Record<string, string>} entries - 占位符到原文的映射。
|
|
117
|
+
*/
|
|
118
|
+
loadMapping(entries) {
|
|
119
|
+
for (const [placeholder, original] of Object.entries(entries ?? {})) {
|
|
120
|
+
this.placeholderMap.set(placeholder, original)
|
|
121
|
+
this.valueMap.set(original, placeholder)
|
|
122
|
+
const match = /^<([A-Z_]+)_(\d+)>$/u.exec(placeholder)
|
|
123
|
+
if (match !== null) {
|
|
124
|
+
const label = match[1]
|
|
125
|
+
const number = Number(match[2])
|
|
126
|
+
const current = this.counter.get(label) ?? 0
|
|
127
|
+
if (number > current) this.counter.set(label, number)
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* 累计统计(审计用):替换总数 + 类型分布。绝不含原文与映射。
|
|
134
|
+
* @returns {{replaced: number, distribution: Record<string, number>}} 统计。
|
|
135
|
+
*/
|
|
136
|
+
stats() {
|
|
137
|
+
return { replaced: this.totalReplaced, distribution: Object.fromEntries(this.distribution) }
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* 检测 + 重叠消解 + 替换,返回脱敏文本(核心管道)。
|
|
142
|
+
* @param {unknown} input - 任意文本(非字符串保守转字符串)。
|
|
143
|
+
* @returns {string} 脱敏文本。
|
|
144
|
+
*/
|
|
145
|
+
_apply(input) {
|
|
146
|
+
const text = typeof input === 'string' ? input : String(input)
|
|
147
|
+
if (text.length === 0 || text.length > LIMITS.MAX_TEXT_LENGTH) return text
|
|
148
|
+
const entities = this._detect(text)
|
|
149
|
+
const resolved = this._resolveOverlaps(entities)
|
|
150
|
+
return this._applyReplacements(text, resolved)
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* 正则检测全部实体。
|
|
155
|
+
* @param {string} text - 文本。
|
|
156
|
+
* @returns {PIIEntity[]} 实体列表。
|
|
157
|
+
*/
|
|
158
|
+
_detect(text) {
|
|
159
|
+
/** @type {PIIEntity[]} */
|
|
160
|
+
const entities = []
|
|
161
|
+
for (const pattern of this.patterns) {
|
|
162
|
+
for (const match of text.matchAll(pattern.source)) {
|
|
163
|
+
entities.push({
|
|
164
|
+
text: match[0],
|
|
165
|
+
entity: pattern.entity,
|
|
166
|
+
label: ENTITY_LABELS[pattern.entity] ?? pattern.entity,
|
|
167
|
+
start: match.index,
|
|
168
|
+
end: match.index + match[0].length,
|
|
169
|
+
score: pattern.score,
|
|
170
|
+
})
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return entities
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* 重叠消解:按起点升序、同起点按置信度降序;与已选实体重叠者丢弃
|
|
178
|
+
* (上游先到先得 + 同位置取高分语义)。
|
|
179
|
+
* @param {PIIEntity[]} entities - 未消解实体。
|
|
180
|
+
* @returns {PIIEntity[]} 消解后实体。
|
|
181
|
+
*/
|
|
182
|
+
_resolveOverlaps(entities) {
|
|
183
|
+
if (entities.length === 0) return []
|
|
184
|
+
const sorted = [...entities].sort((a, b) => a.start - b.start || b.score - a.score)
|
|
185
|
+
const result = []
|
|
186
|
+
let lastEnd = -1
|
|
187
|
+
for (const entity of sorted) {
|
|
188
|
+
if (entity.start >= lastEnd) {
|
|
189
|
+
result.push(entity)
|
|
190
|
+
lastEnd = entity.end
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return result
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* 逐实体替换为占位符(相同原文复用同一占位符),并累计统计。
|
|
198
|
+
* @param {string} text - 原文。
|
|
199
|
+
* @param {PIIEntity[]} entities - 已消解实体(按起点升序)。
|
|
200
|
+
* @returns {string} 脱敏文本。
|
|
201
|
+
*/
|
|
202
|
+
_applyReplacements(text, entities) {
|
|
203
|
+
const parts = []
|
|
204
|
+
let lastEnd = 0
|
|
205
|
+
for (const entity of [...entities].sort((a, b) => a.start - b.start)) {
|
|
206
|
+
parts.push(text.slice(lastEnd, entity.start))
|
|
207
|
+
const original = entity.text
|
|
208
|
+
let placeholder = this.valueMap.get(original)
|
|
209
|
+
if (placeholder === undefined) {
|
|
210
|
+
placeholder = this._makePlaceholder(entity.label)
|
|
211
|
+
this.placeholderMap.set(placeholder, original)
|
|
212
|
+
this.valueMap.set(original, placeholder)
|
|
213
|
+
}
|
|
214
|
+
parts.push(placeholder)
|
|
215
|
+
lastEnd = entity.end
|
|
216
|
+
this.totalReplaced += 1
|
|
217
|
+
this.distribution.set(entity.label, (this.distribution.get(entity.label) ?? 0) + 1)
|
|
218
|
+
}
|
|
219
|
+
parts.push(text.slice(lastEnd))
|
|
220
|
+
this._prune()
|
|
221
|
+
return parts.join('')
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* 生成唯一占位符 `<LABEL_N>`;计数单调递增,跨请求不重复。
|
|
226
|
+
* @param {string} label - 实体标签。
|
|
227
|
+
* @returns {string} 占位符。
|
|
228
|
+
*/
|
|
229
|
+
_makePlaceholder(label) {
|
|
230
|
+
const count = (this.counter.get(label) ?? 0) + 1
|
|
231
|
+
this.counter.set(label, count)
|
|
232
|
+
return `<${label}_${count}>`
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** 超过 maxEntries 时逐出最旧映射(计数仍单调,还原旧占位符会失效,可预期)。 */
|
|
236
|
+
_prune() {
|
|
237
|
+
while (this.placeholderMap.size > this.maxEntries) {
|
|
238
|
+
const oldest = this.placeholderMap.keys().next().value
|
|
239
|
+
const original = this.placeholderMap.get(oldest)
|
|
240
|
+
this.placeholderMap.delete(oldest)
|
|
241
|
+
if (original !== undefined) this.valueMap.delete(original)
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* 按启用的实体集创建脱敏器。
|
|
248
|
+
* @param {object} [options] - {entities?: string[], maxEntries?: number}。
|
|
249
|
+
* @returns {Stripper} 配置好的脱敏器。
|
|
250
|
+
*/
|
|
251
|
+
export function createStripper(options = {}) {
|
|
252
|
+
const enabled = new Set(options.entities ?? Object.keys(ENTITY_LABELS))
|
|
253
|
+
const patterns = BUILTIN_PATTERNS.filter(pattern => enabled.has(pattern.entity))
|
|
254
|
+
return new Stripper({ patterns, maxEntries: options.maxEntries })
|
|
255
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "dsh-mask",
|
|
3
|
+
"description": "PII masking middleware for DeepSeek Harness: anonymize names, phones, emails, ID cards, bank cards, keys, and addresses to placeholders before they reach the model, restore them at the display layer, keep the restore table only in memory and a controlled storage domain, never log plaintext, and expose /mask and the mask_test tool",
|
|
4
|
+
"version": "0.1.1",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "git+https://github.com/PerryLink/dsh-mask.git"
|
|
8
|
+
},
|
|
9
|
+
"homepage": "https://github.com/PerryLink/dsh-mask#readme",
|
|
10
|
+
"bugs": {
|
|
11
|
+
"url": "https://github.com/PerryLink/dsh-mask/issues"
|
|
12
|
+
},
|
|
13
|
+
"author": "PerryLink",
|
|
14
|
+
"type": "module",
|
|
15
|
+
"main": "./index.mjs",
|
|
16
|
+
"types": "./types.d.ts",
|
|
17
|
+
"exports": {
|
|
18
|
+
".": {
|
|
19
|
+
"types": "./types.d.ts",
|
|
20
|
+
"default": "./index.mjs"
|
|
21
|
+
},
|
|
22
|
+
"./cordis.patch.yml": "./cordis.patch.yml",
|
|
23
|
+
"./package.json": "./package.json"
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"index.mjs",
|
|
27
|
+
"types.d.ts",
|
|
28
|
+
"lib",
|
|
29
|
+
"cordis.patch.yml",
|
|
30
|
+
"README.md",
|
|
31
|
+
"README.zh.md",
|
|
32
|
+
"README.es.md",
|
|
33
|
+
"README.pt.md",
|
|
34
|
+
"README.hi.md",
|
|
35
|
+
"ARCHITECTURE.md",
|
|
36
|
+
"CHANGELOG.md",
|
|
37
|
+
"SECURITY.md",
|
|
38
|
+
"LICENSE",
|
|
39
|
+
"THIRD_PARTY_NOTICES.md"
|
|
40
|
+
],
|
|
41
|
+
"dsh": {
|
|
42
|
+
"bundle": {
|
|
43
|
+
"patch": "./cordis.patch.yml"
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"dshWorkshop": {
|
|
47
|
+
"schema": "omdsh-workshop-package/v1",
|
|
48
|
+
"type": "plugin",
|
|
49
|
+
"integration": {
|
|
50
|
+
"protocol": "harness-profile",
|
|
51
|
+
"artifact": "cordis.patch.yml"
|
|
52
|
+
},
|
|
53
|
+
"install": {
|
|
54
|
+
"mode": "transactional",
|
|
55
|
+
"adapter": "profile-bundle",
|
|
56
|
+
"failurePolicy": "generation-rollback",
|
|
57
|
+
"touchesCurrentBeforeActivation": false
|
|
58
|
+
},
|
|
59
|
+
"lifecycle": {
|
|
60
|
+
"activation": "restart-profile",
|
|
61
|
+
"dispose": "supported"
|
|
62
|
+
},
|
|
63
|
+
"permissions": [
|
|
64
|
+
"filesystem:read",
|
|
65
|
+
"storage-domain:read",
|
|
66
|
+
"storage-domain:write",
|
|
67
|
+
"session-log:read",
|
|
68
|
+
"credentials:none",
|
|
69
|
+
"network:none"
|
|
70
|
+
],
|
|
71
|
+
"compatibility": {
|
|
72
|
+
"dshVersions": [
|
|
73
|
+
"0.1.0-rc.6"
|
|
74
|
+
]
|
|
75
|
+
},
|
|
76
|
+
"capability": {
|
|
77
|
+
"id": "mask",
|
|
78
|
+
"kind": "command",
|
|
79
|
+
"invocation": "/mask status",
|
|
80
|
+
"expected": "the /mask command is registered on the commands service: status reports masked counts and type distribution, on/off toggles masking at runtime, restore <text> unmaps placeholders back to the values stored for this session; mask_test tool masks a snippet and reports the placeholder result"
|
|
81
|
+
},
|
|
82
|
+
"evidence": {
|
|
83
|
+
"install": null,
|
|
84
|
+
"failureIsolation": null,
|
|
85
|
+
"hotReload": null,
|
|
86
|
+
"remove": null
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
"peerDependencies": {
|
|
90
|
+
"@deepseek-ai/cordis": "^4.0.1",
|
|
91
|
+
"@deepseek-ai/dsh-session": "0.1.0-rc.6",
|
|
92
|
+
"@deepseek-ai/dsh-storage": "0.1.0-rc.6",
|
|
93
|
+
"@deepseek-ai/dsh-storage-json": "0.1.0-rc.6",
|
|
94
|
+
"@deepseek-ai/dsh-storage-domain": "0.1.0-rc.6",
|
|
95
|
+
"@deepseek-ai/dsh-tools": "0.1.0-rc.6",
|
|
96
|
+
"@deepseek-ai/schemastery": "^3.18.0"
|
|
97
|
+
},
|
|
98
|
+
"dependencies": {
|
|
99
|
+
"typescript": "^5.9.0",
|
|
100
|
+
"zod": "^4.4.3"
|
|
101
|
+
},
|
|
102
|
+
"devDependencies": {
|
|
103
|
+
"@deepseek-ai/cordis": "^4.0.1",
|
|
104
|
+
"@deepseek-ai/dsh-agent": "0.1.0-rc.6",
|
|
105
|
+
"@deepseek-ai/dsh-commands": "0.1.0-rc.6",
|
|
106
|
+
"@deepseek-ai/dsh-session": "0.1.0-rc.6",
|
|
107
|
+
"@deepseek-ai/dsh-storage-domain": "0.1.0-rc.6",
|
|
108
|
+
"@deepseek-ai/dsh-tools": "0.1.0-rc.6",
|
|
109
|
+
"@deepseek-ai/schemastery": "^3.18.0",
|
|
110
|
+
"@types/node": "^22.19.0"
|
|
111
|
+
},
|
|
112
|
+
"scripts": {
|
|
113
|
+
"typecheck": "tsc -p tsconfig.check.json",
|
|
114
|
+
"typecheck:ci": "tsc -p tsconfig.check.json",
|
|
115
|
+
"test": "node --test \"test/*.test.mjs\"",
|
|
116
|
+
"coverage": "node --test --experimental-test-coverage \"test/*.test.mjs\"",
|
|
117
|
+
"verify:self-contained": "node scripts/verify-self-contained.mjs",
|
|
118
|
+
"verify:artifacts": "node scripts/verify-artifacts.mjs",
|
|
119
|
+
"check:readmes": "node scripts/verify-readmes.mjs"
|
|
120
|
+
},
|
|
121
|
+
"keywords": [
|
|
122
|
+
"dsh",
|
|
123
|
+
"dsh-plugin",
|
|
124
|
+
"deepseek-harness",
|
|
125
|
+
"deepseek",
|
|
126
|
+
"cordis",
|
|
127
|
+
"pii",
|
|
128
|
+
"mask",
|
|
129
|
+
"privacy",
|
|
130
|
+
"anonymization",
|
|
131
|
+
"security"
|
|
132
|
+
],
|
|
133
|
+
"engines": {
|
|
134
|
+
"node": "^22.19.0 || >=24.0.0"
|
|
135
|
+
},
|
|
136
|
+
"packageManager": "pnpm@11.7.0",
|
|
137
|
+
"publishConfig": {
|
|
138
|
+
"access": "public"
|
|
139
|
+
},
|
|
140
|
+
"license": "Apache-2.0"
|
|
141
|
+
}
|
package/types.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// types.d.ts — dsh-mask 类型契约(会话事件声明合并 + 配置类型)。
|
|
2
|
+
|
|
3
|
+
declare module '@deepseek-ai/dsh-session' {
|
|
4
|
+
interface SessionEventMap {
|
|
5
|
+
/**
|
|
6
|
+
* 一次请求前脱敏的审计记录(log-only):只记"替换了多少处 + 类型分布",
|
|
7
|
+
* 绝不携带 PII 原文或占位符映射。注意:当前宿主构建(KNOWN_SESSION_EVENT_TYPES)
|
|
8
|
+
* 尚未收录 mask/*,运行时经自适应门跳过 append;宿主收录后自动开启
|
|
9
|
+
* (见 README「会话事件」)。
|
|
10
|
+
*/
|
|
11
|
+
'mask/applied': {
|
|
12
|
+
sessionId: string
|
|
13
|
+
replaced: number
|
|
14
|
+
distribution: Record<string, number>
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface Config {
|
|
20
|
+
/** 总开关;false 时命令、工具与 pre-step 监听器全部卸载。 */
|
|
21
|
+
enabled?: boolean
|
|
22
|
+
/** 检测模式;只有 'regex' 实现,'regex+ner'(姓名/地址识别)预留并响亮失败。 */
|
|
23
|
+
mode?: 'regex' | 'regex+ner'
|
|
24
|
+
/** 启用的实体类型;regex 集为 phone/email/id-card/bank-card/key/ip,person/address 需 NER。 */
|
|
25
|
+
entities?: string[]
|
|
26
|
+
/** 遮罩作用域;只有 'messages'(agent/pre-step 消息)实现,'tools' 预留并响亮失败。 */
|
|
27
|
+
scope?: 'messages' | 'tools'
|
|
28
|
+
/** 注册 /mask 命令(默认 true)。 */
|
|
29
|
+
registerCommand?: boolean
|
|
30
|
+
/** tools 服务存在时注册 mask_test 工具(默认 true)。 */
|
|
31
|
+
registerTools?: boolean
|
|
32
|
+
/** 恢复表持久化到受控 storageDomain(false = 仅内存,重启丢失)。 */
|
|
33
|
+
persistRestoreTable?: boolean
|
|
34
|
+
/** 每会话恢复条目上限(超出逐出最旧)。 */
|
|
35
|
+
maxRestoreEntriesPerSession?: number
|
|
36
|
+
/** 内存会话脱敏器上限(LRU 逐出,映射按需从领域回载)。 */
|
|
37
|
+
maxSessions?: number
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** mask_test 工具规范结果。 */
|
|
41
|
+
export interface MaskTestValue {
|
|
42
|
+
ok: boolean
|
|
43
|
+
masked: string
|
|
44
|
+
replaced: number
|
|
45
|
+
distribution: { label: string; count: number }[]
|
|
46
|
+
error?: string
|
|
47
|
+
}
|