advisers 0.1.0-preview.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +131 -21
- package/NOTICE +34 -0
- package/README-zh.md +40 -9
- package/README.md +53 -10
- package/THIRD-PARTY-NOTICES.md +277 -0
- package/bin/advisers.js +4230 -122
- package/index.js +1 -1
- package/lib/agent.js +414 -0
- package/lib/attachments.js +283 -0
- package/lib/capture.js +168 -0
- package/lib/context-window.js +123 -0
- package/lib/digest.js +27 -0
- package/lib/doc-review.js +30 -0
- package/lib/egress.js +27 -0
- package/lib/evidence.js +1995 -0
- package/lib/ical.js +542 -0
- package/lib/imap.js +181 -0
- package/lib/importers.js +1001 -0
- package/lib/learning.js +46 -0
- package/lib/mcp-server.js +869 -0
- package/lib/model.js +307 -0
- package/lib/notify.js +189 -0
- package/lib/nudge.js +10 -0
- package/lib/ocr.js +47 -0
- package/lib/pdf.js +105 -0
- package/lib/pet-brief-store.js +51 -0
- package/lib/pet-brief.js +644 -0
- package/lib/receipt-verify.js +624 -0
- package/lib/relative-date.js +393 -0
- package/lib/remind.js +690 -0
- package/lib/research.js +193 -0
- package/lib/review.js +88 -0
- package/lib/routine.js +22 -0
- package/lib/safety/classifier.js +392 -0
- package/lib/safety/explain.js +157 -0
- package/lib/safety/index.js +333 -0
- package/lib/safety/redact.js +212 -0
- package/lib/safety/rules.js +824 -0
- package/lib/scheduler.js +399 -0
- package/lib/source.js +294 -0
- package/lib/sqlite.js +1022 -0
- package/lib/sync.js +129 -0
- package/lib/timestamp.js +45 -0
- package/lib/tos/index.js +332 -0
- package/lib/tos/manifest.json +32 -0
- package/lib/tos/privacy.zh-CN.md +136 -0
- package/lib/tos/safety.zh-CN.md +101 -0
- package/lib/tos/summary.zh-CN.md +12 -0
- package/lib/tos/terms.zh-CN.md +439 -0
- package/lib/ui.js +49 -0
- package/lib/undo.js +83 -0
- package/lib/vault.js +51 -0
- package/package.json +26 -4
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content-safety module entry point.
|
|
3
|
+
*
|
|
4
|
+
* Pure ESM, offline, zero third-party dependencies:
|
|
5
|
+
* rules.js 14-category matrix + narrow block allowlist + JSON overrides
|
|
6
|
+
* classifier.js normalize -> match -> surface/context policy -> decision
|
|
7
|
+
* redact.js PII / secret detection + one-way redaction
|
|
8
|
+
* explain.js user-facing "advisers safety explain" copy
|
|
9
|
+
*
|
|
10
|
+
* Honest scope: this is an ASSISTIVE local rule layer. It reduces risk but is
|
|
11
|
+
* not a compliance certification, not an official review, and does not perform
|
|
12
|
+
* algorithm filing / model registration. See docs/CONTENT-SAFETY.md.
|
|
13
|
+
*
|
|
14
|
+
* Auditing: the caller passes the HMAC key and writes the returned structured
|
|
15
|
+
* fields to its own appendAudit(); this module never stores text and never
|
|
16
|
+
* writes audit rows itself.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export * from './rules.js'
|
|
20
|
+
export * from './classifier.js'
|
|
21
|
+
export * from './redact.js'
|
|
22
|
+
export * from './explain.js'
|
|
23
|
+
|
|
24
|
+
import { RULESET_VERSION, actionRank } from './rules.js'
|
|
25
|
+
import { classify } from './classifier.js'
|
|
26
|
+
|
|
27
|
+
export const SAFETY_API_VERSION = 'advisers-safety-api-1'
|
|
28
|
+
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
// audit reason helpers (hash-chain compatible: one string field, no schema change)
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
export const AUDIT_REASON_MAX = 300
|
|
34
|
+
|
|
35
|
+
function reasonPart(value, fallback) {
|
|
36
|
+
if (value == null) return fallback || '-'
|
|
37
|
+
const text = String(value).replace(/[|\r\n\t\u0000-\u001f]/g, ' ').trim()
|
|
38
|
+
return text ? text.slice(0, 40) : (fallback || '-')
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function normalizeRuleIds(ruleIds) {
|
|
42
|
+
if (!Array.isArray(ruleIds)) return []
|
|
43
|
+
const out = []
|
|
44
|
+
for (const raw of ruleIds) {
|
|
45
|
+
if (raw == null) continue
|
|
46
|
+
const id = reasonPart(raw).replace(/[^A-Za-z0-9-]/g, '').slice(0, 24)
|
|
47
|
+
if (id && !out.includes(id)) out.push(id)
|
|
48
|
+
if (out.length >= 8) break
|
|
49
|
+
}
|
|
50
|
+
return out
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function normalizeEvidenceHash(hash) {
|
|
54
|
+
if (hash == null) return '-'
|
|
55
|
+
const text = String(hash)
|
|
56
|
+
return /^[0-9a-f]{8,64}$/i.test(text) ? text.slice(0, 16) : '-'
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Build the audit `reason` string. Format (single line, <=300 chars):
|
|
61
|
+
* rulesetVersion|surface|action|code|ruleIds|evidenceHash
|
|
62
|
+
* Example: advisers-content-1|chat.output|block|CSAM|R-SEX-CSAM-01|9f2c1ab7e3d4c5b6
|
|
63
|
+
*/
|
|
64
|
+
export function buildAuditReason(fields = {}) {
|
|
65
|
+
const version = reasonPart(fields.rulesetVersion == null ? RULESET_VERSION : fields.rulesetVersion, 'unknown')
|
|
66
|
+
const surface = reasonPart(fields.surface, 'unknown')
|
|
67
|
+
const action = reasonPart(fields.action, 'allow')
|
|
68
|
+
const code = reasonPart(fields.code, 'NONE')
|
|
69
|
+
const ids = normalizeRuleIds(fields.ruleIds)
|
|
70
|
+
const evidenceHash = normalizeEvidenceHash(fields.evidenceHash)
|
|
71
|
+
const reason = [version, surface, action, code, ids.length ? ids.join(',') : '-', evidenceHash].join('|')
|
|
72
|
+
return reason.slice(0, AUDIT_REASON_MAX)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Parse an audit reason back into structured fields (for explain/report). */
|
|
76
|
+
export function parseAuditReason(reason) {
|
|
77
|
+
if (typeof reason !== 'string') return null
|
|
78
|
+
const parts = reason.slice(0, AUDIT_REASON_MAX).split('|')
|
|
79
|
+
if (parts.length < 6) return null
|
|
80
|
+
const ruleIds = parts[4] === '-' ? [] : parts[4].split(',').filter(Boolean)
|
|
81
|
+
return {
|
|
82
|
+
rulesetVersion: parts[0],
|
|
83
|
+
surface: parts[1],
|
|
84
|
+
action: parts[2],
|
|
85
|
+
code: parts[3],
|
|
86
|
+
ruleIds,
|
|
87
|
+
evidenceHash: parts[5] === '-' ? null : parts[5],
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Build the audit reason for a classify() result. */
|
|
92
|
+
export function auditReasonForResult(result = {}, options = {}) {
|
|
93
|
+
return buildAuditReason({
|
|
94
|
+
rulesetVersion: result.rulesetVersion,
|
|
95
|
+
surface: result.surface || options.surface,
|
|
96
|
+
action: result.action,
|
|
97
|
+
code: result.code,
|
|
98
|
+
ruleIds: result.ruleIds,
|
|
99
|
+
evidenceHash: result.evidenceHash,
|
|
100
|
+
})
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ---------------------------------------------------------------------------
|
|
104
|
+
// typed error (no original text, no upstream text)
|
|
105
|
+
// ---------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
export class ContentSafetyError extends Error {
|
|
108
|
+
constructor(result = {}, options = {}) {
|
|
109
|
+
super('内容安全审核未通过(CONTENT_SAFETY_BLOCKED)')
|
|
110
|
+
this.name = 'ContentSafetyError'
|
|
111
|
+
this.code = result.code || options.code || 'CONTENT_SAFETY_BLOCKED'
|
|
112
|
+
this.action = result.action || 'block'
|
|
113
|
+
this.provider = options.provider || 'local'
|
|
114
|
+
this.surface = result.surface || options.surface || 'unknown'
|
|
115
|
+
this.categories = Array.isArray(result.categories) ? result.categories : []
|
|
116
|
+
this.ruleIds = Array.isArray(result.ruleIds) ? result.ruleIds : []
|
|
117
|
+
this.crisis = Boolean(result.crisis)
|
|
118
|
+
this.evidenceHash = result.evidenceHash || null
|
|
119
|
+
this.rulesetVersion = result.rulesetVersion || options.rulesetVersion || RULESET_VERSION
|
|
120
|
+
this.auditReason = options.auditReason || auditReasonForResult(result, options)
|
|
121
|
+
this.retryable = false
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
toJSON() {
|
|
125
|
+
return {
|
|
126
|
+
name: this.name,
|
|
127
|
+
code: this.code,
|
|
128
|
+
action: this.action,
|
|
129
|
+
provider: this.provider,
|
|
130
|
+
surface: this.surface,
|
|
131
|
+
categories: this.categories,
|
|
132
|
+
ruleIds: this.ruleIds,
|
|
133
|
+
crisis: this.crisis,
|
|
134
|
+
evidenceHash: this.evidenceHash,
|
|
135
|
+
rulesetVersion: this.rulesetVersion,
|
|
136
|
+
auditReason: this.auditReason,
|
|
137
|
+
retryable: this.retryable,
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Convenience constructor; only block/confirm results should be thrown. */
|
|
143
|
+
export function safetyErrorFrom(result = {}, options = {}) {
|
|
144
|
+
return new ContentSafetyError(result, options)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
// provider refusal shapes (DeepSeek docs: Messages / Responses / Chat + 400)
|
|
149
|
+
// ---------------------------------------------------------------------------
|
|
150
|
+
|
|
151
|
+
const PROVIDER_MESSAGES = Object.freeze({
|
|
152
|
+
CONTENT_RISK_400: '模型服务商以内容风险为由拒绝了本次请求。',
|
|
153
|
+
PROVIDER_REFUSAL: '模型服务商以拒答结束了本次请求。',
|
|
154
|
+
CONTENT_FILTER: '模型服务商的内容过滤拦截了本次请求。',
|
|
155
|
+
IMAGE_BLOCKED: '模型服务商拦截了请求中的图片。',
|
|
156
|
+
NONE: null,
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
function jsonOf(payload) {
|
|
160
|
+
try {
|
|
161
|
+
return JSON.stringify(payload == null ? {} : payload)
|
|
162
|
+
} catch {
|
|
163
|
+
return ''
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function pickSafetyTokens(payload) {
|
|
168
|
+
const candidates = [
|
|
169
|
+
payload && payload.usage && payload.usage.output_tokens_details && payload.usage.output_tokens_details.safety_tokens,
|
|
170
|
+
payload && payload.usage && payload.usage.completion_tokens_details && payload.usage.completion_tokens_details.safety_tokens,
|
|
171
|
+
payload && payload.usage && payload.usage.input_tokens_details && payload.usage.input_tokens_details.safety_tokens,
|
|
172
|
+
payload && payload.usage && payload.usage.safety_tokens,
|
|
173
|
+
payload && payload.safety_tokens,
|
|
174
|
+
payload && payload.safetyTokens,
|
|
175
|
+
]
|
|
176
|
+
for (const candidate of candidates) {
|
|
177
|
+
if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) return candidate
|
|
178
|
+
}
|
|
179
|
+
return 0
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function detectProtocol(payload, status) {
|
|
183
|
+
if (status >= 400 || (payload && payload.error)) return 'http'
|
|
184
|
+
if (payload && (payload.stop_reason != null || payload.stop_details != null)) return 'messages'
|
|
185
|
+
if (payload && (payload.status != null || payload.incomplete_details != null)) return 'responses'
|
|
186
|
+
if (payload && Array.isArray(payload.choices)) return 'chat'
|
|
187
|
+
return 'unknown'
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function imagePaths(json) {
|
|
191
|
+
const out = []
|
|
192
|
+
const matches = json.match(/messages\.\d+\.content\.\d+/g) || []
|
|
193
|
+
for (const match of matches) if (!out.includes(match)) out.push(match)
|
|
194
|
+
return out
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Map a provider response/payload to a unified refusal shape WITHOUT copying
|
|
199
|
+
* upstream text. Returns:
|
|
200
|
+
* { blocked, provider, protocol, code, message, retryable, safetyTokens, imagesBlocked }
|
|
201
|
+
* Never includes the upstream explanation/refusal text (only stable codes).
|
|
202
|
+
*/
|
|
203
|
+
export function normalizeProviderRefusal(payload, options = {}) {
|
|
204
|
+
const status = Number.isInteger(options.status) ? options.status : 0
|
|
205
|
+
const json = jsonOf(payload)
|
|
206
|
+
const lower = json.toLowerCase()
|
|
207
|
+
const safetyTokens = pickSafetyTokens(payload)
|
|
208
|
+
const protocol = detectProtocol(payload, status)
|
|
209
|
+
const images = imagePaths(json)
|
|
210
|
+
|
|
211
|
+
let code = 'NONE'
|
|
212
|
+
if (images.length || lower.includes('blocked by content moderation')) {
|
|
213
|
+
code = 'IMAGE_BLOCKED'
|
|
214
|
+
} else if (payload && (payload.stop_reason === 'refusal' || (payload.stop_details && payload.stop_details.type === 'refusal'))) {
|
|
215
|
+
code = 'PROVIDER_REFUSAL'
|
|
216
|
+
} else if (payload && payload.status === 'incomplete' && payload.incomplete_details && payload.incomplete_details.reason === 'content_filter') {
|
|
217
|
+
code = 'CONTENT_FILTER'
|
|
218
|
+
} else if (payload && Array.isArray(payload.choices) && payload.choices.some((choice) => (
|
|
219
|
+
(choice && choice.finish_reason === 'content_filter')
|
|
220
|
+
|| (choice && choice.message && typeof choice.message.refusal === 'string' && choice.message.refusal.length > 0)
|
|
221
|
+
))) {
|
|
222
|
+
code = 'CONTENT_FILTER'
|
|
223
|
+
} else if (status >= 400 || lower.includes('content exists risk') || lower.includes('content_exists_risk')) {
|
|
224
|
+
if (lower.includes('content exists risk') || lower.includes('content_exists_risk') || lower.includes('content risk')) {
|
|
225
|
+
code = 'CONTENT_RISK_400'
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const blocked = code !== 'NONE'
|
|
230
|
+
return {
|
|
231
|
+
blocked,
|
|
232
|
+
provider: 'deepseek',
|
|
233
|
+
protocol,
|
|
234
|
+
code,
|
|
235
|
+
message: PROVIDER_MESSAGES[code],
|
|
236
|
+
retryable: false,
|
|
237
|
+
safetyTokens,
|
|
238
|
+
imagesBlocked: images,
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// ---------------------------------------------------------------------------
|
|
243
|
+
// message-level convenience
|
|
244
|
+
// ---------------------------------------------------------------------------
|
|
245
|
+
|
|
246
|
+
function textPartsOf(message) {
|
|
247
|
+
const texts = []
|
|
248
|
+
if (!message || typeof message !== 'object') {
|
|
249
|
+
if (typeof message === 'string') texts.push(message)
|
|
250
|
+
return texts
|
|
251
|
+
}
|
|
252
|
+
const content = message.content
|
|
253
|
+
if (typeof content === 'string') texts.push(content)
|
|
254
|
+
if (Array.isArray(content)) {
|
|
255
|
+
for (const part of content) {
|
|
256
|
+
if (part && typeof part === 'object' && typeof part.text === 'string' && (part.type == null || part.type === 'text' || part.type === 'input_text' || part.type === 'output_text')) {
|
|
257
|
+
texts.push(part.text)
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (typeof message.refusal === 'string') texts.push(message.refusal)
|
|
262
|
+
if (Array.isArray(message.tool_calls)) {
|
|
263
|
+
for (const call of message.tool_calls) {
|
|
264
|
+
const args = call && call.function && call.function.arguments
|
|
265
|
+
if (typeof args === 'string') texts.push(args)
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return texts
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Classify every user/assistant text in an OpenAI-style messages array and
|
|
273
|
+
* aggregate the worst action. Output messages use chat.output policy.
|
|
274
|
+
*/
|
|
275
|
+
export function inspectMessages(messages, options = {}) {
|
|
276
|
+
const list = Array.isArray(messages) ? messages : []
|
|
277
|
+
const results = []
|
|
278
|
+
let worst = null
|
|
279
|
+
for (let index = 0; index < list.length; index += 1) {
|
|
280
|
+
const message = list[index]
|
|
281
|
+
const role = message && typeof message === 'object' && message.role ? message.role : 'unknown'
|
|
282
|
+
const surface = options.surface || (role === 'assistant' ? 'chat.output' : 'chat.input')
|
|
283
|
+
for (const text of textPartsOf(message)) {
|
|
284
|
+
const result = classify(text, { ...options, surface })
|
|
285
|
+
results.push({
|
|
286
|
+
index,
|
|
287
|
+
role,
|
|
288
|
+
surface,
|
|
289
|
+
action: result.action,
|
|
290
|
+
code: result.code,
|
|
291
|
+
categories: result.categories,
|
|
292
|
+
ruleIds: result.ruleIds,
|
|
293
|
+
crisis: result.crisis,
|
|
294
|
+
evidenceHash: result.evidenceHash,
|
|
295
|
+
})
|
|
296
|
+
if (!worst || actionRank(result.action) > actionRank(worst.action)) worst = result
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
if (!worst) {
|
|
300
|
+
const result = classify('', { ...options, surface: options.surface || 'chat.input' })
|
|
301
|
+
return {
|
|
302
|
+
action: 'allow',
|
|
303
|
+
code: 'NONE',
|
|
304
|
+
categories: [],
|
|
305
|
+
ruleIds: [],
|
|
306
|
+
crisis: false,
|
|
307
|
+
confidence: 0,
|
|
308
|
+
evidenceHash: null,
|
|
309
|
+
surface: result.surface,
|
|
310
|
+
rulesetVersion: result.rulesetVersion,
|
|
311
|
+
rulesetDigest: result.rulesetDigest,
|
|
312
|
+
results: [],
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
return {
|
|
316
|
+
action: worst.action,
|
|
317
|
+
code: worst.code,
|
|
318
|
+
categories: worst.categories,
|
|
319
|
+
ruleIds: worst.ruleIds,
|
|
320
|
+
crisis: results.some((item) => item.crisis),
|
|
321
|
+
confidence: worst.confidence,
|
|
322
|
+
evidenceHash: worst.evidenceHash,
|
|
323
|
+
surface: worst.surface,
|
|
324
|
+
rulesetVersion: worst.rulesetVersion,
|
|
325
|
+
rulesetDigest: worst.rulesetDigest,
|
|
326
|
+
results,
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** Alias kept for callers that read the design doc naming. */
|
|
331
|
+
export function inspectText(text, options = {}) {
|
|
332
|
+
return classify(text, options)
|
|
333
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PII / secret detection and redaction (local, offline, zero-dependency).
|
|
3
|
+
*
|
|
4
|
+
* Single-value personal data is never a block reason: it is REDACTED so logs,
|
|
5
|
+
* audit rows, Pet Brief inputs and model-adjacent copies do not carry it.
|
|
6
|
+
* Only bulk exfiltration rules (in rules.js) may block.
|
|
7
|
+
*
|
|
8
|
+
* Rules are the `detector: 'pii'` entries of the ruleset, so a JSON ruleset
|
|
9
|
+
* override can tune them like any other rule. Redaction is one-way here: the
|
|
10
|
+
* original value is never returned in the result object.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { DEFAULT_RULESET, PII_KINDS, resolveRuleset } from './rules.js'
|
|
14
|
+
|
|
15
|
+
/** Full-width ASCII -> half-width, same length (so offsets stay valid). */
|
|
16
|
+
function toAsciiView(text) {
|
|
17
|
+
return text.replace(/[\uFF01-\uFF5E\u3000]/g, (char) => (
|
|
18
|
+
char === '\u3000' ? ' ' : String.fromCharCode(char.charCodeAt(0) - 0xFEE0)
|
|
19
|
+
))
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function digitsOf(value) {
|
|
23
|
+
return value.replace(/[^0-9Xx]/g, '').toUpperCase()
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function isLuhn(number) {
|
|
27
|
+
const digits = number.replace(/\D/g, '')
|
|
28
|
+
if (digits.length < 12) return false
|
|
29
|
+
let sum = 0
|
|
30
|
+
let double = false
|
|
31
|
+
for (let index = digits.length - 1; index >= 0; index -= 1) {
|
|
32
|
+
let value = Number(digits[index])
|
|
33
|
+
if (double) {
|
|
34
|
+
value *= 2
|
|
35
|
+
if (value > 9) value -= 9
|
|
36
|
+
}
|
|
37
|
+
sum += value
|
|
38
|
+
double = !double
|
|
39
|
+
}
|
|
40
|
+
return sum % 10 === 0
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function isValidIdCard(value) {
|
|
44
|
+
const id = digitsOf(value)
|
|
45
|
+
if (id.length !== 18) return false
|
|
46
|
+
const weights = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]
|
|
47
|
+
const checks = '10X98765432'
|
|
48
|
+
let sum = 0
|
|
49
|
+
for (let index = 0; index < 17; index += 1) sum += Number(id[index]) * weights[index]
|
|
50
|
+
return checks[sum % 11] === id[17]
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function labelNear(prepared, start, labels) {
|
|
54
|
+
const window = prepared.slice(Math.max(0, start - 16), start)
|
|
55
|
+
return labels.test(window)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const ID_LABELS = /(身份证|证件号|证件号码|公民身份号码)/
|
|
59
|
+
const BANK_LABELS = /(银行卡|卡号|信用卡|储蓄卡|借记卡|账户|账号)/
|
|
60
|
+
const BANK_BIN = /^(4|5|62|34|37|6011|65)/
|
|
61
|
+
|
|
62
|
+
function validatorAccept(validator, prepared, start, end) {
|
|
63
|
+
const raw = prepared.slice(start, end)
|
|
64
|
+
if (validator === 'idcard') {
|
|
65
|
+
return isValidIdCard(raw) || labelNear(prepared, start, ID_LABELS)
|
|
66
|
+
}
|
|
67
|
+
if (validator === 'bankcard') {
|
|
68
|
+
const digits = raw.replace(/\D/g, '')
|
|
69
|
+
if (!isLuhn(digits)) return false
|
|
70
|
+
return labelNear(prepared, start, BANK_LABELS) || BANK_BIN.test(digits)
|
|
71
|
+
}
|
|
72
|
+
return true
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function addDetectionFlags(pattern) {
|
|
76
|
+
const flags = new Set((pattern.flags || '').split(''))
|
|
77
|
+
flags.add('g')
|
|
78
|
+
flags.add('d')
|
|
79
|
+
return new RegExp(pattern.source, [...flags].join(''))
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function piiRulesOf(ruleset) {
|
|
83
|
+
return ruleset.rules.filter((rule) => rule.detector === 'pii')
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Internal detector: returns hits WITH the matched raw value so the classifier
|
|
88
|
+
* can key an HMAC evidence hash. Never export this object to users/UI/logs.
|
|
89
|
+
*/
|
|
90
|
+
export function detectPiiInternal(input, options = {}) {
|
|
91
|
+
const source = String(input == null ? '' : input)
|
|
92
|
+
if (!source) return []
|
|
93
|
+
let ruleset = DEFAULT_RULESET
|
|
94
|
+
if (options.ruleset) {
|
|
95
|
+
try {
|
|
96
|
+
ruleset = options.ruleset.__compiled ? options.ruleset : resolveRuleset(options.ruleset)
|
|
97
|
+
} catch {
|
|
98
|
+
ruleset = DEFAULT_RULESET
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const prepared = toAsciiView(source)
|
|
102
|
+
const hits = []
|
|
103
|
+
for (const rule of piiRulesOf(ruleset)) {
|
|
104
|
+
for (const pattern of rule.patterns) {
|
|
105
|
+
let regex
|
|
106
|
+
try {
|
|
107
|
+
regex = addDetectionFlags(pattern)
|
|
108
|
+
} catch {
|
|
109
|
+
continue
|
|
110
|
+
}
|
|
111
|
+
let match
|
|
112
|
+
while ((match = regex.exec(prepared)) !== null) {
|
|
113
|
+
if (match[0].length === 0) {
|
|
114
|
+
regex.lastIndex += 1
|
|
115
|
+
continue
|
|
116
|
+
}
|
|
117
|
+
let start = match.index
|
|
118
|
+
let end = match.index + match[0].length
|
|
119
|
+
if (rule.group != null && match.indices && match.indices[rule.group]) {
|
|
120
|
+
start = match.indices[rule.group][0]
|
|
121
|
+
end = match.indices[rule.group][1]
|
|
122
|
+
}
|
|
123
|
+
if (start == null || end == null || end <= start) continue
|
|
124
|
+
if (!validatorAccept(rule.validator, prepared, start, end)) continue
|
|
125
|
+
hits.push({
|
|
126
|
+
ruleId: rule.id,
|
|
127
|
+
kind: rule.piiKind || 'PII',
|
|
128
|
+
start,
|
|
129
|
+
end,
|
|
130
|
+
raw: source.slice(start, end),
|
|
131
|
+
confidence: rule.confidence,
|
|
132
|
+
severity: rule.severity,
|
|
133
|
+
})
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return resolveOverlaps(hits)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function resolveOverlaps(hits) {
|
|
141
|
+
const sorted = [...hits].sort((a, b) => (a.start - b.start) || (b.end - a.end))
|
|
142
|
+
const kept = []
|
|
143
|
+
let lastEnd = -1
|
|
144
|
+
for (const hit of sorted) {
|
|
145
|
+
if (hit.start < lastEnd) continue
|
|
146
|
+
kept.push(hit)
|
|
147
|
+
lastEnd = hit.end
|
|
148
|
+
}
|
|
149
|
+
return kept
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Public detector: same as detectPiiInternal but strips the raw value. Safe to
|
|
154
|
+
* print or store; only rule ids, kinds and offsets remain.
|
|
155
|
+
*/
|
|
156
|
+
export function detectPii(input, options = {}) {
|
|
157
|
+
return detectPiiInternal(input, options).map((hit) => ({
|
|
158
|
+
ruleId: hit.ruleId,
|
|
159
|
+
kind: hit.kind,
|
|
160
|
+
start: hit.start,
|
|
161
|
+
end: hit.end,
|
|
162
|
+
confidence: hit.confidence,
|
|
163
|
+
severity: hit.severity,
|
|
164
|
+
}))
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Redact PII / secrets in place. Idempotent: redacting redacted text is a no-op.
|
|
169
|
+
* Returns { text, redacted, count, kinds, categories, hits } and never returns
|
|
170
|
+
* the original values. `kinds` filters by PII kind (e.g. ['PHONE']).
|
|
171
|
+
*/
|
|
172
|
+
export function redact(input, options = {}) {
|
|
173
|
+
const source = String(input == null ? '' : input)
|
|
174
|
+
let hits = detectPiiInternal(source, options)
|
|
175
|
+
const only = options.kinds == null
|
|
176
|
+
? null
|
|
177
|
+
: new Set(Array.isArray(options.kinds) ? options.kinds : [options.kinds])
|
|
178
|
+
if (only) hits = hits.filter((hit) => only.has(hit.kind))
|
|
179
|
+
if (!hits.length) {
|
|
180
|
+
return { text: source, redacted: false, count: 0, kinds: [], categories: [], hits: [] }
|
|
181
|
+
}
|
|
182
|
+
const baseToken = options.token || 'REDACTED'
|
|
183
|
+
let out = ''
|
|
184
|
+
let cursor = 0
|
|
185
|
+
const publicHits = []
|
|
186
|
+
for (const hit of hits) {
|
|
187
|
+
const replacement = '[' + baseToken + ':' + hit.kind + ']'
|
|
188
|
+
out += source.slice(cursor, hit.start) + replacement
|
|
189
|
+
cursor = hit.end
|
|
190
|
+
publicHits.push({
|
|
191
|
+
ruleId: hit.ruleId,
|
|
192
|
+
kind: hit.kind,
|
|
193
|
+
start: hit.start,
|
|
194
|
+
end: hit.end,
|
|
195
|
+
replacement,
|
|
196
|
+
})
|
|
197
|
+
}
|
|
198
|
+
out += source.slice(cursor)
|
|
199
|
+
return {
|
|
200
|
+
text: out,
|
|
201
|
+
redacted: true,
|
|
202
|
+
count: publicHits.length,
|
|
203
|
+
kinds: [...new Set(publicHits.map((hit) => hit.kind))],
|
|
204
|
+
categories: ['pii'],
|
|
205
|
+
hits: publicHits,
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Human-readable label for a PII kind (safe to print, no values). */
|
|
210
|
+
export function piiKindLabel(kind) {
|
|
211
|
+
return PII_KINDS[kind] || PII_KINDS.SECRET
|
|
212
|
+
}
|