@link-foundation/sensitive-data-sanitizer 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/docs/API.md +31 -0
- package/docs/COVERAGE.md +65 -0
- package/examples/presidio-bridge.py +15 -9
- package/package.json +30 -19
- package/src/context.js +5 -1
- package/src/detection.js +13 -1
- package/src/document-entities.js +4 -2
- package/src/document-extras.js +170 -0
- package/src/document-fields.js +19 -0
- package/src/document-patterns.js +5 -8
- package/src/engines.js +15 -4
- package/src/index.d.ts +3 -5
- package/src/log-bytes.js +1 -1
- package/src/sanitizer.js +29 -4
- package/src/structured.js +96 -1
- package/src/transforms.js +7 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.17.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 8029dd8: Refresh dependencies and supported development tooling, and repair upgrade regressions in workflow validation, formatting and reproducible example installs.
|
|
8
|
+
|
|
9
|
+
## 0.17.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- Add synchronous sensitiveFields schema redaction and consistent document field aliases. Protect percent-encoded URL contacts, personal paths, opaque share links, CV employers and contextual company names; preserve multiline empty attributes and redact escaped fragments without losing structured records.
|
|
14
|
+
|
|
15
|
+
Install and verify a checksum-pinned GitHub CLI in the preview screenshot container, and exercise the same publishing tools on pull requests before generated images reach main.
|
|
16
|
+
|
|
17
|
+
Check maintained Python helpers with Ruff and CodeQL, and audit every tracked Python requirements file, including transitive benchmark dependencies, without installing NLP packages.
|
|
18
|
+
|
|
3
19
|
## 0.16.2
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
package/docs/API.md
CHANGED
|
@@ -176,6 +176,7 @@ new document recognizers; `documentEntities: false` turns them off explicitly.
|
|
|
176
176
|
| `fakeIdentity` | Off; `'specimen-and-synthetic'` retains explicitly fake identity values |
|
|
177
177
|
| `fakeValues` | `[]`; exact caller-reviewed identity values to retain, regardless of fakeIdentity |
|
|
178
178
|
| `structured` | Off; `'json'` or `'jsonl'` validates and sanitizes string keys/values separately |
|
|
179
|
+
| `sensitiveFields` | `[]`; exact JSON/JSONL property names whose complete values become a redaction string; supports synchronous calls |
|
|
179
180
|
| `structuralFields` | `[]`; declared property names suppress weak name/ID/entropy heuristics in their string values |
|
|
180
181
|
|
|
181
182
|
Unknown options and invalid limits are errors. `knownSecretsFromEnv(env, names?)` discovers credential-named variables (excluding auth configuration names) when names are omitted, or uses an explicit selection, deduplicating nonempty values. It never invokes shell commands or reads auth files. `personalVariants(entries)` generates numeric and ISO/day-first date spellings; name case variants are already handled by detection. Neither helper invents transliterations.
|
|
@@ -474,3 +475,33 @@ restores IP detection; explicit `knownSecrets` and `knownPersonal` retain
|
|
|
474
475
|
priority. No shell command or workflow expression is executed.
|
|
475
476
|
|
|
476
477
|
Authorization headers (including Proxy-Authorization) redact the scheme and one credential token. Whitespace, C0 controls (including NUL), DEL, quotes, backslashes and commas end the value, preserving subsequent log text in shell and escaped JSON prompts. Matching brackets opened within a token are included in its redaction; unmatched closing brackets remain part of the surrounding log. Already-redacted tokens and known credential references remain exempt after an optional scheme, including `<token>` placeholders. The debug callback receives value-free `gitleaks-rule` events with rule ID, input length and elapsed milliseconds; tracing is off by default.
|
|
478
|
+
|
|
479
|
+
## Caller-declared sensitive fields
|
|
480
|
+
|
|
481
|
+
In structured JSON/JSONL, `sensitiveFields: string[]` matches exact,
|
|
482
|
+
case-sensitive property names at any depth. The complete property value becomes
|
|
483
|
+
`"[REDACTED]"`, including empty strings, numbers, booleans, null, arrays and
|
|
484
|
+
objects. This deliberate schema policy can change value types. It takes
|
|
485
|
+
priority over `structuralFields` and transformations, preserves property names
|
|
486
|
+
and surrounding whitespace, and works in both `sanitize` and `createSanitizer`.
|
|
487
|
+
It has no effect without `structured`. The personal-data switch
|
|
488
|
+
`credentialsOnly` still suppresses personal findings.
|
|
489
|
+
|
|
490
|
+
```js
|
|
491
|
+
sanitize('{"privateNote":"arbitrary free text","count":2}', {
|
|
492
|
+
structured: 'json',
|
|
493
|
+
sensitiveFields: ['privateNote'],
|
|
494
|
+
}).text;
|
|
495
|
+
// {"privateNote":"[REDACTED]","count":2}
|
|
496
|
+
|
|
497
|
+
sanitize('Lyutik работал', {
|
|
498
|
+
profile: 'documents',
|
|
499
|
+
knownPersonal: [{ type: 'ORGANIZATION', value: 'Lyutik' }],
|
|
500
|
+
}).text;
|
|
501
|
+
// [REDACTED] работал
|
|
502
|
+
```
|
|
503
|
+
|
|
504
|
+
`knownPersonal` is the synchronous custom-value list, including names without
|
|
505
|
+
context. For computed synchronous detection, callers can supply validated
|
|
506
|
+
`findings` for the exact input. `detectors` remains an asynchronous-engine
|
|
507
|
+
option; `sanitize` does not invoke these callbacks.
|
package/docs/COVERAGE.md
CHANGED
|
@@ -337,3 +337,68 @@ further work. These numbers do not describe `createSanitizer` with Secretlint.
|
|
|
337
337
|
|
|
338
338
|
The [complete issue 89 requirement inventory](case-studies/issue-89/README.md)
|
|
339
339
|
records every row, root cause, alternative and regression mapping.
|
|
340
|
+
|
|
341
|
+
## Document URL and field refinements (issues 91 and 94)
|
|
342
|
+
|
|
343
|
+
The document profile detects `email=` and `phone=` query values in absolute and
|
|
344
|
+
relative URLs and access-log text, retaining the label while percent-decoding.
|
|
345
|
+
It replaces the same complete value for plain and percent-encoded contacts,
|
|
346
|
+
without reserializing unaffected URL bytes. Invalid percent escapes do not
|
|
347
|
+
qualify through this decoding rule. Personal `/u/`, `/in/`, `/user(s)/`,
|
|
348
|
+
`/profile(s)/`, `/people/`, `/member(s)/` and `/home/` paths apply independently
|
|
349
|
+
of the containing field label. These paths can also describe public accounts;
|
|
350
|
+
the rule cannot establish identity or access permissions.
|
|
351
|
+
|
|
352
|
+
An absolute HTTP(S) URL with a 24–128-character ASCII alphanumeric/underscore/
|
|
353
|
+
hyphen path segment is treated as a possible share link and replaced, including
|
|
354
|
+
Google Drive/Docs and generic `/s/<opaque-id>` links. This is a conservative
|
|
355
|
+
shape rule, not a check of sharing permissions or entropy; public URLs with
|
|
356
|
+
long segments may also match. Ordinary documentation paths and provider roots
|
|
357
|
+
remain unchanged. Personal site roots such as
|
|
358
|
+
`портфолио https://ivan-petrov.example.org/` remain a documented limit: a root
|
|
359
|
+
alone cannot establish that the site belongs to a person.
|
|
360
|
+
|
|
361
|
+
Structured document field aliases remove `_`/`-` and ignore case before matching
|
|
362
|
+
this explicit vocabulary: `firstName`, `lastName`, `middleName`, `patronymic`,
|
|
363
|
+
`birthday`, `birthDate`, `sex`, `citizenship`, `nationality`, `company`,
|
|
364
|
+
`companyName`, `employer`, `university`, `school`, `contact`, `resumeUrl`,
|
|
365
|
+
`alternateUrl`, `profileUrl`, `companyUrl`. Their nonempty string values are
|
|
366
|
+
sensitive in full, including ISO birth timestamps. Thus `first_name` and
|
|
367
|
+
`firstName`, or `birth_date` and `birthDate`, follow the same policy. These are
|
|
368
|
+
schema assumptions: `contact` or `company` can hold operational data too.
|
|
369
|
+
Existing native labels still apply. Unlisted URL-named fields have ordinary
|
|
370
|
+
content detection; arbitrary names do not imply sensitivity. Declare them with
|
|
371
|
+
[`sensitiveFields`](API.md#caller-declared-sensitive-fields) to replace complete
|
|
372
|
+
values, including non-string values. Generic numeric primitives otherwise keep
|
|
373
|
+
the existing type-preservation policy.
|
|
374
|
+
|
|
375
|
+
Employer labels consume multi-word values to a comma or line/quote boundary.
|
|
376
|
+
Dated CV entries additionally support compound titles, title parentheses/slashes,
|
|
377
|
+
quoted group names, legal-form/location suffixes, empty/`null` range ends,
|
|
378
|
+
multiple entries, employer-first role entries and pipe-separated entries.
|
|
379
|
+
The supplemental grammar uses finite title cues and bounded entries. Undated
|
|
380
|
+
`Менеджер, Ромашка Лютик`, `- Ромашка Лютик 5 лет` and
|
|
381
|
+
`Последнее место работы Ромашка Лютик` remain explicit ambiguous limits, as
|
|
382
|
+
permitted by #94. Use known values or schema declarations for them.
|
|
383
|
+
|
|
384
|
+
Company prose recognizes `компании «…»`, `опыт в`, `Роль в`, `проекты для`,
|
|
385
|
+
capitalized lists after those cues, `опыт (Company Name)`, and names followed
|
|
386
|
+
by `(2 года)`/`(3 года)`-style durations. Arbitrary uncued company words remain
|
|
387
|
+
ambiguous; synchronous callers can supply `knownPersonal` lists or precomputed
|
|
388
|
+
`findings`. Custom `detectors` callbacks are still used by `createSanitizer`
|
|
389
|
+
only. No model or network service is required for these native rules.
|
|
390
|
+
|
|
391
|
+
`контакт:` and `пишите,` introduce domestic mobile numbers. A complete line
|
|
392
|
+
consisting of an 11-digit `79…` or `89…` mobile shape is also sensitive. Arbitrary
|
|
393
|
+
prose numbers such as `order 89161234567 shipped` and `id 79161234567` remain
|
|
394
|
+
unchanged. This shape rule does not prove that a number is assigned.
|
|
395
|
+
|
|
396
|
+
Additional forms cover Cyrillic acronyms after `Education:`, `учился в`,
|
|
397
|
+
`Соискатель:`/`Сотрудник:` names and split-line `С уважением,` signatures.
|
|
398
|
+
`Дети: нет` remains an empty attribute with later JSON string lines preserved.
|
|
399
|
+
When mapped escaped fragments lose detecting context on recursive decoding,
|
|
400
|
+
redaction replaces the fragment instead of repeating an unchanged result until
|
|
401
|
+
`ERR_RESIDUAL`; the surrounding JSON remains parseable.
|
|
402
|
+
|
|
403
|
+
The [requirement inventory and research](case-studies/issue-100/README.md)
|
|
404
|
+
records the complete scope and regression evidence.
|
|
@@ -5,28 +5,34 @@ Example: python -m pip install presidio-analyzer
|
|
|
5
5
|
Configure NlpEngineProvider here for other installed language models.
|
|
6
6
|
This script does not download models or contact a remote analysis service.
|
|
7
7
|
"""
|
|
8
|
+
|
|
8
9
|
import argparse
|
|
9
10
|
import json
|
|
10
11
|
import sys
|
|
11
12
|
|
|
13
|
+
import spacy
|
|
12
14
|
from presidio_analyzer import AnalyzerEngine
|
|
13
15
|
from presidio_analyzer.nlp_engine import NlpEngineProvider
|
|
14
|
-
import spacy
|
|
15
16
|
|
|
16
17
|
parser = argparse.ArgumentParser()
|
|
17
|
-
parser.add_argument(
|
|
18
|
-
parser.add_argument(
|
|
18
|
+
parser.add_argument("--language", default="en")
|
|
19
|
+
parser.add_argument("--model", default="en_core_web_lg")
|
|
19
20
|
options = parser.parse_args()
|
|
20
21
|
if not spacy.util.is_package(options.model):
|
|
21
|
-
raise SystemExit(
|
|
22
|
+
raise SystemExit("Install the selected model before running this offline bridge.")
|
|
22
23
|
|
|
23
24
|
text = sys.stdin.read(10 * 1024 * 1024 + 1)
|
|
24
25
|
if len(text) > 10 * 1024 * 1024:
|
|
25
26
|
raise SystemExit(2)
|
|
26
|
-
nlp = NlpEngineProvider(
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
}
|
|
27
|
+
nlp = NlpEngineProvider(
|
|
28
|
+
nlp_configuration={
|
|
29
|
+
"nlp_engine_name": "spacy",
|
|
30
|
+
"models": [{"lang_code": options.language, "model_name": options.model}],
|
|
31
|
+
}
|
|
32
|
+
).create_engine()
|
|
30
33
|
analyzer = AnalyzerEngine(nlp_engine=nlp, supported_languages=[options.language])
|
|
31
34
|
results = analyzer.analyze(text=text, language=options.language)
|
|
32
|
-
json.dump(
|
|
35
|
+
json.dump(
|
|
36
|
+
[{"start": r.start, "end": r.end, "entity_type": r.entity_type} for r in results],
|
|
37
|
+
sys.stdout,
|
|
38
|
+
)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.1",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -77,7 +77,9 @@
|
|
|
77
77
|
"license": "Unlicense",
|
|
78
78
|
"devEngines": {
|
|
79
79
|
"packageManager": {
|
|
80
|
-
"name": "npm"
|
|
80
|
+
"name": "npm",
|
|
81
|
+
"version": ">=12.2.2",
|
|
82
|
+
"onFail": "error"
|
|
81
83
|
}
|
|
82
84
|
},
|
|
83
85
|
"publishConfig": {
|
|
@@ -92,19 +94,23 @@
|
|
|
92
94
|
},
|
|
93
95
|
"devDependencies": {
|
|
94
96
|
"@changesets/cli": "^3.0.3",
|
|
95
|
-
"eslint": "^10.
|
|
97
|
+
"@eslint/js": "^10.0.1",
|
|
98
|
+
"browser-commander": "0.34.1",
|
|
99
|
+
"cmd-shim": "^9.0.2",
|
|
100
|
+
"command-stream": "2.6.0",
|
|
101
|
+
"eslint": "^10.14.0",
|
|
96
102
|
"eslint-config-prettier": "^10.1.8",
|
|
97
|
-
"eslint-plugin-prettier": "^5.5.
|
|
103
|
+
"eslint-plugin-prettier": "^5.5.7",
|
|
98
104
|
"husky": "^9.1.7",
|
|
99
|
-
"jscpd": "^5.4.
|
|
100
|
-
"
|
|
105
|
+
"jscpd": "^5.4.2",
|
|
106
|
+
"lino-arguments": "0.4.0",
|
|
107
|
+
"lint-staged": "^17.6.0",
|
|
101
108
|
"mrz": "^5.0.2",
|
|
102
|
-
"
|
|
103
|
-
"
|
|
104
|
-
"
|
|
105
|
-
"
|
|
106
|
-
"
|
|
107
|
-
"lino-arguments": "0.4.0"
|
|
109
|
+
"playwright": "1.65.0",
|
|
110
|
+
"prettier": "^3.9.10",
|
|
111
|
+
"secretlint": "13.1.0",
|
|
112
|
+
"semver": "^7.8.6",
|
|
113
|
+
"test-anywhere": "^0.9.1"
|
|
108
114
|
},
|
|
109
115
|
"lint-staged": {
|
|
110
116
|
"*.{js,mjs,cjs}": [
|
|
@@ -119,25 +125,30 @@
|
|
|
119
125
|
"*.sh": [
|
|
120
126
|
"pipx run --spec shellcheck-py==0.11.0.1 shellcheck"
|
|
121
127
|
],
|
|
128
|
+
"*.py": [
|
|
129
|
+
"pipx run --spec ruff==0.17.4 ruff check --fix --force-exclude",
|
|
130
|
+
"pipx run --spec ruff==0.17.4 ruff format --force-exclude",
|
|
131
|
+
"pipx run --spec ruff==0.17.4 ruff format --check --force-exclude"
|
|
132
|
+
],
|
|
122
133
|
".husky/*": [
|
|
123
134
|
"pipx run --spec shellcheck-py==0.11.0.1 shellcheck"
|
|
124
135
|
]
|
|
125
136
|
},
|
|
126
137
|
"dependencies": {
|
|
127
138
|
"@faker-js/faker": "^10.6.0",
|
|
128
|
-
"@secretlint/core": "13.0
|
|
129
|
-
"@secretlint/profiler": "13.0
|
|
130
|
-
"@secretlint/secretlint-rule-preset-recommend": "13.0
|
|
131
|
-
"libphonenumber-js": "1.13.
|
|
132
|
-
"links-notation": "0.
|
|
139
|
+
"@secretlint/core": "13.1.0",
|
|
140
|
+
"@secretlint/profiler": "13.1.0",
|
|
141
|
+
"@secretlint/secretlint-rule-preset-recommend": "13.1.0",
|
|
142
|
+
"libphonenumber-js": "1.13.15",
|
|
143
|
+
"links-notation": "0.31.0",
|
|
133
144
|
"re2-wasm": "1.0.2",
|
|
134
|
-
"re2js": "^2.9.
|
|
145
|
+
"re2js": "^2.9.1",
|
|
135
146
|
"yaml": "^2.9.1"
|
|
136
147
|
},
|
|
137
148
|
"allowScripts": {
|
|
138
149
|
"node-pty": false
|
|
139
150
|
},
|
|
140
151
|
"overrides": {
|
|
141
|
-
"minimatch@>=7 <11": "^11.0.
|
|
152
|
+
"minimatch@>=7 <11": "^11.0.5"
|
|
142
153
|
}
|
|
143
154
|
}
|
package/src/context.js
CHANGED
|
@@ -421,7 +421,11 @@ function detectPersonalLabels(text, emit) {
|
|
|
421
421
|
const range = valueRange(
|
|
422
422
|
text,
|
|
423
423
|
match.index + match[0].length + suffix.length,
|
|
424
|
-
|
|
424
|
+
/[?&]$/.test(text.slice(0, match.index))
|
|
425
|
+
? 'query'
|
|
426
|
+
: identity
|
|
427
|
+
? 'token'
|
|
428
|
+
: 'line',
|
|
425
429
|
512
|
|
426
430
|
);
|
|
427
431
|
if (range && personalValueAllowed(text, range, type, match.index)) {
|
package/src/detection.js
CHANGED
|
@@ -273,7 +273,19 @@ function detectPersonalFormats(text, options, emit) {
|
|
|
273
273
|
collectMatches(
|
|
274
274
|
text,
|
|
275
275
|
/(?<!\d)\+\d[\d ()-]{6,24}\d(?!\d)/g,
|
|
276
|
-
|
|
276
|
+
(finding) => {
|
|
277
|
+
if (
|
|
278
|
+
/[?&]phone=$/i.test(
|
|
279
|
+
text.slice(Math.max(0, finding.start - 8), finding.start)
|
|
280
|
+
)
|
|
281
|
+
) {
|
|
282
|
+
const token = /^[^\s&#]+/.exec(
|
|
283
|
+
text.slice(finding.start, finding.end)
|
|
284
|
+
)[0];
|
|
285
|
+
finding.end = finding.start + token.length;
|
|
286
|
+
}
|
|
287
|
+
emit(finding);
|
|
288
|
+
},
|
|
277
289
|
{ type: 'PHONE', category: 'personal', rule: 'international-phone' },
|
|
278
290
|
0,
|
|
279
291
|
(v) => {
|
package/src/document-entities.js
CHANGED
|
@@ -3,6 +3,7 @@ import { firstNames } from './names.js';
|
|
|
3
3
|
import { organizationCore, personalRegex } from './personal-variants.js';
|
|
4
4
|
import { documentVocabulary } from './document-vocabulary.js';
|
|
5
5
|
import { months } from './personal-context.js';
|
|
6
|
+
import { detectDocumentExtras } from './document-extras.js';
|
|
6
7
|
import { detectDocumentPatterns } from './document-patterns.js';
|
|
7
8
|
|
|
8
9
|
const words = (column) =>
|
|
@@ -47,7 +48,7 @@ const ruAcronym =
|
|
|
47
48
|
const ruResidence =
|
|
48
49
|
/(?:[Гг]ород|[Пп]роживаю|[Пп]роживание|[Мм]есто жительства)[ \t]*:[ \t]*([А-ЯЁ][а-яё-]{2,47})|([А-ЯЁ][а-яё-]{2,47})(?=,[ \t]*(?:готов[а]?[ \t]+к[ \t]+переезду|Россия|Беларусь|Казахстан|Украина))/gu;
|
|
49
50
|
const ruIntroduction = new RegExp(
|
|
50
|
-
`(?:[Мм]еня зовут|[Сс] уважением)[, \t]+(${firstNames.ru.split(' ').join('|')})(?!${boundary})`,
|
|
51
|
+
`(?:[Мм]еня зовут|[Сс] уважением)[, \t\r\n]+(${firstNames.ru.split(' ').join('|')})(?!${boundary})`,
|
|
51
52
|
'gu'
|
|
52
53
|
);
|
|
53
54
|
const groupOrganization = new RegExp(
|
|
@@ -67,7 +68,7 @@ const rangeOpen = ['по настоящее время', 'настоящее в
|
|
|
67
68
|
.flatMap((word) => [word, capitalized(word)])
|
|
68
69
|
.map(escapePattern)
|
|
69
70
|
.join('|');
|
|
70
|
-
const range = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen})(?!${boundary})`;
|
|
71
|
+
const range = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen}|null|(?=\\)))(?!${boundary})`;
|
|
71
72
|
const titleWord = '[\\p{L}\\p{N}][\\p{L}\\p{N}/+#.&-]{0,39}';
|
|
72
73
|
// A segment starts a line or follows `|`. `Опыт:` or `Опыт -` before the title
|
|
73
74
|
// reads like a list marker: the date range remains the anchor of the rule.
|
|
@@ -89,6 +90,7 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
89
90
|
return;
|
|
90
91
|
}
|
|
91
92
|
detectDocumentPatterns(text, emit);
|
|
93
|
+
detectDocumentExtras(text, emit, range);
|
|
92
94
|
const found = [];
|
|
93
95
|
const capture = (finding) => {
|
|
94
96
|
found.push(finding);
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { collectMatches } from './detection.js';
|
|
2
|
+
|
|
3
|
+
const boundary = '[\\p{L}\\p{N}_]';
|
|
4
|
+
const proper = '[\\p{Lu}][\\p{L}\\p{M}\\p{N}.-]{0,46}[\\p{L}\\p{M}\\p{N}]';
|
|
5
|
+
const name = `${proper}(?:[ \\t]+${proper}){0,5}`;
|
|
6
|
+
const orgList = `(?:«[^«»\\r\\n]{1,160}»|${name})(?:[ \\t]*(?:,|и|and)[ \\t]+${name}){0,5}`;
|
|
7
|
+
const cues = new RegExp(
|
|
8
|
+
`(?<!${boundary})(?:компании|опыт[ \\t]+в|проекты[ \\t]+для|[Рр]оль[ \\t]+в)[ \\t]+(${orgList})(?!${boundary})`,
|
|
9
|
+
'gu'
|
|
10
|
+
);
|
|
11
|
+
const duration = new RegExp(
|
|
12
|
+
`(?<!${boundary})(${name})(?=[ \\t]*\\(\\d{1,2}[ \\t]+(?:год|года|лет|years?)\\))`,
|
|
13
|
+
'gu'
|
|
14
|
+
);
|
|
15
|
+
// JavaScript's \b is ASCII-only; Cyrillic titles use a Unicode lookahead.
|
|
16
|
+
const role =
|
|
17
|
+
/^(?:менеджер|директор|руководитель|инженер|аналитик|разработчик)(?![\p{L}])|^(?:manager|director|engineer|developer|senior|qa)\b/iu;
|
|
18
|
+
|
|
19
|
+
export function detectDocumentExtras(text, emit, range) {
|
|
20
|
+
for (const [type, rule, pattern, group] of [
|
|
21
|
+
['PHONE', 'document-mobile-bare', /^[78]9\d{9}$/gm, 0],
|
|
22
|
+
[
|
|
23
|
+
'PHONE',
|
|
24
|
+
'document-mobile-context',
|
|
25
|
+
/(?<![\p{L}\p{N}_])(?:пишите|контакт)[:,]?[ \t]+([78]9\d{9})(?!\d)/giu,
|
|
26
|
+
1,
|
|
27
|
+
],
|
|
28
|
+
[
|
|
29
|
+
'ORGANIZATION',
|
|
30
|
+
'organization-experience-prose',
|
|
31
|
+
new RegExp(`опыт[ \\t]*\\((${name})\\)`, 'gu'),
|
|
32
|
+
1,
|
|
33
|
+
],
|
|
34
|
+
['ORGANIZATION', 'organization-duration', duration, 1],
|
|
35
|
+
[
|
|
36
|
+
'PERSON',
|
|
37
|
+
'document-role',
|
|
38
|
+
/(?<![\p{L}\p{N}_])(?:Соискатель|Сотрудник)[ \t]*:[ \t]*([\p{Lu}][\p{L}-]{1,47}(?:[ \t]+[\p{Lu}][\p{L}-]{1,47}){0,4})/gu,
|
|
39
|
+
1,
|
|
40
|
+
],
|
|
41
|
+
[
|
|
42
|
+
'URL',
|
|
43
|
+
'document-personal-path',
|
|
44
|
+
/(?<![\p{L}\p{N}_])(?:https?:\/\/[^\s/"<>]+)?\/(?:u|in|users?|profiles?|people|members?|home)\/[^\s?&#"<>\\]{1,128}/giu,
|
|
45
|
+
0,
|
|
46
|
+
],
|
|
47
|
+
[
|
|
48
|
+
'URL',
|
|
49
|
+
'document-share-link',
|
|
50
|
+
/https?:\/\/[^\s/"<>]+\/(?:[^\s/?#"<>]{1,64}\/){0,5}[A-Za-z0-9_-]{24,128}(?:\/[^\s?&#"<>\\]{0,64})?/gu,
|
|
51
|
+
0,
|
|
52
|
+
],
|
|
53
|
+
]) {
|
|
54
|
+
collectMatches(
|
|
55
|
+
text,
|
|
56
|
+
pattern,
|
|
57
|
+
emit,
|
|
58
|
+
{ type, rule, category: 'personal', confidence: 0.85 },
|
|
59
|
+
group
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
for (const match of text.matchAll(cues)) {
|
|
63
|
+
const offset = match.index + match[0].lastIndexOf(match[1]);
|
|
64
|
+
for (const company of match[1].matchAll(
|
|
65
|
+
new RegExp(`«[^«»\\r\\n]{1,160}»|${name}`, 'gu')
|
|
66
|
+
)) {
|
|
67
|
+
emit({
|
|
68
|
+
start: offset + company.index,
|
|
69
|
+
end: offset + company.index + company[0].length,
|
|
70
|
+
type: 'ORGANIZATION',
|
|
71
|
+
category: 'personal',
|
|
72
|
+
rule: 'organization-cue',
|
|
73
|
+
confidence: 0.85,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
detectQueryContacts(text, emit);
|
|
78
|
+
detectExperience(text, emit, range);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function detectQueryContacts(text, emit) {
|
|
82
|
+
// Retain query context while decoding; standalone contacts may be public specimens.
|
|
83
|
+
for (const match of text.matchAll(
|
|
84
|
+
/[?&](email|phone)=([^\s&#"<>\\]{1,512})/giu
|
|
85
|
+
)) {
|
|
86
|
+
let value;
|
|
87
|
+
try {
|
|
88
|
+
value = decodeURIComponent(match[2]);
|
|
89
|
+
} catch {
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
const email = match[1].toLowerCase() === 'email';
|
|
93
|
+
if (
|
|
94
|
+
!(
|
|
95
|
+
email ? /^[^\s@]+@[^\s@]+\.[\p{L}]{2,63}$/u : /^\+?[\d () .-]{7,40}$/
|
|
96
|
+
).test(value)
|
|
97
|
+
) {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (value === '[REDACTED]') {
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
const start = match.index + match[0].length - match[2].length;
|
|
104
|
+
emit({
|
|
105
|
+
start,
|
|
106
|
+
end: start + match[2].length,
|
|
107
|
+
type: email ? 'EMAIL' : 'PHONE',
|
|
108
|
+
category: 'personal',
|
|
109
|
+
rule: 'label',
|
|
110
|
+
confidence: 0.95,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function detectExperience(text, emit, range) {
|
|
116
|
+
// Date anchors bound each entry; do not guess at undated employer/title fragments.
|
|
117
|
+
collectMatches(
|
|
118
|
+
text,
|
|
119
|
+
new RegExp(
|
|
120
|
+
`(?:^|[\\r\\n])[ \\t]*[^|\\r\\n]{1,120}\\|[ \\t]*(${name})[ \\t]*\\|[ \\t]*${range}`,
|
|
121
|
+
'gu'
|
|
122
|
+
),
|
|
123
|
+
emit,
|
|
124
|
+
{
|
|
125
|
+
type: 'ORGANIZATION',
|
|
126
|
+
rule: 'organization-experience',
|
|
127
|
+
category: 'personal',
|
|
128
|
+
confidence: 0.85,
|
|
129
|
+
},
|
|
130
|
+
1
|
|
131
|
+
);
|
|
132
|
+
const dated = new RegExp(`([^\\r\\n;|]{1,320}?)\\((${range})\\)`, 'gu');
|
|
133
|
+
for (const match of text.matchAll(dated)) {
|
|
134
|
+
const prefix = match[1];
|
|
135
|
+
const content = prefix
|
|
136
|
+
.replace(/^.*\(\d[^)]*\)[, \t]*/, '')
|
|
137
|
+
.replace(/^[, \t]+/, '');
|
|
138
|
+
const chunks = [...content.matchAll(/[^,]+/g)];
|
|
139
|
+
const parts = chunks.map((part) => part[0].trim());
|
|
140
|
+
let value;
|
|
141
|
+
if (parts.length > 1 && role.test(parts[0])) {
|
|
142
|
+
let index = 1;
|
|
143
|
+
while (index < parts.length - 1 && /^[\p{Ll}]/u.test(parts[index])) {
|
|
144
|
+
index++;
|
|
145
|
+
}
|
|
146
|
+
value = content.slice(chunks[index].index).trim();
|
|
147
|
+
} else if (parts.length === 2 && role.test(parts[1])) {
|
|
148
|
+
value = parts[0].trim();
|
|
149
|
+
}
|
|
150
|
+
if (
|
|
151
|
+
!value ||
|
|
152
|
+
!/^[\p{Lu}«“"]/u.test(value) ||
|
|
153
|
+
value.includes('[REDACTED]')
|
|
154
|
+
) {
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const start = match.index + prefix.lastIndexOf(value);
|
|
158
|
+
if (start < match.index) {
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
emit({
|
|
162
|
+
start,
|
|
163
|
+
end: start + value.length,
|
|
164
|
+
type: 'ORGANIZATION',
|
|
165
|
+
category: 'personal',
|
|
166
|
+
rule: 'organization-experience',
|
|
167
|
+
confidence: 0.85,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// One explicit field vocabulary; camelCase, snake_case and kebab-case normalize alike.
|
|
2
|
+
const groups = {
|
|
3
|
+
PERSON: ['firstname', 'lastname', 'middlename', 'patronymic'],
|
|
4
|
+
DATE_OF_BIRTH: ['birthday', 'birthdate'],
|
|
5
|
+
GENDER: ['sex'],
|
|
6
|
+
PERSONAL_ATTRIBUTE: ['citizenship', 'nationality'],
|
|
7
|
+
ORGANIZATION: ['company', 'companyname', 'employer'],
|
|
8
|
+
EDUCATION: ['university', 'school'],
|
|
9
|
+
PHONE: ['contact'],
|
|
10
|
+
URL: ['resumeurl', 'alternateurl', 'profileurl', 'companyurl'],
|
|
11
|
+
};
|
|
12
|
+
const types = new Map(
|
|
13
|
+
Object.entries(groups).flatMap(([type, fields]) =>
|
|
14
|
+
fields.map((field) => [field, type])
|
|
15
|
+
)
|
|
16
|
+
);
|
|
17
|
+
export function documentFieldType(field) {
|
|
18
|
+
return types.get(field.replace(/[_-]/g, '').toLowerCase());
|
|
19
|
+
}
|
package/src/document-patterns.js
CHANGED
|
@@ -30,7 +30,7 @@ const rules = [
|
|
|
30
30
|
[
|
|
31
31
|
'PHONE',
|
|
32
32
|
'document-phone-label',
|
|
33
|
-
/(?<![\p{L}\p{N}_])(?:(?:контактный[ \t]+)
|
|
33
|
+
/(?<![\p{L}\p{N}_])(?:(?:контактный[ \t]+)?телефон|контакт|тел\.?|моб\.?|сот\.?|WhatsApp)(?:[ \t]*:[ \t]*|[ \t]+)((?:[78][ () .-]*)?9\d{2}[ () .-]*\d{3}[ .-]*\d{2}[ .-]*\d{2})(?!\d)/giu,
|
|
34
34
|
1,
|
|
35
35
|
],
|
|
36
36
|
[
|
|
@@ -78,13 +78,13 @@ const rules = [
|
|
|
78
78
|
[
|
|
79
79
|
'EDUCATION',
|
|
80
80
|
'school-acronym-context',
|
|
81
|
-
/(?<![\p{L}\p{N}_])(
|
|
81
|
+
/(?<![\p{L}\p{N}_])(?:образование|ВУЗ|университет|учил(?:ся|ась)[ \t]+в|окончил[аи]?)(?:[ \t]*:[ \t]*|[ \t]+)([А-ЯЁ]{2,12}(?:[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{2,40})?)(?![\p{L}\p{N}_])/giu,
|
|
82
82
|
1,
|
|
83
83
|
],
|
|
84
84
|
[
|
|
85
85
|
'EDUCATION',
|
|
86
86
|
'education-label',
|
|
87
|
-
/(?<![\p{L}\p{N}_])Education[ \t]*:[ \t]*([
|
|
87
|
+
/(?<![\p{L}\p{N}_])Education[ \t]*:[ \t]*([\p{Lu}][\p{L}.-]{1,47})(?![\p{L}\p{N}_])/gu,
|
|
88
88
|
1,
|
|
89
89
|
],
|
|
90
90
|
[
|
|
@@ -124,16 +124,13 @@ const rules = [
|
|
|
124
124
|
[
|
|
125
125
|
'PERSONAL_ATTRIBUTE',
|
|
126
126
|
'profile-attribute',
|
|
127
|
-
/(?<![\p{L}\p{N}_])(?:Гражданство|Citizenship|Семейное положение|Marital status|Дети|национальность|Вероисповедание|инвалидность|судимость)[ \t]*:[ \t]*([
|
|
127
|
+
/(?<![\p{L}\p{N}_])(?:Гражданство|Citizenship|Семейное положение|Marital status|Дети|национальность|Вероисповедание|инвалидность|судимость)[ \t]*:[ \t]*([^\\\r\n";|]{1,160})/giu,
|
|
128
128
|
1,
|
|
129
129
|
],
|
|
130
130
|
[
|
|
131
131
|
'ORGANIZATION',
|
|
132
132
|
'employer-label',
|
|
133
|
-
|
|
134
|
-
`(?<!${boundary})(?:[Кк]омпания|[Рр]аботодатель|[Мм]есто работы|[Cc]ompany|[Ee]mployer)[ \\t]*:[ \\t]*([\\p{Lu}][\\p{L}\\p{M}.-]{1,47})(?!${boundary})`,
|
|
135
|
-
'gu'
|
|
136
|
-
),
|
|
133
|
+
/(?<![\p{L}\p{N}_])(?:Компания|Работодатель|Место работы|Company|Employer)[ \t]*:[ \t]*([\p{Lu}][^,;|\\\r\n":]{0,159})/giu,
|
|
137
134
|
1,
|
|
138
135
|
],
|
|
139
136
|
[
|
package/src/engines.js
CHANGED
|
@@ -24,6 +24,8 @@ import {
|
|
|
24
24
|
} from './placeholders.js';
|
|
25
25
|
import {
|
|
26
26
|
structuredBatches,
|
|
27
|
+
structuredFieldFindings,
|
|
28
|
+
declaredFieldFindings,
|
|
27
29
|
structuredValueViews,
|
|
28
30
|
mapStructuredFindings,
|
|
29
31
|
verifyStructured,
|
|
@@ -131,6 +133,7 @@ async function analyzeStructured(
|
|
|
131
133
|
) {
|
|
132
134
|
const findings = [];
|
|
133
135
|
for (const batch of structuredBatches(text, options)) {
|
|
136
|
+
findings.push(...structuredFieldFindings(batch, options));
|
|
134
137
|
for (const finding of mapStructuredFindings(
|
|
135
138
|
batch,
|
|
136
139
|
await analyze(
|
|
@@ -162,10 +165,18 @@ async function analyzeStructured(
|
|
|
162
165
|
}
|
|
163
166
|
// Native structured inspection validates, locates and reapplies the
|
|
164
167
|
// same fake-value/public policy to source offsets from all engines.
|
|
165
|
-
return inspectFindingsOnly(
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
168
|
+
return inspectFindingsOnly(
|
|
169
|
+
text,
|
|
170
|
+
[
|
|
171
|
+
...findings,
|
|
172
|
+
...declaredFieldFindings(text, options),
|
|
173
|
+
...(nativeFindings ?? []),
|
|
174
|
+
],
|
|
175
|
+
{
|
|
176
|
+
...options,
|
|
177
|
+
findings: initial,
|
|
178
|
+
}
|
|
179
|
+
);
|
|
169
180
|
}
|
|
170
181
|
|
|
171
182
|
export function createSanitizer(options = {}) {
|
package/src/index.d.ts
CHANGED
|
@@ -9,11 +9,7 @@ export interface Finding {
|
|
|
9
9
|
/** Detector confidence, not a cross-engine probability calibration. */
|
|
10
10
|
confidence?: number;
|
|
11
11
|
likelihood?:
|
|
12
|
-
| '
|
|
13
|
-
| 'UNLIKELY'
|
|
14
|
-
| 'POSSIBLE'
|
|
15
|
-
| 'LIKELY'
|
|
16
|
-
| 'VERY_LIKELY';
|
|
12
|
+
'VERY_UNLIKELY' | 'UNLIKELY' | 'POSSIBLE' | 'LIKELY' | 'VERY_LIKELY';
|
|
17
13
|
/** One-based line and UTF-16 column, populated by inspect. */
|
|
18
14
|
line?: number;
|
|
19
15
|
column?: number;
|
|
@@ -67,6 +63,8 @@ export interface SanitizerOptions {
|
|
|
67
63
|
structured?: 'json' | 'jsonl';
|
|
68
64
|
/** Property names whose metadata heuristics may be suppressed. */
|
|
69
65
|
structuralFields?: string[];
|
|
66
|
+
/** Exact, case-sensitive property names whose entire values are redacted in JSON/JSONL. */
|
|
67
|
+
sensitiveFields?: string[];
|
|
70
68
|
/** Personal replacement threshold, default 0.5. Inspect retains lower scores. */
|
|
71
69
|
minConfidence?: number;
|
|
72
70
|
/** Alias for minConfidence; specifying both requires equal values. */
|
package/src/log-bytes.js
CHANGED
|
@@ -13,7 +13,7 @@ export function decodeLogBytes(input) {
|
|
|
13
13
|
} catch {
|
|
14
14
|
const parts = [];
|
|
15
15
|
let start = 0;
|
|
16
|
-
for (let i = 0; i < bytes.length;
|
|
16
|
+
for (let i = 0; i < bytes.length;) {
|
|
17
17
|
const width = sequenceWidth(bytes[i]);
|
|
18
18
|
if (validSequence(bytes, i, width)) {
|
|
19
19
|
i += width;
|
package/src/sanitizer.js
CHANGED
|
@@ -31,6 +31,8 @@ import {
|
|
|
31
31
|
} from './placeholders.js';
|
|
32
32
|
import {
|
|
33
33
|
structuredBatches,
|
|
34
|
+
structuredFieldFindings,
|
|
35
|
+
declaredFieldFindings,
|
|
34
36
|
inspectStructuredValues,
|
|
35
37
|
mapStructuredFindings,
|
|
36
38
|
preserveIdentifierKeys,
|
|
@@ -71,6 +73,7 @@ const optionNames = new Set([
|
|
|
71
73
|
'fakeValues',
|
|
72
74
|
'structured',
|
|
73
75
|
'structuralFields',
|
|
76
|
+
'sensitiveFields',
|
|
74
77
|
'minConfidence',
|
|
75
78
|
'threshold',
|
|
76
79
|
'profile',
|
|
@@ -101,12 +104,18 @@ function validateCollections(options) {
|
|
|
101
104
|
'detectors',
|
|
102
105
|
'fakeValues',
|
|
103
106
|
'structuralFields',
|
|
107
|
+
'sensitiveFields',
|
|
104
108
|
]) {
|
|
105
109
|
if (options[key] !== undefined && !Array.isArray(options[key])) {
|
|
106
110
|
throw failure('ERR_CONFIG');
|
|
107
111
|
}
|
|
108
112
|
}
|
|
109
|
-
for (const key of [
|
|
113
|
+
for (const key of [
|
|
114
|
+
'knownSecrets',
|
|
115
|
+
'fakeValues',
|
|
116
|
+
'structuralFields',
|
|
117
|
+
'sensitiveFields',
|
|
118
|
+
]) {
|
|
110
119
|
if (
|
|
111
120
|
(options[key] ?? []).some((value) => typeof value !== 'string' || !value)
|
|
112
121
|
) {
|
|
@@ -394,6 +403,7 @@ function inspectText(text, options = {}) {
|
|
|
394
403
|
validateInput(text, options);
|
|
395
404
|
if (options.structured) {
|
|
396
405
|
const findings = structuredBatches(text, options).flatMap((batch) => [
|
|
406
|
+
...structuredFieldFindings(batch, options),
|
|
397
407
|
...mapStructuredFindings(
|
|
398
408
|
batch,
|
|
399
409
|
inspect(batch.text, {
|
|
@@ -407,7 +417,11 @@ function inspectText(text, options = {}) {
|
|
|
407
417
|
inspect(value, { ...options, structured: undefined, findings: [] })
|
|
408
418
|
),
|
|
409
419
|
]);
|
|
410
|
-
return inspectFindingsOnly(
|
|
420
|
+
return inspectFindingsOnly(
|
|
421
|
+
text,
|
|
422
|
+
[...findings, ...declaredFieldFindings(text, options)],
|
|
423
|
+
options
|
|
424
|
+
);
|
|
411
425
|
}
|
|
412
426
|
const findings = [];
|
|
413
427
|
const emit = (f) => {
|
|
@@ -602,6 +616,9 @@ export function markFake(text, findings, options) {
|
|
|
602
616
|
),
|
|
603
617
|
];
|
|
604
618
|
return findings.map((f) => {
|
|
619
|
+
if (f.type === 'JSON_VALUE') {
|
|
620
|
+
return { ...f, kept: undefined };
|
|
621
|
+
}
|
|
605
622
|
if (
|
|
606
623
|
trusted.some(
|
|
607
624
|
(span) =>
|
|
@@ -660,7 +677,9 @@ function validatedSpans(text, findings, options) {
|
|
|
660
677
|
if (f.category === 'credential') {
|
|
661
678
|
last.category = 'credential';
|
|
662
679
|
}
|
|
663
|
-
if (
|
|
680
|
+
if (last.type === 'JSON_VALUE' || f.type === 'JSON_VALUE') {
|
|
681
|
+
last.type = 'JSON_VALUE';
|
|
682
|
+
} else if (credentialRun) {
|
|
664
683
|
last.type = 'SECRET';
|
|
665
684
|
} else if (preferredType(last.type, f.type, options)) {
|
|
666
685
|
last.type = f.type;
|
|
@@ -673,6 +692,12 @@ function validatedSpans(text, findings, options) {
|
|
|
673
692
|
}
|
|
674
693
|
|
|
675
694
|
function preferredType(current, candidate, options) {
|
|
695
|
+
if (current === 'JSON_VALUE') {
|
|
696
|
+
return false;
|
|
697
|
+
}
|
|
698
|
+
if (candidate === 'JSON_VALUE') {
|
|
699
|
+
return true;
|
|
700
|
+
}
|
|
676
701
|
if (current === 'ENCODED_SENSITIVE') {
|
|
677
702
|
return false;
|
|
678
703
|
}
|
|
@@ -810,7 +835,7 @@ export async function redactResultAsync(
|
|
|
810
835
|
);
|
|
811
836
|
replacements.set(
|
|
812
837
|
`${span.start}:${span.end}`,
|
|
813
|
-
encodeRun(run, result.text)
|
|
838
|
+
encodeRun(run, result.text === run.text ? REDACTED : result.text)
|
|
814
839
|
);
|
|
815
840
|
}
|
|
816
841
|
}
|
package/src/structured.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { failure } from './detection.js';
|
|
2
|
+
import { documentFieldType } from './document-fields.js';
|
|
2
3
|
import { hasPlaceholders } from './placeholders.js';
|
|
3
4
|
|
|
4
5
|
// Session metadata is opt-in at the CLI; library callers choose their own
|
|
@@ -114,6 +115,33 @@ export function structuredBatches(text, options) {
|
|
|
114
115
|
return batches;
|
|
115
116
|
}
|
|
116
117
|
|
|
118
|
+
// Schema declarations take priority over metadata/public/fake heuristics.
|
|
119
|
+
export function structuredFieldFindings(batch, options) {
|
|
120
|
+
return batch.entries.flatMap((entry) => {
|
|
121
|
+
const type = options.documentEntities
|
|
122
|
+
? documentFieldType(entry.field)
|
|
123
|
+
: undefined;
|
|
124
|
+
if (
|
|
125
|
+
entry.key ||
|
|
126
|
+
!type ||
|
|
127
|
+
entry.raw.length <= 2 ||
|
|
128
|
+
JSON.parse(entry.raw) === '[REDACTED]'
|
|
129
|
+
) {
|
|
130
|
+
return [];
|
|
131
|
+
}
|
|
132
|
+
return [
|
|
133
|
+
{
|
|
134
|
+
start: entry.start + 1,
|
|
135
|
+
end: entry.start + entry.raw.length - 1,
|
|
136
|
+
type,
|
|
137
|
+
category: 'personal',
|
|
138
|
+
rule: 'known-personal',
|
|
139
|
+
confidence: 1,
|
|
140
|
+
},
|
|
141
|
+
];
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
117
145
|
export function structuredValueRanges(text, options) {
|
|
118
146
|
return structuredBatches(text, options).flatMap((batch) =>
|
|
119
147
|
batch.entries
|
|
@@ -356,7 +384,7 @@ function stringOffsets(raw) {
|
|
|
356
384
|
}
|
|
357
385
|
const starts = [],
|
|
358
386
|
ends = [];
|
|
359
|
-
for (let i = 1; i < raw.length - 1;
|
|
387
|
+
for (let i = 1; i < raw.length - 1;) {
|
|
360
388
|
const start = i;
|
|
361
389
|
i += raw[i] === '\\' ? (raw[i + 1] === 'u' ? 6 : 2) : 1;
|
|
362
390
|
starts.push(start);
|
|
@@ -364,3 +392,70 @@ function stringOffsets(raw) {
|
|
|
364
392
|
}
|
|
365
393
|
return { starts, ends };
|
|
366
394
|
}
|
|
395
|
+
|
|
396
|
+
// A caller-declared property replaces its entire JSON value, including containers
|
|
397
|
+
// and primitives. Source token offsets preserve the surrounding record.
|
|
398
|
+
export function declaredFieldFindings(text, options) {
|
|
399
|
+
if (!options.sensitiveFields?.length) {
|
|
400
|
+
return [];
|
|
401
|
+
}
|
|
402
|
+
const fields = new Set(options.sensitiveFields);
|
|
403
|
+
const findings = [];
|
|
404
|
+
for (const token of lexemes(text)) {
|
|
405
|
+
if (
|
|
406
|
+
token.index < (findings.at(-1)?.end ?? 0) ||
|
|
407
|
+
!token.key ||
|
|
408
|
+
!fields.has(JSON.parse(token.raw))
|
|
409
|
+
) {
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
let start = token.index + token.raw.length;
|
|
413
|
+
while (/[\s:]/.test(text[start] ?? '') && start < text.length) {
|
|
414
|
+
start++;
|
|
415
|
+
}
|
|
416
|
+
const end = jsonValueEnd(text, start);
|
|
417
|
+
if (text.slice(start, end) === '"[REDACTED]"') {
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
findings.push({
|
|
421
|
+
start,
|
|
422
|
+
end,
|
|
423
|
+
type: 'JSON_VALUE',
|
|
424
|
+
category: 'personal',
|
|
425
|
+
rule: 'known-personal',
|
|
426
|
+
confidence: 1,
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
return findings;
|
|
430
|
+
}
|
|
431
|
+
function jsonValueEnd(text, start) {
|
|
432
|
+
let depth = 0,
|
|
433
|
+
quoted = false;
|
|
434
|
+
for (let i = start; i < text.length; i++) {
|
|
435
|
+
const char = text[i];
|
|
436
|
+
if (quoted) {
|
|
437
|
+
if (char === '\\') {
|
|
438
|
+
i++;
|
|
439
|
+
} else if (char === '"') {
|
|
440
|
+
quoted = false;
|
|
441
|
+
if (!depth) {
|
|
442
|
+
return i + 1;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
} else if (char === '"') {
|
|
446
|
+
quoted = true;
|
|
447
|
+
} else if (char === '[' || char === '{') {
|
|
448
|
+
depth++;
|
|
449
|
+
} else if (char === ']' || char === '}') {
|
|
450
|
+
if (!depth) {
|
|
451
|
+
return i;
|
|
452
|
+
}
|
|
453
|
+
if (!--depth) {
|
|
454
|
+
return i + 1;
|
|
455
|
+
}
|
|
456
|
+
} else if (!depth && /[\s,]/.test(char)) {
|
|
457
|
+
return i;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
return text.length;
|
|
461
|
+
}
|
package/src/transforms.js
CHANGED
|
@@ -183,6 +183,9 @@ function transformed(value, finding, transform, options) {
|
|
|
183
183
|
return `${lower}–${lower + transform.size - 1}`;
|
|
184
184
|
}
|
|
185
185
|
export function renderReplacement(text, finding, options, encodedSanitize) {
|
|
186
|
+
if (finding.type === 'JSON_VALUE') {
|
|
187
|
+
return JSON.stringify(REDACTED);
|
|
188
|
+
}
|
|
186
189
|
const transform =
|
|
187
190
|
options.transformations?.[finding.type] ??
|
|
188
191
|
options.transformation ??
|
|
@@ -193,7 +196,10 @@ export function renderReplacement(text, finding, options, encodedSanitize) {
|
|
|
193
196
|
const raw = text.slice(finding.start, finding.end);
|
|
194
197
|
const run = encodedValue(raw);
|
|
195
198
|
if (run && (options.preserveEncoding || run.encoding === 'json-content')) {
|
|
196
|
-
|
|
199
|
+
const sanitized = encodedSanitize(run.text).text;
|
|
200
|
+
// Mapped escaped fragments can lose their detecting context on decode.
|
|
201
|
+
// Redact the fragment if recursive scanning makes no progress.
|
|
202
|
+
return encodeRun(run, sanitized === run.text ? REDACTED : sanitized);
|
|
197
203
|
}
|
|
198
204
|
if (options.preserveEncoding) {
|
|
199
205
|
throw failure('ERR_ENCODING');
|