@link-foundation/sensitive-data-sanitizer 0.19.0 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/docs/API.md +11 -6
- package/docs/COVERAGE.md +47 -8
- package/docs/RULES.md +2 -0
- package/package.json +1 -1
- package/src/document-entities.js +2 -0
- package/src/document-fields.js +30 -3
- package/src/document-patterns.js +22 -2
- package/src/document-person-cues.js +38 -0
- package/src/policy.js +1 -1
- package/src/propagation.js +2 -1
- package/src/proper-names.js +26 -3
- package/src/rule-catalog.js +3 -0
- package/src/rules.js +5 -0
- package/src/strict.js +51 -4
- package/src/structured.js +47 -26
- package/src/vocabulary.js +13 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.19.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 4afc88d: Read `Mr`, `Ms` and `Dr` as person cues only when capitalized, so lowercase `ms` (milliseconds) and `dr` before a capitalized log word stay. Treat compound JSON keys ending in a personal field, such as `recruiterDisplayName`, `authorFullName` and `resumePdfOriginalFileName`, as person fields.
|
|
8
|
+
|
|
9
|
+
## 0.19.1
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- Redact complete obfuscated email addresses and military IDs with capitalized
|
|
14
|
+
proper-name detection, recognize lowercase employers after employment cues and
|
|
15
|
+
unknown Cyrillic names before colons, and preserve work and skill durations in
|
|
16
|
+
strict mode while keeping ages sensitive.
|
|
17
|
+
|
|
18
|
+
Detect bare Yandex Cloud credentials, Latin names and filenames in explicit personal JSON fields, numeric ages and birth years, and single Latin names after English person cues while preserving workflow headings.
|
|
19
|
+
|
|
3
20
|
## 0.19.0
|
|
4
21
|
|
|
5
22
|
### Minor Changes
|
package/docs/API.md
CHANGED
|
@@ -544,9 +544,11 @@ are case-sensitive runtime Unicode property aliases, not language codes.
|
|
|
544
544
|
Inside activated values, the rule replaces words containing an uppercase letter
|
|
545
545
|
anywhere (`iPhone`, `A.B`, `Санкт-Петербург`), typographic quoted spans (including
|
|
546
546
|
lowercase `«ромашка лютик»`), links including `example.org/path`, and bare domestic
|
|
547
|
-
mobile shapes.
|
|
548
|
-
|
|
549
|
-
|
|
547
|
+
mobile shapes. Known Cyrillic labels before a colon (`Опыт:`, `Стаж:`,
|
|
548
|
+
`Навыки:`) and non-Cyrillic labels (also `Label :`) stay. Unknown capitalized
|
|
549
|
+
Cyrillic words such as `Ромашка: менеджер` are replaced before the colon.
|
|
550
|
+
Lowercase words, numbers, date ranges and path tokens stay. A path can still match
|
|
551
|
+
existing personal-path or credential rules. Numeric primitives and JSON keys stay; string
|
|
550
552
|
escapes and surrounding source formatting are retained. JSON/JSONL activation
|
|
551
553
|
also applies independently within serialized JSON strings, to the existing
|
|
552
554
|
bounded decoding depth. Findings do not seed document-wide name propagation.
|
|
@@ -555,8 +557,10 @@ produces stable tokens and exact restoration; the existing placeholder-namespace
|
|
|
555
557
|
collision checks still reject input that already contains typed placeholders.
|
|
556
558
|
|
|
557
559
|
This is a structural heuristic, so capitalized ordinary words also disappear.
|
|
558
|
-
Lowercase organizations
|
|
559
|
-
|
|
560
|
+
Lowercase organizations without employment context stay; the document rules
|
|
561
|
+
replace a lowercase employer after `работал в` / `работала в`. Complete contact
|
|
562
|
+
and identifier findings take priority over overlapping proper-name fragments.
|
|
563
|
+
English prose without a trigger (`Worked at Romashka for three years.`), isolated neutral-key values
|
|
560
564
|
`{"note":"Romashka"}` / `{"note":"1991-03-12"}`, and a personal website root
|
|
561
565
|
without a trigger stay under this rule. The existing detectors and explicit
|
|
562
566
|
known/sensitive-field policies still run: an unactivated value is byte-for-byte
|
|
@@ -567,7 +571,8 @@ The new scanner uses a fixed number of forward passes over maximal tokens,
|
|
|
567
571
|
including long tokens without spaces. This does not impose a linear-time bound
|
|
568
572
|
on optional external engines or every preexisting detection rule. Debugging is
|
|
569
573
|
off by default; `debug` receives value-free `proper-name-prose` and
|
|
570
|
-
`document-experience` hit counts when those scans run.
|
|
574
|
+
`document-experience` hit counts when those scans run. `document-prose-overlap`
|
|
575
|
+
reports complete-contact/identifier and suppressed broad-span counts.
|
|
571
576
|
|
|
572
577
|
CLI equivalents: `--proper-names capitalized --scripts Cyrillic,Greek`.
|
|
573
578
|
Configuration files accept the same `properNames` and `scripts` API options;
|
package/docs/COVERAGE.md
CHANGED
|
@@ -218,7 +218,9 @@ The `documents` profile additionally recognizes:
|
|
|
218
218
|
- Russian mobile numbers with dashed, dotted, spaced or parenthesized forms,
|
|
219
219
|
including ten-digit `9xx` forms and domestic `8` prefixes; `тел.`, `моб.` and
|
|
220
220
|
WhatsApp labels; `wa.me` contact links.
|
|
221
|
-
- Obfuscated email separators `(at)`, `[at]`, `собака`, and English `dot`/`at
|
|
221
|
+
- Obfuscated email separators `(at)`, `[at]`, `собака`, and English `dot`/`at`,
|
|
222
|
+
including `[dot]`, `(dot)`, `(точка)` and `[точка]` with optional spaces.
|
|
223
|
+
Complete addresses remain protected alongside capitalized proper-name matches.
|
|
222
224
|
This is a finite spelling grammar, not arbitrary email deobfuscation.
|
|
223
225
|
- Personal paths on LinkedIn (`/in/`), GitHub, VK, Instagram and Telegram,
|
|
224
226
|
including scheme-less links; Telegram/телеграм/tg/skype/ник labels without `@`.
|
|
@@ -361,17 +363,24 @@ alone cannot establish that the site belongs to a person.
|
|
|
361
363
|
|
|
362
364
|
Structured document field aliases remove `_`/`-` and ignore case before matching
|
|
363
365
|
this explicit vocabulary: `firstName`, `lastName`, `middleName`, `patronymic`,
|
|
364
|
-
`
|
|
366
|
+
`candidateName`, `displayName`, `fullName`, `originalFileName`,
|
|
367
|
+
`age`, `birthYear`, `birthday`, `birthDate`, `sex`, `citizenship`, `nationality`, `company`,
|
|
365
368
|
`companyName`, `employer`, `university`, `school`, `contact`, `resumeUrl`,
|
|
366
369
|
`alternateUrl`, `profileUrl`, `companyUrl`. Their nonempty string values are
|
|
367
370
|
sensitive in full, including ISO birth timestamps. Thus `first_name` and
|
|
368
371
|
`firstName`, or `birth_date` and `birthDate`, follow the same policy. These are
|
|
369
372
|
schema assumptions: `contact` or `company` can hold operational data too.
|
|
373
|
+
Person fields also match as a word-boundary suffix of a compound key, so
|
|
374
|
+
`recruiterDisplayName`, `author_full_name` and `resumePdfOriginalFileName` are
|
|
375
|
+
person fields. The same suffix rule applies to `vocabulary.personalFields`.
|
|
376
|
+
Other types match only the whole key (`maxAge` and `fileName` stay), and a
|
|
377
|
+
bare `name` is not in the list.
|
|
370
378
|
Existing native labels still apply. Unlisted URL-named fields have ordinary
|
|
371
379
|
content detection; arbitrary names do not imply sensitivity. Declare them with
|
|
372
380
|
[`sensitiveFields`](API.md#caller-declared-sensitive-fields) to replace complete
|
|
373
|
-
values, including non-string values.
|
|
374
|
-
|
|
381
|
+
values, including non-string values. Numeric `age` and `birthYear` values are replaced by the JSON string
|
|
382
|
+
`"[REDACTED]"`; unrelated numeric primitives retain their types. Bare `name`,
|
|
383
|
+
`jobName` and `productName` remain ambiguous and use content detection.
|
|
375
384
|
|
|
376
385
|
Employer labels consume multi-word values to a comma or line/quote boundary.
|
|
377
386
|
Dated CV entries additionally support compound titles, title parentheses/slashes,
|
|
@@ -438,9 +447,11 @@ now use the same case/underscore/hyphen normalization as built-in document names
|
|
|
438
447
|
For no-cue prose, enable [`properNames: 'capitalized'`](API.md#structural-prose-redaction).
|
|
439
448
|
It activates per value, defaults to Cyrillic script letters or structural data
|
|
440
449
|
triggers, and supports configurable Unicode scripts. It intentionally loses
|
|
441
|
-
capitalized ordinary words.
|
|
442
|
-
untriggered values are outside this additional
|
|
443
|
-
|
|
450
|
+
capitalized ordinary words. Known Cyrillic labels, non-Cyrillic labels, lowercase
|
|
451
|
+
words, dates, paths, JSON keys and untriggered values are outside this additional
|
|
452
|
+
rule. Unknown capitalized Cyrillic words before colons are protected. English
|
|
453
|
+
untriggered prose and lowercase names without context remain documented limits;
|
|
454
|
+
an explicit `работал в` / `работала в` cue also protects lowercase employers.
|
|
444
455
|
The API describes the interaction with existing detectors and placeholder modes.
|
|
445
456
|
|
|
446
457
|
The date-anchor scan visits each preceding entry once; the optional prose scanner
|
|
@@ -465,7 +476,14 @@ proper-name mode. Romashka Inc keeps its existing legal-form treatment.
|
|
|
465
476
|
`strict: true` independently opts into personal attributes (gender, family,
|
|
466
477
|
ethnicity, religion, health, criminal/political status) and bare dates, ages,
|
|
467
478
|
passport/mobile/postal shapes. It is intended for stored personal prose and can
|
|
468
|
-
remove ordinary uses of these words. It preserves ISO timestamps, work durations
|
|
479
|
+
remove ordinary uses of these words. It preserves ISO timestamps, work durations
|
|
480
|
+
(including `Стаж: 5 лет`, `работал там 5 лет`, and duration cues from issue 87),
|
|
481
|
+
and parenthesized durations after known technologies such as `Kubernetes (5 лет)`
|
|
482
|
+
and `Power BI (5 лет)`.
|
|
483
|
+
Explicit ages (`кандидату 34 года`, `мне 34 года`, `возраст 41`) and bare strict
|
|
484
|
+
ages remain sensitive. Work and skill duration context is retained through
|
|
485
|
+
redaction, placeholders and generated fake values for stable verification.
|
|
486
|
+
It also preserves
|
|
469
487
|
unit-bearing request timings and `Дети: нет`. In JSON/JSONL, numeric primitives
|
|
470
488
|
stay numeric, and `structuralFields` suppresses strict findings in declared
|
|
471
489
|
string metadata. Credentials/known values still take priority. Strict is off by
|
|
@@ -524,3 +542,24 @@ Reproduce with `node --max-old-space-size=384 experiments/issue-118/measure.mjs`
|
|
|
524
542
|
The [raw results](case-studies/issue-118/measurements.json) include environment,
|
|
525
543
|
exact-span counts and measurement method. This fixed experiment uses no large
|
|
526
544
|
corpus, unbounded input, parallel builds or external analysis service.
|
|
545
|
+
|
|
546
|
+
## Single Latin names and Yandex Cloud credentials
|
|
547
|
+
|
|
548
|
+
Document rules recognize a capitalized Latin name after `candidate`, `recruiter`,
|
|
549
|
+
`applicant`, `Dear`, `Hi`, `Mr`/`Mr.`, `Ms`/`Ms.`, `Dr`/`Dr.`, or `message from`.
|
|
550
|
+
Honorifics count only when capitalized: lowercase `ms` (milliseconds), `mr` and
|
|
551
|
+
`dr` are not cues, and `Ms` right after a number is read as a unit, so
|
|
552
|
+
`took 30 ms Retry scheduled` and `see dr Drive mount` stay unchanged.
|
|
553
|
+
Only the name is replaced; punctuation and the cue remain. These rules work
|
|
554
|
+
without enabling Latin script matching. Finite generic heading nouns such as
|
|
555
|
+
`Stage` and `Status` are preserved: `Candidate Stage changed` and
|
|
556
|
+
`Request Failed: POST /api` stay unchanged. This is contextual recognition,
|
|
557
|
+
not universal English NER; unknown cue-free single names can remain.
|
|
558
|
+
Use `rules: { 'document-person-cue': false }` to disable this recognizer.
|
|
559
|
+
|
|
560
|
+
Native credential detection recognizes bare Yandex Cloud API keys (`AQVN`),
|
|
561
|
+
IAM tokens (`t1.` including optional base64 padding), static access key IDs
|
|
562
|
+
(`YCAJE`) and static secrets (`YC`). These provider rules apply in every profile
|
|
563
|
+
and do not require an assignment label. See the official
|
|
564
|
+
[token grammar](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token)
|
|
565
|
+
and [credential formats](https://yandex.cloud/en/docs/security/standard/authentication).
|
package/docs/RULES.md
CHANGED
|
@@ -47,6 +47,7 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
47
47
|
- `document-mobile-context`
|
|
48
48
|
- `document-msu`
|
|
49
49
|
- `document-patronymic`
|
|
50
|
+
- `document-person-cue`
|
|
50
51
|
- `document-personal-path`
|
|
51
52
|
- `document-phone`
|
|
52
53
|
- `document-phone-label`
|
|
@@ -401,3 +402,4 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
401
402
|
- `vocabulary-role`
|
|
402
403
|
- `webhook`
|
|
403
404
|
- `wordpress`
|
|
405
|
+
- `yandex`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.2",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/document-entities.js
CHANGED
|
@@ -7,6 +7,7 @@ import { detectDocumentExtras } from './document-extras.js';
|
|
|
7
7
|
import { detectDocumentPatterns } from './document-patterns.js';
|
|
8
8
|
import { isDocumentTechnology } from './document-experience.js';
|
|
9
9
|
import { detectVocabulary } from './vocabulary.js';
|
|
10
|
+
import { detectDocumentPersonCues } from './document-person-cues.js';
|
|
10
11
|
import { detectDocumentCatalog } from './document-catalog.js';
|
|
11
12
|
|
|
12
13
|
const words = (column) =>
|
|
@@ -82,6 +83,7 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
82
83
|
return;
|
|
83
84
|
}
|
|
84
85
|
detectVocabulary(text, emit, options);
|
|
86
|
+
detectDocumentPersonCues(text, emit);
|
|
85
87
|
detectDocumentCatalog(text, emit);
|
|
86
88
|
detectDocumentPatterns(text, emit);
|
|
87
89
|
detectDocumentExtras(text, emit, range, options.debug, options);
|
package/src/document-fields.js
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
// One explicit field vocabulary; camelCase, snake_case and kebab-case normalize alike.
|
|
2
2
|
const groups = {
|
|
3
|
-
PERSON: [
|
|
4
|
-
|
|
3
|
+
PERSON: [
|
|
4
|
+
'firstname',
|
|
5
|
+
'lastname',
|
|
6
|
+
'middlename',
|
|
7
|
+
'patronymic',
|
|
8
|
+
'candidatename',
|
|
9
|
+
'displayname',
|
|
10
|
+
'fullname',
|
|
11
|
+
'originalfilename',
|
|
12
|
+
],
|
|
13
|
+
AGE: ['age'],
|
|
14
|
+
DATE_OF_BIRTH: ['birthday', 'birthdate', 'birthyear'],
|
|
5
15
|
GENDER: ['sex'],
|
|
6
16
|
PERSONAL_ATTRIBUTE: ['citizenship', 'nationality'],
|
|
7
17
|
ORGANIZATION: ['company', 'companyname', 'employer'],
|
|
@@ -15,7 +25,24 @@ const types = new Map(
|
|
|
15
25
|
)
|
|
16
26
|
);
|
|
17
27
|
export function documentFieldType(field) {
|
|
18
|
-
|
|
28
|
+
const type = types.get(normalizeFieldName(field));
|
|
29
|
+
if (type) {
|
|
30
|
+
return type;
|
|
31
|
+
}
|
|
32
|
+
// Compound person keys (`recruiterDisplayName`, `author_full_name`) end in
|
|
33
|
+
// a listed person field. Only PERSON entries extend this way, so generic
|
|
34
|
+
// suffixes such as `maxAge` or `homeSchool` keep their exact-match meaning,
|
|
35
|
+
// and a bare `name` is never listed.
|
|
36
|
+
return fieldSuffixes(field).some((suffix) => types.get(suffix) === 'PERSON')
|
|
37
|
+
? 'PERSON'
|
|
38
|
+
: undefined;
|
|
39
|
+
}
|
|
40
|
+
// Word-boundary suffixes of a key, normalized; the whole key comes first.
|
|
41
|
+
export function fieldSuffixes(field) {
|
|
42
|
+
const words = field
|
|
43
|
+
.split(/[\s_.-]+|(?<=[\p{Ll}\p{N}])(?=\p{Lu})|(?<=\p{Lu})(?=\p{Lu}\p{Ll})/u)
|
|
44
|
+
.filter(Boolean);
|
|
45
|
+
return words.map((_, i) => words.slice(i).join('').toLowerCase());
|
|
19
46
|
}
|
|
20
47
|
export const normalizeFieldName = (field) =>
|
|
21
48
|
field.replace(/[_-]/g, '').toLowerCase();
|
package/src/document-patterns.js
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { firstNames } from './names.js';
|
|
2
2
|
import { isNameForm } from './russian-names.js';
|
|
3
|
+
import { isDocumentTechnology } from './document-experience.js';
|
|
3
4
|
|
|
4
5
|
const boundary = '[\\p{L}\\p{N}_]';
|
|
5
6
|
const word = '[А-ЯЁ][а-яё-]{1,47}';
|
|
7
|
+
const emailDot =
|
|
8
|
+
'(?:\\.|[ \\t]{0,8}(?:\\((?:dot|точка)\\)|\\[(?:dot|точка)\\])[ \\t]{0,8}|[ \\t]{1,8}(?:dot|точка)[ \\t]{1,8})';
|
|
9
|
+
const emailAt =
|
|
10
|
+
'(?:[ \\t]{0,8}(?:\\((?:at|собака)\\)|\\[(?:at|собака)\\])[ \\t]{0,8}|[ \\t]{1,8}(?:at|собака)[ \\t]{1,8})';
|
|
6
11
|
const surname =
|
|
7
12
|
'[А-ЯЁа-яё][А-ЯЁа-яё-]{1,28}(?:ов|ев|ин|ын|ск|цк)(?:а|ой|ою|у|ым|ом|е|ы|и|их|ий|ая|ого|ому|ую|ие)?';
|
|
8
13
|
const initials = '(?:[А-ЯЁA-Z]\\.[ \\t]*){1,2}';
|
|
@@ -36,7 +41,10 @@ const rules = [
|
|
|
36
41
|
[
|
|
37
42
|
'EMAIL',
|
|
38
43
|
'obfuscated-email',
|
|
39
|
-
|
|
44
|
+
new RegExp(
|
|
45
|
+
`(?<![\\p{L}\\p{N}_.+\\\\-])[\\p{L}\\p{N}_+-]{1,64}(?:${emailDot}[\\p{L}\\p{N}_+-]{1,64}){0,4}${emailAt}[\\p{L}\\p{N}-]{1,63}(?:${emailDot}[\\p{L}\\p{N}-]{1,63}){1,8}(?![\\p{L}\\p{N}_])`,
|
|
46
|
+
'giu'
|
|
47
|
+
),
|
|
40
48
|
0,
|
|
41
49
|
],
|
|
42
50
|
[
|
|
@@ -142,6 +150,12 @@ const rules = [
|
|
|
142
150
|
),
|
|
143
151
|
1,
|
|
144
152
|
],
|
|
153
|
+
[
|
|
154
|
+
'ORGANIZATION',
|
|
155
|
+
'employer-prose',
|
|
156
|
+
/(?<![\p{L}\p{N}_])[Рр]аботал[аи]?[ \t]+в[ \t]+([\p{Ll}][\p{Ll}-]{1,47})(?![\p{L}\p{N}_])/gu,
|
|
157
|
+
1,
|
|
158
|
+
],
|
|
145
159
|
[
|
|
146
160
|
'ORGANIZATION',
|
|
147
161
|
'employer-years',
|
|
@@ -237,6 +251,12 @@ function documentValueAllowed(rule, value) {
|
|
|
237
251
|
rule === 'profile-attribute' &&
|
|
238
252
|
/^(?:не указан[оаы]?|not specified|unknown|n\/a)$/iu.test(value)
|
|
239
253
|
) &&
|
|
240
|
-
!(
|
|
254
|
+
!(
|
|
255
|
+
rule === 'messenger-label' && /^(?:недоступен|unavailable)$/i.test(value)
|
|
256
|
+
) &&
|
|
257
|
+
!(
|
|
258
|
+
rule === 'employer-prose' &&
|
|
259
|
+
(value === 'компании' || isDocumentTechnology(value))
|
|
260
|
+
)
|
|
241
261
|
);
|
|
242
262
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { collectMatches } from './detection.js';
|
|
2
|
+
|
|
3
|
+
// A cue supplies evidence for a single Latin name without activating broad
|
|
4
|
+
// capitalized-word matching for the rest of an English log or document.
|
|
5
|
+
const boundary = '[\\p{L}\\p{M}\\p{N}_]';
|
|
6
|
+
const name = "[A-Z][a-z]{1,47}(?:['’-][A-Z]?[a-z]{1,47})?";
|
|
7
|
+
// Honorifics count only when capitalized: lowercase `ms` is the millisecond
|
|
8
|
+
// unit in logs and lowercase `dr` is usually a word or path fragment. A
|
|
9
|
+
// capitalized `Ms` right after a number is still a unit (`took 30 Ms Retry`).
|
|
10
|
+
const cue =
|
|
11
|
+
'(?:[Cc]andidate|[Rr]ecruiter|[Aa]pplicant|[Dd]ear|[Hh]i|Mr\\.?|(?<!\\d[ \\t]*)Ms\\.?|Dr\\.?|[Mm]essage[ \\t]+from)';
|
|
12
|
+
const pattern = new RegExp(
|
|
13
|
+
`(?<!${boundary})${cue}[ \\t]+(${name})(?!${boundary})`,
|
|
14
|
+
'gu'
|
|
15
|
+
);
|
|
16
|
+
// Workflow headings use the same person-role cues. Do not treat these finite
|
|
17
|
+
// generic nouns as names merely because a heading is in title case.
|
|
18
|
+
const labels = new Set(
|
|
19
|
+
'stage status name profile details information summary experience application tracking interview selection pipeline request failed'.split(
|
|
20
|
+
' '
|
|
21
|
+
)
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
export function detectDocumentPersonCues(text, emit) {
|
|
25
|
+
collectMatches(
|
|
26
|
+
text,
|
|
27
|
+
pattern,
|
|
28
|
+
emit,
|
|
29
|
+
{
|
|
30
|
+
type: 'PERSON',
|
|
31
|
+
rule: 'document-person-cue',
|
|
32
|
+
category: 'personal',
|
|
33
|
+
confidence: 0.85,
|
|
34
|
+
},
|
|
35
|
+
1,
|
|
36
|
+
(value) => !labels.has(value.toLowerCase())
|
|
37
|
+
);
|
|
38
|
+
}
|
package/src/policy.js
CHANGED
|
@@ -31,7 +31,7 @@ const vocabularyRules = {
|
|
|
31
31
|
'education-type',
|
|
32
32
|
'education-russian',
|
|
33
33
|
],
|
|
34
|
-
personalLabels: ['document-role'],
|
|
34
|
+
personalLabels: ['document-role', 'document-person-cue'],
|
|
35
35
|
};
|
|
36
36
|
export function validatePolicyControls(options) {
|
|
37
37
|
for (const key of ['allowCredentialDisable', 'strict', 'exactFields']) {
|
package/src/propagation.js
CHANGED
|
@@ -35,7 +35,8 @@ function eligibleSeed(finding, options) {
|
|
|
35
35
|
return (
|
|
36
36
|
!options.knownOnly &&
|
|
37
37
|
finding.category === 'personal' &&
|
|
38
|
-
|
|
38
|
+
// Whole JSON values include syntax; they cannot seed string-content spans.
|
|
39
|
+
!['ENCODED_SENSITIVE', 'JSON_VALUE'].includes(finding.type) &&
|
|
39
40
|
![
|
|
40
41
|
'confirmed-personal',
|
|
41
42
|
'document-propagation',
|
package/src/proper-names.js
CHANGED
|
@@ -7,6 +7,19 @@ const commaMobile = /,[ \t]*[78]9\d{9}(?![\p{L}\p{N}_])/u;
|
|
|
7
7
|
const mobile = /^[78]9\d{9}$/u;
|
|
8
8
|
const tokenPattern = /[\p{L}\p{M}\p{N}_./:\\?&#%=@+'’~+-]+/gu;
|
|
9
9
|
const upper = /[\p{Lu}\p{Lt}]/u;
|
|
10
|
+
// Only known labels are exempt before colons in activated Cyrillic prose.
|
|
11
|
+
const proseLabels = new Set(
|
|
12
|
+
'опыт стаж навыки образование компания работодатель должность обязанности достижения проекты контакты телефон почта адрес возраст пол гражданство дети национальность вероисповедание инвалидность судимость резюме имя фамилия отчество город статус дата время уровень языки интересы о себе место работы семейное положение'.split(
|
|
13
|
+
' '
|
|
14
|
+
)
|
|
15
|
+
);
|
|
16
|
+
const completeTypes = new Set([
|
|
17
|
+
'EMAIL',
|
|
18
|
+
'PHONE',
|
|
19
|
+
'ID',
|
|
20
|
+
'VIN',
|
|
21
|
+
'PASSPORT_NUMBER',
|
|
22
|
+
]);
|
|
10
23
|
const quotePairs = new Map([
|
|
11
24
|
['«', '»'],
|
|
12
25
|
['“', '”'],
|
|
@@ -109,6 +122,8 @@ export function detectProseAndDocuments(text, emit, options, detectDocuments) {
|
|
|
109
122
|
return;
|
|
110
123
|
}
|
|
111
124
|
const runs = [[]];
|
|
125
|
+
let complete = 0,
|
|
126
|
+
suppressed = 0;
|
|
112
127
|
detectProperNames(
|
|
113
128
|
text,
|
|
114
129
|
(finding) => {
|
|
@@ -123,12 +138,20 @@ export function detectProseAndDocuments(text, emit, options, detectDocuments) {
|
|
|
123
138
|
detectDocuments(
|
|
124
139
|
text,
|
|
125
140
|
(finding) => {
|
|
126
|
-
if (
|
|
141
|
+
if (completeTypes.has(finding.type)) {
|
|
142
|
+
// Contacts and identifiers are indivisible: a domain or series alone
|
|
143
|
+
// cannot safely replace the complete value detected by document rules.
|
|
144
|
+
emit(finding);
|
|
145
|
+
complete++;
|
|
146
|
+
} else if (!runs.some((run) => overlapsRun(run, finding))) {
|
|
127
147
|
emit(finding);
|
|
148
|
+
} else {
|
|
149
|
+
suppressed++;
|
|
128
150
|
}
|
|
129
151
|
},
|
|
130
152
|
options
|
|
131
153
|
);
|
|
154
|
+
options.debug?.({ event: 'document-prose-overlap', complete, suppressed });
|
|
132
155
|
}
|
|
133
156
|
function overlapsRun(run, finding) {
|
|
134
157
|
let low = 0,
|
|
@@ -197,8 +220,8 @@ function preserveToken(text, raw, value, start, end, range) {
|
|
|
197
220
|
!value ||
|
|
198
221
|
(range && start >= range.index && end <= range.index + range[0].length) ||
|
|
199
222
|
(value === 'REDACTED' && text[start - 1] === '[' && text[end] === ']') ||
|
|
200
|
-
raw.endsWith(':') ||
|
|
201
|
-
|
|
223
|
+
((raw.endsWith(':') || text[next] === ':') &&
|
|
224
|
+
(!/^[А-ЯЁа-яё-]+$/u.test(value) || proseLabels.has(value.toLowerCase())))
|
|
202
225
|
);
|
|
203
226
|
}
|
|
204
227
|
function tokenType(value) {
|
package/src/rule-catalog.js
CHANGED
|
@@ -39,6 +39,7 @@ export const builtInRules = Object.freeze([
|
|
|
39
39
|
'document-mobile-context',
|
|
40
40
|
'document-msu',
|
|
41
41
|
'document-patronymic',
|
|
42
|
+
'document-person-cue',
|
|
42
43
|
'document-personal-path',
|
|
43
44
|
'document-phone',
|
|
44
45
|
'document-phone-label',
|
|
@@ -393,6 +394,7 @@ export const builtInRules = Object.freeze([
|
|
|
393
394
|
'vocabulary-role',
|
|
394
395
|
'webhook',
|
|
395
396
|
'wordpress',
|
|
397
|
+
'yandex',
|
|
396
398
|
]);
|
|
397
399
|
export const credentialRules = new Set([
|
|
398
400
|
'anthropic',
|
|
@@ -666,4 +668,5 @@ export const credentialRules = new Set([
|
|
|
666
668
|
'vercel',
|
|
667
669
|
'webhook',
|
|
668
670
|
'wordpress',
|
|
671
|
+
'yandex',
|
|
669
672
|
]);
|
package/src/rules.js
CHANGED
|
@@ -22,6 +22,11 @@ export const serviceRules = [
|
|
|
22
22
|
'anthropic',
|
|
23
23
|
/(?<![A-Za-z0-9_])sk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g,
|
|
24
24
|
],
|
|
25
|
+
// Yandex Cloud provider prefixes work without an assignment or auth label.
|
|
26
|
+
[
|
|
27
|
+
'yandex',
|
|
28
|
+
/(?<![A-Za-z0-9_-])(?:AQVN[A-Za-z0-9_-]{35,38}|YCAJE[A-Za-z0-9_-]{20}|YC[A-Za-z0-9_-]{38}|t1\.[A-Za-z0-9_-]+={0,2}\.[A-Za-z0-9_-]{86}={0,2})(?![A-Za-z0-9_=-])/g,
|
|
29
|
+
],
|
|
25
30
|
['aws', /(?:AKIA|ASIA|A3T[A-Z0-9])[A-Z0-9]{16}/g],
|
|
26
31
|
[
|
|
27
32
|
'google',
|
package/src/strict.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { collectMatches } from './detection.js';
|
|
2
2
|
import { months } from './personal-context.js';
|
|
3
|
+
import { isDocumentTechnology } from './document-experience.js';
|
|
4
|
+
import { placeholderValues } from './placeholders.js';
|
|
5
|
+
import { generatedValues } from './fake.js';
|
|
3
6
|
|
|
4
7
|
const boundary = '[\\p{L}\\p{N}_]';
|
|
5
8
|
const attributes =
|
|
@@ -7,6 +10,49 @@ const attributes =
|
|
|
7
10
|
const additionalAttributes =
|
|
8
11
|
/(?<![\p{L}\p{N}_])(?:диабет(?:ом|а|у|е)?|ВИЧ(?:-положительн(?:ый|ая|ого|ой))?|болен|больна|больны|католик(?:ом|а|и)?|католичк(?:а|ой|и)|буддист(?:ка|кой|ом|а|ы)?|атеист(?:ка|кой|ом|а|ы)?|украин(?:ец|ка|цы|цем|кой)|белорус(?:ка|кой|ом|а|ы)?|несудим(?:а|ы)?|судим(?:а|ы)?|член[ \t]+политической[ \t]+партии|холост(?:а|ы)?|развед[её]н(?:а|ы)?|в[ \t]+разводе|diabetes|diabetic|HIV(?:[ \t-]+positive)?|cancer|buddhist|catholic|atheist|ukrainian|belarusian|previous[ \t]+conviction|member[ \t]+of[ \t]+(?:a[ \t]+)?(?:political[ \t]+)?party|single|unmarried|widowed)(?![\p{L}\p{N}_])/giu;
|
|
9
12
|
|
|
13
|
+
function isDuration(text, match, replacements) {
|
|
14
|
+
// Keep cues on the same clause; an age on the next line is independent.
|
|
15
|
+
let prefix = text.slice(Math.max(0, match.index - 128), match.index);
|
|
16
|
+
// Verification scans transformed text too. Keep its duration cues equivalent
|
|
17
|
+
// to the default redaction view, including custom placeholder spellings.
|
|
18
|
+
for (const { value } of replacements) {
|
|
19
|
+
prefix = prefix.replaceAll(value, '[REDACTED]');
|
|
20
|
+
}
|
|
21
|
+
prefix = prefix.split(/[\r\n;|]/u).at(-1);
|
|
22
|
+
const suffix = text.slice(match.index + match[0].length);
|
|
23
|
+
if (
|
|
24
|
+
/(?<![\p{L}\p{N}_])(?:мне|ему|ей|кандидату|кандидатке|исполнилось|возраст)[ \t]*:?[ \t]*$/iu.test(
|
|
25
|
+
prefix
|
|
26
|
+
)
|
|
27
|
+
) {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
if (
|
|
31
|
+
/(?<![\p{L}\p{N}_])(?:опыт(?:[ \t]+работы)?|стаж|experience|duration)[ \t]*:?[ \t]*$/iu.test(
|
|
32
|
+
prefix
|
|
33
|
+
) ||
|
|
34
|
+
/(?<![\p{L}\p{N}_])(?:за(?:[ \t]+последние)?|через|спустя|более|на|длился|длилась)[ \t]+$/iu.test(
|
|
35
|
+
prefix
|
|
36
|
+
) ||
|
|
37
|
+
/(?<![\p{L}\p{N}_])(?:работал[аи]?|проработал[аи]?|\[REDACTED\](?=[ \t]+(?:там|в)(?![\p{L}\p{N}_])))(?:[ \t]+(?:[\p{L}-]{1,48}|\[REDACTED\])){0,3}[ \t]+$/iu.test(
|
|
38
|
+
prefix
|
|
39
|
+
) ||
|
|
40
|
+
/^[ \t]+(?:опыта|стажа|работы)(?![\p{L}\p{N}_])/iu.test(suffix)
|
|
41
|
+
) {
|
|
42
|
+
return true;
|
|
43
|
+
}
|
|
44
|
+
const skill =
|
|
45
|
+
/([\p{L}\p{N}+#.-]+(?:[ \t]+[\p{L}\p{N}+#.-]+)?)[ \t]+\([ \t]*$/u
|
|
46
|
+
.exec(prefix)?.[1]
|
|
47
|
+
?.replace(/[ \t]+/gu, ' ') ?? '';
|
|
48
|
+
return (
|
|
49
|
+
/^[ \t]*\)/u.test(suffix) &&
|
|
50
|
+
(isDocumentTechnology(skill) ||
|
|
51
|
+
isDocumentTechnology(skill.split(' ').at(-1)) ||
|
|
52
|
+
/\[REDACTED\][ \t]+\([ \t]*$/u.test(prefix))
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
10
56
|
export function detectStrict(text, emit, options) {
|
|
11
57
|
if (!options.strict) {
|
|
12
58
|
return;
|
|
@@ -29,16 +75,17 @@ export function detectStrict(text, emit, options) {
|
|
|
29
75
|
category: 'personal',
|
|
30
76
|
confidence: 0.9,
|
|
31
77
|
});
|
|
78
|
+
const replacements = [
|
|
79
|
+
...placeholderValues(options),
|
|
80
|
+
...generatedValues(options),
|
|
81
|
+
].filter((entry) => !entry.credential);
|
|
32
82
|
collectMatches(
|
|
33
83
|
text,
|
|
34
84
|
/(?<![\p{L}\p{N}_])\d{1,3}[ \t]+(?:год|года|лет|years?[ \t]+old)(?![\p{L}\p{N}_])/giu,
|
|
35
85
|
emit,
|
|
36
86
|
{ type: 'AGE', rule: 'strict-age', category: 'personal', confidence: 0.9 },
|
|
37
87
|
0,
|
|
38
|
-
(_value, match) =>
|
|
39
|
-
!/(?:опыт(?:[ \t]+работы)?|стаж|experience|duration|за|через|спустя)[ \t]*$/iu.test(
|
|
40
|
-
text.slice(Math.max(0, match.index - 64), match.index)
|
|
41
|
-
)
|
|
88
|
+
(_value, match) => !isDuration(text, match, replacements)
|
|
42
89
|
);
|
|
43
90
|
for (const [type, pattern] of [
|
|
44
91
|
[
|
package/src/structured.js
CHANGED
|
@@ -459,7 +459,7 @@ function stringOffsets(raw) {
|
|
|
459
459
|
// A caller-declared property replaces its entire JSON value, including containers
|
|
460
460
|
// and primitives. Source token offsets preserve the surrounding record.
|
|
461
461
|
export function declaredFieldFindings(text, options) {
|
|
462
|
-
if (!options.sensitiveFields?.length) {
|
|
462
|
+
if (!options.sensitiveFields?.length && !options.documentEntities) {
|
|
463
463
|
return [];
|
|
464
464
|
}
|
|
465
465
|
const findings = [];
|
|
@@ -467,39 +467,60 @@ export function declaredFieldFindings(text, options) {
|
|
|
467
467
|
let offset = 0;
|
|
468
468
|
for (const record of records) {
|
|
469
469
|
for (const token of tokens(record, offset)) {
|
|
470
|
-
if (
|
|
471
|
-
token.start < (findings.at(-1)?.end ?? 0) ||
|
|
472
|
-
!token.key ||
|
|
473
|
-
!fieldMatches(
|
|
474
|
-
options.sensitiveFields,
|
|
475
|
-
token.property,
|
|
476
|
-
[...token.path.slice(0, -1), token.property],
|
|
477
|
-
options.exactFields
|
|
478
|
-
)
|
|
479
|
-
) {
|
|
470
|
+
if (token.start < (findings.at(-1)?.end ?? 0) || !token.key) {
|
|
480
471
|
continue;
|
|
481
472
|
}
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
473
|
+
const finding = declaredValueFinding(text, token, options);
|
|
474
|
+
if (finding) {
|
|
475
|
+
findings.push(finding);
|
|
485
476
|
}
|
|
486
|
-
const end = jsonValueEnd(text, start);
|
|
487
|
-
if (text.slice(start, end) === '"[REDACTED]"') {
|
|
488
|
-
continue;
|
|
489
|
-
}
|
|
490
|
-
findings.push({
|
|
491
|
-
start,
|
|
492
|
-
end,
|
|
493
|
-
type: 'JSON_VALUE',
|
|
494
|
-
category: 'personal',
|
|
495
|
-
rule: 'known-personal',
|
|
496
|
-
confidence: 1,
|
|
497
|
-
});
|
|
498
477
|
}
|
|
499
478
|
offset += record.length + 1;
|
|
500
479
|
}
|
|
501
480
|
return findings;
|
|
502
481
|
}
|
|
482
|
+
function declaredValueFinding(text, token, options) {
|
|
483
|
+
const declared = fieldMatches(
|
|
484
|
+
options.sensitiveFields,
|
|
485
|
+
token.property,
|
|
486
|
+
[...token.path.slice(0, -1), token.property],
|
|
487
|
+
options.exactFields
|
|
488
|
+
);
|
|
489
|
+
const type = configuredFieldType(token.property, options);
|
|
490
|
+
if (!declared && !['AGE', 'DATE_OF_BIRTH'].includes(type)) {
|
|
491
|
+
return undefined;
|
|
492
|
+
}
|
|
493
|
+
let start = token.start + token.raw.length;
|
|
494
|
+
while (/[\s:]/.test(text[start] ?? '') && start < text.length) {
|
|
495
|
+
start++;
|
|
496
|
+
}
|
|
497
|
+
const end = jsonValueEnd(text, start);
|
|
498
|
+
// Strings use structuredFieldFindings. Numeric personal attributes need
|
|
499
|
+
// a whole-value replacement so the resulting JSON stays valid.
|
|
500
|
+
if (
|
|
501
|
+
!declared &&
|
|
502
|
+
!(
|
|
503
|
+
/^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(text.slice(start, end)) &&
|
|
504
|
+
builtInEnabled(
|
|
505
|
+
{ type, rule: 'known-personal', category: 'personal' },
|
|
506
|
+
options
|
|
507
|
+
)
|
|
508
|
+
)
|
|
509
|
+
) {
|
|
510
|
+
return undefined;
|
|
511
|
+
}
|
|
512
|
+
if (text.slice(start, end) === '"[REDACTED]"') {
|
|
513
|
+
return undefined;
|
|
514
|
+
}
|
|
515
|
+
return {
|
|
516
|
+
start,
|
|
517
|
+
end,
|
|
518
|
+
type: 'JSON_VALUE',
|
|
519
|
+
category: 'personal',
|
|
520
|
+
rule: 'known-personal',
|
|
521
|
+
confidence: 1,
|
|
522
|
+
};
|
|
523
|
+
}
|
|
503
524
|
function jsonValueEnd(text, start) {
|
|
504
525
|
let depth = 0,
|
|
505
526
|
quoted = false;
|
package/src/vocabulary.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { failure, collectMatches } from './detection.js';
|
|
2
2
|
import { wordPattern } from './personal-variants.js';
|
|
3
|
-
import { normalizeFieldName } from './document-fields.js';
|
|
3
|
+
import { fieldSuffixes, normalizeFieldName } from './document-fields.js';
|
|
4
4
|
|
|
5
5
|
export const defaultVocabulary = Object.freeze(
|
|
6
6
|
Object.fromEntries(
|
|
@@ -40,7 +40,16 @@ export const defaultVocabulary = Object.freeze(
|
|
|
40
40
|
'graduated from',
|
|
41
41
|
],
|
|
42
42
|
personalLabels: ['соискатель', 'сотрудник', 'кандидат'],
|
|
43
|
-
personalFields: [
|
|
43
|
+
personalFields: [
|
|
44
|
+
'firstname',
|
|
45
|
+
'lastname',
|
|
46
|
+
'middlename',
|
|
47
|
+
'patronymic',
|
|
48
|
+
'candidatename',
|
|
49
|
+
'displayname',
|
|
50
|
+
'fullname',
|
|
51
|
+
'originalfilename',
|
|
52
|
+
],
|
|
44
53
|
}).map(([key, values]) => [key, Object.freeze(values)])
|
|
45
54
|
)
|
|
46
55
|
);
|
|
@@ -161,7 +170,8 @@ export function vocabularyField(field, options) {
|
|
|
161
170
|
if (!vocabulary) {
|
|
162
171
|
return undefined;
|
|
163
172
|
}
|
|
164
|
-
return vocabulary.fields.has(normalizeFieldName(field))
|
|
173
|
+
return vocabulary.fields.has(normalizeFieldName(field)) ||
|
|
174
|
+
fieldSuffixes(field).some((suffix) => vocabulary.fields.has(suffix))
|
|
165
175
|
? true
|
|
166
176
|
: options.vocabulary.personalFields?.replace
|
|
167
177
|
? false
|