@link-foundation/sensitive-data-sanitizer 0.19.0 → 0.19.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,22 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.19.2
4
+
5
+ ### Patch Changes
6
+
7
+ - 4afc88d: Read `Mr`, `Ms` and `Dr` as person cues only when capitalized, so lowercase `ms` (milliseconds) and `dr` before a capitalized log word stay. Treat compound JSON keys ending in a personal field, such as `recruiterDisplayName`, `authorFullName` and `resumePdfOriginalFileName`, as person fields.
8
+
9
+ ## 0.19.1
10
+
11
+ ### Patch Changes
12
+
13
+ - Redact complete obfuscated email addresses and military IDs with capitalized
14
+ proper-name detection, recognize lowercase employers after employment cues and
15
+ unknown Cyrillic names before colons, and preserve work and skill durations in
16
+ strict mode while keeping ages sensitive.
17
+
18
+ Detect bare Yandex Cloud credentials, Latin names and filenames in explicit personal JSON fields, numeric ages and birth years, and single Latin names after English person cues while preserving workflow headings.
19
+
3
20
  ## 0.19.0
4
21
 
5
22
  ### Minor Changes
package/docs/API.md CHANGED
@@ -544,9 +544,11 @@ are case-sensitive runtime Unicode property aliases, not language codes.
544
544
  Inside activated values, the rule replaces words containing an uppercase letter
545
545
  anywhere (`iPhone`, `A.B`, `Санкт-Петербург`), typographic quoted spans (including
546
546
  lowercase `«ромашка лютик»`), links including `example.org/path`, and bare domestic
547
- mobile shapes. Labels immediately before a colon (also `Label :`), lowercase
548
- words, numbers, date ranges and path tokens stay. A path can still match existing
549
- personal-path or credential rules. Numeric primitives and JSON keys stay; string
547
+ mobile shapes. Known Cyrillic labels before a colon (`Опыт:`, `Стаж:`,
548
+ `Навыки:`) and non-Cyrillic labels (also `Label :`) stay. Unknown capitalized
549
+ Cyrillic words such as `Ромашка: менеджер` are replaced before the colon.
550
+ Lowercase words, numbers, date ranges and path tokens stay. A path can still match
551
+ existing personal-path or credential rules. Numeric primitives and JSON keys stay; string
550
552
  escapes and surrounding source formatting are retained. JSON/JSONL activation
551
553
  also applies independently within serialized JSON strings, to the existing
552
554
  bounded decoding depth. Findings do not seed document-wide name propagation.
@@ -555,8 +557,10 @@ produces stable tokens and exact restoration; the existing placeholder-namespace
555
557
  collision checks still reject input that already contains typed placeholders.
556
558
 
557
559
  This is a structural heuristic, so capitalized ordinary words also disappear.
558
- Lowercase organizations and `Ромашка: менеджер` stay. English prose without a
559
- trigger (`Worked at Romashka for three years.`), isolated neutral-key values
560
+ Lowercase organizations without employment context stay; the document rules
561
+ replace a lowercase employer after `работал в` / `работала в`. Complete contact
562
+ and identifier findings take priority over overlapping proper-name fragments.
563
+ English prose without a trigger (`Worked at Romashka for three years.`), isolated neutral-key values
560
564
  `{"note":"Romashka"}` / `{"note":"1991-03-12"}`, and a personal website root
561
565
  without a trigger stay under this rule. The existing detectors and explicit
562
566
  known/sensitive-field policies still run: an unactivated value is byte-for-byte
@@ -567,7 +571,8 @@ The new scanner uses a fixed number of forward passes over maximal tokens,
567
571
  including long tokens without spaces. This does not impose a linear-time bound
568
572
  on optional external engines or every preexisting detection rule. Debugging is
569
573
  off by default; `debug` receives value-free `proper-name-prose` and
570
- `document-experience` hit counts when those scans run.
574
+ `document-experience` hit counts when those scans run. `document-prose-overlap`
575
+ reports complete-contact/identifier and suppressed broad-span counts.
571
576
 
572
577
  CLI equivalents: `--proper-names capitalized --scripts Cyrillic,Greek`.
573
578
  Configuration files accept the same `properNames` and `scripts` API options;
package/docs/COVERAGE.md CHANGED
@@ -218,7 +218,9 @@ The `documents` profile additionally recognizes:
218
218
  - Russian mobile numbers with dashed, dotted, spaced or parenthesized forms,
219
219
  including ten-digit `9xx` forms and domestic `8` prefixes; `тел.`, `моб.` and
220
220
  WhatsApp labels; `wa.me` contact links.
221
- - Obfuscated email separators `(at)`, `[at]`, `собака`, and English `dot`/`at`.
221
+ - Obfuscated email separators `(at)`, `[at]`, `собака`, and English `dot`/`at`,
222
+ including `[dot]`, `(dot)`, `(точка)` and `[точка]` with optional spaces.
223
+ Complete addresses remain protected alongside capitalized proper-name matches.
222
224
  This is a finite spelling grammar, not arbitrary email deobfuscation.
223
225
  - Personal paths on LinkedIn (`/in/`), GitHub, VK, Instagram and Telegram,
224
226
  including scheme-less links; Telegram/телеграм/tg/skype/ник labels without `@`.
@@ -361,17 +363,24 @@ alone cannot establish that the site belongs to a person.
361
363
 
362
364
  Structured document field aliases remove `_`/`-` and ignore case before matching
363
365
  this explicit vocabulary: `firstName`, `lastName`, `middleName`, `patronymic`,
364
- `birthday`, `birthDate`, `sex`, `citizenship`, `nationality`, `company`,
366
+ `candidateName`, `displayName`, `fullName`, `originalFileName`,
367
+ `age`, `birthYear`, `birthday`, `birthDate`, `sex`, `citizenship`, `nationality`, `company`,
365
368
  `companyName`, `employer`, `university`, `school`, `contact`, `resumeUrl`,
366
369
  `alternateUrl`, `profileUrl`, `companyUrl`. Their nonempty string values are
367
370
  sensitive in full, including ISO birth timestamps. Thus `first_name` and
368
371
  `firstName`, or `birth_date` and `birthDate`, follow the same policy. These are
369
372
  schema assumptions: `contact` or `company` can hold operational data too.
373
+ Person fields also match as a word-boundary suffix of a compound key, so
374
+ `recruiterDisplayName`, `author_full_name` and `resumePdfOriginalFileName` are
375
+ person fields. The same suffix rule applies to `vocabulary.personalFields`.
376
+ Other types match only the whole key (`maxAge` and `fileName` stay), and a
377
+ bare `name` is not in the list.
370
378
  Existing native labels still apply. Unlisted URL-named fields have ordinary
371
379
  content detection; arbitrary names do not imply sensitivity. Declare them with
372
380
  [`sensitiveFields`](API.md#caller-declared-sensitive-fields) to replace complete
373
- values, including non-string values. Generic numeric primitives otherwise keep
374
- the existing type-preservation policy.
381
+ values, including non-string values. Numeric `age` and `birthYear` values are replaced by the JSON string
382
+ `"[REDACTED]"`; unrelated numeric primitives retain their types. Bare `name`,
383
+ `jobName` and `productName` remain ambiguous and use content detection.
375
384
 
376
385
  Employer labels consume multi-word values to a comma or line/quote boundary.
377
386
  Dated CV entries additionally support compound titles, title parentheses/slashes,
@@ -438,9 +447,11 @@ now use the same case/underscore/hyphen normalization as built-in document names
438
447
  For no-cue prose, enable [`properNames: 'capitalized'`](API.md#structural-prose-redaction).
439
448
  It activates per value, defaults to Cyrillic script letters or structural data
440
449
  triggers, and supports configurable Unicode scripts. It intentionally loses
441
- capitalized ordinary words. Labels, lowercase words, dates, paths, JSON keys and
442
- untriggered values are outside this additional rule. English untriggered prose,
443
- lowercase names and a name immediately before a colon remain documented limits.
450
+ capitalized ordinary words. Known Cyrillic labels, non-Cyrillic labels, lowercase
451
+ words, dates, paths, JSON keys and untriggered values are outside this additional
452
+ rule. Unknown capitalized Cyrillic words before colons are protected. English
453
+ untriggered prose and lowercase names without context remain documented limits;
454
+ an explicit `работал в` / `работала в` cue also protects lowercase employers.
444
455
  The API describes the interaction with existing detectors and placeholder modes.
445
456
 
446
457
  The date-anchor scan visits each preceding entry once; the optional prose scanner
@@ -465,7 +476,14 @@ proper-name mode. Romashka Inc keeps its existing legal-form treatment.
465
476
  `strict: true` independently opts into personal attributes (gender, family,
466
477
  ethnicity, religion, health, criminal/political status) and bare dates, ages,
467
478
  passport/mobile/postal shapes. It is intended for stored personal prose and can
468
- remove ordinary uses of these words. It preserves ISO timestamps, work durations,
479
+ remove ordinary uses of these words. It preserves ISO timestamps, work durations
480
+ (including `Стаж: 5 лет`, `работал там 5 лет`, and duration cues from issue 87),
481
+ and parenthesized durations after known technologies such as `Kubernetes (5 лет)`
482
+ and `Power BI (5 лет)`.
483
+ Explicit ages (`кандидату 34 года`, `мне 34 года`, `возраст 41`) and bare strict
484
+ ages remain sensitive. Work and skill duration context is retained through
485
+ redaction, placeholders and generated fake values for stable verification.
486
+ It also preserves
469
487
  unit-bearing request timings and `Дети: нет`. In JSON/JSONL, numeric primitives
470
488
  stay numeric, and `structuralFields` suppresses strict findings in declared
471
489
  string metadata. Credentials/known values still take priority. Strict is off by
@@ -524,3 +542,24 @@ Reproduce with `node --max-old-space-size=384 experiments/issue-118/measure.mjs`
524
542
  The [raw results](case-studies/issue-118/measurements.json) include environment,
525
543
  exact-span counts and measurement method. This fixed experiment uses no large
526
544
  corpus, unbounded input, parallel builds or external analysis service.
545
+
546
+ ## Single Latin names and Yandex Cloud credentials
547
+
548
+ Document rules recognize a capitalized Latin name after `candidate`, `recruiter`,
549
+ `applicant`, `Dear`, `Hi`, `Mr`/`Mr.`, `Ms`/`Ms.`, `Dr`/`Dr.`, or `message from`.
550
+ Honorifics count only when capitalized: lowercase `ms` (milliseconds), `mr` and
551
+ `dr` are not cues, and `Ms` right after a number is read as a unit, so
552
+ `took 30 ms Retry scheduled` and `see dr Drive mount` stay unchanged.
553
+ Only the name is replaced; punctuation and the cue remain. These rules work
554
+ without enabling Latin script matching. Finite generic heading nouns such as
555
+ `Stage` and `Status` are preserved: `Candidate Stage changed` and
556
+ `Request Failed: POST /api` stay unchanged. This is contextual recognition,
557
+ not universal English NER; unknown cue-free single names can remain.
558
+ Use `rules: { 'document-person-cue': false }` to disable this recognizer.
559
+
560
+ Native credential detection recognizes bare Yandex Cloud API keys (`AQVN`),
561
+ IAM tokens (`t1.` including optional base64 padding), static access key IDs
562
+ (`YCAJE`) and static secrets (`YC`). These provider rules apply in every profile
563
+ and do not require an assignment label. See the official
564
+ [token grammar](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token)
565
+ and [credential formats](https://yandex.cloud/en/docs/security/standard/authentication).
package/docs/RULES.md CHANGED
@@ -47,6 +47,7 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
47
47
  - `document-mobile-context`
48
48
  - `document-msu`
49
49
  - `document-patronymic`
50
+ - `document-person-cue`
50
51
  - `document-personal-path`
51
52
  - `document-phone`
52
53
  - `document-phone-label`
@@ -401,3 +402,4 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
401
402
  - `vocabulary-role`
402
403
  - `webhook`
403
404
  - `wordpress`
405
+ - `yandex`
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@link-foundation/sensitive-data-sanitizer",
3
- "version": "0.19.0",
3
+ "version": "0.19.2",
4
4
  "description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
@@ -7,6 +7,7 @@ import { detectDocumentExtras } from './document-extras.js';
7
7
  import { detectDocumentPatterns } from './document-patterns.js';
8
8
  import { isDocumentTechnology } from './document-experience.js';
9
9
  import { detectVocabulary } from './vocabulary.js';
10
+ import { detectDocumentPersonCues } from './document-person-cues.js';
10
11
  import { detectDocumentCatalog } from './document-catalog.js';
11
12
 
12
13
  const words = (column) =>
@@ -82,6 +83,7 @@ export function detectDocumentEntities(text, emit, options) {
82
83
  return;
83
84
  }
84
85
  detectVocabulary(text, emit, options);
86
+ detectDocumentPersonCues(text, emit);
85
87
  detectDocumentCatalog(text, emit);
86
88
  detectDocumentPatterns(text, emit);
87
89
  detectDocumentExtras(text, emit, range, options.debug, options);
@@ -1,7 +1,17 @@
1
1
  // One explicit field vocabulary; camelCase, snake_case and kebab-case normalize alike.
2
2
  const groups = {
3
- PERSON: ['firstname', 'lastname', 'middlename', 'patronymic'],
4
- DATE_OF_BIRTH: ['birthday', 'birthdate'],
3
+ PERSON: [
4
+ 'firstname',
5
+ 'lastname',
6
+ 'middlename',
7
+ 'patronymic',
8
+ 'candidatename',
9
+ 'displayname',
10
+ 'fullname',
11
+ 'originalfilename',
12
+ ],
13
+ AGE: ['age'],
14
+ DATE_OF_BIRTH: ['birthday', 'birthdate', 'birthyear'],
5
15
  GENDER: ['sex'],
6
16
  PERSONAL_ATTRIBUTE: ['citizenship', 'nationality'],
7
17
  ORGANIZATION: ['company', 'companyname', 'employer'],
@@ -15,7 +25,24 @@ const types = new Map(
15
25
  )
16
26
  );
17
27
  export function documentFieldType(field) {
18
- return types.get(normalizeFieldName(field));
28
+ const type = types.get(normalizeFieldName(field));
29
+ if (type) {
30
+ return type;
31
+ }
32
+ // Compound person keys (`recruiterDisplayName`, `author_full_name`) end in
33
+ // a listed person field. Only PERSON entries extend this way, so generic
34
+ // suffixes such as `maxAge` or `homeSchool` keep their exact-match meaning,
35
+ // and a bare `name` is never listed.
36
+ return fieldSuffixes(field).some((suffix) => types.get(suffix) === 'PERSON')
37
+ ? 'PERSON'
38
+ : undefined;
39
+ }
40
+ // Word-boundary suffixes of a key, normalized; the whole key comes first.
41
+ export function fieldSuffixes(field) {
42
+ const words = field
43
+ .split(/[\s_.-]+|(?<=[\p{Ll}\p{N}])(?=\p{Lu})|(?<=\p{Lu})(?=\p{Lu}\p{Ll})/u)
44
+ .filter(Boolean);
45
+ return words.map((_, i) => words.slice(i).join('').toLowerCase());
19
46
  }
20
47
  export const normalizeFieldName = (field) =>
21
48
  field.replace(/[_-]/g, '').toLowerCase();
@@ -1,8 +1,13 @@
1
1
  import { firstNames } from './names.js';
2
2
  import { isNameForm } from './russian-names.js';
3
+ import { isDocumentTechnology } from './document-experience.js';
3
4
 
4
5
  const boundary = '[\\p{L}\\p{N}_]';
5
6
  const word = '[А-ЯЁ][а-яё-]{1,47}';
7
+ const emailDot =
8
+ '(?:\\.|[ \\t]{0,8}(?:\\((?:dot|точка)\\)|\\[(?:dot|точка)\\])[ \\t]{0,8}|[ \\t]{1,8}(?:dot|точка)[ \\t]{1,8})';
9
+ const emailAt =
10
+ '(?:[ \\t]{0,8}(?:\\((?:at|собака)\\)|\\[(?:at|собака)\\])[ \\t]{0,8}|[ \\t]{1,8}(?:at|собака)[ \\t]{1,8})';
6
11
  const surname =
7
12
  '[А-ЯЁа-яё][А-ЯЁа-яё-]{1,28}(?:ов|ев|ин|ын|ск|цк)(?:а|ой|ою|у|ым|ом|е|ы|и|их|ий|ая|ого|ому|ую|ие)?';
8
13
  const initials = '(?:[А-ЯЁA-Z]\\.[ \\t]*){1,2}';
@@ -36,7 +41,10 @@ const rules = [
36
41
  [
37
42
  'EMAIL',
38
43
  'obfuscated-email',
39
- /(?<![\p{L}\p{N}_.+\\-])[\p{L}\p{N}_+-]{1,64}(?:(?:\.|[ \t]{1,8}dot[ \t]{1,8})[\p{L}\p{N}_+-]{1,64}){0,4}(?:[ \t]{0,8}\((?:at|собака)\)[ \t]{0,8}|[ \t]{0,8}\[at\][ \t]{0,8}|[ \t]{1,8}(?:at|собака)[ \t]{1,8})[\p{L}\p{N}-]{1,63}(?:(?:\.|[ \t]{1,8}dot[ \t]{1,8})[\p{L}\p{N}-]{1,63}){1,8}(?![\p{L}\p{N}_])/giu,
44
+ new RegExp(
45
+ `(?<![\\p{L}\\p{N}_.+\\\\-])[\\p{L}\\p{N}_+-]{1,64}(?:${emailDot}[\\p{L}\\p{N}_+-]{1,64}){0,4}${emailAt}[\\p{L}\\p{N}-]{1,63}(?:${emailDot}[\\p{L}\\p{N}-]{1,63}){1,8}(?![\\p{L}\\p{N}_])`,
46
+ 'giu'
47
+ ),
40
48
  0,
41
49
  ],
42
50
  [
@@ -142,6 +150,12 @@ const rules = [
142
150
  ),
143
151
  1,
144
152
  ],
153
+ [
154
+ 'ORGANIZATION',
155
+ 'employer-prose',
156
+ /(?<![\p{L}\p{N}_])[Рр]аботал[аи]?[ \t]+в[ \t]+([\p{Ll}][\p{Ll}-]{1,47})(?![\p{L}\p{N}_])/gu,
157
+ 1,
158
+ ],
145
159
  [
146
160
  'ORGANIZATION',
147
161
  'employer-years',
@@ -237,6 +251,12 @@ function documentValueAllowed(rule, value) {
237
251
  rule === 'profile-attribute' &&
238
252
  /^(?:не указан[оаы]?|not specified|unknown|n\/a)$/iu.test(value)
239
253
  ) &&
240
- !(rule === 'messenger-label' && /^(?:недоступен|unavailable)$/i.test(value))
254
+ !(
255
+ rule === 'messenger-label' && /^(?:недоступен|unavailable)$/i.test(value)
256
+ ) &&
257
+ !(
258
+ rule === 'employer-prose' &&
259
+ (value === 'компании' || isDocumentTechnology(value))
260
+ )
241
261
  );
242
262
  }
@@ -0,0 +1,38 @@
1
+ import { collectMatches } from './detection.js';
2
+
3
+ // A cue supplies evidence for a single Latin name without activating broad
4
+ // capitalized-word matching for the rest of an English log or document.
5
+ const boundary = '[\\p{L}\\p{M}\\p{N}_]';
6
+ const name = "[A-Z][a-z]{1,47}(?:['’-][A-Z]?[a-z]{1,47})?";
7
+ // Honorifics count only when capitalized: lowercase `ms` is the millisecond
8
+ // unit in logs and lowercase `dr` is usually a word or path fragment. A
9
+ // capitalized `Ms` right after a number is still a unit (`took 30 Ms Retry`).
10
+ const cue =
11
+ '(?:[Cc]andidate|[Rr]ecruiter|[Aa]pplicant|[Dd]ear|[Hh]i|Mr\\.?|(?<!\\d[ \\t]*)Ms\\.?|Dr\\.?|[Mm]essage[ \\t]+from)';
12
+ const pattern = new RegExp(
13
+ `(?<!${boundary})${cue}[ \\t]+(${name})(?!${boundary})`,
14
+ 'gu'
15
+ );
16
+ // Workflow headings use the same person-role cues. Do not treat these finite
17
+ // generic nouns as names merely because a heading is in title case.
18
+ const labels = new Set(
19
+ 'stage status name profile details information summary experience application tracking interview selection pipeline request failed'.split(
20
+ ' '
21
+ )
22
+ );
23
+
24
+ export function detectDocumentPersonCues(text, emit) {
25
+ collectMatches(
26
+ text,
27
+ pattern,
28
+ emit,
29
+ {
30
+ type: 'PERSON',
31
+ rule: 'document-person-cue',
32
+ category: 'personal',
33
+ confidence: 0.85,
34
+ },
35
+ 1,
36
+ (value) => !labels.has(value.toLowerCase())
37
+ );
38
+ }
package/src/policy.js CHANGED
@@ -31,7 +31,7 @@ const vocabularyRules = {
31
31
  'education-type',
32
32
  'education-russian',
33
33
  ],
34
- personalLabels: ['document-role'],
34
+ personalLabels: ['document-role', 'document-person-cue'],
35
35
  };
36
36
  export function validatePolicyControls(options) {
37
37
  for (const key of ['allowCredentialDisable', 'strict', 'exactFields']) {
@@ -35,7 +35,8 @@ function eligibleSeed(finding, options) {
35
35
  return (
36
36
  !options.knownOnly &&
37
37
  finding.category === 'personal' &&
38
- finding.type !== 'ENCODED_SENSITIVE' &&
38
+ // Whole JSON values include syntax; they cannot seed string-content spans.
39
+ !['ENCODED_SENSITIVE', 'JSON_VALUE'].includes(finding.type) &&
39
40
  ![
40
41
  'confirmed-personal',
41
42
  'document-propagation',
@@ -7,6 +7,19 @@ const commaMobile = /,[ \t]*[78]9\d{9}(?![\p{L}\p{N}_])/u;
7
7
  const mobile = /^[78]9\d{9}$/u;
8
8
  const tokenPattern = /[\p{L}\p{M}\p{N}_./:\\?&#%=@+'’~+-]+/gu;
9
9
  const upper = /[\p{Lu}\p{Lt}]/u;
10
+ // Only known labels are exempt before colons in activated Cyrillic prose.
11
+ const proseLabels = new Set(
12
+ 'опыт стаж навыки образование компания работодатель должность обязанности достижения проекты контакты телефон почта адрес возраст пол гражданство дети национальность вероисповедание инвалидность судимость резюме имя фамилия отчество город статус дата время уровень языки интересы о себе место работы семейное положение'.split(
13
+ ' '
14
+ )
15
+ );
16
+ const completeTypes = new Set([
17
+ 'EMAIL',
18
+ 'PHONE',
19
+ 'ID',
20
+ 'VIN',
21
+ 'PASSPORT_NUMBER',
22
+ ]);
10
23
  const quotePairs = new Map([
11
24
  ['«', '»'],
12
25
  ['“', '”'],
@@ -109,6 +122,8 @@ export function detectProseAndDocuments(text, emit, options, detectDocuments) {
109
122
  return;
110
123
  }
111
124
  const runs = [[]];
125
+ let complete = 0,
126
+ suppressed = 0;
112
127
  detectProperNames(
113
128
  text,
114
129
  (finding) => {
@@ -123,12 +138,20 @@ export function detectProseAndDocuments(text, emit, options, detectDocuments) {
123
138
  detectDocuments(
124
139
  text,
125
140
  (finding) => {
126
- if (!runs.some((run) => overlapsRun(run, finding))) {
141
+ if (completeTypes.has(finding.type)) {
142
+ // Contacts and identifiers are indivisible: a domain or series alone
143
+ // cannot safely replace the complete value detected by document rules.
144
+ emit(finding);
145
+ complete++;
146
+ } else if (!runs.some((run) => overlapsRun(run, finding))) {
127
147
  emit(finding);
148
+ } else {
149
+ suppressed++;
128
150
  }
129
151
  },
130
152
  options
131
153
  );
154
+ options.debug?.({ event: 'document-prose-overlap', complete, suppressed });
132
155
  }
133
156
  function overlapsRun(run, finding) {
134
157
  let low = 0,
@@ -197,8 +220,8 @@ function preserveToken(text, raw, value, start, end, range) {
197
220
  !value ||
198
221
  (range && start >= range.index && end <= range.index + range[0].length) ||
199
222
  (value === 'REDACTED' && text[start - 1] === '[' && text[end] === ']') ||
200
- raw.endsWith(':') ||
201
- text[next] === ':'
223
+ ((raw.endsWith(':') || text[next] === ':') &&
224
+ (!/^[А-ЯЁа-яё-]+$/u.test(value) || proseLabels.has(value.toLowerCase())))
202
225
  );
203
226
  }
204
227
  function tokenType(value) {
@@ -39,6 +39,7 @@ export const builtInRules = Object.freeze([
39
39
  'document-mobile-context',
40
40
  'document-msu',
41
41
  'document-patronymic',
42
+ 'document-person-cue',
42
43
  'document-personal-path',
43
44
  'document-phone',
44
45
  'document-phone-label',
@@ -393,6 +394,7 @@ export const builtInRules = Object.freeze([
393
394
  'vocabulary-role',
394
395
  'webhook',
395
396
  'wordpress',
397
+ 'yandex',
396
398
  ]);
397
399
  export const credentialRules = new Set([
398
400
  'anthropic',
@@ -666,4 +668,5 @@ export const credentialRules = new Set([
666
668
  'vercel',
667
669
  'webhook',
668
670
  'wordpress',
671
+ 'yandex',
669
672
  ]);
package/src/rules.js CHANGED
@@ -22,6 +22,11 @@ export const serviceRules = [
22
22
  'anthropic',
23
23
  /(?<![A-Za-z0-9_])sk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g,
24
24
  ],
25
+ // Yandex Cloud provider prefixes work without an assignment or auth label.
26
+ [
27
+ 'yandex',
28
+ /(?<![A-Za-z0-9_-])(?:AQVN[A-Za-z0-9_-]{35,38}|YCAJE[A-Za-z0-9_-]{20}|YC[A-Za-z0-9_-]{38}|t1\.[A-Za-z0-9_-]+={0,2}\.[A-Za-z0-9_-]{86}={0,2})(?![A-Za-z0-9_=-])/g,
29
+ ],
25
30
  ['aws', /(?:AKIA|ASIA|A3T[A-Z0-9])[A-Z0-9]{16}/g],
26
31
  [
27
32
  'google',
package/src/strict.js CHANGED
@@ -1,5 +1,8 @@
1
1
  import { collectMatches } from './detection.js';
2
2
  import { months } from './personal-context.js';
3
+ import { isDocumentTechnology } from './document-experience.js';
4
+ import { placeholderValues } from './placeholders.js';
5
+ import { generatedValues } from './fake.js';
3
6
 
4
7
  const boundary = '[\\p{L}\\p{N}_]';
5
8
  const attributes =
@@ -7,6 +10,49 @@ const attributes =
7
10
  const additionalAttributes =
8
11
  /(?<![\p{L}\p{N}_])(?:диабет(?:ом|а|у|е)?|ВИЧ(?:-положительн(?:ый|ая|ого|ой))?|болен|больна|больны|католик(?:ом|а|и)?|католичк(?:а|ой|и)|буддист(?:ка|кой|ом|а|ы)?|атеист(?:ка|кой|ом|а|ы)?|украин(?:ец|ка|цы|цем|кой)|белорус(?:ка|кой|ом|а|ы)?|несудим(?:а|ы)?|судим(?:а|ы)?|член[ \t]+политической[ \t]+партии|холост(?:а|ы)?|развед[её]н(?:а|ы)?|в[ \t]+разводе|diabetes|diabetic|HIV(?:[ \t-]+positive)?|cancer|buddhist|catholic|atheist|ukrainian|belarusian|previous[ \t]+conviction|member[ \t]+of[ \t]+(?:a[ \t]+)?(?:political[ \t]+)?party|single|unmarried|widowed)(?![\p{L}\p{N}_])/giu;
9
12
 
13
+ function isDuration(text, match, replacements) {
14
+ // Keep cues on the same clause; an age on the next line is independent.
15
+ let prefix = text.slice(Math.max(0, match.index - 128), match.index);
16
+ // Verification scans transformed text too. Keep its duration cues equivalent
17
+ // to the default redaction view, including custom placeholder spellings.
18
+ for (const { value } of replacements) {
19
+ prefix = prefix.replaceAll(value, '[REDACTED]');
20
+ }
21
+ prefix = prefix.split(/[\r\n;|]/u).at(-1);
22
+ const suffix = text.slice(match.index + match[0].length);
23
+ if (
24
+ /(?<![\p{L}\p{N}_])(?:мне|ему|ей|кандидату|кандидатке|исполнилось|возраст)[ \t]*:?[ \t]*$/iu.test(
25
+ prefix
26
+ )
27
+ ) {
28
+ return false;
29
+ }
30
+ if (
31
+ /(?<![\p{L}\p{N}_])(?:опыт(?:[ \t]+работы)?|стаж|experience|duration)[ \t]*:?[ \t]*$/iu.test(
32
+ prefix
33
+ ) ||
34
+ /(?<![\p{L}\p{N}_])(?:за(?:[ \t]+последние)?|через|спустя|более|на|длился|длилась)[ \t]+$/iu.test(
35
+ prefix
36
+ ) ||
37
+ /(?<![\p{L}\p{N}_])(?:работал[аи]?|проработал[аи]?|\[REDACTED\](?=[ \t]+(?:там|в)(?![\p{L}\p{N}_])))(?:[ \t]+(?:[\p{L}-]{1,48}|\[REDACTED\])){0,3}[ \t]+$/iu.test(
38
+ prefix
39
+ ) ||
40
+ /^[ \t]+(?:опыта|стажа|работы)(?![\p{L}\p{N}_])/iu.test(suffix)
41
+ ) {
42
+ return true;
43
+ }
44
+ const skill =
45
+ /([\p{L}\p{N}+#.-]+(?:[ \t]+[\p{L}\p{N}+#.-]+)?)[ \t]+\([ \t]*$/u
46
+ .exec(prefix)?.[1]
47
+ ?.replace(/[ \t]+/gu, ' ') ?? '';
48
+ return (
49
+ /^[ \t]*\)/u.test(suffix) &&
50
+ (isDocumentTechnology(skill) ||
51
+ isDocumentTechnology(skill.split(' ').at(-1)) ||
52
+ /\[REDACTED\][ \t]+\([ \t]*$/u.test(prefix))
53
+ );
54
+ }
55
+
10
56
  export function detectStrict(text, emit, options) {
11
57
  if (!options.strict) {
12
58
  return;
@@ -29,16 +75,17 @@ export function detectStrict(text, emit, options) {
29
75
  category: 'personal',
30
76
  confidence: 0.9,
31
77
  });
78
+ const replacements = [
79
+ ...placeholderValues(options),
80
+ ...generatedValues(options),
81
+ ].filter((entry) => !entry.credential);
32
82
  collectMatches(
33
83
  text,
34
84
  /(?<![\p{L}\p{N}_])\d{1,3}[ \t]+(?:год|года|лет|years?[ \t]+old)(?![\p{L}\p{N}_])/giu,
35
85
  emit,
36
86
  { type: 'AGE', rule: 'strict-age', category: 'personal', confidence: 0.9 },
37
87
  0,
38
- (_value, match) =>
39
- !/(?:опыт(?:[ \t]+работы)?|стаж|experience|duration|за|через|спустя)[ \t]*$/iu.test(
40
- text.slice(Math.max(0, match.index - 64), match.index)
41
- )
88
+ (_value, match) => !isDuration(text, match, replacements)
42
89
  );
43
90
  for (const [type, pattern] of [
44
91
  [
package/src/structured.js CHANGED
@@ -459,7 +459,7 @@ function stringOffsets(raw) {
459
459
  // A caller-declared property replaces its entire JSON value, including containers
460
460
  // and primitives. Source token offsets preserve the surrounding record.
461
461
  export function declaredFieldFindings(text, options) {
462
- if (!options.sensitiveFields?.length) {
462
+ if (!options.sensitiveFields?.length && !options.documentEntities) {
463
463
  return [];
464
464
  }
465
465
  const findings = [];
@@ -467,39 +467,60 @@ export function declaredFieldFindings(text, options) {
467
467
  let offset = 0;
468
468
  for (const record of records) {
469
469
  for (const token of tokens(record, offset)) {
470
- if (
471
- token.start < (findings.at(-1)?.end ?? 0) ||
472
- !token.key ||
473
- !fieldMatches(
474
- options.sensitiveFields,
475
- token.property,
476
- [...token.path.slice(0, -1), token.property],
477
- options.exactFields
478
- )
479
- ) {
470
+ if (token.start < (findings.at(-1)?.end ?? 0) || !token.key) {
480
471
  continue;
481
472
  }
482
- let start = token.start + token.raw.length;
483
- while (/[\s:]/.test(text[start] ?? '') && start < text.length) {
484
- start++;
473
+ const finding = declaredValueFinding(text, token, options);
474
+ if (finding) {
475
+ findings.push(finding);
485
476
  }
486
- const end = jsonValueEnd(text, start);
487
- if (text.slice(start, end) === '"[REDACTED]"') {
488
- continue;
489
- }
490
- findings.push({
491
- start,
492
- end,
493
- type: 'JSON_VALUE',
494
- category: 'personal',
495
- rule: 'known-personal',
496
- confidence: 1,
497
- });
498
477
  }
499
478
  offset += record.length + 1;
500
479
  }
501
480
  return findings;
502
481
  }
482
+ function declaredValueFinding(text, token, options) {
483
+ const declared = fieldMatches(
484
+ options.sensitiveFields,
485
+ token.property,
486
+ [...token.path.slice(0, -1), token.property],
487
+ options.exactFields
488
+ );
489
+ const type = configuredFieldType(token.property, options);
490
+ if (!declared && !['AGE', 'DATE_OF_BIRTH'].includes(type)) {
491
+ return undefined;
492
+ }
493
+ let start = token.start + token.raw.length;
494
+ while (/[\s:]/.test(text[start] ?? '') && start < text.length) {
495
+ start++;
496
+ }
497
+ const end = jsonValueEnd(text, start);
498
+ // Strings use structuredFieldFindings. Numeric personal attributes need
499
+ // a whole-value replacement so the resulting JSON stays valid.
500
+ if (
501
+ !declared &&
502
+ !(
503
+ /^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(text.slice(start, end)) &&
504
+ builtInEnabled(
505
+ { type, rule: 'known-personal', category: 'personal' },
506
+ options
507
+ )
508
+ )
509
+ ) {
510
+ return undefined;
511
+ }
512
+ if (text.slice(start, end) === '"[REDACTED]"') {
513
+ return undefined;
514
+ }
515
+ return {
516
+ start,
517
+ end,
518
+ type: 'JSON_VALUE',
519
+ category: 'personal',
520
+ rule: 'known-personal',
521
+ confidence: 1,
522
+ };
523
+ }
503
524
  function jsonValueEnd(text, start) {
504
525
  let depth = 0,
505
526
  quoted = false;
package/src/vocabulary.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { failure, collectMatches } from './detection.js';
2
2
  import { wordPattern } from './personal-variants.js';
3
- import { normalizeFieldName } from './document-fields.js';
3
+ import { fieldSuffixes, normalizeFieldName } from './document-fields.js';
4
4
 
5
5
  export const defaultVocabulary = Object.freeze(
6
6
  Object.fromEntries(
@@ -40,7 +40,16 @@ export const defaultVocabulary = Object.freeze(
40
40
  'graduated from',
41
41
  ],
42
42
  personalLabels: ['соискатель', 'сотрудник', 'кандидат'],
43
- personalFields: ['firstname', 'lastname', 'middlename', 'patronymic'],
43
+ personalFields: [
44
+ 'firstname',
45
+ 'lastname',
46
+ 'middlename',
47
+ 'patronymic',
48
+ 'candidatename',
49
+ 'displayname',
50
+ 'fullname',
51
+ 'originalfilename',
52
+ ],
44
53
  }).map(([key, values]) => [key, Object.freeze(values)])
45
54
  )
46
55
  );
@@ -161,7 +170,8 @@ export function vocabularyField(field, options) {
161
170
  if (!vocabulary) {
162
171
  return undefined;
163
172
  }
164
- return vocabulary.fields.has(normalizeFieldName(field))
173
+ return vocabulary.fields.has(normalizeFieldName(field)) ||
174
+ fieldSuffixes(field).some((suffix) => vocabulary.fields.has(suffix))
165
175
  ? true
166
176
  : options.vocabulary.personalFields?.replace
167
177
  ? false