@link-foundation/sensitive-data-sanitizer 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,304 @@
1
+ import { createHmac } from 'node:crypto';
2
+ import { fakeName, fakeWord, wordProfile } from './fake-names.js';
3
+ import { canonicalPerson } from './placeholders.js';
4
+ import {
5
+ isGivenName,
6
+ isPatronymic,
7
+ russianNameRoles,
8
+ } from './russian-names.js';
9
+ import {
10
+ casesOf,
11
+ commonCase,
12
+ formOf,
13
+ governedCases,
14
+ russianNameForms,
15
+ russianNoun,
16
+ } from './russian-declension.js';
17
+ import {
18
+ organizationCore,
19
+ samePersonal,
20
+ wordPattern,
21
+ } from './personal-variants.js';
22
+
23
+ // Fictional but ordinary names. The lists are finite on purpose; callers can
24
+ // replace them with `transformation.pools`.
25
+ const vocabulary = {
26
+ ORGANIZATION: {
27
+ ru: 'Северный ветер|Альфа-Строй|Бета-Сервис|Гамма-Трейд|Меридиан|Горизонт|Гранит|Рассвет|Прогресс|Импульс|Орион|Азимут|Кристалл|Арсенал|Квант|Заря|Радуга|Волна|Орбита|Вершина|Искра|Тихая гавань|Новый горизонт|Белый парус|Золотой колос|Синяя птица',
28
+ latin:
29
+ 'Northwind Trading|Bluebird Logistics|Silver Birch Partners|Harbor Lane Studio|Summit Ridge Supply|Clearwater Works|Redwood Systems|Maple Street Foods',
30
+ },
31
+ LOCATION: {
32
+ ru: 'Зареченск|Приморск|Светлогорск|Лесногорск|Березовка|Сосновка|Каменка|Покровка|Ольховка|Заречье|Краснополье',
33
+ latin: 'Riverton|Oakdale|Fairview|Lakewood|Brookfield|Greenville|Ashford',
34
+ },
35
+ EDUCATION: {
36
+ ru: 'Северный государственный университет|Приморский технический институт|Лесногорский гуманитарный университет|Зареченский экономический колледж|Светлогорская городская школа',
37
+ latin:
38
+ 'Riverside University|Lakeview College|Oakdale Institute of Technology|Fairview State University',
39
+ },
40
+ };
41
+ const cyrillic = /\p{Script=Cyrillic}/u;
42
+ const surfacesKey = Symbol('fakeSurfaces');
43
+ const textKey = Symbol('fakeText');
44
+ const surfaceLimit = 1000;
45
+
46
+ // Lets each span see the other mentions of its document, as placeholders do.
47
+ export function prepareFakeText(text, findings, options) {
48
+ const surfaces = new Map();
49
+ for (const finding of findings) {
50
+ if (
51
+ ['PERSON', 'ORGANIZATION', 'LOCATION', 'EDUCATION'].includes(finding.type)
52
+ ) {
53
+ const values = surfaces.get(finding.type) ?? new Set();
54
+ if (values.size < surfaceLimit) {
55
+ values.add(text.slice(finding.start, finding.end));
56
+ }
57
+ surfaces.set(finding.type, values);
58
+ }
59
+ }
60
+ return { ...options, [surfacesKey]: surfaces, [textKey]: text };
61
+ }
62
+ const wordCount = (value) => value.trim().split(/\s+/u).length;
63
+ // The fullest form this mention belongs to, or the mention itself.
64
+ function entityLemma(value, type, options) {
65
+ const known = [
66
+ ...(options.knownPersonal ?? []),
67
+ ...(options.confirmedPersonal ?? []),
68
+ ]
69
+ .filter((entry) => entry.type === type)
70
+ .map((entry) => entry.value);
71
+ const mentions = [...(options[surfacesKey]?.get(type) ?? [])];
72
+ for (const group of [known, mentions]) {
73
+ const matches = group.filter((other) => samePersonal(value, other, type));
74
+ const longest = Math.max(0, ...matches.map(wordCount));
75
+ const fullest = [
76
+ ...new Set(matches.filter((other) => wordCount(other) === longest)),
77
+ ];
78
+ // `Гранит` covers its own form `Граните`, so several forms share one lemma.
79
+ const lemma = fullest.filter((other) =>
80
+ fullest.every((form) => samePersonal(form, other, type))
81
+ );
82
+ if (lemma.length === 1) {
83
+ return lemma[0];
84
+ }
85
+ }
86
+ return type === 'PERSON' ? canonicalPerson(value, mentions) : value;
87
+ }
88
+
89
+ const applyStyle = (surface, output) =>
90
+ surface.length > 1 && surface === surface.toUpperCase()
91
+ ? output.toUpperCase()
92
+ : surface === surface.toLowerCase()
93
+ ? output.toLowerCase()
94
+ : output;
95
+ const capitalize = (word) =>
96
+ word[0].toUpperCase() + word.slice(1).toLowerCase();
97
+ function nameSlots(parts) {
98
+ const roles = russianNameRoles(parts);
99
+ if (roles) {
100
+ return parts.map((_part, i) =>
101
+ i === roles.surname
102
+ ? 'lastName'
103
+ : i === roles.patronymic
104
+ ? 'middleName'
105
+ : 'firstName'
106
+ );
107
+ }
108
+ if (parts.length === 1) {
109
+ return [wordProfile(parts[0]).role];
110
+ }
111
+ const given = parts.map((part) => isGivenName(part));
112
+ if (parts.length === 2 && given[0] !== given[1]) {
113
+ return given.map((isGiven) => (isGiven ? 'firstName' : 'lastName'));
114
+ }
115
+ return parts.map((_part, i) =>
116
+ i === 0 ? 'firstName' : i === parts.length - 1 ? 'lastName' : 'middleName'
117
+ );
118
+ }
119
+ function nameSex(parts, slots) {
120
+ const at = (slot) => parts[slots.indexOf(slot)];
121
+ const middle = at('middleName');
122
+ if (middle && (isPatronymic(middle) || /(?:вна|чна)$/iu.test(middle))) {
123
+ return /(?:вна|чна)$/iu.test(middle) ? 'female' : 'male';
124
+ }
125
+ const last = at('lastName');
126
+ if (last && /(?:ова|ева|ёва|ина|ына|ая|яя)$/iu.test(last)) {
127
+ return 'female';
128
+ }
129
+ if (last && /(?:ов|ев|ёв|ин|ын|ий|ой|ый)$/iu.test(last)) {
130
+ return 'male';
131
+ }
132
+ return wordProfile(at('firstName') ?? parts[0]).sex;
133
+ }
134
+ const letters = /[\p{L}\p{M}]+/gu;
135
+ // One identity per lemma: each lemma word has one seeded fake, and the matched
136
+ // surface only decides its case, its order and its letter case.
137
+ function russianFakeName(surface, lemma, key, context) {
138
+ const parts = lemma.trim().split(/\s+/u);
139
+ const slots = nameSlots(parts);
140
+ const sex = nameSex(parts, slots);
141
+ const lemmaWords = parts.flatMap((part, i) =>
142
+ part.match(letters).map((word) => {
143
+ const slot = slots[i];
144
+ return {
145
+ slot,
146
+ lemmaForms: russianNameForms(word, slot, sex),
147
+ pattern: new RegExp(`^(?:${wordPattern(word, 'PERSON')})$`, 'iu'),
148
+ first: word[0].toLowerCase().replace('ё', 'е'),
149
+ fake: fakeWord(capitalize(word), key, slot, undefined, sex),
150
+ };
151
+ })
152
+ );
153
+ const matched = surface.match(letters).map((word) => {
154
+ const initial = word.length === 1;
155
+ const entry = lemmaWords.find(
156
+ (candidate) =>
157
+ !candidate.used &&
158
+ (initial
159
+ ? candidate.slot !== 'lastName' &&
160
+ candidate.first === word.toLowerCase().replace('ё', 'е')
161
+ : candidate.pattern.test(word))
162
+ );
163
+ if (entry) {
164
+ entry.used = true;
165
+ }
166
+ return { word, entry, initial };
167
+ });
168
+ const index = commonCase(
169
+ matched
170
+ .filter(({ entry, initial }) => entry && !initial)
171
+ .map(({ word, entry }) => casesOf(word, entry.lemmaForms)),
172
+ context
173
+ );
174
+ let position = 0;
175
+ return surface.replace(letters, () => {
176
+ const { word, entry, initial } = matched[position++];
177
+ if (!entry) {
178
+ return fakeWord(word, key);
179
+ }
180
+ if (initial) {
181
+ return entry.fake[0].toUpperCase();
182
+ }
183
+ const forms = russianNameForms(entry.fake, entry.slot, sex);
184
+ return applyStyle(word, capitalize(formOf(forms, index)));
185
+ });
186
+ }
187
+ const contextOf = (finding, options) =>
188
+ governedCases(options[textKey] ?? '', finding.start);
189
+ export function fakePerson(value, finding, transform, options) {
190
+ const lemma = entityLemma(value, 'PERSON', options);
191
+ return cyrillic.test(lemma) &&
192
+ cyrillic.test(value) &&
193
+ !/\p{Script=Latin}/u.test(`${lemma}${value}`)
194
+ ? russianFakeName(value, lemma, transform.key, contextOf(finding, options))
195
+ : fakeName(value, transform.key);
196
+ }
197
+
198
+ const assigned = new WeakMap();
199
+ const lemmaKey = (core) => core.toLowerCase().replaceAll('ё', 'е');
200
+ const poolFor = (type, core, transform) =>
201
+ transform.pools?.[type] ??
202
+ vocabulary[type][cyrillic.test(core) ? 'ru' : 'latin'].split('|');
203
+ function isReal(candidate, type, options) {
204
+ return [
205
+ ...(options.knownPersonal ?? []),
206
+ ...(options.confirmedPersonal ?? []),
207
+ ...[...(options[surfacesKey]?.get(type) ?? [])].map((value) => ({
208
+ type,
209
+ value,
210
+ })),
211
+ ].some(
212
+ (entry) => entry.type === type && samePersonal(candidate, entry.value, type)
213
+ );
214
+ }
215
+ // A deterministic start, then the next free name so that two entities of one
216
+ // run never share a fake. An exhausted list continues with numbered names.
217
+ function assignFake(core, type, transform, options) {
218
+ let state = assigned.get(transform);
219
+ if (!state) {
220
+ state = new Map();
221
+ assigned.set(transform, state);
222
+ }
223
+ const entityKey = `${type}\0${lemmaKey(core)}`;
224
+ if (state.has(entityKey)) {
225
+ return state.get(entityKey);
226
+ }
227
+ const taken = new Set(state.values());
228
+ const pool = poolFor(type, core, transform);
229
+ const start = createHmac('sha256', transform.key)
230
+ .update(`entity\0${entityKey}`)
231
+ .digest()
232
+ .readUInt32BE(0);
233
+ for (let attempt = 0; ; attempt++) {
234
+ const name = pool[(start + attempt) % pool.length];
235
+ const round = Math.floor(attempt / pool.length);
236
+ const candidate = round ? `${name}-${round + 1}` : name;
237
+ if (!taken.has(candidate) && !isReal(candidate, type, options)) {
238
+ state.set(entityKey, candidate);
239
+ return candidate;
240
+ }
241
+ }
242
+ }
243
+ const nounOptions = (word, phrase, position) => ({
244
+ animate: false,
245
+ sex: phrase.some((other) => /[ая]я$/u.test(other)) ? 'female' : undefined,
246
+ adjective:
247
+ !word.includes('-') &&
248
+ (/(?:ый|ая|яя)$/u.test(word) ||
249
+ (/(?:ий|ой|ое|ее)$/u.test(word) && position < phrase.length - 1)),
250
+ });
251
+ // Only the last part of a hyphenated name declines: в Альфа-Строе.
252
+ const phraseForms = (phrase) => {
253
+ const words = phrase.toLowerCase().split(/\s+/u);
254
+ return words.map((word, position) => {
255
+ const head = word.slice(0, word.lastIndexOf('-') + 1);
256
+ const tail = word.slice(head.length);
257
+ return {
258
+ head,
259
+ forms: russianNoun(tail, nounOptions(word, words, position)),
260
+ };
261
+ });
262
+ };
263
+ function phraseCase(surface, lemma, context) {
264
+ const words = surface.split(/\s+/u);
265
+ const lemmaForms = phraseForms(lemma);
266
+ if (words.length !== lemmaForms.length) {
267
+ return 0;
268
+ }
269
+ return commonCase(
270
+ words.map((word, i) =>
271
+ casesOf(word.slice(word.lastIndexOf('-') + 1), lemmaForms[i].forms)
272
+ ),
273
+ context
274
+ );
275
+ }
276
+ function declinedPhrase(phrase, index) {
277
+ if (!cyrillic.test(phrase)) {
278
+ return phrase;
279
+ }
280
+ const original = phrase.split(/\s+/u);
281
+ return phraseForms(phrase)
282
+ .map(({ head, forms }, i) => {
283
+ const word = head + formOf(forms, index);
284
+ return original[i][0] === original[i][0].toUpperCase()
285
+ ? word.replace(
286
+ /(^|-)(\p{L})/gu,
287
+ (_m, gap, letter) => `${gap}${letter.toUpperCase()}`
288
+ )
289
+ : word;
290
+ })
291
+ .join(' ');
292
+ }
293
+ const decoration = /^((?:(?:ООО|АО|ПАО|ИП|ЗАО)[ \t]+)?[«“"]?)(.*?)([»”"]?)$/su;
294
+ // `ООО «Ромашка»` keeps its legal form and quotes; only the name is replaced.
295
+ export function fakeEntity(value, finding, transform, options) {
296
+ const { type } = finding;
297
+ const [, before, core, after] = decoration.exec(value);
298
+ const lemma = organizationCore(entityLemma(value, type, options));
299
+ const fake = assignFake(lemma, type, transform, options);
300
+ const output = cyrillic.test(core)
301
+ ? declinedPhrase(fake, phraseCase(core, lemma, contextOf(finding, options)))
302
+ : fake;
303
+ return `${before}${applyStyle(core, output)}${after}`;
304
+ }
package/src/fake-names.js CHANGED
@@ -146,6 +146,58 @@ function scriptLocale(value) {
146
146
  }
147
147
  export function fakeName(value, key) {
148
148
  indexNames();
149
+ const words = value.match(/[\p{L}\p{M}]+/gu) ?? [];
150
+ const roles = new Map();
151
+ let nameSex;
152
+ if (
153
+ /\p{Script=Cyrillic}/u.test(value) &&
154
+ words.length >= 2 &&
155
+ words.length <= 3
156
+ ) {
157
+ const middle = words.findIndex((w) =>
158
+ /(?:ович|евич|ична|овна|евна)$/iu.test(w)
159
+ );
160
+ const surname = words.findIndex(
161
+ (w, i) =>
162
+ i !== middle &&
163
+ /(?:ов|ев|ин|ский|ова|ева|ина|ская)$/iu.test(w) &&
164
+ !given.has(latinName(w))
165
+ );
166
+ const genderWord = words[middle >= 0 ? middle : surname];
167
+ if (genderWord) {
168
+ nameSex = /(?:ова|ева|ина|ская|овна|евна|ична)$/iu.test(genderWord)
169
+ ? 'female'
170
+ : 'male';
171
+ }
172
+ if (middle > 0) {
173
+ words.forEach((w, i) =>
174
+ roles.set(
175
+ w,
176
+ i === middle
177
+ ? 'middleName'
178
+ : i === middle - 1
179
+ ? 'firstName'
180
+ : 'lastName'
181
+ )
182
+ );
183
+ } else if (surname >= 0) {
184
+ words.forEach((w, i) =>
185
+ roles.set(w, i === surname ? 'lastName' : 'firstName')
186
+ );
187
+ } else {
188
+ // Ambiguous spans receive a complete name with the same token count.
189
+ words.forEach((w, i) =>
190
+ roles.set(
191
+ w,
192
+ i === 0
193
+ ? 'firstName'
194
+ : i === words.length - 1
195
+ ? 'lastName'
196
+ : 'middleName'
197
+ )
198
+ );
199
+ }
200
+ }
149
201
  return value.replace(/[\p{L}\p{M}]+/gu, (word) => {
150
202
  for (const locale of ['ja', 'zh', 'ko']) {
151
203
  const family = firstNames[locale]
@@ -158,10 +210,11 @@ export function fakeName(value, key) {
158
210
  );
159
211
  }
160
212
  }
161
- return fakeWord(word, key);
213
+ return fakeWord(word, key, roles.get(word), undefined, nameSex);
162
214
  });
163
215
  }
164
- function wordProfile(word, forcedRole, forcedLocale) {
216
+ export function wordProfile(word, forcedRole, forcedLocale) {
217
+ indexNames();
165
218
  const canonical = latinName(word);
166
219
  const patronymic = /(?:vich|vna|ichna|ogly|kyzy)$/.test(canonical);
167
220
  const slavic = /(?:ova|eva|ov|ev|in|ina|enko|sky|ski|skaya)$/.test(canonical);
@@ -188,12 +241,14 @@ function wordProfile(word, forcedRole, forcedLocale) {
188
241
  : 'lastName');
189
242
  return { canonical, locale, sex, role };
190
243
  }
191
- function fakeWord(word, key, forcedRole, forcedLocale) {
192
- const { canonical, locale, sex, role } = wordProfile(
193
- word,
194
- forcedRole,
195
- forcedLocale
196
- );
244
+ export function fakeWord(word, key, forcedRole, forcedLocale, forcedSex) {
245
+ const {
246
+ canonical,
247
+ locale,
248
+ sex: detectedSex,
249
+ role,
250
+ } = wordProfile(word, forcedRole, forcedLocale);
251
+ const sex = forcedSex ?? detectedSex;
197
252
  const faker = new Faker({ locale: [locales[locale] ?? en, en] });
198
253
  faker.seed(
199
254
  createHmac('sha256', key)
package/src/fake.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { fakeEntity, fakePerson } from './fake-entities.js';
1
2
  import { createHmac } from 'node:crypto';
2
3
  import { parsePhoneNumberFromString } from 'libphonenumber-js/max';
3
4
  import { mrzCheck } from './identity.js';
@@ -487,7 +488,7 @@ function fakeTd1(parts, transform) {
487
488
  .padEnd(30, '<');
488
489
  return parts.join('');
489
490
  }
490
- export function realisticFake(value, finding, transform) {
491
+ export function realisticFake(value, finding, transform, options = {}) {
491
492
  let output;
492
493
  if (finding.category === 'credential') {
493
494
  if (!transform.credentials) {
@@ -500,14 +501,9 @@ export function realisticFake(value, finding, transform) {
500
501
  .toString('hex')
501
502
  .slice(0, 8)}!`;
502
503
  } else if (finding.type === 'PERSON') {
503
- output = fakeName(value, transform.key);
504
+ output = fakePerson(value, finding, transform, options);
504
505
  } else if (['ORGANIZATION', 'EDUCATION', 'LOCATION'].includes(finding.type)) {
505
- const label = {
506
- ORGANIZATION: 'Organization',
507
- EDUCATION: 'School',
508
- LOCATION: 'Location',
509
- }[finding.type];
510
- output = `Example ${label} ${digest(value, transform.key, finding.type).readUInt32BE(0)}`;
506
+ output = fakeEntity(value, finding, transform, options);
511
507
  } else if (/DATE|BIRTH/.test(finding.type)) {
512
508
  output = fakeDate(value, transform.key);
513
509
  } else if (finding.type === 'PASSPORT_MRZ') {
package/src/index.d.ts CHANGED
@@ -42,6 +42,8 @@ export interface Detector {
42
42
  export interface SanitizerOptions {
43
43
  /** Preserve all personal/account data; credential findings still redact. */
44
44
  credentialsOnly?: boolean;
45
+ /** Replace only knownPersonal/knownSecrets and reused map values; skip automatic detectors. */
46
+ knownOnly?: boolean;
45
47
  profile?: 'publication' | 'documents';
46
48
  /** Opt in to bounded employer, school, city/region heuristics. */
47
49
  documentEntities?: boolean;
@@ -97,6 +99,8 @@ export interface SanitizedText {
97
99
  findings: Finding[];
98
100
  /** Number of disjoint replacements after overlapping detections are merged. */
99
101
  redactions: number;
102
+ /** Placeholder-mode replacements absent from the restore map (e.g. credentials). */
103
+ irreversibleRedactions?: number;
100
104
  /** Private personal data. Absent for map:false and non-placeholder modes. */
101
105
  map?: RestoreMap;
102
106
  }
@@ -246,6 +250,10 @@ export type Transformation =
246
250
  credentials?: boolean;
247
251
  /** Keep original MRZ issuer/nationality unless Utopia is requested. */
248
252
  mrzCountry?: 'UTO';
253
+ /** Caller-reviewed names that replace the generated entity vocabulary. */
254
+ pools?: Partial<
255
+ Record<'ORGANIZATION' | 'LOCATION' | 'EDUCATION', string[]>
256
+ >;
249
257
  }
250
258
  | { mode: 'date-shift'; days: number }
251
259
  | { mode: 'bucket'; size: number };
package/src/keywords.js CHANGED
@@ -27,10 +27,27 @@ export function keywordMatcher(words) {
27
27
  queue.push(child);
28
28
  }
29
29
  }
30
+ // At the root, characters that cannot start any keyword need no state
31
+ // transitions. A native search skips large padding runs without Map work
32
+ // per character. Escape UTF-16 units so metacharacters retain literal meaning.
33
+ const starters = new RegExp(
34
+ `[${[...nodes[0].next.keys()]
35
+ .map((char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`)
36
+ .join('')}]`,
37
+ 'g'
38
+ );
30
39
  return (text) => {
31
40
  const found = new Set();
32
41
  let state = 0;
33
42
  for (let i = 0; i < text.length; i++) {
43
+ if (!state && !nodes[0].output.length && !nodes[0].next.has(text[i])) {
44
+ starters.lastIndex = i + 1;
45
+ const next = starters.exec(text);
46
+ if (!next) {
47
+ break;
48
+ }
49
+ i = next.index;
50
+ }
34
51
  const char = text[i];
35
52
  while (state && !nodes[state].next.has(char)) {
36
53
  state = nodes[state].fail;
package/src/names.js CHANGED
@@ -12,7 +12,7 @@ export const firstNames = {
12
12
  jaLatn: 'Yamada Sato Suzuki Takahashi Tanaka Ito Watanabe Nakamura Kobayashi',
13
13
  zhLatn:
14
14
  'Zhang Wang Li Zhao Liu Chen Yang Huang Zhou Wu Xu Sun Hu Zhu Gao Lin',
15
- ru: 'Иван Пётр Петр Сергей Александр Алексей Дмитрий Михаил Николай Андрей Владимир Павел Виктор Олег Борис Евгений Василий Юрий Марина Анна Мария Елена Ольга Наталья Ирина Светлана Татьяна Екатерина Анастасия Юлия Ивану Ивана Иваном Сергея Сергею Александра Александру Анне Анну Елены',
15
+ ru: 'Данил Максим Алина Иван Пётр Петр Сергей Александр Алексей Дмитрий Михаил Николай Андрей Владимир Павел Виктор Олег Борис Евгений Василий Юрий Марина Анна Мария Елена Ольга Наталья Ирина Светлана Татьяна Екатерина Анастасия Юлия Ивану Ивана Иваном Сергея Сергею Александра Александру Анне Анну Елены',
16
16
  ja: '山田 佐藤 鈴木 高橋 田中 伊藤 渡辺 渡邊 中村 小林 加藤 吉田 松本 山本 井上 木村 林 清水 斎藤 斉藤',
17
17
  zh: '张 王 李 赵 刘 陈 杨 黄 周 吴 徐 孙 胡 朱 高 林 何 郭 马 罗 張 趙 劉 陳 楊 黃 吳 孫 馬 羅',
18
18
  ko: '김 이 박 최 정 강 조 윤 장 임 한 오 서 신 권 황 안 송 전 홍 유 고 문 양 손 배 백',
@@ -2,7 +2,7 @@ import { collectMatches } from './detection.js';
2
2
 
3
3
  const birth =
4
4
  /(?<![\p{L}\p{N}_])(?:born|birth[ _-]?date|date[ _-]of[ _-]birth|DOB|дата рождения|д\.р\.|родился|родилась|Geburtsdatum|fecha de nacimiento|date de naissance)(?![\p{L}\p{N}_])/giu;
5
- const months =
5
+ export const months =
6
6
  'January February March April May June July August September October November December Jan Feb Mar Apr Jun Jul Aug Sep Sept Oct Nov Dec января февраля марта апреля мая июня июля августа сентября октября ноября декабря январь февраль март апрель май июнь июль август сентябрь октябрь ноябрь декабрь'
7
7
  .split(' ')
8
8
  .join('|');
@@ -0,0 +1,184 @@
1
+ import { escapePattern, failure } from './detection.js';
2
+ import { russianNameRoles } from './russian-names.js';
3
+
4
+ const boundary = '[\\p{L}\\p{M}\\p{N}_]';
5
+ export function organizationCore(value) {
6
+ return value
7
+ .replace(/^(?:ООО|АО|ПАО|ИП|ЗАО)[ \t]+/u, '')
8
+ .replace(/^[«“"]|[»”"]$/gu, '')
9
+ .trim();
10
+ }
11
+ // `Вектор-1` also identifies its base name and numbered siblings like `Вектор-2`.
12
+ export function organizationFamily(core) {
13
+ const base = /^(.{3,})-\d{1,9}$/u.exec(core)?.[1];
14
+ return base
15
+ ? {
16
+ base,
17
+ pattern: new RegExp(
18
+ `(?<!${boundary})${escapePattern(base)}(?:-\\d{1,9})?(?!${boundary}|-\\d)`,
19
+ 'giu'
20
+ ),
21
+ }
22
+ : undefined;
23
+ }
24
+ // ё and е are one letter in ordinary Russian writing; the i flag covers Ё and Е.
25
+ const foldYo = (pattern) => pattern.replace(/[еёЕЁ]/gu, '[её]');
26
+ const fleeting = { павел: 'Павл', лев: 'Льв' };
27
+ const hardCases = '(?:а|у|ом|е)';
28
+ // Only caller-confirmed Russian words receive finite case alternatives. A
29
+ // minimum three-letter stem avoids turning short names into broad matches.
30
+ function cyrillicPattern(word, type) {
31
+ const endings = [
32
+ [/(?:ова|ева|ина)$/u, 1, '(?:а|ой|ою|у|е|ы)'],
33
+ [/(?:ая)$/u, 2, '(?:ая|ой|ою|ую)'],
34
+ [
35
+ /(?:ий|ый|ой)$/u,
36
+ 2,
37
+ '(?:ий|ый|ой|ого|его|ому|ему|им|ым|ом|ем|ия|ию|ием|ии)',
38
+ ],
39
+ [/[аеуюя]й$/u, 1, '(?:й|я|ю|ем|е)'],
40
+ [/(?:ов|ев|ин)$/u, 0, '(?:а|у|ым|ом|е|ой)?'],
41
+ [/[жчшщц]$/u, 0, '(?:а|у|ем|ом|е)?'],
42
+ [/[жчшщц]а$/u, 1, '(?:а|и|е|у|ей|ею|ой|ою)'],
43
+ [/а$/u, 1, '(?:а|ы|и|е|у|ой|ою)'],
44
+ [/я$/u, 1, '(?:я|и|е|ю|ей|ею)'],
45
+ [/ь$/u, 1, '(?:ь|я|ю|ем|е|и|ью)'],
46
+ ];
47
+ const lower = word.toLowerCase();
48
+ if (fleeting[lower]) {
49
+ return `(?:${word}|${fleeting[lower]}${hardCases})`;
50
+ }
51
+ for (const [ending, remove, suffix] of endings) {
52
+ if (ending.test(lower) && word.length - remove >= 3) {
53
+ return escapePattern(remove ? word.slice(0, -remove) : word) + suffix;
54
+ }
55
+ }
56
+ if (/^[а-яё]{3,}[бвгджзклмнпрстфхцчшщ]$/u.test(lower)) {
57
+ // Plural forms would turn a given name into a surname: Петр and Петров.
58
+ return `${escapePattern(word)}${type === 'PERSON' ? `${hardCases}?` : '(?:а|у|ом|е|ы|ов|ам|ами|ах)?'}`;
59
+ }
60
+ return escapePattern(word);
61
+ }
62
+ export function wordPattern(word, type) {
63
+ return foldYo(
64
+ word
65
+ .split('-')
66
+ .map((part) =>
67
+ /^[а-яё]+$/iu.test(part)
68
+ ? cyrillicPattern(part, type)
69
+ : escapePattern(part)
70
+ )
71
+ .join('-')
72
+ );
73
+ }
74
+ const initialGap = '[ \\t\\u00a0]*';
75
+ const initialPattern = (word) => `${foldYo(escapePattern(word[0]))}\\.`;
76
+ // Everyday orders, partial forms and initials of a Russian full name, longest
77
+ // first so that alternation prefers the complete form at each position.
78
+ function russianForms(words, roles, type) {
79
+ const name = Object.fromEntries(
80
+ Object.entries(roles).map(([role, i]) => [role, words[i]])
81
+ );
82
+ const gap = '[ \\t]+';
83
+ const full = (...parts) =>
84
+ parts.map((part) => wordPattern(part, type)).join(gap);
85
+ if (name.surname === undefined) {
86
+ return [];
87
+ }
88
+ const initial = initialPattern(name.given);
89
+ const last = wordPattern(name.surname, type);
90
+ const short = [
91
+ full(name.surname, name.given),
92
+ full(name.given, name.surname),
93
+ `${last}${gap}${initial}`,
94
+ `${initial}[ \\t\\u00a0]+${last}`,
95
+ ];
96
+ if (name.patronymic === undefined) {
97
+ return short;
98
+ }
99
+ const initials = `${initial}${initialGap}${initialPattern(name.patronymic)}`;
100
+ return [
101
+ full(name.surname, name.given, name.patronymic),
102
+ full(name.given, name.patronymic, name.surname),
103
+ `${last}[ \\t\\u00a0]+${initials}`,
104
+ `${initials}[ \\t\\u00a0]+${last}`,
105
+ ...short,
106
+ full(name.given, name.patronymic),
107
+ ];
108
+ }
109
+ export function personalPatterns(value, type) {
110
+ if (value.length > 256) {
111
+ // Large explicit values still support literal matching without expansion.
112
+ return [escapePattern(value)];
113
+ }
114
+ const values = new Set([value]);
115
+ if (type === 'ORGANIZATION') {
116
+ const core = organizationCore(value);
117
+ if (core.length >= 3) {
118
+ values.add(core);
119
+ }
120
+ }
121
+ const patterns = new Set();
122
+ for (const surface of values) {
123
+ const parts = surface.split(/([ \t]+)/u);
124
+ const pattern = parts
125
+ .map((part, i) => (i % 2 ? '[ \\t]+' : wordPattern(part, type)))
126
+ .join('');
127
+ patterns.add(pattern);
128
+ if (type === 'PERSON') {
129
+ const words = parts.filter((_part, i) => i % 2 === 0);
130
+ const roles = russianNameRoles(words);
131
+ for (const form of roles ? russianForms(words, roles, type) : []) {
132
+ patterns.add(form);
133
+ }
134
+ if (words.length === 2) {
135
+ patterns.add(
136
+ [wordPattern(words[1], type), wordPattern(words[0], type)].join(
137
+ '[ \\t]+'
138
+ )
139
+ );
140
+ }
141
+ }
142
+ }
143
+ return [...patterns];
144
+ }
145
+ export function personalRegex(value, type) {
146
+ return new RegExp(
147
+ `(?<!${boundary})(?:${personalPatterns(value, type).join('|')})(?!${boundary})`,
148
+ 'giu'
149
+ );
150
+ }
151
+ export function samePersonal(value, original, type) {
152
+ return personalPatterns(original, type).some((pattern) =>
153
+ new RegExp(`^(?:${pattern})$`, 'iu').test(value)
154
+ );
155
+ }
156
+ export function detectKnownPersonal(text, entries, rule, emit, options) {
157
+ let count = 0;
158
+ for (const entry of entries ?? []) {
159
+ const family =
160
+ rule === 'confirmed-personal' &&
161
+ options.documentEntities &&
162
+ !options.knownOnly &&
163
+ entry.type === 'ORGANIZATION' &&
164
+ organizationFamily(organizationCore(entry.value));
165
+ for (const pattern of [
166
+ personalRegex(entry.value, entry.type),
167
+ ...(family ? [family.pattern] : []),
168
+ ]) {
169
+ for (const match of text.matchAll(pattern)) {
170
+ if (++count > (options.maxFindings ?? 100000)) {
171
+ throw failure('ERR_LIMIT');
172
+ }
173
+ emit({
174
+ start: match.index,
175
+ end: match.index + match[0].length,
176
+ type: entry.type,
177
+ category: 'personal',
178
+ rule,
179
+ confidence: rule === 'known-personal' ? 0.85 : 0.9,
180
+ });
181
+ }
182
+ }
183
+ }
184
+ }