@link-foundation/sensitive-data-sanitizer 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +7 -1
- package/docs/API.md +13 -1
- package/docs/COVERAGE.md +13 -1
- package/package.json +1 -1
- package/src/detection.js +19 -1
- package/src/document-entities.js +29 -0
- package/src/fake-entities.js +304 -0
- package/src/fake-names.js +3 -2
- package/src/fake.js +4 -8
- package/src/index.d.ts +4 -0
- package/src/personal-context.js +1 -1
- package/src/personal-variants.js +115 -25
- package/src/personal.js +12 -6
- package/src/placeholders.js +23 -4
- package/src/propagation.js +7 -3
- package/src/russian-declension.js +167 -0
- package/src/russian-names.js +36 -0
- package/src/sanitizer.js +7 -1
- package/src/transforms.js +21 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.15.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 7e977ac: Match known Russian names in all six cases and both word orders, with `ё`/`е` variants, initials and surname-plus-given forms. Give all case forms of one known person, organization, school or location one declined fake identity with natural names and an optional `pools` override. Keep `@handle/email` and `Name-99` organization spans whole so placeholders stay restorable and are never glued to digits. Detect employers in CV experience lines of the `documents` profile.
|
|
8
|
+
|
|
3
9
|
## 0.14.0
|
|
4
10
|
|
|
5
11
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -60,6 +60,10 @@ sensitive-data-sanitizer redact cv.txt --profile documents --placeholder --no-ma
|
|
|
60
60
|
Placeholder results report `irreversibleRedactions`; a nonzero count means some
|
|
61
61
|
replacements, such as credentials excluded from the map, cannot be restored.
|
|
62
62
|
|
|
63
|
+
Known Russian names match in all six cases and in any word order, with `ё`/`е`
|
|
64
|
+
spelling variants and initials (`с Петровым Данилом Сергеевичем`, `Петров Д. С.`).
|
|
65
|
+
A part of a longer name is matched alone only when it is listed itself.
|
|
66
|
+
|
|
63
67
|
For irreversible output, choose `transformation: { mode: 'placeholder', map:
|
|
64
68
|
false }`: no restore map is returned or written. Detection limits still apply
|
|
65
69
|
in either mode. See the [placeholder API](docs/API.md#reversible-placeholders),
|
|
@@ -144,7 +148,9 @@ See [the API](docs/API.md) and [all issue 13 requirements and research](docs/cas
|
|
|
144
148
|
`transformation: {mode: 'fake', key}` generates deterministic names, valid
|
|
145
149
|
dates/MRZs/checksummed IDs and reserved email/domain destinations. Credentials
|
|
146
150
|
remain fully redacted by default. Keep the key private: this preserves relations
|
|
147
|
-
and structure and is pseudonymisation.
|
|
151
|
+
and structure and is pseudonymisation. Russian mentions of one known person or
|
|
152
|
+
entity share one fake identity declined to match the source (`Петров … С Петровым`
|
|
153
|
+
becomes `Баранов … С Барановым`), and `pools` overrides the entity names. See [realistic fakes](examples/realistic-fakes.mjs)
|
|
148
154
|
and [the complete issue 24 requirements, research and solution plans](docs/case-studies/issue-24/REQUIREMENTS.md).
|
|
149
155
|
|
|
150
156
|
Keep policies containing real private values outside Git. Built-in exact public knowledge and narrowly scoped role/domain heuristics are enabled offline. Explicit private literals override public exemptions. Keep custom public policies reviewed against evidence; `--verify-public` separately opts into Wikidata name lookup:
|
package/docs/API.md
CHANGED
|
@@ -104,6 +104,16 @@ in reverse order, and organization cores without legal forms/quotes. Inflected
|
|
|
104
104
|
forms consume whole Unicode words; minimum stem lengths limit overmatching.
|
|
105
105
|
This is conservative heuristic identity linking, not general coreference:
|
|
106
106
|
ambiguous surnames/initials and unrelated people are not intentionally merged.
|
|
107
|
+
|
|
108
|
+
A listed Russian PERSON value also matches all six grammatical cases, including
|
|
109
|
+
the instrumental (`Петровым Данилом Сергеевичем`), every word order, `ё`/`е`
|
|
110
|
+
spelling differences in either direction, upper case, and initials with or
|
|
111
|
+
without spaces in both positions and all cases (`Петрову Д. С.`, `Д.С. Петрова`).
|
|
112
|
+
The contiguous name parts of a listed three-part name match together
|
|
113
|
+
(`Петров Данил`, `Данил Сергеевич`, `с Петровым Данилом`). A part on its own does
|
|
114
|
+
not: a bare `Данил` or `Сергеевич` is too ambiguous to imply from a longer value,
|
|
115
|
+
so list each part you want matched alone as its own `knownPersonal` entry. Listing
|
|
116
|
+
parts next to the full name keeps one token and one fake identity.
|
|
107
117
|
Every replaced occurrence retains its exact original surface privately. With
|
|
108
118
|
one identical token, a derived answer cannot encode the original grammatical
|
|
109
119
|
case; it receives the canonical spelling. Two distinct originals that produce
|
|
@@ -132,7 +142,7 @@ new document recognizers; `documentEntities: false` turns them off explicitly.
|
|
|
132
142
|
| `profile` | Off; `'publication'` sets a 0.3 threshold and identity masking; `'documents'` enables document entities |
|
|
133
143
|
| `knownOnly` | `false`; disable all automatic detection (including credentials, Secretlint and custom detectors); use only explicit known values and reused map identities |
|
|
134
144
|
| `knownSecrets` | `[]`; exact literal credential values, regardless of length or format |
|
|
135
|
-
| `knownPersonal` | `[]`; `{type, value}` values with whole-word Unicode case-insensitive matching and bounded Russian
|
|
145
|
+
| `knownPersonal` | `[]`; `{type, value}` values with whole-word Unicode case-insensitive matching and bounded Russian forms; list parts to match them alone |
|
|
136
146
|
| `publicEntities` | `[]`; exact PERSON/ORGANIZATION/EDUCATION/LOCATION/EMAIL exception with HTTPS evidence and review date |
|
|
137
147
|
| `findings` | `[]`; validated detections for the original input, supplied by the caller |
|
|
138
148
|
| `detectors` | `[]`; additional required callbacks in the asynchronous engine |
|
|
@@ -205,6 +215,8 @@ All keyed modes require a key of at least 16 UTF-8 bytes. `format-preserving` is
|
|
|
205
215
|
|
|
206
216
|
`{mode: 'fake', key}` infers name locale, script and gender from the supported gazetteer/datasets and recognizable surname/patronymic endings. Independent HMAC-seeded Faker instances make results independent of processing order. Russian Cyrillic/transliterated spelling, name order and filename separators share component identities. Ambiguous names can use a fallback locale; this is not universal name/gender inference. Dates use one keyed backward offset of 1–365 days and preserve ISO, slash and dot formats. Complete TD1/TD2/TD3 MRZs regenerate individual/composite checks and match separately faked name/date fields. Issuer/nationality remain original unless `mrzCountry: 'UTO'` is supplied. Unsupported/damaged formats fall back to full redaction.
|
|
207
217
|
|
|
218
|
+
Russian mentions of one person, organization, school or location share one fake identity. The fake follows the source word order, initials, capitalization, grammatical case and, for people, sex: `Петров пришёл. Позвонили Петрову. С Петровым.` becomes `Баранов пришёл. Позвонили Баранову. С Барановым.`, and `ООО «Ромашка» … в Ромашке` becomes `ООО «Импульс» … в Импульсе`. Declension uses finite offline rules for the generated vocabulary. The grammatical case comes from the source surface and the preceding preposition, so a source preposition stays as written even where the fake would call for another spelling (`о Осипове`). Organizations, schools and locations draw from an authored Russian or Latin vocabulary according to the script of the original, skip any name that is itself private in the text, and take a numeric suffix (`Заря-2`) once the vocabulary is exhausted. Each transformation object remembers its entity names so that two entities never share one, which makes a name depend on first appearance when two entities start on the same vocabulary entry; reuse the transformation object across documents to keep names stable, and note that this memory is not shared across worker threads or processes. `pools: {ORGANIZATION: [...], LOCATION: [...], EDUCATION: [...]}` on the fake transformation replaces a vocabulary with caller-reviewed non-empty names. Autodetected mentions are linked as in placeholder mode: a mention that appears only in oblique forms can remain a separate entity unless the value is listed in `knownPersonal`.
|
|
219
|
+
|
|
208
220
|
Implemented national checksums, IBANs and payment-card Luhn checks remain valid. Phones retain calling code, length and separators; NANP uses 555-0100–0199 and UK mobile uses 07700 900000–900999. Email/domain outputs use example.com/.test. Other national identities and phone numbers have no universal reserved range: generated arithmetic does not prove that a value is unassigned.
|
|
209
221
|
|
|
210
222
|
Generated replacements carry `faked: true`. A private bounded registry recognizes values produced by the same transformation context as `kept: 'fake'` during later inspection/verification. Caller-supplied flags and arbitrary FAKE-looking strings grant no exemption. A fresh sanitizer does not inherit that registry; reviewed persisted fake identity values can be configured through `fakeValues`. Credentials remain fully redacted unless `credentials: true` requests provider-shaped values containing FAKE and punctuation that violates token grammar. Generated personal values cannot exempt credential spans.
|
package/docs/COVERAGE.md
CHANGED
|
@@ -43,7 +43,9 @@ probabilities. `minConfidence` still controls replacement. Findings retain
|
|
|
43
43
|
their confidence even below threshold. `publicEntities` supports exact reviewed
|
|
44
44
|
exceptions for all three types; `knownPersonal` always forces removal of a
|
|
45
45
|
listed private value. All types support placeholder mode; fake mode uses stable,
|
|
46
|
-
|
|
46
|
+
plausible organization/school/location names from an authored vocabulary or
|
|
47
|
+
caller-supplied `pools`, declined to match Russian source mentions. These names
|
|
48
|
+
are plausible and can coincide with real entities that the text does not mention.
|
|
47
49
|
|
|
48
50
|
`src/document-vocabulary.js` is the complete authored vocabulary: English,
|
|
49
51
|
Spanish, French, German, Portuguese, Russian, Japanese, Simplified/Traditional
|
|
@@ -54,6 +56,16 @@ case); they do not implement grammatical parsing. Other lowercase connectors,
|
|
|
54
56
|
unusual typography, unsupported legal forms and unfamiliar institutions can be
|
|
55
57
|
missed. Legal-form words and uncased scripts can also yield false positives.
|
|
56
58
|
|
|
59
|
+
The `documents` profile also reads employers from CV experience lines anchored on
|
|
60
|
+
a date range: `Title, Employer (2019-05 – 2021-03)`, `Title — Employer, 2019–2021`
|
|
61
|
+
and `Title (Employer), 2019–2021`. Ranges use years, `YYYY-MM`, `MM.YYYY`, full
|
|
62
|
+
dates or `month YYYY`, with `по настоящее время`, `н.в.`, `present` and similar
|
|
63
|
+
open ends. Confidence is 0.60, and the employer then propagates through the
|
|
64
|
+
document like other organizations. The default profile ignores these lines. Lines
|
|
65
|
+
with more fields (`Title, Employer, City (dates)`) and titles containing a comma
|
|
66
|
+
are not targeted and can leave the employer in place; date-anchored prose with
|
|
67
|
+
the same shape can be flagged.
|
|
68
|
+
|
|
57
69
|
The gazetteer has only the cities/regions explicitly listed in that module,
|
|
58
70
|
including Springfield and Riverton. It is not a worldwide geocoder. Location
|
|
59
71
|
labels can find unlisted proper names; unlabelled places outside the vocabulary
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/detection.js
CHANGED
|
@@ -240,11 +240,29 @@ function detectAuthorization(text, emit, credential) {
|
|
|
240
240
|
}
|
|
241
241
|
}
|
|
242
242
|
|
|
243
|
+
// `/` and `|` are valid local-part characters, so `@handle/jane@example.com`
|
|
244
|
+
// reads as one address that starts after `@`. A mention keeps its own finding;
|
|
245
|
+
// any other leading `@` joins the address so that it cannot survive as a handle.
|
|
246
|
+
function emailAfterMention(text, finding) {
|
|
247
|
+
if (
|
|
248
|
+
text[finding.start - 1] !== '@' ||
|
|
249
|
+
/[\p{L}\p{N}_.\]]/u.test(text[finding.start - 2] ?? '')
|
|
250
|
+
) {
|
|
251
|
+
return finding;
|
|
252
|
+
}
|
|
253
|
+
const handle = /^[\p{L}\p{N}_]{2,32}[/|]/u.exec(
|
|
254
|
+
text.slice(finding.start, finding.end)
|
|
255
|
+
);
|
|
256
|
+
return handle
|
|
257
|
+
? { ...finding, start: finding.start + handle[0].length }
|
|
258
|
+
: { ...finding, start: finding.start - 1 };
|
|
259
|
+
}
|
|
260
|
+
|
|
243
261
|
function detectPersonalFormats(text, options, emit) {
|
|
244
262
|
collectMatches(
|
|
245
263
|
text,
|
|
246
264
|
/(?<![\p{L}\p{N}_.!#$%&'*+/?^`{|}~-])[\p{L}\p{N}_.!#$%&'*+/?^`{|}~-]+@[\p{L}\p{N}](?:[\p{L}\p{N}.-]{0,252})\.[\p{L}]{2,63}(?![\p{L}\p{N}_-])/gu,
|
|
247
|
-
emit,
|
|
265
|
+
(finding) => emit(emailAfterMention(text, finding)),
|
|
248
266
|
{ type: 'EMAIL', category: 'personal', rule: 'email' }
|
|
249
267
|
);
|
|
250
268
|
collectMatches(
|
package/src/document-entities.js
CHANGED
|
@@ -2,6 +2,7 @@ import { collectMatches, escapePattern } from './detection.js';
|
|
|
2
2
|
import { firstNames } from './names.js';
|
|
3
3
|
import { organizationCore, personalRegex } from './personal-variants.js';
|
|
4
4
|
import { documentVocabulary } from './document-vocabulary.js';
|
|
5
|
+
import { months } from './personal-context.js';
|
|
5
6
|
|
|
6
7
|
const words = (column) =>
|
|
7
8
|
[...new Set(documentVocabulary.flatMap((row) => row[column].split('|')))]
|
|
@@ -52,6 +53,32 @@ const groupOrganization = new RegExp(
|
|
|
52
53
|
`(?<!${boundary})${latinName}[ \t]+(?:Групп|Group|Холдинг|Holding)(?!${boundary})`,
|
|
53
54
|
'gu'
|
|
54
55
|
);
|
|
56
|
+
// CV experience lines: the date range anchors the employer slot, so prose with
|
|
57
|
+
// a comma is untouched. Titles are not required to be lowercase because English
|
|
58
|
+
// ones are often title case (`Senior QA Engineer`).
|
|
59
|
+
const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
|
|
60
|
+
const month = [
|
|
61
|
+
...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
|
|
62
|
+
].join('|');
|
|
63
|
+
const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
|
|
64
|
+
const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'н.в.']
|
|
65
|
+
.concat(['н/в', 'present', 'now', 'current'])
|
|
66
|
+
.flatMap((word) => [word, capitalized(word)])
|
|
67
|
+
.map(escapePattern)
|
|
68
|
+
.join('|');
|
|
69
|
+
const range = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen})(?!${boundary})`;
|
|
70
|
+
const titleWord = '[\\p{L}\\p{N}][\\p{L}\\p{N}/+#.&-]{0,39}';
|
|
71
|
+
const lineTitle = `^[ \\t]*(?:[-–—•*·][ \\t]+|\\d{1,2}[.)][ \\t]+)?${titleWord}(?:[ \\t]+${titleWord}){0,5}`;
|
|
72
|
+
const employerWord = "[\\p{L}\\p{M}\\p{N}&'’.+-]{1,40}";
|
|
73
|
+
const employer = `«[^«»\\r\\n]{1,160}»|"[^"\\r\\n]{1,160}"|“[^“”\\r\\n]{1,160}”|(?:\\p{Lu}|\\d{1,4}-?\\p{L})${employerWord}?(?:[ \\t]+${employerWord}){0,5}`;
|
|
74
|
+
const experienceLine = new RegExp(
|
|
75
|
+
`${lineTitle}(?:,[ \\t]+|[ \\t]+[-–—][ \\t]+)(${employer})(?:[ \\t]*\\(${range}\\)|[ \\t]*[,–—-][ \\t]*${range})`,
|
|
76
|
+
'gmu'
|
|
77
|
+
);
|
|
78
|
+
const experienceParenthesis = new RegExp(
|
|
79
|
+
`${lineTitle}[ \\t]*\\((${employer})\\)(?:,?[ \\t]*${range}|[ \\t]*\\(${range}\\))`,
|
|
80
|
+
'gmu'
|
|
81
|
+
);
|
|
55
82
|
|
|
56
83
|
export function detectDocumentEntities(text, emit, options) {
|
|
57
84
|
if (!options.documentEntities) {
|
|
@@ -71,6 +98,8 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
71
98
|
['EDUCATION', 'education-type', schools, 0, 0.65],
|
|
72
99
|
['LOCATION', 'location-gazetteer', places, 0, 0.6],
|
|
73
100
|
['LOCATION', 'location-label', labelledPlace, 1, 0.75],
|
|
101
|
+
['ORGANIZATION', 'organization-experience', experienceLine, 1, 0.6],
|
|
102
|
+
['ORGANIZATION', 'organization-experience', experienceParenthesis, 1, 0.6],
|
|
74
103
|
]) {
|
|
75
104
|
collectMatches(
|
|
76
105
|
text,
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import { createHmac } from 'node:crypto';
|
|
2
|
+
import { fakeName, fakeWord, wordProfile } from './fake-names.js';
|
|
3
|
+
import { canonicalPerson } from './placeholders.js';
|
|
4
|
+
import {
|
|
5
|
+
isGivenName,
|
|
6
|
+
isPatronymic,
|
|
7
|
+
russianNameRoles,
|
|
8
|
+
} from './russian-names.js';
|
|
9
|
+
import {
|
|
10
|
+
casesOf,
|
|
11
|
+
commonCase,
|
|
12
|
+
formOf,
|
|
13
|
+
governedCases,
|
|
14
|
+
russianNameForms,
|
|
15
|
+
russianNoun,
|
|
16
|
+
} from './russian-declension.js';
|
|
17
|
+
import {
|
|
18
|
+
organizationCore,
|
|
19
|
+
samePersonal,
|
|
20
|
+
wordPattern,
|
|
21
|
+
} from './personal-variants.js';
|
|
22
|
+
|
|
23
|
+
// Fictional but ordinary names. The lists are finite on purpose; callers can
|
|
24
|
+
// replace them with `transformation.pools`.
|
|
25
|
+
const vocabulary = {
|
|
26
|
+
ORGANIZATION: {
|
|
27
|
+
ru: 'Северный ветер|Альфа-Строй|Бета-Сервис|Гамма-Трейд|Меридиан|Горизонт|Гранит|Рассвет|Прогресс|Импульс|Орион|Азимут|Кристалл|Арсенал|Квант|Заря|Радуга|Волна|Орбита|Вершина|Искра|Тихая гавань|Новый горизонт|Белый парус|Золотой колос|Синяя птица',
|
|
28
|
+
latin:
|
|
29
|
+
'Northwind Trading|Bluebird Logistics|Silver Birch Partners|Harbor Lane Studio|Summit Ridge Supply|Clearwater Works|Redwood Systems|Maple Street Foods',
|
|
30
|
+
},
|
|
31
|
+
LOCATION: {
|
|
32
|
+
ru: 'Зареченск|Приморск|Светлогорск|Лесногорск|Березовка|Сосновка|Каменка|Покровка|Ольховка|Заречье|Краснополье',
|
|
33
|
+
latin: 'Riverton|Oakdale|Fairview|Lakewood|Brookfield|Greenville|Ashford',
|
|
34
|
+
},
|
|
35
|
+
EDUCATION: {
|
|
36
|
+
ru: 'Северный государственный университет|Приморский технический институт|Лесногорский гуманитарный университет|Зареченский экономический колледж|Светлогорская городская школа',
|
|
37
|
+
latin:
|
|
38
|
+
'Riverside University|Lakeview College|Oakdale Institute of Technology|Fairview State University',
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
const cyrillic = /\p{Script=Cyrillic}/u;
|
|
42
|
+
const surfacesKey = Symbol('fakeSurfaces');
|
|
43
|
+
const textKey = Symbol('fakeText');
|
|
44
|
+
const surfaceLimit = 1000;
|
|
45
|
+
|
|
46
|
+
// Lets each span see the other mentions of its document, as placeholders do.
|
|
47
|
+
export function prepareFakeText(text, findings, options) {
|
|
48
|
+
const surfaces = new Map();
|
|
49
|
+
for (const finding of findings) {
|
|
50
|
+
if (
|
|
51
|
+
['PERSON', 'ORGANIZATION', 'LOCATION', 'EDUCATION'].includes(finding.type)
|
|
52
|
+
) {
|
|
53
|
+
const values = surfaces.get(finding.type) ?? new Set();
|
|
54
|
+
if (values.size < surfaceLimit) {
|
|
55
|
+
values.add(text.slice(finding.start, finding.end));
|
|
56
|
+
}
|
|
57
|
+
surfaces.set(finding.type, values);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return { ...options, [surfacesKey]: surfaces, [textKey]: text };
|
|
61
|
+
}
|
|
62
|
+
const wordCount = (value) => value.trim().split(/\s+/u).length;
|
|
63
|
+
// The fullest form this mention belongs to, or the mention itself.
|
|
64
|
+
function entityLemma(value, type, options) {
|
|
65
|
+
const known = [
|
|
66
|
+
...(options.knownPersonal ?? []),
|
|
67
|
+
...(options.confirmedPersonal ?? []),
|
|
68
|
+
]
|
|
69
|
+
.filter((entry) => entry.type === type)
|
|
70
|
+
.map((entry) => entry.value);
|
|
71
|
+
const mentions = [...(options[surfacesKey]?.get(type) ?? [])];
|
|
72
|
+
for (const group of [known, mentions]) {
|
|
73
|
+
const matches = group.filter((other) => samePersonal(value, other, type));
|
|
74
|
+
const longest = Math.max(0, ...matches.map(wordCount));
|
|
75
|
+
const fullest = [
|
|
76
|
+
...new Set(matches.filter((other) => wordCount(other) === longest)),
|
|
77
|
+
];
|
|
78
|
+
// `Гранит` covers its own form `Граните`, so several forms share one lemma.
|
|
79
|
+
const lemma = fullest.filter((other) =>
|
|
80
|
+
fullest.every((form) => samePersonal(form, other, type))
|
|
81
|
+
);
|
|
82
|
+
if (lemma.length === 1) {
|
|
83
|
+
return lemma[0];
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return type === 'PERSON' ? canonicalPerson(value, mentions) : value;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const applyStyle = (surface, output) =>
|
|
90
|
+
surface.length > 1 && surface === surface.toUpperCase()
|
|
91
|
+
? output.toUpperCase()
|
|
92
|
+
: surface === surface.toLowerCase()
|
|
93
|
+
? output.toLowerCase()
|
|
94
|
+
: output;
|
|
95
|
+
const capitalize = (word) =>
|
|
96
|
+
word[0].toUpperCase() + word.slice(1).toLowerCase();
|
|
97
|
+
function nameSlots(parts) {
|
|
98
|
+
const roles = russianNameRoles(parts);
|
|
99
|
+
if (roles) {
|
|
100
|
+
return parts.map((_part, i) =>
|
|
101
|
+
i === roles.surname
|
|
102
|
+
? 'lastName'
|
|
103
|
+
: i === roles.patronymic
|
|
104
|
+
? 'middleName'
|
|
105
|
+
: 'firstName'
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
if (parts.length === 1) {
|
|
109
|
+
return [wordProfile(parts[0]).role];
|
|
110
|
+
}
|
|
111
|
+
const given = parts.map((part) => isGivenName(part));
|
|
112
|
+
if (parts.length === 2 && given[0] !== given[1]) {
|
|
113
|
+
return given.map((isGiven) => (isGiven ? 'firstName' : 'lastName'));
|
|
114
|
+
}
|
|
115
|
+
return parts.map((_part, i) =>
|
|
116
|
+
i === 0 ? 'firstName' : i === parts.length - 1 ? 'lastName' : 'middleName'
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
function nameSex(parts, slots) {
|
|
120
|
+
const at = (slot) => parts[slots.indexOf(slot)];
|
|
121
|
+
const middle = at('middleName');
|
|
122
|
+
if (middle && (isPatronymic(middle) || /(?:вна|чна)$/iu.test(middle))) {
|
|
123
|
+
return /(?:вна|чна)$/iu.test(middle) ? 'female' : 'male';
|
|
124
|
+
}
|
|
125
|
+
const last = at('lastName');
|
|
126
|
+
if (last && /(?:ова|ева|ёва|ина|ына|ая|яя)$/iu.test(last)) {
|
|
127
|
+
return 'female';
|
|
128
|
+
}
|
|
129
|
+
if (last && /(?:ов|ев|ёв|ин|ын|ий|ой|ый)$/iu.test(last)) {
|
|
130
|
+
return 'male';
|
|
131
|
+
}
|
|
132
|
+
return wordProfile(at('firstName') ?? parts[0]).sex;
|
|
133
|
+
}
|
|
134
|
+
const letters = /[\p{L}\p{M}]+/gu;
|
|
135
|
+
// One identity per lemma: each lemma word has one seeded fake, and the matched
|
|
136
|
+
// surface only decides its case, its order and its letter case.
|
|
137
|
+
function russianFakeName(surface, lemma, key, context) {
|
|
138
|
+
const parts = lemma.trim().split(/\s+/u);
|
|
139
|
+
const slots = nameSlots(parts);
|
|
140
|
+
const sex = nameSex(parts, slots);
|
|
141
|
+
const lemmaWords = parts.flatMap((part, i) =>
|
|
142
|
+
part.match(letters).map((word) => {
|
|
143
|
+
const slot = slots[i];
|
|
144
|
+
return {
|
|
145
|
+
slot,
|
|
146
|
+
lemmaForms: russianNameForms(word, slot, sex),
|
|
147
|
+
pattern: new RegExp(`^(?:${wordPattern(word, 'PERSON')})$`, 'iu'),
|
|
148
|
+
first: word[0].toLowerCase().replace('ё', 'е'),
|
|
149
|
+
fake: fakeWord(capitalize(word), key, slot, undefined, sex),
|
|
150
|
+
};
|
|
151
|
+
})
|
|
152
|
+
);
|
|
153
|
+
const matched = surface.match(letters).map((word) => {
|
|
154
|
+
const initial = word.length === 1;
|
|
155
|
+
const entry = lemmaWords.find(
|
|
156
|
+
(candidate) =>
|
|
157
|
+
!candidate.used &&
|
|
158
|
+
(initial
|
|
159
|
+
? candidate.slot !== 'lastName' &&
|
|
160
|
+
candidate.first === word.toLowerCase().replace('ё', 'е')
|
|
161
|
+
: candidate.pattern.test(word))
|
|
162
|
+
);
|
|
163
|
+
if (entry) {
|
|
164
|
+
entry.used = true;
|
|
165
|
+
}
|
|
166
|
+
return { word, entry, initial };
|
|
167
|
+
});
|
|
168
|
+
const index = commonCase(
|
|
169
|
+
matched
|
|
170
|
+
.filter(({ entry, initial }) => entry && !initial)
|
|
171
|
+
.map(({ word, entry }) => casesOf(word, entry.lemmaForms)),
|
|
172
|
+
context
|
|
173
|
+
);
|
|
174
|
+
let position = 0;
|
|
175
|
+
return surface.replace(letters, () => {
|
|
176
|
+
const { word, entry, initial } = matched[position++];
|
|
177
|
+
if (!entry) {
|
|
178
|
+
return fakeWord(word, key);
|
|
179
|
+
}
|
|
180
|
+
if (initial) {
|
|
181
|
+
return entry.fake[0].toUpperCase();
|
|
182
|
+
}
|
|
183
|
+
const forms = russianNameForms(entry.fake, entry.slot, sex);
|
|
184
|
+
return applyStyle(word, capitalize(formOf(forms, index)));
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
const contextOf = (finding, options) =>
|
|
188
|
+
governedCases(options[textKey] ?? '', finding.start);
|
|
189
|
+
export function fakePerson(value, finding, transform, options) {
|
|
190
|
+
const lemma = entityLemma(value, 'PERSON', options);
|
|
191
|
+
return cyrillic.test(lemma) &&
|
|
192
|
+
cyrillic.test(value) &&
|
|
193
|
+
!/\p{Script=Latin}/u.test(`${lemma}${value}`)
|
|
194
|
+
? russianFakeName(value, lemma, transform.key, contextOf(finding, options))
|
|
195
|
+
: fakeName(value, transform.key);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const assigned = new WeakMap();
|
|
199
|
+
const lemmaKey = (core) => core.toLowerCase().replaceAll('ё', 'е');
|
|
200
|
+
const poolFor = (type, core, transform) =>
|
|
201
|
+
transform.pools?.[type] ??
|
|
202
|
+
vocabulary[type][cyrillic.test(core) ? 'ru' : 'latin'].split('|');
|
|
203
|
+
function isReal(candidate, type, options) {
|
|
204
|
+
return [
|
|
205
|
+
...(options.knownPersonal ?? []),
|
|
206
|
+
...(options.confirmedPersonal ?? []),
|
|
207
|
+
...[...(options[surfacesKey]?.get(type) ?? [])].map((value) => ({
|
|
208
|
+
type,
|
|
209
|
+
value,
|
|
210
|
+
})),
|
|
211
|
+
].some(
|
|
212
|
+
(entry) => entry.type === type && samePersonal(candidate, entry.value, type)
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
// A deterministic start, then the next free name so that two entities of one
|
|
216
|
+
// run never share a fake. An exhausted list continues with numbered names.
|
|
217
|
+
function assignFake(core, type, transform, options) {
|
|
218
|
+
let state = assigned.get(transform);
|
|
219
|
+
if (!state) {
|
|
220
|
+
state = new Map();
|
|
221
|
+
assigned.set(transform, state);
|
|
222
|
+
}
|
|
223
|
+
const entityKey = `${type}\0${lemmaKey(core)}`;
|
|
224
|
+
if (state.has(entityKey)) {
|
|
225
|
+
return state.get(entityKey);
|
|
226
|
+
}
|
|
227
|
+
const taken = new Set(state.values());
|
|
228
|
+
const pool = poolFor(type, core, transform);
|
|
229
|
+
const start = createHmac('sha256', transform.key)
|
|
230
|
+
.update(`entity\0${entityKey}`)
|
|
231
|
+
.digest()
|
|
232
|
+
.readUInt32BE(0);
|
|
233
|
+
for (let attempt = 0; ; attempt++) {
|
|
234
|
+
const name = pool[(start + attempt) % pool.length];
|
|
235
|
+
const round = Math.floor(attempt / pool.length);
|
|
236
|
+
const candidate = round ? `${name}-${round + 1}` : name;
|
|
237
|
+
if (!taken.has(candidate) && !isReal(candidate, type, options)) {
|
|
238
|
+
state.set(entityKey, candidate);
|
|
239
|
+
return candidate;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
const nounOptions = (word, phrase, position) => ({
|
|
244
|
+
animate: false,
|
|
245
|
+
sex: phrase.some((other) => /[ая]я$/u.test(other)) ? 'female' : undefined,
|
|
246
|
+
adjective:
|
|
247
|
+
!word.includes('-') &&
|
|
248
|
+
(/(?:ый|ая|яя)$/u.test(word) ||
|
|
249
|
+
(/(?:ий|ой|ое|ее)$/u.test(word) && position < phrase.length - 1)),
|
|
250
|
+
});
|
|
251
|
+
// Only the last part of a hyphenated name declines: в Альфа-Строе.
|
|
252
|
+
const phraseForms = (phrase) => {
|
|
253
|
+
const words = phrase.toLowerCase().split(/\s+/u);
|
|
254
|
+
return words.map((word, position) => {
|
|
255
|
+
const head = word.slice(0, word.lastIndexOf('-') + 1);
|
|
256
|
+
const tail = word.slice(head.length);
|
|
257
|
+
return {
|
|
258
|
+
head,
|
|
259
|
+
forms: russianNoun(tail, nounOptions(word, words, position)),
|
|
260
|
+
};
|
|
261
|
+
});
|
|
262
|
+
};
|
|
263
|
+
function phraseCase(surface, lemma, context) {
|
|
264
|
+
const words = surface.split(/\s+/u);
|
|
265
|
+
const lemmaForms = phraseForms(lemma);
|
|
266
|
+
if (words.length !== lemmaForms.length) {
|
|
267
|
+
return 0;
|
|
268
|
+
}
|
|
269
|
+
return commonCase(
|
|
270
|
+
words.map((word, i) =>
|
|
271
|
+
casesOf(word.slice(word.lastIndexOf('-') + 1), lemmaForms[i].forms)
|
|
272
|
+
),
|
|
273
|
+
context
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
function declinedPhrase(phrase, index) {
|
|
277
|
+
if (!cyrillic.test(phrase)) {
|
|
278
|
+
return phrase;
|
|
279
|
+
}
|
|
280
|
+
const original = phrase.split(/\s+/u);
|
|
281
|
+
return phraseForms(phrase)
|
|
282
|
+
.map(({ head, forms }, i) => {
|
|
283
|
+
const word = head + formOf(forms, index);
|
|
284
|
+
return original[i][0] === original[i][0].toUpperCase()
|
|
285
|
+
? word.replace(
|
|
286
|
+
/(^|-)(\p{L})/gu,
|
|
287
|
+
(_m, gap, letter) => `${gap}${letter.toUpperCase()}`
|
|
288
|
+
)
|
|
289
|
+
: word;
|
|
290
|
+
})
|
|
291
|
+
.join(' ');
|
|
292
|
+
}
|
|
293
|
+
const decoration = /^((?:(?:ООО|АО|ПАО|ИП|ЗАО)[ \t]+)?[«“"]?)(.*?)([»”"]?)$/su;
|
|
294
|
+
// `ООО «Ромашка»` keeps its legal form and quotes; only the name is replaced.
|
|
295
|
+
export function fakeEntity(value, finding, transform, options) {
|
|
296
|
+
const { type } = finding;
|
|
297
|
+
const [, before, core, after] = decoration.exec(value);
|
|
298
|
+
const lemma = organizationCore(entityLemma(value, type, options));
|
|
299
|
+
const fake = assignFake(lemma, type, transform, options);
|
|
300
|
+
const output = cyrillic.test(core)
|
|
301
|
+
? declinedPhrase(fake, phraseCase(core, lemma, contextOf(finding, options)))
|
|
302
|
+
: fake;
|
|
303
|
+
return `${before}${applyStyle(core, output)}${after}`;
|
|
304
|
+
}
|
package/src/fake-names.js
CHANGED
|
@@ -213,7 +213,8 @@ export function fakeName(value, key) {
|
|
|
213
213
|
return fakeWord(word, key, roles.get(word), undefined, nameSex);
|
|
214
214
|
});
|
|
215
215
|
}
|
|
216
|
-
function wordProfile(word, forcedRole, forcedLocale) {
|
|
216
|
+
export function wordProfile(word, forcedRole, forcedLocale) {
|
|
217
|
+
indexNames();
|
|
217
218
|
const canonical = latinName(word);
|
|
218
219
|
const patronymic = /(?:vich|vna|ichna|ogly|kyzy)$/.test(canonical);
|
|
219
220
|
const slavic = /(?:ova|eva|ov|ev|in|ina|enko|sky|ski|skaya)$/.test(canonical);
|
|
@@ -240,7 +241,7 @@ function wordProfile(word, forcedRole, forcedLocale) {
|
|
|
240
241
|
: 'lastName');
|
|
241
242
|
return { canonical, locale, sex, role };
|
|
242
243
|
}
|
|
243
|
-
function fakeWord(word, key, forcedRole, forcedLocale, forcedSex) {
|
|
244
|
+
export function fakeWord(word, key, forcedRole, forcedLocale, forcedSex) {
|
|
244
245
|
const {
|
|
245
246
|
canonical,
|
|
246
247
|
locale,
|
package/src/fake.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fakeEntity, fakePerson } from './fake-entities.js';
|
|
1
2
|
import { createHmac } from 'node:crypto';
|
|
2
3
|
import { parsePhoneNumberFromString } from 'libphonenumber-js/max';
|
|
3
4
|
import { mrzCheck } from './identity.js';
|
|
@@ -487,7 +488,7 @@ function fakeTd1(parts, transform) {
|
|
|
487
488
|
.padEnd(30, '<');
|
|
488
489
|
return parts.join('');
|
|
489
490
|
}
|
|
490
|
-
export function realisticFake(value, finding, transform) {
|
|
491
|
+
export function realisticFake(value, finding, transform, options = {}) {
|
|
491
492
|
let output;
|
|
492
493
|
if (finding.category === 'credential') {
|
|
493
494
|
if (!transform.credentials) {
|
|
@@ -500,14 +501,9 @@ export function realisticFake(value, finding, transform) {
|
|
|
500
501
|
.toString('hex')
|
|
501
502
|
.slice(0, 8)}!`;
|
|
502
503
|
} else if (finding.type === 'PERSON') {
|
|
503
|
-
output =
|
|
504
|
+
output = fakePerson(value, finding, transform, options);
|
|
504
505
|
} else if (['ORGANIZATION', 'EDUCATION', 'LOCATION'].includes(finding.type)) {
|
|
505
|
-
|
|
506
|
-
ORGANIZATION: 'Organization',
|
|
507
|
-
EDUCATION: 'School',
|
|
508
|
-
LOCATION: 'Location',
|
|
509
|
-
}[finding.type];
|
|
510
|
-
output = `Example ${label} ${digest(value, transform.key, finding.type).readUInt32BE(0)}`;
|
|
506
|
+
output = fakeEntity(value, finding, transform, options);
|
|
511
507
|
} else if (/DATE|BIRTH/.test(finding.type)) {
|
|
512
508
|
output = fakeDate(value, transform.key);
|
|
513
509
|
} else if (finding.type === 'PASSPORT_MRZ') {
|
package/src/index.d.ts
CHANGED
|
@@ -250,6 +250,10 @@ export type Transformation =
|
|
|
250
250
|
credentials?: boolean;
|
|
251
251
|
/** Keep original MRZ issuer/nationality unless Utopia is requested. */
|
|
252
252
|
mrzCountry?: 'UTO';
|
|
253
|
+
/** Caller-reviewed names that replace the generated entity vocabulary. */
|
|
254
|
+
pools?: Partial<
|
|
255
|
+
Record<'ORGANIZATION' | 'LOCATION' | 'EDUCATION', string[]>
|
|
256
|
+
>;
|
|
253
257
|
}
|
|
254
258
|
| { mode: 'date-shift'; days: number }
|
|
255
259
|
| { mode: 'bucket'; size: number };
|
package/src/personal-context.js
CHANGED
|
@@ -2,7 +2,7 @@ import { collectMatches } from './detection.js';
|
|
|
2
2
|
|
|
3
3
|
const birth =
|
|
4
4
|
/(?<![\p{L}\p{N}_])(?:born|birth[ _-]?date|date[ _-]of[ _-]birth|DOB|дата рождения|д\.р\.|родился|родилась|Geburtsdatum|fecha de nacimiento|date de naissance)(?![\p{L}\p{N}_])/giu;
|
|
5
|
-
const months =
|
|
5
|
+
export const months =
|
|
6
6
|
'January February March April May June July August September October November December Jan Feb Mar Apr Jun Jul Aug Sep Sept Oct Nov Dec января февраля марта апреля мая июня июля августа сентября октября ноября декабря январь февраль март апрель май июнь июль август сентябрь октябрь ноябрь декабрь'
|
|
7
7
|
.split(' ')
|
|
8
8
|
.join('|');
|
package/src/personal-variants.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { escapePattern, failure } from './detection.js';
|
|
2
|
+
import { russianNameRoles } from './russian-names.js';
|
|
2
3
|
|
|
3
4
|
const boundary = '[\\p{L}\\p{M}\\p{N}_]';
|
|
4
5
|
export function organizationCore(value) {
|
|
@@ -7,9 +8,26 @@ export function organizationCore(value) {
|
|
|
7
8
|
.replace(/^[«“"]|[»”"]$/gu, '')
|
|
8
9
|
.trim();
|
|
9
10
|
}
|
|
11
|
+
// `Вектор-1` also identifies its base name and numbered siblings like `Вектор-2`.
|
|
12
|
+
export function organizationFamily(core) {
|
|
13
|
+
const base = /^(.{3,})-\d{1,9}$/u.exec(core)?.[1];
|
|
14
|
+
return base
|
|
15
|
+
? {
|
|
16
|
+
base,
|
|
17
|
+
pattern: new RegExp(
|
|
18
|
+
`(?<!${boundary})${escapePattern(base)}(?:-\\d{1,9})?(?!${boundary}|-\\d)`,
|
|
19
|
+
'giu'
|
|
20
|
+
),
|
|
21
|
+
}
|
|
22
|
+
: undefined;
|
|
23
|
+
}
|
|
24
|
+
// ё and е are one letter in ordinary Russian writing; the i flag covers Ё and Е.
|
|
25
|
+
const foldYo = (pattern) => pattern.replace(/[еёЕЁ]/gu, '[её]');
|
|
26
|
+
const fleeting = { павел: 'Павл', лев: 'Льв' };
|
|
27
|
+
const hardCases = '(?:а|у|ом|е)';
|
|
10
28
|
// Only caller-confirmed Russian words receive finite case alternatives. A
|
|
11
29
|
// minimum three-letter stem avoids turning short names into broad matches.
|
|
12
|
-
function
|
|
30
|
+
function cyrillicPattern(word, type) {
|
|
13
31
|
const endings = [
|
|
14
32
|
[/(?:ова|ева|ина)$/u, 1, '(?:а|ой|ою|у|е|ы)'],
|
|
15
33
|
[/(?:ая)$/u, 2, '(?:ая|ой|ою|ую)'],
|
|
@@ -18,24 +36,76 @@ function wordPattern(word) {
|
|
|
18
36
|
2,
|
|
19
37
|
'(?:ий|ый|ой|ого|его|ому|ему|им|ым|ом|ем|ия|ию|ием|ии)',
|
|
20
38
|
],
|
|
39
|
+
[/[аеуюя]й$/u, 1, '(?:й|я|ю|ем|е)'],
|
|
21
40
|
[/(?:ов|ев|ин)$/u, 0, '(?:а|у|ым|ом|е|ой)?'],
|
|
41
|
+
[/[жчшщц]$/u, 0, '(?:а|у|ем|ом|е)?'],
|
|
42
|
+
[/[жчшщц]а$/u, 1, '(?:а|и|е|у|ей|ею|ой|ою)'],
|
|
22
43
|
[/а$/u, 1, '(?:а|ы|и|е|у|ой|ою)'],
|
|
23
44
|
[/я$/u, 1, '(?:я|и|е|ю|ей|ею)'],
|
|
24
|
-
[/ь$/u, 1, '(
|
|
45
|
+
[/ь$/u, 1, '(?:ь|я|ю|ем|е|и|ью)'],
|
|
25
46
|
];
|
|
26
47
|
const lower = word.toLowerCase();
|
|
27
|
-
if (
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
48
|
+
if (fleeting[lower]) {
|
|
49
|
+
return `(?:${word}|${fleeting[lower]}${hardCases})`;
|
|
50
|
+
}
|
|
51
|
+
for (const [ending, remove, suffix] of endings) {
|
|
52
|
+
if (ending.test(lower) && word.length - remove >= 3) {
|
|
53
|
+
return escapePattern(remove ? word.slice(0, -remove) : word) + suffix;
|
|
32
54
|
}
|
|
33
55
|
}
|
|
34
|
-
if (/^[а-яё]{
|
|
35
|
-
|
|
56
|
+
if (/^[а-яё]{3,}[бвгджзклмнпрстфхцчшщ]$/u.test(lower)) {
|
|
57
|
+
// Plural forms would turn a given name into a surname: Петр and Петров.
|
|
58
|
+
return `${escapePattern(word)}${type === 'PERSON' ? `${hardCases}?` : '(?:а|у|ом|е|ы|ов|ам|ами|ах)?'}`;
|
|
36
59
|
}
|
|
37
60
|
return escapePattern(word);
|
|
38
61
|
}
|
|
62
|
+
export function wordPattern(word, type) {
|
|
63
|
+
return foldYo(
|
|
64
|
+
word
|
|
65
|
+
.split('-')
|
|
66
|
+
.map((part) =>
|
|
67
|
+
/^[а-яё]+$/iu.test(part)
|
|
68
|
+
? cyrillicPattern(part, type)
|
|
69
|
+
: escapePattern(part)
|
|
70
|
+
)
|
|
71
|
+
.join('-')
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
const initialGap = '[ \\t\\u00a0]*';
|
|
75
|
+
const initialPattern = (word) => `${foldYo(escapePattern(word[0]))}\\.`;
|
|
76
|
+
// Everyday orders, partial forms and initials of a Russian full name, longest
|
|
77
|
+
// first so that alternation prefers the complete form at each position.
|
|
78
|
+
function russianForms(words, roles, type) {
|
|
79
|
+
const name = Object.fromEntries(
|
|
80
|
+
Object.entries(roles).map(([role, i]) => [role, words[i]])
|
|
81
|
+
);
|
|
82
|
+
const gap = '[ \\t]+';
|
|
83
|
+
const full = (...parts) =>
|
|
84
|
+
parts.map((part) => wordPattern(part, type)).join(gap);
|
|
85
|
+
if (name.surname === undefined) {
|
|
86
|
+
return [];
|
|
87
|
+
}
|
|
88
|
+
const initial = initialPattern(name.given);
|
|
89
|
+
const last = wordPattern(name.surname, type);
|
|
90
|
+
const short = [
|
|
91
|
+
full(name.surname, name.given),
|
|
92
|
+
full(name.given, name.surname),
|
|
93
|
+
`${last}${gap}${initial}`,
|
|
94
|
+
`${initial}[ \\t\\u00a0]+${last}`,
|
|
95
|
+
];
|
|
96
|
+
if (name.patronymic === undefined) {
|
|
97
|
+
return short;
|
|
98
|
+
}
|
|
99
|
+
const initials = `${initial}${initialGap}${initialPattern(name.patronymic)}`;
|
|
100
|
+
return [
|
|
101
|
+
full(name.surname, name.given, name.patronymic),
|
|
102
|
+
full(name.given, name.patronymic, name.surname),
|
|
103
|
+
`${last}[ \\t\\u00a0]+${initials}`,
|
|
104
|
+
`${initials}[ \\t\\u00a0]+${last}`,
|
|
105
|
+
...short,
|
|
106
|
+
full(name.given, name.patronymic),
|
|
107
|
+
];
|
|
108
|
+
}
|
|
39
109
|
export function personalPatterns(value, type) {
|
|
40
110
|
if (value.length > 256) {
|
|
41
111
|
// Large explicit values still support literal matching without expansion.
|
|
@@ -52,13 +122,22 @@ export function personalPatterns(value, type) {
|
|
|
52
122
|
for (const surface of values) {
|
|
53
123
|
const parts = surface.split(/([ \t]+)/u);
|
|
54
124
|
const pattern = parts
|
|
55
|
-
.map((part, i) => (i % 2 ? '[ \\t]+' : wordPattern(part)))
|
|
125
|
+
.map((part, i) => (i % 2 ? '[ \\t]+' : wordPattern(part, type)))
|
|
56
126
|
.join('');
|
|
57
127
|
patterns.add(pattern);
|
|
58
|
-
if (type === 'PERSON'
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
)
|
|
128
|
+
if (type === 'PERSON') {
|
|
129
|
+
const words = parts.filter((_part, i) => i % 2 === 0);
|
|
130
|
+
const roles = russianNameRoles(words);
|
|
131
|
+
for (const form of roles ? russianForms(words, roles, type) : []) {
|
|
132
|
+
patterns.add(form);
|
|
133
|
+
}
|
|
134
|
+
if (words.length === 2) {
|
|
135
|
+
patterns.add(
|
|
136
|
+
[wordPattern(words[1], type), wordPattern(words[0], type)].join(
|
|
137
|
+
'[ \\t]+'
|
|
138
|
+
)
|
|
139
|
+
);
|
|
140
|
+
}
|
|
62
141
|
}
|
|
63
142
|
}
|
|
64
143
|
return [...patterns];
|
|
@@ -77,18 +156,29 @@ export function samePersonal(value, original, type) {
|
|
|
77
156
|
export function detectKnownPersonal(text, entries, rule, emit, options) {
|
|
78
157
|
let count = 0;
|
|
79
158
|
for (const entry of entries ?? []) {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
159
|
+
const family =
|
|
160
|
+
rule === 'confirmed-personal' &&
|
|
161
|
+
options.documentEntities &&
|
|
162
|
+
!options.knownOnly &&
|
|
163
|
+
entry.type === 'ORGANIZATION' &&
|
|
164
|
+
organizationFamily(organizationCore(entry.value));
|
|
165
|
+
for (const pattern of [
|
|
166
|
+
personalRegex(entry.value, entry.type),
|
|
167
|
+
...(family ? [family.pattern] : []),
|
|
168
|
+
]) {
|
|
169
|
+
for (const match of text.matchAll(pattern)) {
|
|
170
|
+
if (++count > (options.maxFindings ?? 100000)) {
|
|
171
|
+
throw failure('ERR_LIMIT');
|
|
172
|
+
}
|
|
173
|
+
emit({
|
|
174
|
+
start: match.index,
|
|
175
|
+
end: match.index + match[0].length,
|
|
176
|
+
type: entry.type,
|
|
177
|
+
category: 'personal',
|
|
178
|
+
rule,
|
|
179
|
+
confidence: rule === 'known-personal' ? 0.85 : 0.9,
|
|
180
|
+
});
|
|
83
181
|
}
|
|
84
|
-
emit({
|
|
85
|
-
start: match.index,
|
|
86
|
-
end: match.index + match[0].length,
|
|
87
|
-
type: entry.type,
|
|
88
|
-
category: 'personal',
|
|
89
|
-
rule,
|
|
90
|
-
confidence: rule === 'known-personal' ? 0.85 : 0.9,
|
|
91
|
-
});
|
|
92
182
|
}
|
|
93
183
|
}
|
|
94
184
|
}
|
package/src/personal.js
CHANGED
|
@@ -29,6 +29,9 @@ const ibanLengths = Object.fromEntries(
|
|
|
29
29
|
.map((entry) => [entry.slice(0, 2), Number(entry.slice(2))])
|
|
30
30
|
);
|
|
31
31
|
|
|
32
|
+
// `@handle/jane@example.com` is a mention and an address, not a package path.
|
|
33
|
+
const mentionedEmail =
|
|
34
|
+
/^\/[\p{L}\p{N}_.!#$%&'*+?^`{|}~-]+@[\p{L}\p{N}][\p{L}\p{N}.-]{0,252}\.\p{L}{2,63}(?![\p{L}\p{N}_-])/u;
|
|
32
35
|
export function iban(value) {
|
|
33
36
|
const compact = value.replace(/\s/g, '').toUpperCase();
|
|
34
37
|
if (
|
|
@@ -207,10 +210,11 @@ export function detectPersonal(text, emit) {
|
|
|
207
210
|
}
|
|
208
211
|
collectMatches(
|
|
209
212
|
text,
|
|
210
|
-
/(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&'
|
|
213
|
+
/(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&']{0,80}(?:-[\p{L}\p{M}\p{N}&']{1,40}){0,4})/gu,
|
|
211
214
|
emit,
|
|
212
215
|
details('ORGANIZATION', 'organization-context', 0.7),
|
|
213
|
-
1
|
|
216
|
+
1,
|
|
217
|
+
(value) => value.length >= 2
|
|
214
218
|
);
|
|
215
219
|
collectMatches(
|
|
216
220
|
text,
|
|
@@ -236,10 +240,12 @@ export function detectPersonal(text, emit) {
|
|
|
236
240
|
emit,
|
|
237
241
|
details('USERNAME', 'messenger-handle', 0.65),
|
|
238
242
|
0,
|
|
239
|
-
(_value, match) =>
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
+
(_value, match) => {
|
|
244
|
+
const rest = text.slice(match.index + match[0].length);
|
|
245
|
+
return (
|
|
246
|
+
!/^\/[a-z0-9][a-z0-9_.-]*/i.test(rest) || mentionedEmail.test(rest)
|
|
247
|
+
);
|
|
248
|
+
}
|
|
243
249
|
);
|
|
244
250
|
collectMatches(
|
|
245
251
|
text,
|
package/src/placeholders.js
CHANGED
|
@@ -107,6 +107,14 @@ function aliases(value, type) {
|
|
|
107
107
|
),
|
|
108
108
|
];
|
|
109
109
|
}
|
|
110
|
+
// A bare name is the same entity as its numbered form, including maps saved
|
|
111
|
+
// when the span stopped at the hyphen of `Вектор-99`.
|
|
112
|
+
function numberedOrganization(value, entry) {
|
|
113
|
+
return (entry.values ?? []).some((original) => {
|
|
114
|
+
const base = original.replace(/-\d*$/u, '');
|
|
115
|
+
return base !== original && samePersonal(value, base, entry.type);
|
|
116
|
+
});
|
|
117
|
+
}
|
|
110
118
|
function related(value, entry) {
|
|
111
119
|
if (
|
|
112
120
|
(entry.values ?? []).some((original) =>
|
|
@@ -119,6 +127,9 @@ function related(value, entry) {
|
|
|
119
127
|
if (candidates.some((alias) => entry.aliases.includes(alias))) {
|
|
120
128
|
return true;
|
|
121
129
|
}
|
|
130
|
+
if (entry.type === 'ORGANIZATION') {
|
|
131
|
+
return numberedOrganization(value, entry);
|
|
132
|
+
}
|
|
122
133
|
if (entry.type !== 'PERSON') {
|
|
123
134
|
return false;
|
|
124
135
|
}
|
|
@@ -257,7 +268,7 @@ function namespaceCollision(text, pattern) {
|
|
|
257
268
|
return false;
|
|
258
269
|
}
|
|
259
270
|
|
|
260
|
-
function canonicalPerson(value, people) {
|
|
271
|
+
export function canonicalPerson(value, people) {
|
|
261
272
|
// Resolve components only when there is a unique compatible full name.
|
|
262
273
|
const matches = people.filter((person) => {
|
|
263
274
|
const words = person.match(/\p{L}+/gu) ?? [];
|
|
@@ -279,8 +290,11 @@ function canonicalKnown(value, type, options, state) {
|
|
|
279
290
|
const known = (options.knownPersonal ?? []).filter(
|
|
280
291
|
(e) => e.type === type && samePersonal(value, e.value, type)
|
|
281
292
|
);
|
|
282
|
-
|
|
283
|
-
|
|
293
|
+
// `Петров Данил` and `Петров Данил Сергеевич` are one person: the fullest wins.
|
|
294
|
+
const longest = Math.max(0, ...known.map((e) => e.value.length));
|
|
295
|
+
const fullest = known.filter((e) => e.value.length === longest);
|
|
296
|
+
if (fullest.length === 1) {
|
|
297
|
+
return fullest[0].value;
|
|
284
298
|
}
|
|
285
299
|
return type === 'PERSON' ? canonicalPerson(value, state.people ?? []) : value;
|
|
286
300
|
}
|
|
@@ -303,7 +317,12 @@ export function placeholderReplacement(value, finding, transform, options) {
|
|
|
303
317
|
const candidates = map.entries.filter(
|
|
304
318
|
(entry) => entry.type === type && related(canonical, entry)
|
|
305
319
|
);
|
|
306
|
-
|
|
320
|
+
// Several related entities are ambiguous, unless one holds this exact value.
|
|
321
|
+
const exact = candidates.filter((entry) =>
|
|
322
|
+
entry.values.some((original) => samePersonal(canonical, original, type))
|
|
323
|
+
);
|
|
324
|
+
const matches = candidates.length === 1 ? candidates : exact;
|
|
325
|
+
let entry = matches.length === 1 ? matches[0] : undefined;
|
|
307
326
|
if (!entry) {
|
|
308
327
|
if (map.entries.length >= (options.maxFindings ?? 100000)) {
|
|
309
328
|
throw failure('ERR_LIMIT');
|
package/src/propagation.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { personalPattern, maxPropagationLength, failure } from './detection.js';
|
|
2
2
|
import { actionable } from './confidence.js';
|
|
3
|
-
import { organizationCore } from './personal-variants.js';
|
|
3
|
+
import { organizationCore, organizationFamily } from './personal-variants.js';
|
|
4
4
|
import { nameAliases } from './transliteration.js';
|
|
5
5
|
|
|
6
6
|
function mrzSeeds(value, finding, seeds) {
|
|
@@ -50,12 +50,16 @@ function eligibleSeed(finding, options) {
|
|
|
50
50
|
].includes(finding.type))
|
|
51
51
|
);
|
|
52
52
|
}
|
|
53
|
-
function organizationSeed(value, finding, seeds, options) {
|
|
53
|
+
function organizationSeed(text, value, finding, seeds, options) {
|
|
54
54
|
if (options.documentEntities && finding.type === 'ORGANIZATION') {
|
|
55
55
|
const core = organizationCore(value);
|
|
56
56
|
if (core.length >= 3) {
|
|
57
57
|
seeds.set(core, finding);
|
|
58
58
|
}
|
|
59
|
+
const family = organizationFamily(core);
|
|
60
|
+
for (const match of family ? text.matchAll(family.pattern) : []) {
|
|
61
|
+
seeds.set(match[0], finding);
|
|
62
|
+
}
|
|
59
63
|
}
|
|
60
64
|
}
|
|
61
65
|
// Only confirmed values seed propagation. A low-score numeric coincidence
|
|
@@ -82,7 +86,7 @@ export function confirmedSeeds(text, findings, options) {
|
|
|
82
86
|
mrzSeeds(value, finding, seeds);
|
|
83
87
|
} else if (value.length >= 2) {
|
|
84
88
|
seeds.set(value, finding);
|
|
85
|
-
organizationSeed(value, finding, seeds, options);
|
|
89
|
+
organizationSeed(text, value, finding, seeds, options);
|
|
86
90
|
if (finding.type === 'PERSON') {
|
|
87
91
|
for (const name of value.match(/\p{L}{2,}/gu) ?? []) {
|
|
88
92
|
seeds.set(name, finding);
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// Finite Russian declension shared by fake names and fake entities. Case order:
|
|
2
|
+
// nominative, genitive, dative, accusative, instrumental, prepositional.
|
|
3
|
+
// `|` lists a rarer spelling, accepted when reading a form, never generated.
|
|
4
|
+
const fleeting = { павел: 'павл', лев: 'льв', ветер: 'ветр' };
|
|
5
|
+
const hush = /[жчшщц]$/u;
|
|
6
|
+
const a = (stem) => [
|
|
7
|
+
'а',
|
|
8
|
+
/[гкхжчшщ]$/u.test(stem) ? 'и' : 'ы',
|
|
9
|
+
'е',
|
|
10
|
+
'у',
|
|
11
|
+
/[жчшщц]$/u.test(stem) ? 'ей|ею' : 'ой|ою',
|
|
12
|
+
'е',
|
|
13
|
+
];
|
|
14
|
+
const adjectives = [
|
|
15
|
+
[
|
|
16
|
+
/[ый]й$|ой$/u,
|
|
17
|
+
2,
|
|
18
|
+
(w) => [w.slice(-2), 'ого', 'ому', w.slice(-2), 'ым', 'ом'],
|
|
19
|
+
],
|
|
20
|
+
[/[кгхжчшщ]ий$/u, 2, () => ['ий', 'ого', 'ому', 'ий', 'им', 'ом']],
|
|
21
|
+
[/ий$/u, 2, () => ['ий', 'его', 'ему', 'ий', 'им', 'ем']],
|
|
22
|
+
[/ая$/u, 2, () => ['ая', 'ой', 'ой', 'ую', 'ой|ою', 'ой']],
|
|
23
|
+
[/яя$/u, 2, () => ['яя', 'ей', 'ей', 'юю', 'ей|ею', 'ей']],
|
|
24
|
+
[/ое$/u, 2, () => ['ое', 'ого', 'ому', 'ое', 'ым', 'ом']],
|
|
25
|
+
[/ее$/u, 2, () => ['ее', 'его', 'ему', 'ее', 'им', 'ем']],
|
|
26
|
+
];
|
|
27
|
+
function noun(word, sex, animate) {
|
|
28
|
+
const accusative = (genitive, nominative = '') =>
|
|
29
|
+
animate ? genitive : nominative;
|
|
30
|
+
if (fleeting[word]) {
|
|
31
|
+
const hard = ['а', 'у', accusative('а'), 'ом', 'е'];
|
|
32
|
+
return [word, ...hard.map((ending) => fleeting[word] + ending)];
|
|
33
|
+
}
|
|
34
|
+
const table = [
|
|
35
|
+
[/ия$/u, 2, ['ия', 'ии', 'ии', 'ию', 'ией|иею', 'ии']],
|
|
36
|
+
[/ье$/u, 2, ['ье', 'ья', 'ью', 'ье', 'ьем', 'ье']],
|
|
37
|
+
[/ие$/u, 2, ['ие', 'ия', 'ию', 'ие', 'ием', 'ии']],
|
|
38
|
+
[/ий$/u, 2, ['ий', 'ия', 'ию', accusative('ия', 'ий'), 'ием', 'ии']],
|
|
39
|
+
[/й$/u, 1, ['й', 'я', 'ю', accusative('я', 'й'), 'ем', 'е']],
|
|
40
|
+
[/я$/u, 1, ['я', 'и', 'е', 'ю', 'ей|ею', 'е']],
|
|
41
|
+
[/а$/u, 1, a(word.slice(0, -1))],
|
|
42
|
+
[/о$/u, 1, ['о', 'а', 'у', 'о', 'ом', 'е']],
|
|
43
|
+
];
|
|
44
|
+
for (const [ending, remove, cases] of table) {
|
|
45
|
+
if (ending.test(word)) {
|
|
46
|
+
return cases.map((suffix) => attach(word.slice(0, -remove), suffix));
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (word.endsWith('ь')) {
|
|
50
|
+
const stem = word.slice(0, -1);
|
|
51
|
+
const feminine = ['ь', 'и', 'и', 'ь', 'ью', 'и'];
|
|
52
|
+
const masculine = ['ь', 'я', 'ю', accusative('я', 'ь'), 'ем', 'е'];
|
|
53
|
+
const cases =
|
|
54
|
+
sex === 'female'
|
|
55
|
+
? feminine
|
|
56
|
+
: sex === 'male'
|
|
57
|
+
? masculine
|
|
58
|
+
: masculine.map((suffix, i) =>
|
|
59
|
+
suffix === feminine[i] ? suffix : `${suffix}|${feminine[i]}`
|
|
60
|
+
);
|
|
61
|
+
return cases.map((suffix) => attach(stem, suffix));
|
|
62
|
+
}
|
|
63
|
+
if (!/[бвгджзклмнпрстфхцчшщ]$/u.test(word)) {
|
|
64
|
+
return Array(6).fill(word); // Indeclinable: Шевченко, Сочи.
|
|
65
|
+
}
|
|
66
|
+
return [
|
|
67
|
+
'',
|
|
68
|
+
'а',
|
|
69
|
+
'у',
|
|
70
|
+
accusative('а'),
|
|
71
|
+
hush.test(word) ? 'ем' : 'ом',
|
|
72
|
+
'е',
|
|
73
|
+
].map((suffix) => attach(word, suffix));
|
|
74
|
+
}
|
|
75
|
+
const attach = (stem, suffix) =>
|
|
76
|
+
suffix
|
|
77
|
+
.split('|')
|
|
78
|
+
.map((ending) => stem + ending)
|
|
79
|
+
.join('|');
|
|
80
|
+
function adjective(word) {
|
|
81
|
+
for (const [ending, remove, cases] of adjectives) {
|
|
82
|
+
if (ending.test(word)) {
|
|
83
|
+
return cases(word).map((suffix) =>
|
|
84
|
+
attach(word.slice(0, -remove), suffix)
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
function surname(word, sex) {
|
|
91
|
+
if (/(?:ов|ев|ёв|ин|ын)$/u.test(word)) {
|
|
92
|
+
return ['', 'а', 'у', 'а', 'ым|ом', 'е'].map((s) => attach(word, s));
|
|
93
|
+
}
|
|
94
|
+
if (/(?:ова|ева|ёва|ина|ына)$/u.test(word)) {
|
|
95
|
+
const stem = word.slice(0, -1);
|
|
96
|
+
return ['а', 'ой', 'ой', 'у', 'ой|ою', 'ой'].map((s) => attach(stem, s));
|
|
97
|
+
}
|
|
98
|
+
const forms = adjective(word);
|
|
99
|
+
if (forms) {
|
|
100
|
+
return forms;
|
|
101
|
+
}
|
|
102
|
+
return sex === 'female' && !/[ая]$/u.test(word)
|
|
103
|
+
? Array(6).fill(word)
|
|
104
|
+
: noun(word, sex, true);
|
|
105
|
+
}
|
|
106
|
+
export const russianNoun = (word, options) =>
|
|
107
|
+
options.adjective && adjective(word)
|
|
108
|
+
? adjective(word)
|
|
109
|
+
: noun(word, options.sex, options.animate);
|
|
110
|
+
// `kind` is the name slot: lastName, firstName or middleName.
|
|
111
|
+
export function russianNameForms(word, kind, sex) {
|
|
112
|
+
const lower = word.toLowerCase();
|
|
113
|
+
if (kind === 'lastName') {
|
|
114
|
+
return surname(lower, sex);
|
|
115
|
+
}
|
|
116
|
+
if (kind === 'middleName' && /ич$/u.test(lower)) {
|
|
117
|
+
return ['', 'а', 'у', 'а', 'ем', 'е'].map((s) => attach(lower, s));
|
|
118
|
+
}
|
|
119
|
+
return noun(lower, sex, true);
|
|
120
|
+
}
|
|
121
|
+
const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
|
|
122
|
+
// Every case a surface can stand for, in case order.
|
|
123
|
+
export function casesOf(surface, forms) {
|
|
124
|
+
const wanted = fold(surface);
|
|
125
|
+
return forms.flatMap((form, index) =>
|
|
126
|
+
fold(form).split('|').includes(wanted) ? [index] : []
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
export const formOf = (forms, index) => forms[index].split('|')[0];
|
|
130
|
+
// Cases governed by a preposition, in order of preference. Bare forms like
|
|
131
|
+
// `Ромашке` (dative or prepositional) cannot be told apart without it.
|
|
132
|
+
const governed = [
|
|
133
|
+
[/^(?:в|во|на|о|об|обо|при)$/u, [5, 3]],
|
|
134
|
+
[/^(?:к|ко|по)$/u, [2]],
|
|
135
|
+
[/^(?:с|со|над|под|перед|за)$/u, [4, 1, 3]],
|
|
136
|
+
[/^(?:у|от|до|из|без|для|около|после|кроме|вокруг|против)$/u, [1]],
|
|
137
|
+
];
|
|
138
|
+
const lastWord = /(?<![\p{L}\p{M}\p{N}_-])(\p{L}{1,7})[ \t\u00a0]+([«“"]?)$/u;
|
|
139
|
+
// `в Лютике и Ромашке`: a conjunct takes the case of the word before it.
|
|
140
|
+
const conjunct =
|
|
141
|
+
/[\p{L}\p{M}-]{1,32}[»”"]?(?:,|[ \t\u00a0]+(?:и|или))[ \t\u00a0]+[«“"]?$/u;
|
|
142
|
+
export function governedCases(text, start) {
|
|
143
|
+
const before = text.slice(Math.max(0, start - 64), start);
|
|
144
|
+
const conjunction = conjunct.exec(before);
|
|
145
|
+
const source = conjunction ? before.slice(0, conjunction.index) : before;
|
|
146
|
+
const match = lastWord.exec(source.slice(-12));
|
|
147
|
+
if (!match) {
|
|
148
|
+
return { cases: [] };
|
|
149
|
+
}
|
|
150
|
+
const word = match[1].toLowerCase();
|
|
151
|
+
const rule = governed.find(([pattern]) => pattern.test(word));
|
|
152
|
+
return { cases: rule?.[1] ?? [], quoted: Boolean(match[2]) };
|
|
153
|
+
}
|
|
154
|
+
// Intersects the candidates of each word; a word that fits no case is ignored.
|
|
155
|
+
// A quoted name in the nominative stays there: в «Вектор».
|
|
156
|
+
export function commonCase(candidates, context = { cases: [] }) {
|
|
157
|
+
const present = candidates.filter((cases) => cases.length);
|
|
158
|
+
const common = [0, 1, 2, 3, 4, 5].filter((index) =>
|
|
159
|
+
present.every((cases) => cases.includes(index))
|
|
160
|
+
);
|
|
161
|
+
if (context.quoted && common.includes(0)) {
|
|
162
|
+
return 0;
|
|
163
|
+
}
|
|
164
|
+
return (
|
|
165
|
+
context.cases.find((index) => common.includes(index)) ?? common[0] ?? 0
|
|
166
|
+
);
|
|
167
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { firstNames } from './names.js';
|
|
2
|
+
|
|
3
|
+
// Finite Russian name grammar shared by known-value matching and fake mode.
|
|
4
|
+
const patronymic = /(?:ич|овна|евна|ична|vich|ovna|evna)$/iu;
|
|
5
|
+
const surname = /(?:ов|ев|ин|ын|ский|цкий|ова|ева|ина|ына|ская|цкая)$/iu;
|
|
6
|
+
const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
|
|
7
|
+
const givenNames = new Set(firstNames.ru.split(' ').map(fold));
|
|
8
|
+
|
|
9
|
+
export const isPatronymic = (word) => patronymic.test(word);
|
|
10
|
+
export const isGivenName = (word) => givenNames.has(fold(word));
|
|
11
|
+
|
|
12
|
+
// Returns word indexes by role, or undefined when the roles are not evident.
|
|
13
|
+
// `Имя Отчество Фамилия` is checked first so that surnames in -ич, as in
|
|
14
|
+
// `Шостакович Дмитрий Дмитриевич`, are not mistaken for patronymics.
|
|
15
|
+
export function russianNameRoles(words, known = isGivenName) {
|
|
16
|
+
if (words.length === 3) {
|
|
17
|
+
if (isPatronymic(words[1])) {
|
|
18
|
+
return { given: 0, patronymic: 1, surname: 2 };
|
|
19
|
+
}
|
|
20
|
+
return isPatronymic(words[2])
|
|
21
|
+
? { surname: 0, given: 1, patronymic: 2 }
|
|
22
|
+
: undefined;
|
|
23
|
+
}
|
|
24
|
+
if (words.length !== 2) {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
if (isPatronymic(words[1])) {
|
|
28
|
+
return { given: 0, patronymic: 1 };
|
|
29
|
+
}
|
|
30
|
+
const candidates = [0, 1].filter(
|
|
31
|
+
(i) => surname.test(words[i]) && !known(words[i])
|
|
32
|
+
);
|
|
33
|
+
return candidates.length === 1
|
|
34
|
+
? { surname: candidates[0], given: 1 - candidates[0] }
|
|
35
|
+
: undefined;
|
|
36
|
+
}
|
package/src/sanitizer.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { generatedValues } from './fake.js';
|
|
2
|
+
import { prepareFakeText } from './fake-entities.js';
|
|
2
3
|
import { alignEscapes } from './escape-spans.js';
|
|
3
4
|
import { actionable } from './confidence.js';
|
|
4
5
|
import { propagate, confirmationOptions } from './propagation.js';
|
|
@@ -692,7 +693,7 @@ export function redactResult(
|
|
|
692
693
|
) {
|
|
693
694
|
options = applyProfile(options);
|
|
694
695
|
const spans = validatedSpans(text, findings, options);
|
|
695
|
-
options = initializePlaceholders(options);
|
|
696
|
+
options = prepareFakeText(text, findings, initializePlaceholders(options));
|
|
696
697
|
preparePlaceholderText(text, findings, options);
|
|
697
698
|
const parts = [];
|
|
698
699
|
const faked = [];
|
|
@@ -719,6 +720,11 @@ export function redactResult(
|
|
|
719
720
|
const transform =
|
|
720
721
|
options.transformations?.[span.type] ?? options.transformation;
|
|
721
722
|
if (retainedReplacement(transform, 'placeholder', span, replacement)) {
|
|
723
|
+
// `Organization-1` followed by `99` would read as a token that does not
|
|
724
|
+
// exist, so a span that ends there fails the run instead.
|
|
725
|
+
if (/\d$/.test(replacement) && /\d/.test(text[span.end] ?? '')) {
|
|
726
|
+
throw failure('ERR_RESIDUAL');
|
|
727
|
+
}
|
|
722
728
|
occurrences.push({
|
|
723
729
|
start: outputLength,
|
|
724
730
|
end: outputLength + replacement.length,
|
package/src/transforms.js
CHANGED
|
@@ -41,6 +41,22 @@ export function validateTransforms(options) {
|
|
|
41
41
|
function positiveInteger(value) {
|
|
42
42
|
return Number.isSafeInteger(value) && value > 0;
|
|
43
43
|
}
|
|
44
|
+
const poolTypes = new Set(['ORGANIZATION', 'LOCATION', 'EDUCATION']);
|
|
45
|
+
function validPools(pools) {
|
|
46
|
+
return (
|
|
47
|
+
pools === undefined ||
|
|
48
|
+
(Boolean(pools) &&
|
|
49
|
+
typeof pools === 'object' &&
|
|
50
|
+
!Array.isArray(pools) &&
|
|
51
|
+
Object.entries(pools).every(
|
|
52
|
+
([type, names]) =>
|
|
53
|
+
poolTypes.has(type) &&
|
|
54
|
+
Array.isArray(names) &&
|
|
55
|
+
names.length > 0 &&
|
|
56
|
+
names.every((name) => typeof name === 'string' && name.trim())
|
|
57
|
+
))
|
|
58
|
+
);
|
|
59
|
+
}
|
|
44
60
|
function validateTransform(transform) {
|
|
45
61
|
if (!transform || !modes.has(transform.mode)) {
|
|
46
62
|
throw failure('ERR_CONFIG');
|
|
@@ -54,7 +70,8 @@ function validateTransform(transform) {
|
|
|
54
70
|
Buffer.byteLength(transform.key) >= 16 &&
|
|
55
71
|
(transform.credentials === undefined ||
|
|
56
72
|
typeof transform.credentials === 'boolean') &&
|
|
57
|
-
(transform.mrzCountry === undefined || transform.mrzCountry === 'UTO')
|
|
73
|
+
(transform.mrzCountry === undefined || transform.mrzCountry === 'UTO') &&
|
|
74
|
+
validPools(transform.pools),
|
|
58
75
|
pseudonym: () =>
|
|
59
76
|
typeof transform.key === 'string' &&
|
|
60
77
|
Buffer.byteLength(transform.key) >= 16,
|
|
@@ -130,7 +147,7 @@ function masked(value, finding, transform) {
|
|
|
130
147
|
? REDACTED
|
|
131
148
|
: `${chars.slice(0, start).join('')}${transform.marker ?? '*'.repeat(chars.length - start - end)}${end ? chars.slice(-end).join('') : ''}`;
|
|
132
149
|
}
|
|
133
|
-
function transformed(value, finding, transform) {
|
|
150
|
+
function transformed(value, finding, transform, options) {
|
|
134
151
|
if (!transform || transform.mode === 'redact') {
|
|
135
152
|
return REDACTED;
|
|
136
153
|
}
|
|
@@ -141,7 +158,7 @@ function transformed(value, finding, transform) {
|
|
|
141
158
|
: REDACTED;
|
|
142
159
|
}
|
|
143
160
|
if (transform.mode === 'fake') {
|
|
144
|
-
return realisticFake(value, finding, transform);
|
|
161
|
+
return realisticFake(value, finding, transform, options);
|
|
145
162
|
}
|
|
146
163
|
if (finding.category === 'credential') {
|
|
147
164
|
return REDACTED;
|
|
@@ -198,6 +215,6 @@ function renderValue(text, finding, transform, options) {
|
|
|
198
215
|
const result =
|
|
199
216
|
transform?.mode === 'placeholder'
|
|
200
217
|
? placeholderReplacement(raw, finding, transform, options)
|
|
201
|
-
: transformed(raw, finding, transform);
|
|
218
|
+
: transformed(raw, finding, transform, options);
|
|
202
219
|
return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
|
|
203
220
|
}
|