@link-foundation/sensitive-data-sanitizer 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +7 -1
- package/docs/API.md +31 -1
- package/docs/COVERAGE.md +19 -1
- package/package.json +1 -1
- package/src/detection.js +19 -1
- package/src/document-entities.js +32 -0
- package/src/fake-entities.js +304 -0
- package/src/fake-names.js +3 -2
- package/src/fake.js +4 -8
- package/src/index.d.ts +6 -0
- package/src/personal-context.js +1 -1
- package/src/personal-variants.js +119 -25
- package/src/personal.js +12 -6
- package/src/placeholders.js +192 -50
- package/src/propagation.js +7 -3
- package/src/restore-map.js +2 -1
- package/src/russian-declension.js +167 -0
- package/src/russian-names.js +64 -0
- package/src/sanitizer.js +13 -2
- package/src/transforms.js +21 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.16.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 75fd2d9: Match a reused restore map PERSON entry like an explicit `knownPersonal` value, so a bare or lower-case part no longer merges people or fails with `ERR_RESIDUAL`, keep parts listed as their own values in an optional `standalone` entry field, cover the adjacent given name or patronymic when a bare part matches, and read labelled CV experience lines (`Опыт: Менеджер, Ромашка (2018 – 2020)`), titles of up to nine words and the `по н.в.` open range in the `documents` profile.
|
|
8
|
+
|
|
9
|
+
## 0.15.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- 7e977ac: Match known Russian names in all six cases and both word orders, with `ё`/`е` variants, initials and surname-plus-given forms. Give all case forms of one known person, organization, school or location one declined fake identity with natural names and an optional `pools` override. Keep `@handle/email` and `Name-99` organization spans whole so placeholders stay restorable and are never glued to digits. Detect employers in CV experience lines of the `documents` profile.
|
|
14
|
+
|
|
3
15
|
## 0.14.0
|
|
4
16
|
|
|
5
17
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -60,6 +60,10 @@ sensitive-data-sanitizer redact cv.txt --profile documents --placeholder --no-ma
|
|
|
60
60
|
Placeholder results report `irreversibleRedactions`; a nonzero count means some
|
|
61
61
|
replacements, such as credentials excluded from the map, cannot be restored.
|
|
62
62
|
|
|
63
|
+
Known Russian names match in all six cases and in any word order, with `ё`/`е`
|
|
64
|
+
spelling variants and initials (`с Петровым Данилом Сергеевичем`, `Петров Д. С.`).
|
|
65
|
+
A part of a longer name is matched alone only when it is listed itself.
|
|
66
|
+
|
|
63
67
|
For irreversible output, choose `transformation: { mode: 'placeholder', map:
|
|
64
68
|
false }`: no restore map is returned or written. Detection limits still apply
|
|
65
69
|
in either mode. See the [placeholder API](docs/API.md#reversible-placeholders),
|
|
@@ -144,7 +148,9 @@ See [the API](docs/API.md) and [all issue 13 requirements and research](docs/cas
|
|
|
144
148
|
`transformation: {mode: 'fake', key}` generates deterministic names, valid
|
|
145
149
|
dates/MRZs/checksummed IDs and reserved email/domain destinations. Credentials
|
|
146
150
|
remain fully redacted by default. Keep the key private: this preserves relations
|
|
147
|
-
and structure and is pseudonymisation.
|
|
151
|
+
and structure and is pseudonymisation. Russian mentions of one known person or
|
|
152
|
+
entity share one fake identity declined to match the source (`Петров … С Петровым`
|
|
153
|
+
becomes `Баранов … С Барановым`), and `pools` overrides the entity names. See [realistic fakes](examples/realistic-fakes.mjs)
|
|
148
154
|
and [the complete issue 24 requirements, research and solution plans](docs/case-studies/issue-24/REQUIREMENTS.md).
|
|
149
155
|
|
|
150
156
|
Keep policies containing real private values outside Git. Built-in exact public knowledge and narrowly scoped role/domain heuristics are enabled offline. Explicit private literals override public exemptions. Keep custom public policies reviewed against evidence; `--verify-public` separately opts into Wikidata name lookup:
|
package/docs/API.md
CHANGED
|
@@ -104,6 +104,34 @@ in reverse order, and organization cores without legal forms/quotes. Inflected
|
|
|
104
104
|
forms consume whole Unicode words; minimum stem lengths limit overmatching.
|
|
105
105
|
This is conservative heuristic identity linking, not general coreference:
|
|
106
106
|
ambiguous surnames/initials and unrelated people are not intentionally merged.
|
|
107
|
+
|
|
108
|
+
A PERSON entry of a reused map matches the same forms as an explicit
|
|
109
|
+
`knownPersonal` value: the full value and its contiguous parts together, all
|
|
110
|
+
cases, both orders and initials. A bare part does not match in the next document
|
|
111
|
+
unless it is stored as its own value, so `Петров Данил` from one CV leaves
|
|
112
|
+
`петров день` and `Петров пришёл.` in the next one untouched. A single stored word
|
|
113
|
+
(`Мороз`) matches its capitalized and upper-case forms but not lower-case text
|
|
114
|
+
(`сильный мороз`); a multi-word value also matches in lower case (`петров данил`).
|
|
115
|
+
A part listed in `knownPersonal` is kept in the entry's `standalone` list once it
|
|
116
|
+
has been replaced, and bare surfaces recorded by earlier versions are ignored on
|
|
117
|
+
reuse. The evidence of the current document is separate: when its text names
|
|
118
|
+
`Петров Данил`, a bare `Петров` or `петрову` in that document, including its stream
|
|
119
|
+
batches and decoded values, links to that person exactly as with a fresh map.
|
|
120
|
+
|
|
121
|
+
When a bare part matches, the span in placeholder mode takes the capitalized
|
|
122
|
+
given name or patronymic next to it on the same line, so `Алина Мороз` becomes one
|
|
123
|
+
new token instead of `Алина Person-1`. Given names come from a bounded list with
|
|
124
|
+
Russian case endings, so an unlisted name can still stay beside the placeholder.
|
|
125
|
+
|
|
126
|
+
A listed Russian PERSON value also matches all six grammatical cases, including
|
|
127
|
+
the instrumental (`Петровым Данилом Сергеевичем`), every word order, `ё`/`е`
|
|
128
|
+
spelling differences in either direction, upper case, and initials with or
|
|
129
|
+
without spaces in both positions and all cases (`Петрову Д. С.`, `Д.С. Петрова`).
|
|
130
|
+
The contiguous name parts of a listed three-part name match together
|
|
131
|
+
(`Петров Данил`, `Данил Сергеевич`, `с Петровым Данилом`). A part on its own does
|
|
132
|
+
not: a bare `Данил` or `Сергеевич` is too ambiguous to imply from a longer value,
|
|
133
|
+
so list each part you want matched alone as its own `knownPersonal` entry. Listing
|
|
134
|
+
parts next to the full name keeps one token and one fake identity.
|
|
107
135
|
Every replaced occurrence retains its exact original surface privately. With
|
|
108
136
|
one identical token, a derived answer cannot encode the original grammatical
|
|
109
137
|
case; it receives the canonical spelling. Two distinct originals that produce
|
|
@@ -132,7 +160,7 @@ new document recognizers; `documentEntities: false` turns them off explicitly.
|
|
|
132
160
|
| `profile` | Off; `'publication'` sets a 0.3 threshold and identity masking; `'documents'` enables document entities |
|
|
133
161
|
| `knownOnly` | `false`; disable all automatic detection (including credentials, Secretlint and custom detectors); use only explicit known values and reused map identities |
|
|
134
162
|
| `knownSecrets` | `[]`; exact literal credential values, regardless of length or format |
|
|
135
|
-
| `knownPersonal` | `[]`; `{type, value}` values with whole-word Unicode case-insensitive matching and bounded Russian
|
|
163
|
+
| `knownPersonal` | `[]`; `{type, value}` values with whole-word Unicode case-insensitive matching and bounded Russian forms; list parts to match them alone |
|
|
136
164
|
| `publicEntities` | `[]`; exact PERSON/ORGANIZATION/EDUCATION/LOCATION/EMAIL exception with HTTPS evidence and review date |
|
|
137
165
|
| `findings` | `[]`; validated detections for the original input, supplied by the caller |
|
|
138
166
|
| `detectors` | `[]`; additional required callbacks in the asynchronous engine |
|
|
@@ -205,6 +233,8 @@ All keyed modes require a key of at least 16 UTF-8 bytes. `format-preserving` is
|
|
|
205
233
|
|
|
206
234
|
`{mode: 'fake', key}` infers name locale, script and gender from the supported gazetteer/datasets and recognizable surname/patronymic endings. Independent HMAC-seeded Faker instances make results independent of processing order. Russian Cyrillic/transliterated spelling, name order and filename separators share component identities. Ambiguous names can use a fallback locale; this is not universal name/gender inference. Dates use one keyed backward offset of 1–365 days and preserve ISO, slash and dot formats. Complete TD1/TD2/TD3 MRZs regenerate individual/composite checks and match separately faked name/date fields. Issuer/nationality remain original unless `mrzCountry: 'UTO'` is supplied. Unsupported/damaged formats fall back to full redaction.
|
|
207
235
|
|
|
236
|
+
Russian mentions of one person, organization, school or location share one fake identity. The fake follows the source word order, initials, capitalization, grammatical case and, for people, sex: `Петров пришёл. Позвонили Петрову. С Петровым.` becomes `Баранов пришёл. Позвонили Баранову. С Барановым.`, and `ООО «Ромашка» … в Ромашке` becomes `ООО «Импульс» … в Импульсе`. Declension uses finite offline rules for the generated vocabulary. The grammatical case comes from the source surface and the preceding preposition, so a source preposition stays as written even where the fake would call for another spelling (`о Осипове`). Organizations, schools and locations draw from an authored Russian or Latin vocabulary according to the script of the original, skip any name that is itself private in the text, and take a numeric suffix (`Заря-2`) once the vocabulary is exhausted. Each transformation object remembers its entity names so that two entities never share one, which makes a name depend on first appearance when two entities start on the same vocabulary entry; reuse the transformation object across documents to keep names stable, and note that this memory is not shared across worker threads or processes. `pools: {ORGANIZATION: [...], LOCATION: [...], EDUCATION: [...]}` on the fake transformation replaces a vocabulary with caller-reviewed non-empty names. Autodetected mentions are linked as in placeholder mode: a mention that appears only in oblique forms can remain a separate entity unless the value is listed in `knownPersonal`.
|
|
237
|
+
|
|
208
238
|
Implemented national checksums, IBANs and payment-card Luhn checks remain valid. Phones retain calling code, length and separators; NANP uses 555-0100–0199 and UK mobile uses 07700 900000–900999. Email/domain outputs use example.com/.test. Other national identities and phone numbers have no universal reserved range: generated arithmetic does not prove that a value is unassigned.
|
|
209
239
|
|
|
210
240
|
Generated replacements carry `faked: true`. A private bounded registry recognizes values produced by the same transformation context as `kept: 'fake'` during later inspection/verification. Caller-supplied flags and arbitrary FAKE-looking strings grant no exemption. A fresh sanitizer does not inherit that registry; reviewed persisted fake identity values can be configured through `fakeValues`. Credentials remain fully redacted unless `credentials: true` requests provider-shaped values containing FAKE and punctuation that violates token grammar. Generated personal values cannot exempt credential spans.
|
package/docs/COVERAGE.md
CHANGED
|
@@ -43,7 +43,9 @@ probabilities. `minConfidence` still controls replacement. Findings retain
|
|
|
43
43
|
their confidence even below threshold. `publicEntities` supports exact reviewed
|
|
44
44
|
exceptions for all three types; `knownPersonal` always forces removal of a
|
|
45
45
|
listed private value. All types support placeholder mode; fake mode uses stable,
|
|
46
|
-
|
|
46
|
+
plausible organization/school/location names from an authored vocabulary or
|
|
47
|
+
caller-supplied `pools`, declined to match Russian source mentions. These names
|
|
48
|
+
are plausible and can coincide with real entities that the text does not mention.
|
|
47
49
|
|
|
48
50
|
`src/document-vocabulary.js` is the complete authored vocabulary: English,
|
|
49
51
|
Spanish, French, German, Portuguese, Russian, Japanese, Simplified/Traditional
|
|
@@ -54,6 +56,22 @@ case); they do not implement grammatical parsing. Other lowercase connectors,
|
|
|
54
56
|
unusual typography, unsupported legal forms and unfamiliar institutions can be
|
|
55
57
|
missed. Legal-form words and uncased scripts can also yield false positives.
|
|
56
58
|
|
|
59
|
+
The `documents` profile also reads employers from CV experience lines anchored on
|
|
60
|
+
a date range: `Title, Employer (2019-05 – 2021-03)`, `Title — Employer, 2019–2021`
|
|
61
|
+
and `Title (Employer), 2019–2021`. Ranges use years, `YYYY-MM`, `MM.YYYY`, full
|
|
62
|
+
dates or `month YYYY`, with `по настоящее время`, `н.в.`, `по н.в.`, `present` and
|
|
63
|
+
similar open ends. Titles have one to nine words. A label of one or two words
|
|
64
|
+
before the title (`Опыт: `, `Опыт работы - `, `Опыт — `) is read like a list marker,
|
|
65
|
+
and a segment after `|` is read like a line start, so
|
|
66
|
+
`Должность: Менеджер | Опыт: Аналитик, Ромашка (2018 – 2020)` also yields the
|
|
67
|
+
employer. The date range stays the anchor and the employer must still start with a
|
|
68
|
+
capital letter, a digit or a quote. Confidence is 0.60, and the employer then
|
|
69
|
+
propagates through the document like other organizations. The default profile
|
|
70
|
+
ignores these lines. Lines
|
|
71
|
+
with more fields (`Title, Employer, City (dates)`) and titles containing a comma
|
|
72
|
+
are not targeted and can leave the employer in place; date-anchored prose with
|
|
73
|
+
the same shape can be flagged.
|
|
74
|
+
|
|
57
75
|
The gazetteer has only the cities/regions explicitly listed in that module,
|
|
58
76
|
including Springfield and Riverton. It is not a worldwide geocoder. Location
|
|
59
77
|
labels can find unlisted proper names; unlabelled places outside the vocabulary
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/detection.js
CHANGED
|
@@ -240,11 +240,29 @@ function detectAuthorization(text, emit, credential) {
|
|
|
240
240
|
}
|
|
241
241
|
}
|
|
242
242
|
|
|
243
|
+
// `/` and `|` are valid local-part characters, so `@handle/jane@example.com`
|
|
244
|
+
// reads as one address that starts after `@`. A mention keeps its own finding;
|
|
245
|
+
// any other leading `@` joins the address so that it cannot survive as a handle.
|
|
246
|
+
function emailAfterMention(text, finding) {
|
|
247
|
+
if (
|
|
248
|
+
text[finding.start - 1] !== '@' ||
|
|
249
|
+
/[\p{L}\p{N}_.\]]/u.test(text[finding.start - 2] ?? '')
|
|
250
|
+
) {
|
|
251
|
+
return finding;
|
|
252
|
+
}
|
|
253
|
+
const handle = /^[\p{L}\p{N}_]{2,32}[/|]/u.exec(
|
|
254
|
+
text.slice(finding.start, finding.end)
|
|
255
|
+
);
|
|
256
|
+
return handle
|
|
257
|
+
? { ...finding, start: finding.start + handle[0].length }
|
|
258
|
+
: { ...finding, start: finding.start - 1 };
|
|
259
|
+
}
|
|
260
|
+
|
|
243
261
|
function detectPersonalFormats(text, options, emit) {
|
|
244
262
|
collectMatches(
|
|
245
263
|
text,
|
|
246
264
|
/(?<![\p{L}\p{N}_.!#$%&'*+/?^`{|}~-])[\p{L}\p{N}_.!#$%&'*+/?^`{|}~-]+@[\p{L}\p{N}](?:[\p{L}\p{N}.-]{0,252})\.[\p{L}]{2,63}(?![\p{L}\p{N}_-])/gu,
|
|
247
|
-
emit,
|
|
265
|
+
(finding) => emit(emailAfterMention(text, finding)),
|
|
248
266
|
{ type: 'EMAIL', category: 'personal', rule: 'email' }
|
|
249
267
|
);
|
|
250
268
|
collectMatches(
|
package/src/document-entities.js
CHANGED
|
@@ -2,6 +2,7 @@ import { collectMatches, escapePattern } from './detection.js';
|
|
|
2
2
|
import { firstNames } from './names.js';
|
|
3
3
|
import { organizationCore, personalRegex } from './personal-variants.js';
|
|
4
4
|
import { documentVocabulary } from './document-vocabulary.js';
|
|
5
|
+
import { months } from './personal-context.js';
|
|
5
6
|
|
|
6
7
|
const words = (column) =>
|
|
7
8
|
[...new Set(documentVocabulary.flatMap((row) => row[column].split('|')))]
|
|
@@ -52,6 +53,35 @@ const groupOrganization = new RegExp(
|
|
|
52
53
|
`(?<!${boundary})${latinName}[ \t]+(?:Групп|Group|Холдинг|Holding)(?!${boundary})`,
|
|
53
54
|
'gu'
|
|
54
55
|
);
|
|
56
|
+
// CV experience lines: the date range anchors the employer slot, so prose with
|
|
57
|
+
// a comma is untouched. Titles are not required to be lowercase because English
|
|
58
|
+
// ones are often title case (`Senior QA Engineer`).
|
|
59
|
+
const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
|
|
60
|
+
const month = [
|
|
61
|
+
...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
|
|
62
|
+
].join('|');
|
|
63
|
+
const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
|
|
64
|
+
const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'по н.в.']
|
|
65
|
+
.concat(['н.в.', 'по н/в', 'н/в', 'present', 'now', 'current'])
|
|
66
|
+
.flatMap((word) => [word, capitalized(word)])
|
|
67
|
+
.map(escapePattern)
|
|
68
|
+
.join('|');
|
|
69
|
+
const range = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen})(?!${boundary})`;
|
|
70
|
+
const titleWord = '[\\p{L}\\p{N}][\\p{L}\\p{N}/+#.&-]{0,39}';
|
|
71
|
+
// A segment starts a line or follows `|`. `Опыт:` or `Опыт -` before the title
|
|
72
|
+
// reads like a list marker: the date range remains the anchor of the rule.
|
|
73
|
+
const lineLabel = `\\p{L}[\\p{L}\\p{N}/+#.&-]{0,39}(?:[ \\t]+${titleWord})?(?:[ \\t]*:[ \\t]*|[ \\t]+[-–—][ \\t]+)`;
|
|
74
|
+
const lineTitle = `(?:^|(?<=\\|))[ \\t]*(?:[-–—•*·][ \\t]+|\\d{1,2}[.)][ \\t]+)?(?:${lineLabel}){0,2}${titleWord}(?:[ \\t]+${titleWord}){0,8}`;
|
|
75
|
+
const employerWord = "[\\p{L}\\p{M}\\p{N}&'’.+-]{1,40}";
|
|
76
|
+
const employer = `«[^«»\\r\\n]{1,160}»|"[^"\\r\\n]{1,160}"|“[^“”\\r\\n]{1,160}”|(?:\\p{Lu}|\\d{1,4}-?\\p{L})${employerWord}?(?:[ \\t]+${employerWord}){0,5}`;
|
|
77
|
+
const experienceLine = new RegExp(
|
|
78
|
+
`${lineTitle}(?:,[ \\t]+|[ \\t]+[-–—][ \\t]+)(${employer})(?:[ \\t]*\\(${range}\\)|[ \\t]*[,–—-][ \\t]*${range})`,
|
|
79
|
+
'gmu'
|
|
80
|
+
);
|
|
81
|
+
const experienceParenthesis = new RegExp(
|
|
82
|
+
`${lineTitle}[ \\t]*\\((${employer})\\)(?:,?[ \\t]*${range}|[ \\t]*\\(${range}\\))`,
|
|
83
|
+
'gmu'
|
|
84
|
+
);
|
|
55
85
|
|
|
56
86
|
export function detectDocumentEntities(text, emit, options) {
|
|
57
87
|
if (!options.documentEntities) {
|
|
@@ -71,6 +101,8 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
71
101
|
['EDUCATION', 'education-type', schools, 0, 0.65],
|
|
72
102
|
['LOCATION', 'location-gazetteer', places, 0, 0.6],
|
|
73
103
|
['LOCATION', 'location-label', labelledPlace, 1, 0.75],
|
|
104
|
+
['ORGANIZATION', 'organization-experience', experienceLine, 1, 0.6],
|
|
105
|
+
['ORGANIZATION', 'organization-experience', experienceParenthesis, 1, 0.6],
|
|
74
106
|
]) {
|
|
75
107
|
collectMatches(
|
|
76
108
|
text,
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import { createHmac } from 'node:crypto';
|
|
2
|
+
import { fakeName, fakeWord, wordProfile } from './fake-names.js';
|
|
3
|
+
import { canonicalPerson } from './placeholders.js';
|
|
4
|
+
import {
|
|
5
|
+
isGivenName,
|
|
6
|
+
isPatronymic,
|
|
7
|
+
russianNameRoles,
|
|
8
|
+
} from './russian-names.js';
|
|
9
|
+
import {
|
|
10
|
+
casesOf,
|
|
11
|
+
commonCase,
|
|
12
|
+
formOf,
|
|
13
|
+
governedCases,
|
|
14
|
+
russianNameForms,
|
|
15
|
+
russianNoun,
|
|
16
|
+
} from './russian-declension.js';
|
|
17
|
+
import {
|
|
18
|
+
organizationCore,
|
|
19
|
+
samePersonal,
|
|
20
|
+
wordPattern,
|
|
21
|
+
} from './personal-variants.js';
|
|
22
|
+
|
|
23
|
+
// Fictional but ordinary names. The lists are finite on purpose; callers can
|
|
24
|
+
// replace them with `transformation.pools`.
|
|
25
|
+
const vocabulary = {
|
|
26
|
+
ORGANIZATION: {
|
|
27
|
+
ru: 'Северный ветер|Альфа-Строй|Бета-Сервис|Гамма-Трейд|Меридиан|Горизонт|Гранит|Рассвет|Прогресс|Импульс|Орион|Азимут|Кристалл|Арсенал|Квант|Заря|Радуга|Волна|Орбита|Вершина|Искра|Тихая гавань|Новый горизонт|Белый парус|Золотой колос|Синяя птица',
|
|
28
|
+
latin:
|
|
29
|
+
'Northwind Trading|Bluebird Logistics|Silver Birch Partners|Harbor Lane Studio|Summit Ridge Supply|Clearwater Works|Redwood Systems|Maple Street Foods',
|
|
30
|
+
},
|
|
31
|
+
LOCATION: {
|
|
32
|
+
ru: 'Зареченск|Приморск|Светлогорск|Лесногорск|Березовка|Сосновка|Каменка|Покровка|Ольховка|Заречье|Краснополье',
|
|
33
|
+
latin: 'Riverton|Oakdale|Fairview|Lakewood|Brookfield|Greenville|Ashford',
|
|
34
|
+
},
|
|
35
|
+
EDUCATION: {
|
|
36
|
+
ru: 'Северный государственный университет|Приморский технический институт|Лесногорский гуманитарный университет|Зареченский экономический колледж|Светлогорская городская школа',
|
|
37
|
+
latin:
|
|
38
|
+
'Riverside University|Lakeview College|Oakdale Institute of Technology|Fairview State University',
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
const cyrillic = /\p{Script=Cyrillic}/u;
|
|
42
|
+
const surfacesKey = Symbol('fakeSurfaces');
|
|
43
|
+
const textKey = Symbol('fakeText');
|
|
44
|
+
const surfaceLimit = 1000;
|
|
45
|
+
|
|
46
|
+
// Lets each span see the other mentions of its document, as placeholders do.
|
|
47
|
+
export function prepareFakeText(text, findings, options) {
|
|
48
|
+
const surfaces = new Map();
|
|
49
|
+
for (const finding of findings) {
|
|
50
|
+
if (
|
|
51
|
+
['PERSON', 'ORGANIZATION', 'LOCATION', 'EDUCATION'].includes(finding.type)
|
|
52
|
+
) {
|
|
53
|
+
const values = surfaces.get(finding.type) ?? new Set();
|
|
54
|
+
if (values.size < surfaceLimit) {
|
|
55
|
+
values.add(text.slice(finding.start, finding.end));
|
|
56
|
+
}
|
|
57
|
+
surfaces.set(finding.type, values);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return { ...options, [surfacesKey]: surfaces, [textKey]: text };
|
|
61
|
+
}
|
|
62
|
+
const wordCount = (value) => value.trim().split(/\s+/u).length;
|
|
63
|
+
// The fullest form this mention belongs to, or the mention itself.
|
|
64
|
+
function entityLemma(value, type, options) {
|
|
65
|
+
const known = [
|
|
66
|
+
...(options.knownPersonal ?? []),
|
|
67
|
+
...(options.confirmedPersonal ?? []),
|
|
68
|
+
]
|
|
69
|
+
.filter((entry) => entry.type === type)
|
|
70
|
+
.map((entry) => entry.value);
|
|
71
|
+
const mentions = [...(options[surfacesKey]?.get(type) ?? [])];
|
|
72
|
+
for (const group of [known, mentions]) {
|
|
73
|
+
const matches = group.filter((other) => samePersonal(value, other, type));
|
|
74
|
+
const longest = Math.max(0, ...matches.map(wordCount));
|
|
75
|
+
const fullest = [
|
|
76
|
+
...new Set(matches.filter((other) => wordCount(other) === longest)),
|
|
77
|
+
];
|
|
78
|
+
// `Гранит` covers its own form `Граните`, so several forms share one lemma.
|
|
79
|
+
const lemma = fullest.filter((other) =>
|
|
80
|
+
fullest.every((form) => samePersonal(form, other, type))
|
|
81
|
+
);
|
|
82
|
+
if (lemma.length === 1) {
|
|
83
|
+
return lemma[0];
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return type === 'PERSON' ? canonicalPerson(value, mentions) : value;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const applyStyle = (surface, output) =>
|
|
90
|
+
surface.length > 1 && surface === surface.toUpperCase()
|
|
91
|
+
? output.toUpperCase()
|
|
92
|
+
: surface === surface.toLowerCase()
|
|
93
|
+
? output.toLowerCase()
|
|
94
|
+
: output;
|
|
95
|
+
const capitalize = (word) =>
|
|
96
|
+
word[0].toUpperCase() + word.slice(1).toLowerCase();
|
|
97
|
+
function nameSlots(parts) {
|
|
98
|
+
const roles = russianNameRoles(parts);
|
|
99
|
+
if (roles) {
|
|
100
|
+
return parts.map((_part, i) =>
|
|
101
|
+
i === roles.surname
|
|
102
|
+
? 'lastName'
|
|
103
|
+
: i === roles.patronymic
|
|
104
|
+
? 'middleName'
|
|
105
|
+
: 'firstName'
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
if (parts.length === 1) {
|
|
109
|
+
return [wordProfile(parts[0]).role];
|
|
110
|
+
}
|
|
111
|
+
const given = parts.map((part) => isGivenName(part));
|
|
112
|
+
if (parts.length === 2 && given[0] !== given[1]) {
|
|
113
|
+
return given.map((isGiven) => (isGiven ? 'firstName' : 'lastName'));
|
|
114
|
+
}
|
|
115
|
+
return parts.map((_part, i) =>
|
|
116
|
+
i === 0 ? 'firstName' : i === parts.length - 1 ? 'lastName' : 'middleName'
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
function nameSex(parts, slots) {
|
|
120
|
+
const at = (slot) => parts[slots.indexOf(slot)];
|
|
121
|
+
const middle = at('middleName');
|
|
122
|
+
if (middle && (isPatronymic(middle) || /(?:вна|чна)$/iu.test(middle))) {
|
|
123
|
+
return /(?:вна|чна)$/iu.test(middle) ? 'female' : 'male';
|
|
124
|
+
}
|
|
125
|
+
const last = at('lastName');
|
|
126
|
+
if (last && /(?:ова|ева|ёва|ина|ына|ая|яя)$/iu.test(last)) {
|
|
127
|
+
return 'female';
|
|
128
|
+
}
|
|
129
|
+
if (last && /(?:ов|ев|ёв|ин|ын|ий|ой|ый)$/iu.test(last)) {
|
|
130
|
+
return 'male';
|
|
131
|
+
}
|
|
132
|
+
return wordProfile(at('firstName') ?? parts[0]).sex;
|
|
133
|
+
}
|
|
134
|
+
const letters = /[\p{L}\p{M}]+/gu;
|
|
135
|
+
// One identity per lemma: each lemma word has one seeded fake, and the matched
|
|
136
|
+
// surface only decides its case, its order and its letter case.
|
|
137
|
+
function russianFakeName(surface, lemma, key, context) {
|
|
138
|
+
const parts = lemma.trim().split(/\s+/u);
|
|
139
|
+
const slots = nameSlots(parts);
|
|
140
|
+
const sex = nameSex(parts, slots);
|
|
141
|
+
const lemmaWords = parts.flatMap((part, i) =>
|
|
142
|
+
part.match(letters).map((word) => {
|
|
143
|
+
const slot = slots[i];
|
|
144
|
+
return {
|
|
145
|
+
slot,
|
|
146
|
+
lemmaForms: russianNameForms(word, slot, sex),
|
|
147
|
+
pattern: new RegExp(`^(?:${wordPattern(word, 'PERSON')})$`, 'iu'),
|
|
148
|
+
first: word[0].toLowerCase().replace('ё', 'е'),
|
|
149
|
+
fake: fakeWord(capitalize(word), key, slot, undefined, sex),
|
|
150
|
+
};
|
|
151
|
+
})
|
|
152
|
+
);
|
|
153
|
+
const matched = surface.match(letters).map((word) => {
|
|
154
|
+
const initial = word.length === 1;
|
|
155
|
+
const entry = lemmaWords.find(
|
|
156
|
+
(candidate) =>
|
|
157
|
+
!candidate.used &&
|
|
158
|
+
(initial
|
|
159
|
+
? candidate.slot !== 'lastName' &&
|
|
160
|
+
candidate.first === word.toLowerCase().replace('ё', 'е')
|
|
161
|
+
: candidate.pattern.test(word))
|
|
162
|
+
);
|
|
163
|
+
if (entry) {
|
|
164
|
+
entry.used = true;
|
|
165
|
+
}
|
|
166
|
+
return { word, entry, initial };
|
|
167
|
+
});
|
|
168
|
+
const index = commonCase(
|
|
169
|
+
matched
|
|
170
|
+
.filter(({ entry, initial }) => entry && !initial)
|
|
171
|
+
.map(({ word, entry }) => casesOf(word, entry.lemmaForms)),
|
|
172
|
+
context
|
|
173
|
+
);
|
|
174
|
+
let position = 0;
|
|
175
|
+
return surface.replace(letters, () => {
|
|
176
|
+
const { word, entry, initial } = matched[position++];
|
|
177
|
+
if (!entry) {
|
|
178
|
+
return fakeWord(word, key);
|
|
179
|
+
}
|
|
180
|
+
if (initial) {
|
|
181
|
+
return entry.fake[0].toUpperCase();
|
|
182
|
+
}
|
|
183
|
+
const forms = russianNameForms(entry.fake, entry.slot, sex);
|
|
184
|
+
return applyStyle(word, capitalize(formOf(forms, index)));
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
const contextOf = (finding, options) =>
|
|
188
|
+
governedCases(options[textKey] ?? '', finding.start);
|
|
189
|
+
export function fakePerson(value, finding, transform, options) {
|
|
190
|
+
const lemma = entityLemma(value, 'PERSON', options);
|
|
191
|
+
return cyrillic.test(lemma) &&
|
|
192
|
+
cyrillic.test(value) &&
|
|
193
|
+
!/\p{Script=Latin}/u.test(`${lemma}${value}`)
|
|
194
|
+
? russianFakeName(value, lemma, transform.key, contextOf(finding, options))
|
|
195
|
+
: fakeName(value, transform.key);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const assigned = new WeakMap();
|
|
199
|
+
const lemmaKey = (core) => core.toLowerCase().replaceAll('ё', 'е');
|
|
200
|
+
const poolFor = (type, core, transform) =>
|
|
201
|
+
transform.pools?.[type] ??
|
|
202
|
+
vocabulary[type][cyrillic.test(core) ? 'ru' : 'latin'].split('|');
|
|
203
|
+
function isReal(candidate, type, options) {
|
|
204
|
+
return [
|
|
205
|
+
...(options.knownPersonal ?? []),
|
|
206
|
+
...(options.confirmedPersonal ?? []),
|
|
207
|
+
...[...(options[surfacesKey]?.get(type) ?? [])].map((value) => ({
|
|
208
|
+
type,
|
|
209
|
+
value,
|
|
210
|
+
})),
|
|
211
|
+
].some(
|
|
212
|
+
(entry) => entry.type === type && samePersonal(candidate, entry.value, type)
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
// A deterministic start, then the next free name so that two entities of one
|
|
216
|
+
// run never share a fake. An exhausted list continues with numbered names.
|
|
217
|
+
function assignFake(core, type, transform, options) {
|
|
218
|
+
let state = assigned.get(transform);
|
|
219
|
+
if (!state) {
|
|
220
|
+
state = new Map();
|
|
221
|
+
assigned.set(transform, state);
|
|
222
|
+
}
|
|
223
|
+
const entityKey = `${type}\0${lemmaKey(core)}`;
|
|
224
|
+
if (state.has(entityKey)) {
|
|
225
|
+
return state.get(entityKey);
|
|
226
|
+
}
|
|
227
|
+
const taken = new Set(state.values());
|
|
228
|
+
const pool = poolFor(type, core, transform);
|
|
229
|
+
const start = createHmac('sha256', transform.key)
|
|
230
|
+
.update(`entity\0${entityKey}`)
|
|
231
|
+
.digest()
|
|
232
|
+
.readUInt32BE(0);
|
|
233
|
+
for (let attempt = 0; ; attempt++) {
|
|
234
|
+
const name = pool[(start + attempt) % pool.length];
|
|
235
|
+
const round = Math.floor(attempt / pool.length);
|
|
236
|
+
const candidate = round ? `${name}-${round + 1}` : name;
|
|
237
|
+
if (!taken.has(candidate) && !isReal(candidate, type, options)) {
|
|
238
|
+
state.set(entityKey, candidate);
|
|
239
|
+
return candidate;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
const nounOptions = (word, phrase, position) => ({
|
|
244
|
+
animate: false,
|
|
245
|
+
sex: phrase.some((other) => /[ая]я$/u.test(other)) ? 'female' : undefined,
|
|
246
|
+
adjective:
|
|
247
|
+
!word.includes('-') &&
|
|
248
|
+
(/(?:ый|ая|яя)$/u.test(word) ||
|
|
249
|
+
(/(?:ий|ой|ое|ее)$/u.test(word) && position < phrase.length - 1)),
|
|
250
|
+
});
|
|
251
|
+
// Only the last part of a hyphenated name declines: в Альфа-Строе.
|
|
252
|
+
const phraseForms = (phrase) => {
|
|
253
|
+
const words = phrase.toLowerCase().split(/\s+/u);
|
|
254
|
+
return words.map((word, position) => {
|
|
255
|
+
const head = word.slice(0, word.lastIndexOf('-') + 1);
|
|
256
|
+
const tail = word.slice(head.length);
|
|
257
|
+
return {
|
|
258
|
+
head,
|
|
259
|
+
forms: russianNoun(tail, nounOptions(word, words, position)),
|
|
260
|
+
};
|
|
261
|
+
});
|
|
262
|
+
};
|
|
263
|
+
function phraseCase(surface, lemma, context) {
|
|
264
|
+
const words = surface.split(/\s+/u);
|
|
265
|
+
const lemmaForms = phraseForms(lemma);
|
|
266
|
+
if (words.length !== lemmaForms.length) {
|
|
267
|
+
return 0;
|
|
268
|
+
}
|
|
269
|
+
return commonCase(
|
|
270
|
+
words.map((word, i) =>
|
|
271
|
+
casesOf(word.slice(word.lastIndexOf('-') + 1), lemmaForms[i].forms)
|
|
272
|
+
),
|
|
273
|
+
context
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
function declinedPhrase(phrase, index) {
|
|
277
|
+
if (!cyrillic.test(phrase)) {
|
|
278
|
+
return phrase;
|
|
279
|
+
}
|
|
280
|
+
const original = phrase.split(/\s+/u);
|
|
281
|
+
return phraseForms(phrase)
|
|
282
|
+
.map(({ head, forms }, i) => {
|
|
283
|
+
const word = head + formOf(forms, index);
|
|
284
|
+
return original[i][0] === original[i][0].toUpperCase()
|
|
285
|
+
? word.replace(
|
|
286
|
+
/(^|-)(\p{L})/gu,
|
|
287
|
+
(_m, gap, letter) => `${gap}${letter.toUpperCase()}`
|
|
288
|
+
)
|
|
289
|
+
: word;
|
|
290
|
+
})
|
|
291
|
+
.join(' ');
|
|
292
|
+
}
|
|
293
|
+
const decoration = /^((?:(?:ООО|АО|ПАО|ИП|ЗАО)[ \t]+)?[«“"]?)(.*?)([»”"]?)$/su;
|
|
294
|
+
// `ООО «Ромашка»` keeps its legal form and quotes; only the name is replaced.
|
|
295
|
+
export function fakeEntity(value, finding, transform, options) {
|
|
296
|
+
const { type } = finding;
|
|
297
|
+
const [, before, core, after] = decoration.exec(value);
|
|
298
|
+
const lemma = organizationCore(entityLemma(value, type, options));
|
|
299
|
+
const fake = assignFake(lemma, type, transform, options);
|
|
300
|
+
const output = cyrillic.test(core)
|
|
301
|
+
? declinedPhrase(fake, phraseCase(core, lemma, contextOf(finding, options)))
|
|
302
|
+
: fake;
|
|
303
|
+
return `${before}${applyStyle(core, output)}${after}`;
|
|
304
|
+
}
|
package/src/fake-names.js
CHANGED
|
@@ -213,7 +213,8 @@ export function fakeName(value, key) {
|
|
|
213
213
|
return fakeWord(word, key, roles.get(word), undefined, nameSex);
|
|
214
214
|
});
|
|
215
215
|
}
|
|
216
|
-
function wordProfile(word, forcedRole, forcedLocale) {
|
|
216
|
+
export function wordProfile(word, forcedRole, forcedLocale) {
|
|
217
|
+
indexNames();
|
|
217
218
|
const canonical = latinName(word);
|
|
218
219
|
const patronymic = /(?:vich|vna|ichna|ogly|kyzy)$/.test(canonical);
|
|
219
220
|
const slavic = /(?:ova|eva|ov|ev|in|ina|enko|sky|ski|skaya)$/.test(canonical);
|
|
@@ -240,7 +241,7 @@ function wordProfile(word, forcedRole, forcedLocale) {
|
|
|
240
241
|
: 'lastName');
|
|
241
242
|
return { canonical, locale, sex, role };
|
|
242
243
|
}
|
|
243
|
-
function fakeWord(word, key, forcedRole, forcedLocale, forcedSex) {
|
|
244
|
+
export function fakeWord(word, key, forcedRole, forcedLocale, forcedSex) {
|
|
244
245
|
const {
|
|
245
246
|
canonical,
|
|
246
247
|
locale,
|
package/src/fake.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fakeEntity, fakePerson } from './fake-entities.js';
|
|
1
2
|
import { createHmac } from 'node:crypto';
|
|
2
3
|
import { parsePhoneNumberFromString } from 'libphonenumber-js/max';
|
|
3
4
|
import { mrzCheck } from './identity.js';
|
|
@@ -487,7 +488,7 @@ function fakeTd1(parts, transform) {
|
|
|
487
488
|
.padEnd(30, '<');
|
|
488
489
|
return parts.join('');
|
|
489
490
|
}
|
|
490
|
-
export function realisticFake(value, finding, transform) {
|
|
491
|
+
export function realisticFake(value, finding, transform, options = {}) {
|
|
491
492
|
let output;
|
|
492
493
|
if (finding.category === 'credential') {
|
|
493
494
|
if (!transform.credentials) {
|
|
@@ -500,14 +501,9 @@ export function realisticFake(value, finding, transform) {
|
|
|
500
501
|
.toString('hex')
|
|
501
502
|
.slice(0, 8)}!`;
|
|
502
503
|
} else if (finding.type === 'PERSON') {
|
|
503
|
-
output =
|
|
504
|
+
output = fakePerson(value, finding, transform, options);
|
|
504
505
|
} else if (['ORGANIZATION', 'EDUCATION', 'LOCATION'].includes(finding.type)) {
|
|
505
|
-
|
|
506
|
-
ORGANIZATION: 'Organization',
|
|
507
|
-
EDUCATION: 'School',
|
|
508
|
-
LOCATION: 'Location',
|
|
509
|
-
}[finding.type];
|
|
510
|
-
output = `Example ${label} ${digest(value, transform.key, finding.type).readUInt32BE(0)}`;
|
|
506
|
+
output = fakeEntity(value, finding, transform, options);
|
|
511
507
|
} else if (/DATE|BIRTH/.test(finding.type)) {
|
|
512
508
|
output = fakeDate(value, transform.key);
|
|
513
509
|
} else if (finding.type === 'PASSPORT_MRZ') {
|
package/src/index.d.ts
CHANGED
|
@@ -250,6 +250,10 @@ export type Transformation =
|
|
|
250
250
|
credentials?: boolean;
|
|
251
251
|
/** Keep original MRZ issuer/nationality unless Utopia is requested. */
|
|
252
252
|
mrzCountry?: 'UTO';
|
|
253
|
+
/** Caller-reviewed names that replace the generated entity vocabulary. */
|
|
254
|
+
pools?: Partial<
|
|
255
|
+
Record<'ORGANIZATION' | 'LOCATION' | 'EDUCATION', string[]>
|
|
256
|
+
>;
|
|
253
257
|
}
|
|
254
258
|
| { mode: 'date-shift'; days: number }
|
|
255
259
|
| { mode: 'bucket'; size: number };
|
|
@@ -265,6 +269,8 @@ export interface RestoreMap {
|
|
|
265
269
|
number: number;
|
|
266
270
|
values: string[];
|
|
267
271
|
aliases: string[];
|
|
272
|
+
/** PERSON parts listed by the caller, which also match alone on reuse. */
|
|
273
|
+
standalone?: string[];
|
|
268
274
|
}>;
|
|
269
275
|
documents: Array<{
|
|
270
276
|
digest: string;
|
package/src/personal-context.js
CHANGED
|
@@ -2,7 +2,7 @@ import { collectMatches } from './detection.js';
|
|
|
2
2
|
|
|
3
3
|
const birth =
|
|
4
4
|
/(?<![\p{L}\p{N}_])(?:born|birth[ _-]?date|date[ _-]of[ _-]birth|DOB|дата рождения|д\.р\.|родился|родилась|Geburtsdatum|fecha de nacimiento|date de naissance)(?![\p{L}\p{N}_])/giu;
|
|
5
|
-
const months =
|
|
5
|
+
export const months =
|
|
6
6
|
'January February March April May June July August September October November December Jan Feb Mar Apr Jun Jul Aug Sep Sept Oct Nov Dec января февраля марта апреля мая июня июля августа сентября октября ноября декабря январь февраль март апрель май июнь июль август сентябрь октябрь ноябрь декабрь'
|
|
7
7
|
.split(' ')
|
|
8
8
|
.join('|');
|