@link-foundation/sensitive-data-sanitizer 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +7 -1
- package/docs/API.md +31 -1
- package/docs/COVERAGE.md +19 -1
- package/package.json +1 -1
- package/src/detection.js +19 -1
- package/src/document-entities.js +32 -0
- package/src/fake-entities.js +304 -0
- package/src/fake-names.js +3 -2
- package/src/fake.js +4 -8
- package/src/index.d.ts +6 -0
- package/src/personal-context.js +1 -1
- package/src/personal-variants.js +119 -25
- package/src/personal.js +12 -6
- package/src/placeholders.js +192 -50
- package/src/propagation.js +7 -3
- package/src/restore-map.js +2 -1
- package/src/russian-declension.js +167 -0
- package/src/russian-names.js +64 -0
- package/src/sanitizer.js +13 -2
- package/src/transforms.js +21 -4
package/src/personal-variants.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { escapePattern, failure } from './detection.js';
|
|
2
|
+
import { russianNameRoles } from './russian-names.js';
|
|
2
3
|
|
|
3
4
|
const boundary = '[\\p{L}\\p{M}\\p{N}_]';
|
|
4
5
|
export function organizationCore(value) {
|
|
@@ -7,9 +8,26 @@ export function organizationCore(value) {
|
|
|
7
8
|
.replace(/^[«“"]|[»”"]$/gu, '')
|
|
8
9
|
.trim();
|
|
9
10
|
}
|
|
11
|
+
// `Вектор-1` also identifies its base name and numbered siblings like `Вектор-2`.
|
|
12
|
+
export function organizationFamily(core) {
|
|
13
|
+
const base = /^(.{3,})-\d{1,9}$/u.exec(core)?.[1];
|
|
14
|
+
return base
|
|
15
|
+
? {
|
|
16
|
+
base,
|
|
17
|
+
pattern: new RegExp(
|
|
18
|
+
`(?<!${boundary})${escapePattern(base)}(?:-\\d{1,9})?(?!${boundary}|-\\d)`,
|
|
19
|
+
'giu'
|
|
20
|
+
),
|
|
21
|
+
}
|
|
22
|
+
: undefined;
|
|
23
|
+
}
|
|
24
|
+
// ё and е are one letter in ordinary Russian writing; the i flag covers Ё and Е.
|
|
25
|
+
const foldYo = (pattern) => pattern.replace(/[еёЕЁ]/gu, '[её]');
|
|
26
|
+
const fleeting = { павел: 'Павл', лев: 'Льв' };
|
|
27
|
+
const hardCases = '(?:а|у|ом|е)';
|
|
10
28
|
// Only caller-confirmed Russian words receive finite case alternatives. A
|
|
11
29
|
// minimum three-letter stem avoids turning short names into broad matches.
|
|
12
|
-
function
|
|
30
|
+
function cyrillicPattern(word, type) {
|
|
13
31
|
const endings = [
|
|
14
32
|
[/(?:ова|ева|ина)$/u, 1, '(?:а|ой|ою|у|е|ы)'],
|
|
15
33
|
[/(?:ая)$/u, 2, '(?:ая|ой|ою|ую)'],
|
|
@@ -18,24 +36,76 @@ function wordPattern(word) {
|
|
|
18
36
|
2,
|
|
19
37
|
'(?:ий|ый|ой|ого|его|ому|ему|им|ым|ом|ем|ия|ию|ием|ии)',
|
|
20
38
|
],
|
|
39
|
+
[/[аеуюя]й$/u, 1, '(?:й|я|ю|ем|е)'],
|
|
21
40
|
[/(?:ов|ев|ин)$/u, 0, '(?:а|у|ым|ом|е|ой)?'],
|
|
41
|
+
[/[жчшщц]$/u, 0, '(?:а|у|ем|ом|е)?'],
|
|
42
|
+
[/[жчшщц]а$/u, 1, '(?:а|и|е|у|ей|ею|ой|ою)'],
|
|
22
43
|
[/а$/u, 1, '(?:а|ы|и|е|у|ой|ою)'],
|
|
23
44
|
[/я$/u, 1, '(?:я|и|е|ю|ей|ею)'],
|
|
24
|
-
[/ь$/u, 1, '(
|
|
45
|
+
[/ь$/u, 1, '(?:ь|я|ю|ем|е|и|ью)'],
|
|
25
46
|
];
|
|
26
47
|
const lower = word.toLowerCase();
|
|
27
|
-
if (
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
48
|
+
if (fleeting[lower]) {
|
|
49
|
+
return `(?:${word}|${fleeting[lower]}${hardCases})`;
|
|
50
|
+
}
|
|
51
|
+
for (const [ending, remove, suffix] of endings) {
|
|
52
|
+
if (ending.test(lower) && word.length - remove >= 3) {
|
|
53
|
+
return escapePattern(remove ? word.slice(0, -remove) : word) + suffix;
|
|
32
54
|
}
|
|
33
55
|
}
|
|
34
|
-
if (/^[а-яё]{
|
|
35
|
-
|
|
56
|
+
if (/^[а-яё]{3,}[бвгджзклмнпрстфхцчшщ]$/u.test(lower)) {
|
|
57
|
+
// Plural forms would turn a given name into a surname: Петр and Петров.
|
|
58
|
+
return `${escapePattern(word)}${type === 'PERSON' ? `${hardCases}?` : '(?:а|у|ом|е|ы|ов|ам|ами|ах)?'}`;
|
|
36
59
|
}
|
|
37
60
|
return escapePattern(word);
|
|
38
61
|
}
|
|
62
|
+
export function wordPattern(word, type) {
|
|
63
|
+
return foldYo(
|
|
64
|
+
word
|
|
65
|
+
.split('-')
|
|
66
|
+
.map((part) =>
|
|
67
|
+
/^[а-яё]+$/iu.test(part)
|
|
68
|
+
? cyrillicPattern(part, type)
|
|
69
|
+
: escapePattern(part)
|
|
70
|
+
)
|
|
71
|
+
.join('-')
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
const initialGap = '[ \\t\\u00a0]*';
|
|
75
|
+
const initialPattern = (word) => `${foldYo(escapePattern(word[0]))}\\.`;
|
|
76
|
+
// Everyday orders, partial forms and initials of a Russian full name, longest
|
|
77
|
+
// first so that alternation prefers the complete form at each position.
|
|
78
|
+
function russianForms(words, roles, type) {
|
|
79
|
+
const name = Object.fromEntries(
|
|
80
|
+
Object.entries(roles).map(([role, i]) => [role, words[i]])
|
|
81
|
+
);
|
|
82
|
+
const gap = '[ \\t]+';
|
|
83
|
+
const full = (...parts) =>
|
|
84
|
+
parts.map((part) => wordPattern(part, type)).join(gap);
|
|
85
|
+
if (name.surname === undefined) {
|
|
86
|
+
return [];
|
|
87
|
+
}
|
|
88
|
+
const initial = initialPattern(name.given);
|
|
89
|
+
const last = wordPattern(name.surname, type);
|
|
90
|
+
const short = [
|
|
91
|
+
full(name.surname, name.given),
|
|
92
|
+
full(name.given, name.surname),
|
|
93
|
+
`${last}${gap}${initial}`,
|
|
94
|
+
`${initial}[ \\t\\u00a0]+${last}`,
|
|
95
|
+
];
|
|
96
|
+
if (name.patronymic === undefined) {
|
|
97
|
+
return short;
|
|
98
|
+
}
|
|
99
|
+
const initials = `${initial}${initialGap}${initialPattern(name.patronymic)}`;
|
|
100
|
+
return [
|
|
101
|
+
full(name.surname, name.given, name.patronymic),
|
|
102
|
+
full(name.given, name.patronymic, name.surname),
|
|
103
|
+
`${last}[ \\t\\u00a0]+${initials}`,
|
|
104
|
+
`${initials}[ \\t\\u00a0]+${last}`,
|
|
105
|
+
...short,
|
|
106
|
+
full(name.given, name.patronymic),
|
|
107
|
+
];
|
|
108
|
+
}
|
|
39
109
|
export function personalPatterns(value, type) {
|
|
40
110
|
if (value.length > 256) {
|
|
41
111
|
// Large explicit values still support literal matching without expansion.
|
|
@@ -52,13 +122,22 @@ export function personalPatterns(value, type) {
|
|
|
52
122
|
for (const surface of values) {
|
|
53
123
|
const parts = surface.split(/([ \t]+)/u);
|
|
54
124
|
const pattern = parts
|
|
55
|
-
.map((part, i) => (i % 2 ? '[ \\t]+' : wordPattern(part)))
|
|
125
|
+
.map((part, i) => (i % 2 ? '[ \\t]+' : wordPattern(part, type)))
|
|
56
126
|
.join('');
|
|
57
127
|
patterns.add(pattern);
|
|
58
|
-
if (type === 'PERSON'
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
)
|
|
128
|
+
if (type === 'PERSON') {
|
|
129
|
+
const words = parts.filter((_part, i) => i % 2 === 0);
|
|
130
|
+
const roles = russianNameRoles(words);
|
|
131
|
+
for (const form of roles ? russianForms(words, roles, type) : []) {
|
|
132
|
+
patterns.add(form);
|
|
133
|
+
}
|
|
134
|
+
if (words.length === 2) {
|
|
135
|
+
patterns.add(
|
|
136
|
+
[wordPattern(words[1], type), wordPattern(words[0], type)].join(
|
|
137
|
+
'[ \\t]+'
|
|
138
|
+
)
|
|
139
|
+
);
|
|
140
|
+
}
|
|
62
141
|
}
|
|
63
142
|
}
|
|
64
143
|
return [...patterns];
|
|
@@ -77,18 +156,33 @@ export function samePersonal(value, original, type) {
|
|
|
77
156
|
export function detectKnownPersonal(text, entries, rule, emit, options) {
|
|
78
157
|
let count = 0;
|
|
79
158
|
for (const entry of entries ?? []) {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
159
|
+
const family =
|
|
160
|
+
rule === 'confirmed-personal' &&
|
|
161
|
+
options.documentEntities &&
|
|
162
|
+
!options.knownOnly &&
|
|
163
|
+
entry.type === 'ORGANIZATION' &&
|
|
164
|
+
organizationFamily(organizationCore(entry.value));
|
|
165
|
+
for (const pattern of [
|
|
166
|
+
personalRegex(entry.value, entry.type),
|
|
167
|
+
...(family ? [family.pattern] : []),
|
|
168
|
+
]) {
|
|
169
|
+
for (const match of text.matchAll(pattern)) {
|
|
170
|
+
// A stored capitalized word in lower case is ordinary text: `Мороз`, `мороз`.
|
|
171
|
+
if (entry.properCase && /^\p{Ll}/u.test(match[0])) {
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
if (++count > (options.maxFindings ?? 100000)) {
|
|
175
|
+
throw failure('ERR_LIMIT');
|
|
176
|
+
}
|
|
177
|
+
emit({
|
|
178
|
+
start: match.index,
|
|
179
|
+
end: match.index + match[0].length,
|
|
180
|
+
type: entry.type,
|
|
181
|
+
category: 'personal',
|
|
182
|
+
rule,
|
|
183
|
+
confidence: rule === 'known-personal' ? 0.85 : 0.9,
|
|
184
|
+
});
|
|
83
185
|
}
|
|
84
|
-
emit({
|
|
85
|
-
start: match.index,
|
|
86
|
-
end: match.index + match[0].length,
|
|
87
|
-
type: entry.type,
|
|
88
|
-
category: 'personal',
|
|
89
|
-
rule,
|
|
90
|
-
confidence: rule === 'known-personal' ? 0.85 : 0.9,
|
|
91
|
-
});
|
|
92
186
|
}
|
|
93
187
|
}
|
|
94
188
|
}
|
package/src/personal.js
CHANGED
|
@@ -29,6 +29,9 @@ const ibanLengths = Object.fromEntries(
|
|
|
29
29
|
.map((entry) => [entry.slice(0, 2), Number(entry.slice(2))])
|
|
30
30
|
);
|
|
31
31
|
|
|
32
|
+
// `@handle/jane@example.com` is a mention and an address, not a package path.
|
|
33
|
+
const mentionedEmail =
|
|
34
|
+
/^\/[\p{L}\p{N}_.!#$%&'*+?^`{|}~-]+@[\p{L}\p{N}][\p{L}\p{N}.-]{0,252}\.\p{L}{2,63}(?![\p{L}\p{N}_-])/u;
|
|
32
35
|
export function iban(value) {
|
|
33
36
|
const compact = value.replace(/\s/g, '').toUpperCase();
|
|
34
37
|
if (
|
|
@@ -207,10 +210,11 @@ export function detectPersonal(text, emit) {
|
|
|
207
210
|
}
|
|
208
211
|
collectMatches(
|
|
209
212
|
text,
|
|
210
|
-
/(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&'
|
|
213
|
+
/(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&']{0,80}(?:-[\p{L}\p{M}\p{N}&']{1,40}){0,4})/gu,
|
|
211
214
|
emit,
|
|
212
215
|
details('ORGANIZATION', 'organization-context', 0.7),
|
|
213
|
-
1
|
|
216
|
+
1,
|
|
217
|
+
(value) => value.length >= 2
|
|
214
218
|
);
|
|
215
219
|
collectMatches(
|
|
216
220
|
text,
|
|
@@ -236,10 +240,12 @@ export function detectPersonal(text, emit) {
|
|
|
236
240
|
emit,
|
|
237
241
|
details('USERNAME', 'messenger-handle', 0.65),
|
|
238
242
|
0,
|
|
239
|
-
(_value, match) =>
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
+
(_value, match) => {
|
|
244
|
+
const rest = text.slice(match.index + match[0].length);
|
|
245
|
+
return (
|
|
246
|
+
!/^\/[a-z0-9][a-z0-9_.-]*/i.test(rest) || mentionedEmail.test(rest)
|
|
247
|
+
);
|
|
248
|
+
}
|
|
243
249
|
);
|
|
244
250
|
collectMatches(
|
|
245
251
|
text,
|
package/src/placeholders.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { failure, REDACTED, escapePattern } from './detection.js';
|
|
2
2
|
import { reservedPlaceholderTypes } from './placeholder-types.js';
|
|
3
3
|
import { samePersonal } from './personal-variants.js';
|
|
4
|
+
import { isNameForm } from './russian-names.js';
|
|
4
5
|
import { nameAliases } from './transliteration.js';
|
|
5
6
|
import { createHash } from 'node:crypto';
|
|
6
7
|
import { jsonStringLiterals } from './json-string-runs.js';
|
|
@@ -85,13 +86,50 @@ export function placeholderKnown(options) {
|
|
|
85
86
|
...map.entries
|
|
86
87
|
.filter((entry) => entry.type !== 'CREDENTIAL')
|
|
87
88
|
.flatMap((entry) =>
|
|
88
|
-
entry.values
|
|
89
|
+
(entry.type === 'PERSON' ? reusablePersonValues(entry) : entry.values)
|
|
89
90
|
.filter((value) => value.length <= 256)
|
|
90
|
-
.map((value) => ({
|
|
91
|
+
.map((value) => ({
|
|
92
|
+
type: entry.type,
|
|
93
|
+
value,
|
|
94
|
+
...(entry.type === 'PERSON' && properCaseWord(value)
|
|
95
|
+
? { properCase: true }
|
|
96
|
+
: {}),
|
|
97
|
+
}))
|
|
91
98
|
),
|
|
92
99
|
],
|
|
93
100
|
};
|
|
94
101
|
}
|
|
102
|
+
const singleWord = (value) => !/\s/u.test(value.trim());
|
|
103
|
+
const properCaseWord = (value) =>
|
|
104
|
+
singleWord(value) && /^\p{Lu}/u.test(value.trim());
|
|
105
|
+
// A bare part recorded while replacing a fuller name in its own document is not
|
|
106
|
+
// evidence on its own in the next one, so a reused map offers only the full
|
|
107
|
+
// values, parts listed by the caller and values that were stored alone.
|
|
108
|
+
function reusablePersonValues(entry) {
|
|
109
|
+
const longer = entry.values.filter((value) => !singleWord(value));
|
|
110
|
+
return [
|
|
111
|
+
...new Set([
|
|
112
|
+
...entry.values.filter(
|
|
113
|
+
(value) =>
|
|
114
|
+
!singleWord(value) ||
|
|
115
|
+
!longer.some((full) => partOfPerson(value, full))
|
|
116
|
+
),
|
|
117
|
+
...(entry.standalone ?? []),
|
|
118
|
+
]),
|
|
119
|
+
];
|
|
120
|
+
}
|
|
121
|
+
function partOfPerson(value, full) {
|
|
122
|
+
return (
|
|
123
|
+
related(
|
|
124
|
+
value,
|
|
125
|
+
{ type: 'PERSON', aliases: aliases(full, 'PERSON') },
|
|
126
|
+
true
|
|
127
|
+
) ||
|
|
128
|
+
full
|
|
129
|
+
.split(/\s+/u)
|
|
130
|
+
.some((word) => word && samePersonal(value, word, 'PERSON'))
|
|
131
|
+
);
|
|
132
|
+
}
|
|
95
133
|
const stem = (word) =>
|
|
96
134
|
word.replace(
|
|
97
135
|
/(?:овым|евым|овой|евой|ова|ева|ову|еву|ове|еве|ов|ев|ovym|evym|ovoi|evoi|ova|eva|ovu|evu|ove|eve|ov|ev)$/u,
|
|
@@ -107,7 +145,30 @@ function aliases(value, type) {
|
|
|
107
145
|
),
|
|
108
146
|
];
|
|
109
147
|
}
|
|
110
|
-
|
|
148
|
+
// A bare name is the same entity as its numbered form, including maps saved
|
|
149
|
+
// when the span stopped at the hyphen of `Вектор-99`.
|
|
150
|
+
function numberedOrganization(value, entry) {
|
|
151
|
+
return (entry.values ?? []).some((original) => {
|
|
152
|
+
const base = original.replace(/-\d*$/u, '');
|
|
153
|
+
return base !== original && samePersonal(value, base, entry.type);
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
function wordsMatchAlias(words, full, parts) {
|
|
157
|
+
if (words.length === 1) {
|
|
158
|
+
return parts && words[0].length > 1 && full.includes(words[0]);
|
|
159
|
+
}
|
|
160
|
+
return (
|
|
161
|
+
words.length > 1 &&
|
|
162
|
+
words.length === full.length &&
|
|
163
|
+
words.every(
|
|
164
|
+
(part, i) =>
|
|
165
|
+
part === full[i] || (part.length === 1 && full[i].startsWith(part))
|
|
166
|
+
)
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
// `parts` links a bare word to a fuller alias. Only callers that hold evidence
|
|
170
|
+
// from the current document may ask for it; a stored entry alone is not enough.
|
|
171
|
+
function related(value, entry, parts = false) {
|
|
111
172
|
if (
|
|
112
173
|
(entry.values ?? []).some((original) =>
|
|
113
174
|
samePersonal(value, original, entry.type)
|
|
@@ -119,49 +180,41 @@ function related(value, entry) {
|
|
|
119
180
|
if (candidates.some((alias) => entry.aliases.includes(alias))) {
|
|
120
181
|
return true;
|
|
121
182
|
}
|
|
183
|
+
if (entry.type === 'ORGANIZATION') {
|
|
184
|
+
return numberedOrganization(value, entry);
|
|
185
|
+
}
|
|
122
186
|
if (entry.type !== 'PERSON') {
|
|
123
187
|
return false;
|
|
124
188
|
}
|
|
125
|
-
|
|
126
|
-
const
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
parts[0].length > 1 &&
|
|
132
|
-
full.includes(parts[0])
|
|
133
|
-
) {
|
|
134
|
-
return true;
|
|
135
|
-
}
|
|
136
|
-
if (
|
|
137
|
-
parts.length === full.length &&
|
|
138
|
-
parts.length > 1 &&
|
|
139
|
-
parts.every(
|
|
140
|
-
(part, i) =>
|
|
141
|
-
part === full[i] || (part.length === 1 && full[i].startsWith(part))
|
|
142
|
-
)
|
|
143
|
-
) {
|
|
144
|
-
return true;
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
return false;
|
|
189
|
+
return candidates.some((candidate) => {
|
|
190
|
+
const words = candidate.match(/[\p{L}\p{M}]+/gu) ?? [];
|
|
191
|
+
return entry.aliases.some((alias) =>
|
|
192
|
+
wordsMatchAlias(words, alias.match(/[\p{L}\p{M}]+/gu) ?? [], parts)
|
|
193
|
+
);
|
|
194
|
+
});
|
|
149
195
|
}
|
|
150
196
|
export function placeholderNameFindings(text, findings, options) {
|
|
151
197
|
if (!hasPlaceholders(options) || options.knownOnly) {
|
|
152
198
|
return findings;
|
|
153
199
|
}
|
|
154
|
-
const
|
|
200
|
+
const state = options[placeholderState];
|
|
201
|
+
const patterns = new Set();
|
|
202
|
+
// Bare surname forms need the full name in this document. A stored person
|
|
203
|
+
// keeps only the initial form, which is itself a full mention.
|
|
204
|
+
for (const person of [
|
|
155
205
|
...findings
|
|
156
206
|
.filter((f) => f.type === 'PERSON')
|
|
157
207
|
.map((f) => text.slice(f.start, f.end)),
|
|
158
|
-
...(options
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
208
|
+
...earlierPeople(options),
|
|
209
|
+
]) {
|
|
210
|
+
addNamePatterns(person, patterns, true);
|
|
211
|
+
}
|
|
212
|
+
for (const entry of state?.map.entries ?? []) {
|
|
213
|
+
if (entry.type === 'PERSON') {
|
|
214
|
+
for (const person of entry.values) {
|
|
215
|
+
addNamePatterns(person, patterns, false);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
165
218
|
}
|
|
166
219
|
const result = [...findings];
|
|
167
220
|
for (const pattern of patterns) {
|
|
@@ -183,7 +236,56 @@ export function placeholderNameFindings(text, findings, options) {
|
|
|
183
236
|
}
|
|
184
237
|
return result;
|
|
185
238
|
}
|
|
186
|
-
|
|
239
|
+
// Stream batches and decoded layers are pieces of the document being replaced.
|
|
240
|
+
function earlierPeople(options) {
|
|
241
|
+
const state = options[placeholderState];
|
|
242
|
+
return state?.stream || options[placeholderNested]
|
|
243
|
+
? (state?.people ?? [])
|
|
244
|
+
: [];
|
|
245
|
+
}
|
|
246
|
+
const nameNeighbour = "\\p{Lu}[\\p{L}\\p{M}'’-]{1,31}";
|
|
247
|
+
const nameBefore = new RegExp(
|
|
248
|
+
`(?<![\\p{L}\\p{M}\\p{N}_-])(${nameNeighbour})[ \\t]+$`,
|
|
249
|
+
'u'
|
|
250
|
+
);
|
|
251
|
+
const nameAfter = new RegExp(
|
|
252
|
+
`^[ \\t]+(${nameNeighbour})(?![\\p{L}\\p{M}\\p{N}_])`,
|
|
253
|
+
'u'
|
|
254
|
+
);
|
|
255
|
+
const neighbourWindow = 64;
|
|
256
|
+
// `Алина` beside a bare `петрову` is the rest of that mention: replacing only the
|
|
257
|
+
// surname would leave that given name beside the token of the surname's owner.
|
|
258
|
+
export function coverAdjacentNames(text, findings, options) {
|
|
259
|
+
if (!hasPlaceholders(options)) {
|
|
260
|
+
return findings;
|
|
261
|
+
}
|
|
262
|
+
return findings.map((finding) => {
|
|
263
|
+
if (
|
|
264
|
+
finding.type !== 'PERSON' ||
|
|
265
|
+
!singleWord(text.slice(finding.start, finding.end))
|
|
266
|
+
) {
|
|
267
|
+
return finding;
|
|
268
|
+
}
|
|
269
|
+
let { start, end } = finding;
|
|
270
|
+
// A given name and a patronymic can both stand on one side of a surname.
|
|
271
|
+
for (let step = 0; step < 2; step++) {
|
|
272
|
+
const before = nameBefore.exec(
|
|
273
|
+
text.slice(Math.max(0, start - neighbourWindow), start)
|
|
274
|
+
);
|
|
275
|
+
if (before && isNameForm(before[1])) {
|
|
276
|
+
start -= before[0].length;
|
|
277
|
+
}
|
|
278
|
+
const after = nameAfter.exec(text.slice(end, end + neighbourWindow));
|
|
279
|
+
if (after && isNameForm(after[1])) {
|
|
280
|
+
end += after[0].length;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return start === finding.start && end === finding.end
|
|
284
|
+
? finding
|
|
285
|
+
: { ...finding, start, end, rule: 'name-variant' };
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
function addNamePatterns(person, patterns, bareForms) {
|
|
187
289
|
const words = person.match(/[\p{L}\p{M}]+/gu) ?? [];
|
|
188
290
|
if (words.length < 2 || words.some((word) => word.length < 2)) {
|
|
189
291
|
return;
|
|
@@ -196,7 +298,7 @@ function addNamePatterns(person, patterns) {
|
|
|
196
298
|
patterns.add(
|
|
197
299
|
`${escapePattern(parts[0][0])}\\.[ \\t]+${parts.slice(1).map(escapePattern).join('[ \\t]+')}`
|
|
198
300
|
);
|
|
199
|
-
for (const word of parts) {
|
|
301
|
+
for (const word of bareForms ? parts : []) {
|
|
200
302
|
if (/^[а-яё]+(?:ов|ев)$/u.test(word)) {
|
|
201
303
|
patterns.add(`${escapePattern(word)}(?:ым|ой|а|у|е|ы|ых|ами)?`);
|
|
202
304
|
}
|
|
@@ -230,14 +332,16 @@ export function preparePlaceholderText(text, findings, options) {
|
|
|
230
332
|
if (namespaceCollision(text, pattern)) {
|
|
231
333
|
throw failure('ERR_PLACEHOLDER_COLLISION');
|
|
232
334
|
}
|
|
233
|
-
const people = findings
|
|
234
|
-
.filter((f) => f.type === 'PERSON')
|
|
235
|
-
.map((f) => text.slice(f.start, f.end));
|
|
236
335
|
state.people = [
|
|
237
|
-
...
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
336
|
+
...new Set([
|
|
337
|
+
...earlierPeople(options),
|
|
338
|
+
...findings
|
|
339
|
+
.filter((f) => f.type === 'PERSON')
|
|
340
|
+
.map((f) => text.slice(f.start, f.end)),
|
|
341
|
+
...(options.knownPersonal ?? [])
|
|
342
|
+
.filter((e) => e.type === 'PERSON')
|
|
343
|
+
.map((e) => e.value),
|
|
344
|
+
]),
|
|
241
345
|
];
|
|
242
346
|
}
|
|
243
347
|
function namespaceCollision(text, pattern) {
|
|
@@ -257,14 +361,18 @@ function namespaceCollision(text, pattern) {
|
|
|
257
361
|
return false;
|
|
258
362
|
}
|
|
259
363
|
|
|
260
|
-
function canonicalPerson(value, people) {
|
|
364
|
+
export function canonicalPerson(value, people) {
|
|
261
365
|
// Resolve components only when there is a unique compatible full name.
|
|
262
366
|
const matches = people.filter((person) => {
|
|
263
367
|
const words = person.match(/\p{L}+/gu) ?? [];
|
|
264
368
|
return (
|
|
265
369
|
words.length > 1 &&
|
|
266
370
|
words.every((word) => word.length > 1) &&
|
|
267
|
-
related(
|
|
371
|
+
related(
|
|
372
|
+
value,
|
|
373
|
+
{ type: 'PERSON', aliases: aliases(person, 'PERSON') },
|
|
374
|
+
true
|
|
375
|
+
)
|
|
268
376
|
);
|
|
269
377
|
});
|
|
270
378
|
const distinct = new Map(
|
|
@@ -279,10 +387,36 @@ function canonicalKnown(value, type, options, state) {
|
|
|
279
387
|
const known = (options.knownPersonal ?? []).filter(
|
|
280
388
|
(e) => e.type === type && samePersonal(value, e.value, type)
|
|
281
389
|
);
|
|
282
|
-
|
|
283
|
-
|
|
390
|
+
// `Петров Данил` and `Петров Данил Сергеевич` are one person: the fullest wins.
|
|
391
|
+
const longest = Math.max(0, ...known.map((e) => e.value.length));
|
|
392
|
+
const fullest = known.filter((e) => e.value.length === longest);
|
|
393
|
+
const listed = fullest.length === 1 ? fullest[0].value : value;
|
|
394
|
+
if (type !== 'PERSON') {
|
|
395
|
+
return listed;
|
|
396
|
+
}
|
|
397
|
+
// The caller's list names each person once, while the document may hold the
|
|
398
|
+
// same full name in several cases, so the list settles a listed part first.
|
|
399
|
+
const named = canonicalPerson(
|
|
400
|
+
listed,
|
|
401
|
+
(options.knownPersonal ?? [])
|
|
402
|
+
.filter((e) => e.type === 'PERSON')
|
|
403
|
+
.map((e) => e.value)
|
|
404
|
+
);
|
|
405
|
+
return named === listed ? canonicalPerson(listed, state.people ?? []) : named;
|
|
406
|
+
}
|
|
407
|
+
// A part the caller listed is stored as its own value, unlike a part that was
|
|
408
|
+
// only recorded as a surface of a longer name.
|
|
409
|
+
function rememberListedParts(entry, value, options) {
|
|
410
|
+
for (const known of options.knownPersonal ?? []) {
|
|
411
|
+
if (
|
|
412
|
+
known.type === 'PERSON' &&
|
|
413
|
+
singleWord(known.value) &&
|
|
414
|
+
samePersonal(value, known.value, 'PERSON') &&
|
|
415
|
+
!entry.standalone?.includes(known.value)
|
|
416
|
+
) {
|
|
417
|
+
(entry.standalone ??= []).push(known.value);
|
|
418
|
+
}
|
|
284
419
|
}
|
|
285
|
-
return type === 'PERSON' ? canonicalPerson(value, state.people ?? []) : value;
|
|
286
420
|
}
|
|
287
421
|
export function placeholderReplacement(value, finding, transform, options) {
|
|
288
422
|
if (finding.category === 'credential' && !transform.credentials) {
|
|
@@ -303,7 +437,12 @@ export function placeholderReplacement(value, finding, transform, options) {
|
|
|
303
437
|
const candidates = map.entries.filter(
|
|
304
438
|
(entry) => entry.type === type && related(canonical, entry)
|
|
305
439
|
);
|
|
306
|
-
|
|
440
|
+
// Several related entities are ambiguous, unless one holds this exact value.
|
|
441
|
+
const exact = candidates.filter((entry) =>
|
|
442
|
+
entry.values.some((original) => samePersonal(canonical, original, type))
|
|
443
|
+
);
|
|
444
|
+
const matches = candidates.length === 1 ? candidates : exact;
|
|
445
|
+
let entry = matches.length === 1 ? matches[0] : undefined;
|
|
307
446
|
if (!entry) {
|
|
308
447
|
if (map.entries.length >= (options.maxFindings ?? 100000)) {
|
|
309
448
|
throw failure('ERR_LIMIT');
|
|
@@ -333,6 +472,9 @@ export function placeholderReplacement(value, finding, transform, options) {
|
|
|
333
472
|
if (!entry.values.includes(value)) {
|
|
334
473
|
entry.values.push(value);
|
|
335
474
|
}
|
|
475
|
+
if (type === 'PERSON') {
|
|
476
|
+
rememberListedParts(entry, value, options);
|
|
477
|
+
}
|
|
336
478
|
return entry.token;
|
|
337
479
|
}
|
|
338
480
|
export function finishPlaceholders(result, occurrences, options) {
|
package/src/propagation.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { personalPattern, maxPropagationLength, failure } from './detection.js';
|
|
2
2
|
import { actionable } from './confidence.js';
|
|
3
|
-
import { organizationCore } from './personal-variants.js';
|
|
3
|
+
import { organizationCore, organizationFamily } from './personal-variants.js';
|
|
4
4
|
import { nameAliases } from './transliteration.js';
|
|
5
5
|
|
|
6
6
|
function mrzSeeds(value, finding, seeds) {
|
|
@@ -50,12 +50,16 @@ function eligibleSeed(finding, options) {
|
|
|
50
50
|
].includes(finding.type))
|
|
51
51
|
);
|
|
52
52
|
}
|
|
53
|
-
function organizationSeed(value, finding, seeds, options) {
|
|
53
|
+
function organizationSeed(text, value, finding, seeds, options) {
|
|
54
54
|
if (options.documentEntities && finding.type === 'ORGANIZATION') {
|
|
55
55
|
const core = organizationCore(value);
|
|
56
56
|
if (core.length >= 3) {
|
|
57
57
|
seeds.set(core, finding);
|
|
58
58
|
}
|
|
59
|
+
const family = organizationFamily(core);
|
|
60
|
+
for (const match of family ? text.matchAll(family.pattern) : []) {
|
|
61
|
+
seeds.set(match[0], finding);
|
|
62
|
+
}
|
|
59
63
|
}
|
|
60
64
|
}
|
|
61
65
|
// Only confirmed values seed propagation. A low-score numeric coincidence
|
|
@@ -82,7 +86,7 @@ export function confirmedSeeds(text, findings, options) {
|
|
|
82
86
|
mrzSeeds(value, finding, seeds);
|
|
83
87
|
} else if (value.length >= 2) {
|
|
84
88
|
seeds.set(value, finding);
|
|
85
|
-
organizationSeed(value, finding, seeds, options);
|
|
89
|
+
organizationSeed(text, value, finding, seeds, options);
|
|
86
90
|
if (finding.type === 'PERSON') {
|
|
87
91
|
for (const name of value.match(/\p{L}{2,}/gu) ?? []) {
|
|
88
92
|
seeds.set(name, finding);
|
package/src/restore-map.js
CHANGED
|
@@ -85,7 +85,8 @@ function validateEntries(map) {
|
|
|
85
85
|
tokens.has(entry.token) ||
|
|
86
86
|
!nonemptyStrings(entry.values) ||
|
|
87
87
|
!Array.isArray(entry.aliases) ||
|
|
88
|
-
entry.aliases.some((v) => typeof v !== 'string' || !v)
|
|
88
|
+
entry.aliases.some((v) => typeof v !== 'string' || !v) ||
|
|
89
|
+
(entry.standalone !== undefined && !nonemptyStrings(entry.standalone))
|
|
89
90
|
) {
|
|
90
91
|
throw failure('ERR_MAP');
|
|
91
92
|
}
|