@link-foundation/sensitive-data-sanitizer 0.18.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +8 -0
- package/docs/API.md +216 -39
- package/docs/COVERAGE.md +122 -15
- package/docs/RULES.md +405 -0
- package/examples/custom-log-policy.mjs +23 -0
- package/package.json +17 -2
- package/src/compiled-known.js +55 -0
- package/src/detection.js +9 -0
- package/src/document-catalog.js +246 -0
- package/src/document-entities.js +10 -2
- package/src/document-experience.js +9 -4
- package/src/document-extras.js +2 -2
- package/src/document-fields.js +12 -2
- package/src/document-patterns.js +22 -2
- package/src/document-person-cues.js +35 -0
- package/src/document-vocabulary.js +80 -68
- package/src/engines.js +81 -34
- package/src/index.d.ts +52 -4
- package/src/index.js +5 -0
- package/src/jsonl-stream.js +2 -5
- package/src/ner.d.ts +5 -0
- package/src/ner.js +96 -0
- package/src/personal-variants.js +29 -8
- package/src/policy.js +107 -0
- package/src/propagation.js +2 -1
- package/src/proper-names.js +26 -3
- package/src/record-worker.js +6 -0
- package/src/rule-catalog.js +672 -0
- package/src/rules.js +5 -0
- package/src/sanitizer.js +99 -19
- package/src/stream-worker.js +5 -0
- package/src/stream.js +11 -6
- package/src/strict.js +113 -0
- package/src/structured.js +133 -48
- package/src/sync-detectors.js +38 -0
- package/src/sync-sanitizer.js +16 -0
- package/src/transforms.js +85 -7
- package/src/vocabulary.js +226 -0
- package/src/worker-options.js +12 -0
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { collectMatches } from './detection.js';
|
|
2
|
+
import { isDocumentTechnology } from './document-experience.js';
|
|
3
|
+
|
|
4
|
+
const b = '[\\p{L}\\p{N}_]';
|
|
5
|
+
const proper = '[\\p{Lu}][\\p{L}\\p{M}.-]{1,47}';
|
|
6
|
+
const rules = [
|
|
7
|
+
[
|
|
8
|
+
'PHONE',
|
|
9
|
+
'document-local-phone',
|
|
10
|
+
/(?<![\p{L}\p{N}_])(?:тел\.?|телефон|phone)[ \t]*:?[ \t]*(\d{7})(?!\d)/giu,
|
|
11
|
+
1,
|
|
12
|
+
],
|
|
13
|
+
[
|
|
14
|
+
'PHONE',
|
|
15
|
+
'document-spaced-mobile',
|
|
16
|
+
/(?<![\p{L}\p{N}_])8[ \t]+9(?:[ \t]+\d){9}(?!\d)/gu,
|
|
17
|
+
0,
|
|
18
|
+
],
|
|
19
|
+
[
|
|
20
|
+
'EMAIL',
|
|
21
|
+
'document-email-obfuscation',
|
|
22
|
+
/(?<![\p{L}\p{N}_])[\p{L}\p{N}_.+-]{1,64}\(собака\)[\p{L}\p{N}-]{1,63}(?:\(точка\)[\p{L}\p{N}-]{1,63}){1,6}(?![\p{L}\p{N}_])/giu,
|
|
23
|
+
0,
|
|
24
|
+
],
|
|
25
|
+
[
|
|
26
|
+
'EMAIL',
|
|
27
|
+
'document-email-spaces',
|
|
28
|
+
/(?<![\p{L}\p{N}_])(?:[a-z][ \t]+){1,63}[a-z][ \t]*@[ \t]*[a-z\d-]{1,63}[ \t]*\.[ \t]*[a-z]{2,20}(?![\p{L}\p{N}_])/giu,
|
|
29
|
+
0,
|
|
30
|
+
],
|
|
31
|
+
[
|
|
32
|
+
'USERNAME',
|
|
33
|
+
'document-messenger',
|
|
34
|
+
/(?<![\p{L}\p{N}_])(?:discord|max)[ \t]*:[ \t]*([a-z][a-z\d_.-]{1,63}(?:#\d{4})?)(?![\p{L}\p{N}_])/giu,
|
|
35
|
+
1,
|
|
36
|
+
],
|
|
37
|
+
[
|
|
38
|
+
'URL',
|
|
39
|
+
'document-social-link',
|
|
40
|
+
/(?<![\p{L}\p{N}_./-])(?:https?:\/\/)?(?:www\.)?(?:ok\.ru\/profile|gitlab\.com|habr\.com\/ru\/users|facebook\.com)\/[\p{L}\p{N}_.-]{2,100}(?:\/)?(?:[?#][^\s"<>\\]*)?/giu,
|
|
41
|
+
0,
|
|
42
|
+
],
|
|
43
|
+
[
|
|
44
|
+
'URL',
|
|
45
|
+
'document-resume-link',
|
|
46
|
+
/https?:\/\/[^\s/"<>\\]+\/(?:cv\.pdf(?:[?#][^\s"<>\\]*)?|cv(?:\/[^\s"<>\\]*)?|~[\p{L}\p{N}_.-]{1,100}\/?)/giu,
|
|
47
|
+
0,
|
|
48
|
+
],
|
|
49
|
+
[
|
|
50
|
+
'DATE_OF_BIRTH',
|
|
51
|
+
'document-birthday',
|
|
52
|
+
/(?<![\p{L}\p{N}_])(?:д\.р\.|день рождения)[ \t]*:?[ \t]*(\d{1,2}(?:[/.]\d{1,2}[/.]\d{2,4}|[ \t]+(?:января|февраля|марта|апреля|мая|июня|июля|августа|сентября|октября|ноября|декабря)(?:[ \t]+\d{4})?))(?!\d)/giu,
|
|
53
|
+
1,
|
|
54
|
+
],
|
|
55
|
+
[
|
|
56
|
+
'AGE',
|
|
57
|
+
'document-age-unit',
|
|
58
|
+
/(?<![\p{L}\p{N}_])\d{1,3}[ \t]*(?:y\.o\.|лет[ \t]+от[ \t]+роду)(?![\p{L}\p{N}_])/giu,
|
|
59
|
+
0,
|
|
60
|
+
],
|
|
61
|
+
[
|
|
62
|
+
'ID',
|
|
63
|
+
'document-military-id',
|
|
64
|
+
/(?<![\p{L}\p{N}_])военный билет[ \t]+([А-ЯЁ]{2}[ \t]+\d{7})(?!\d)/giu,
|
|
65
|
+
1,
|
|
66
|
+
],
|
|
67
|
+
[
|
|
68
|
+
'VIN',
|
|
69
|
+
'document-vin',
|
|
70
|
+
/(?<![\p{L}\p{N}_])VIN[ \t]*:?[ \t]*([A-HJ-NPR-Z\d]{17})(?![\p{L}\p{N}_])/giu,
|
|
71
|
+
1,
|
|
72
|
+
],
|
|
73
|
+
[
|
|
74
|
+
'ORGANIZATION',
|
|
75
|
+
'document-legal-lowercase',
|
|
76
|
+
/(?<![\p{L}\p{N}_])(?:ооо|зао|пао|ао)[ \t]+[а-яё][а-яё-]{1,47}(?![\p{L}\p{N}_])/gu,
|
|
77
|
+
0,
|
|
78
|
+
],
|
|
79
|
+
[
|
|
80
|
+
'ORGANIZATION',
|
|
81
|
+
'document-employer-dash',
|
|
82
|
+
new RegExp(
|
|
83
|
+
`(?<!${b})(?:Компания|Место работы)[ \\t]+[-–—][ \\t]+(${proper})(?!${b})`,
|
|
84
|
+
'gu'
|
|
85
|
+
),
|
|
86
|
+
1,
|
|
87
|
+
],
|
|
88
|
+
[
|
|
89
|
+
'ORGANIZATION',
|
|
90
|
+
'document-employer-prefix',
|
|
91
|
+
new RegExp(`(?<!${b})(?:Группа|стартап)[ \\t]+(${proper})(?!${b})`, 'gu'),
|
|
92
|
+
1,
|
|
93
|
+
],
|
|
94
|
+
[
|
|
95
|
+
'ORGANIZATION',
|
|
96
|
+
'document-bank-suffix',
|
|
97
|
+
new RegExp(`(?<!${b})${proper}[ \\t]+Банк(?!${b})`, 'gu'),
|
|
98
|
+
0,
|
|
99
|
+
],
|
|
100
|
+
[
|
|
101
|
+
'ORGANIZATION',
|
|
102
|
+
'document-industry',
|
|
103
|
+
new RegExp(
|
|
104
|
+
`(?<!${b})(${proper})(?=[ \\t]*\\((?:ритейл|retail|финансы|finance|IT)\\))`,
|
|
105
|
+
'gu'
|
|
106
|
+
),
|
|
107
|
+
1,
|
|
108
|
+
],
|
|
109
|
+
[
|
|
110
|
+
'ORGANIZATION',
|
|
111
|
+
'document-reversed-role',
|
|
112
|
+
new RegExp(
|
|
113
|
+
`(?<!${b})(${proper})[ \\t]*(?:,|[-–—])[ \\t]*(?:менеджер|manager|директор|developer|инженер)(?!${b})`,
|
|
114
|
+
'gu'
|
|
115
|
+
),
|
|
116
|
+
1,
|
|
117
|
+
],
|
|
118
|
+
[
|
|
119
|
+
'ORGANIZATION',
|
|
120
|
+
'document-role-employer',
|
|
121
|
+
new RegExp(
|
|
122
|
+
`(?<!${b})(?:менеджер|директор|инженер)[ \\t]+(?:в[ \\t]+)?(${proper})(?!${b})`,
|
|
123
|
+
'gu'
|
|
124
|
+
),
|
|
125
|
+
1,
|
|
126
|
+
],
|
|
127
|
+
[
|
|
128
|
+
'ORGANIZATION',
|
|
129
|
+
'document-at-employer',
|
|
130
|
+
/(?<![\p{L}\p{N}_])at[ \t]+([A-Z][a-z][\p{L}.-]{0,45})(?![\p{L}\p{N}_])/gu,
|
|
131
|
+
1,
|
|
132
|
+
],
|
|
133
|
+
[
|
|
134
|
+
'EDUCATION',
|
|
135
|
+
'document-school-context',
|
|
136
|
+
/(?<![\p{L}\p{N}_])(?:graduated from|бакалавр,?|диплом|колледж|[Кк]урсы|курс на)[ \t]+([\p{Lu}][\p{L}.-]{1,47})(?![\p{L}\p{N}_])/gu,
|
|
137
|
+
1,
|
|
138
|
+
],
|
|
139
|
+
[
|
|
140
|
+
'EDUCATION',
|
|
141
|
+
'document-school-year',
|
|
142
|
+
/(?<![\p{L}\p{N}_])[А-ЯЁA-Z]{2,12}(?=,[ \t]*(?:19|20)\d{2}(?!\d))/gu,
|
|
143
|
+
0,
|
|
144
|
+
],
|
|
145
|
+
[
|
|
146
|
+
'EDUCATION',
|
|
147
|
+
'document-msu',
|
|
148
|
+
/(?<![\p{L}\p{N}_])МГУ(?:[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{1,40})?(?![\p{L}\p{N}_])/gu,
|
|
149
|
+
0,
|
|
150
|
+
],
|
|
151
|
+
[
|
|
152
|
+
'PERSON',
|
|
153
|
+
'document-patronymic',
|
|
154
|
+
/(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё]{2,32}(?:овна|евна|ична|ович|евич)(?![\p{L}\p{N}_])/gu,
|
|
155
|
+
0,
|
|
156
|
+
],
|
|
157
|
+
[
|
|
158
|
+
'PERSON',
|
|
159
|
+
'document-uppercase-initials',
|
|
160
|
+
/(?<![\p{L}\p{N}_])[А-ЯЁ]{2,40}[ \t]+(?:[А-ЯЁA-Z]\.[ \t]*){1,2}(?![\p{L}\p{N}_])/gu,
|
|
161
|
+
0,
|
|
162
|
+
],
|
|
163
|
+
[
|
|
164
|
+
'PERSON',
|
|
165
|
+
'document-unfamiliar-name',
|
|
166
|
+
/(?<![\p{L}\p{N}_])(?:[А-ЯЁ][а-яё]{1,30}[ \t]+(?:Ким|[А-ЯЁ][а-яё]{1,25}(?:чук|енко|швили|дзе))|Ли[ \t]+[А-ЯЁ][а-яё]{1,20})(?![\p{L}\p{N}_])/gu,
|
|
167
|
+
0,
|
|
168
|
+
],
|
|
169
|
+
[
|
|
170
|
+
'ADDRESS',
|
|
171
|
+
'document-street-complete',
|
|
172
|
+
/(?<![\p{L}\p{N}_])(?:(?:ул\.?|улица|пр-т)[ \t]+[А-ЯЁ][а-яё-]{1,40},?[ \t]+(?:д\.[ \t]*)?\d{1,5}(?:-\d{1,5})?|[А-ЯЁ][а-яё-]{2,40}[ \t]+шоссе,[ \t]*\d{1,5})(?:,[ \t]*кв\.?[ \t]*\d{1,5})?(?!\d)/gu,
|
|
173
|
+
0,
|
|
174
|
+
],
|
|
175
|
+
[
|
|
176
|
+
'LOCATION',
|
|
177
|
+
'document-region',
|
|
178
|
+
/(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё-]{2,40}[ \t]+(?:область|район)(?![\p{L}\p{N}_])/gu,
|
|
179
|
+
0,
|
|
180
|
+
],
|
|
181
|
+
[
|
|
182
|
+
'PERSONAL_ATTRIBUTE',
|
|
183
|
+
'document-citizenship',
|
|
184
|
+
/(?<![\p{L}\p{N}_])(?:гражданство|гражданин|гражданка)[ \t]+([А-ЯЁ][А-ЯЁа-яё-]{1,40})(?![\p{L}\p{N}_])/gu,
|
|
185
|
+
1,
|
|
186
|
+
],
|
|
187
|
+
[
|
|
188
|
+
'LICENSE_PLATE',
|
|
189
|
+
'document-vehicle-plate',
|
|
190
|
+
/(?<![\p{L}\p{N}_])[АВЕКМНОРСТУХABEKMHOPCTYX]\d{3}[АВЕКМНОРСТУХABEKMHOPCTYX]{2}[ \t]*\d{2,3}(?:[ \t]+RUS)?(?![\p{L}\p{N}_])/gu,
|
|
191
|
+
0,
|
|
192
|
+
],
|
|
193
|
+
];
|
|
194
|
+
|
|
195
|
+
export function detectDocumentCatalog(text, emit) {
|
|
196
|
+
for (const [type, rule, pattern, group] of rules) {
|
|
197
|
+
collectMatches(
|
|
198
|
+
text,
|
|
199
|
+
pattern,
|
|
200
|
+
emit,
|
|
201
|
+
{ type, rule, category: 'personal', confidence: 0.85 },
|
|
202
|
+
group,
|
|
203
|
+
(value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
// Lookalike substitution preserves UTF-16 offsets and is limited to words
|
|
207
|
+
// containing Cyrillic plus Latin letters. Ordinary English is untouched.
|
|
208
|
+
const lookalikes = {
|
|
209
|
+
A: 'А',
|
|
210
|
+
a: 'а',
|
|
211
|
+
B: 'В',
|
|
212
|
+
C: 'С',
|
|
213
|
+
c: 'с',
|
|
214
|
+
E: 'Е',
|
|
215
|
+
e: 'е',
|
|
216
|
+
H: 'Н',
|
|
217
|
+
K: 'К',
|
|
218
|
+
M: 'М',
|
|
219
|
+
O: 'О',
|
|
220
|
+
o: 'о',
|
|
221
|
+
P: 'Р',
|
|
222
|
+
p: 'р',
|
|
223
|
+
T: 'Т',
|
|
224
|
+
X: 'Х',
|
|
225
|
+
x: 'х',
|
|
226
|
+
y: 'у',
|
|
227
|
+
};
|
|
228
|
+
const folded = text.replace(/[\p{L}]+/gu, (word) =>
|
|
229
|
+
/[А-Яа-яЁё]/u.test(word)
|
|
230
|
+
? word.replace(/[AaBCcEeHKMOoPpTXxy]/g, (c) => lookalikes[c])
|
|
231
|
+
: word
|
|
232
|
+
);
|
|
233
|
+
if (folded !== text) {
|
|
234
|
+
collectMatches(
|
|
235
|
+
folded,
|
|
236
|
+
/(?<![\p{L}\p{N}_])[А-ЯЁ][а-яё]{1,30}[ \t]+[А-ЯЁ][а-яё]{1,28}(?:ова|ева|ина|ов|ев|ин|ский|ская)(?![\p{L}\p{N}_])/gu,
|
|
237
|
+
emit,
|
|
238
|
+
{
|
|
239
|
+
type: 'PERSON',
|
|
240
|
+
rule: 'document-mixed-name',
|
|
241
|
+
category: 'personal',
|
|
242
|
+
confidence: 0.9,
|
|
243
|
+
}
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
}
|
package/src/document-entities.js
CHANGED
|
@@ -6,6 +6,9 @@ import { documentDateRange as range } from './document-dates.js';
|
|
|
6
6
|
import { detectDocumentExtras } from './document-extras.js';
|
|
7
7
|
import { detectDocumentPatterns } from './document-patterns.js';
|
|
8
8
|
import { isDocumentTechnology } from './document-experience.js';
|
|
9
|
+
import { detectVocabulary } from './vocabulary.js';
|
|
10
|
+
import { detectDocumentPersonCues } from './document-person-cues.js';
|
|
11
|
+
import { detectDocumentCatalog } from './document-catalog.js';
|
|
9
12
|
|
|
10
13
|
const words = (column) =>
|
|
11
14
|
[...new Set(documentVocabulary.flatMap((row) => row[column].split('|')))]
|
|
@@ -79,8 +82,11 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
79
82
|
if (!options.documentEntities) {
|
|
80
83
|
return;
|
|
81
84
|
}
|
|
85
|
+
detectVocabulary(text, emit, options);
|
|
86
|
+
detectDocumentPersonCues(text, emit);
|
|
87
|
+
detectDocumentCatalog(text, emit);
|
|
82
88
|
detectDocumentPatterns(text, emit);
|
|
83
|
-
detectDocumentExtras(text, emit, range, options.debug);
|
|
89
|
+
detectDocumentExtras(text, emit, range, options.debug, options);
|
|
84
90
|
collectMatches(
|
|
85
91
|
text,
|
|
86
92
|
new RegExp(
|
|
@@ -120,7 +126,9 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
120
126
|
capture,
|
|
121
127
|
{ type, rule, category: 'personal', confidence },
|
|
122
128
|
group,
|
|
123
|
-
(value) =>
|
|
129
|
+
(value) =>
|
|
130
|
+
type !== 'ORGANIZATION' ||
|
|
131
|
+
(!isDocumentTechnology(value) && value !== 'ОГРНИП')
|
|
124
132
|
);
|
|
125
133
|
}
|
|
126
134
|
for (const match of text.matchAll(ruResidence)) {
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { compiledVocabulary } from './vocabulary.js';
|
|
2
|
+
|
|
1
3
|
const technology = new Set(
|
|
2
4
|
[
|
|
3
5
|
'Python',
|
|
@@ -45,7 +47,7 @@ function commaParts(text) {
|
|
|
45
47
|
return parts;
|
|
46
48
|
}
|
|
47
49
|
|
|
48
|
-
function employerSpan(prefix) {
|
|
50
|
+
function employerSpan(prefix, roles) {
|
|
49
51
|
// An industry suffix belongs to the entry, but is not an employer. A pipe
|
|
50
52
|
// before the title remains a section separator, as in `Опыт | Менеджер, ...`.
|
|
51
53
|
const comma = prefix.indexOf(',');
|
|
@@ -59,7 +61,7 @@ function employerSpan(prefix) {
|
|
|
59
61
|
if (/^[=\s\d]/u.test(parts[0].text)) {
|
|
60
62
|
index++;
|
|
61
63
|
}
|
|
62
|
-
if (parts.length === 2 &&
|
|
64
|
+
if (parts.length === 2 && roles.test(parts[1].text)) {
|
|
63
65
|
index = 0;
|
|
64
66
|
} else {
|
|
65
67
|
while (index < parts.length - 1 && /^\p{Ll}/u.test(parts[index].text)) {
|
|
@@ -77,7 +79,7 @@ function employerSpan(prefix) {
|
|
|
77
79
|
: { start, end };
|
|
78
80
|
}
|
|
79
81
|
|
|
80
|
-
export function detectDatedEmployers(text, emit, range, debug) {
|
|
82
|
+
export function detectDatedEmployers(text, emit, range, debug, options = {}) {
|
|
81
83
|
let cursor = 0,
|
|
82
84
|
count = 0;
|
|
83
85
|
// Match only the anchor, then visit each preceding segment once. No lazy
|
|
@@ -91,7 +93,10 @@ export function detectDatedEmployers(text, emit, range, debug) {
|
|
|
91
93
|
segment.lastIndexOf(';')
|
|
92
94
|
) + 1;
|
|
93
95
|
const prefix = segment.slice(boundary).replace(/^[., \t]+/u, '');
|
|
94
|
-
const span = employerSpan(
|
|
96
|
+
const span = employerSpan(
|
|
97
|
+
prefix,
|
|
98
|
+
compiledVocabulary(options.vocabulary)?.roles ?? reversedRole
|
|
99
|
+
);
|
|
95
100
|
if (span) {
|
|
96
101
|
const offset = match.index - prefix.length;
|
|
97
102
|
emit({
|
package/src/document-extras.js
CHANGED
|
@@ -18,7 +18,7 @@ const duration = new RegExp(
|
|
|
18
18
|
`(?<!${boundary})(${name})(?=[ \\t]*\\(\\d{1,2}[ \\t]+(?:год|года|лет|years?)\\))`,
|
|
19
19
|
'gu'
|
|
20
20
|
);
|
|
21
|
-
export function detectDocumentExtras(text, emit, range, debug) {
|
|
21
|
+
export function detectDocumentExtras(text, emit, range, debug, options) {
|
|
22
22
|
for (const [type, rule, pattern, group] of [
|
|
23
23
|
['PHONE', 'document-mobile-bare', /^[78]9\d{9}$/gm, 0],
|
|
24
24
|
[
|
|
@@ -128,7 +128,7 @@ export function detectDocumentExtras(text, emit, range, debug) {
|
|
|
128
128
|
isDocumentShareLink
|
|
129
129
|
);
|
|
130
130
|
detectEmployerContinuation(text, emit);
|
|
131
|
-
detectDatedEmployers(text, emit, range, debug);
|
|
131
|
+
detectDatedEmployers(text, emit, range, debug, options);
|
|
132
132
|
}
|
|
133
133
|
|
|
134
134
|
function detectEmployerContinuation(text, emit) {
|
package/src/document-fields.js
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
// One explicit field vocabulary; camelCase, snake_case and kebab-case normalize alike.
|
|
2
2
|
const groups = {
|
|
3
|
-
PERSON: [
|
|
4
|
-
|
|
3
|
+
PERSON: [
|
|
4
|
+
'firstname',
|
|
5
|
+
'lastname',
|
|
6
|
+
'middlename',
|
|
7
|
+
'patronymic',
|
|
8
|
+
'candidatename',
|
|
9
|
+
'displayname',
|
|
10
|
+
'fullname',
|
|
11
|
+
'originalfilename',
|
|
12
|
+
],
|
|
13
|
+
AGE: ['age'],
|
|
14
|
+
DATE_OF_BIRTH: ['birthday', 'birthdate', 'birthyear'],
|
|
5
15
|
GENDER: ['sex'],
|
|
6
16
|
PERSONAL_ATTRIBUTE: ['citizenship', 'nationality'],
|
|
7
17
|
ORGANIZATION: ['company', 'companyname', 'employer'],
|
package/src/document-patterns.js
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { firstNames } from './names.js';
|
|
2
2
|
import { isNameForm } from './russian-names.js';
|
|
3
|
+
import { isDocumentTechnology } from './document-experience.js';
|
|
3
4
|
|
|
4
5
|
const boundary = '[\\p{L}\\p{N}_]';
|
|
5
6
|
const word = '[А-ЯЁ][а-яё-]{1,47}';
|
|
7
|
+
const emailDot =
|
|
8
|
+
'(?:\\.|[ \\t]{0,8}(?:\\((?:dot|точка)\\)|\\[(?:dot|точка)\\])[ \\t]{0,8}|[ \\t]{1,8}(?:dot|точка)[ \\t]{1,8})';
|
|
9
|
+
const emailAt =
|
|
10
|
+
'(?:[ \\t]{0,8}(?:\\((?:at|собака)\\)|\\[(?:at|собака)\\])[ \\t]{0,8}|[ \\t]{1,8}(?:at|собака)[ \\t]{1,8})';
|
|
6
11
|
const surname =
|
|
7
12
|
'[А-ЯЁа-яё][А-ЯЁа-яё-]{1,28}(?:ов|ев|ин|ын|ск|цк)(?:а|ой|ою|у|ым|ом|е|ы|и|их|ий|ая|ого|ому|ую|ие)?';
|
|
8
13
|
const initials = '(?:[А-ЯЁA-Z]\\.[ \\t]*){1,2}';
|
|
@@ -36,7 +41,10 @@ const rules = [
|
|
|
36
41
|
[
|
|
37
42
|
'EMAIL',
|
|
38
43
|
'obfuscated-email',
|
|
39
|
-
|
|
44
|
+
new RegExp(
|
|
45
|
+
`(?<![\\p{L}\\p{N}_.+\\\\-])[\\p{L}\\p{N}_+-]{1,64}(?:${emailDot}[\\p{L}\\p{N}_+-]{1,64}){0,4}${emailAt}[\\p{L}\\p{N}-]{1,63}(?:${emailDot}[\\p{L}\\p{N}-]{1,63}){1,8}(?![\\p{L}\\p{N}_])`,
|
|
46
|
+
'giu'
|
|
47
|
+
),
|
|
40
48
|
0,
|
|
41
49
|
],
|
|
42
50
|
[
|
|
@@ -142,6 +150,12 @@ const rules = [
|
|
|
142
150
|
),
|
|
143
151
|
1,
|
|
144
152
|
],
|
|
153
|
+
[
|
|
154
|
+
'ORGANIZATION',
|
|
155
|
+
'employer-prose',
|
|
156
|
+
/(?<![\p{L}\p{N}_])[Рр]аботал[аи]?[ \t]+в[ \t]+([\p{Ll}][\p{Ll}-]{1,47})(?![\p{L}\p{N}_])/gu,
|
|
157
|
+
1,
|
|
158
|
+
],
|
|
145
159
|
[
|
|
146
160
|
'ORGANIZATION',
|
|
147
161
|
'employer-years',
|
|
@@ -237,6 +251,12 @@ function documentValueAllowed(rule, value) {
|
|
|
237
251
|
rule === 'profile-attribute' &&
|
|
238
252
|
/^(?:не указан[оаы]?|not specified|unknown|n\/a)$/iu.test(value)
|
|
239
253
|
) &&
|
|
240
|
-
!(
|
|
254
|
+
!(
|
|
255
|
+
rule === 'messenger-label' && /^(?:недоступен|unavailable)$/i.test(value)
|
|
256
|
+
) &&
|
|
257
|
+
!(
|
|
258
|
+
rule === 'employer-prose' &&
|
|
259
|
+
(value === 'компании' || isDocumentTechnology(value))
|
|
260
|
+
)
|
|
241
261
|
);
|
|
242
262
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { collectMatches } from './detection.js';
|
|
2
|
+
|
|
3
|
+
// A cue supplies evidence for a single Latin name without activating broad
|
|
4
|
+
// capitalized-word matching for the rest of an English log or document.
|
|
5
|
+
const boundary = '[\\p{L}\\p{M}\\p{N}_]';
|
|
6
|
+
const name = "[A-Z][a-z]{1,47}(?:['’-][A-Z]?[a-z]{1,47})?";
|
|
7
|
+
const cue =
|
|
8
|
+
'(?:[Cc]andidate|[Rr]ecruiter|[Aa]pplicant|[Dd]ear|[Hh]i|[Mm]r\\.?|[Mm]s\\.?|[Dd]r\\.?|[Mm]essage[ \\t]+from)';
|
|
9
|
+
const pattern = new RegExp(
|
|
10
|
+
`(?<!${boundary})${cue}[ \\t]+(${name})(?!${boundary})`,
|
|
11
|
+
'gu'
|
|
12
|
+
);
|
|
13
|
+
// Workflow headings use the same person-role cues. Do not treat these finite
|
|
14
|
+
// generic nouns as names merely because a heading is in title case.
|
|
15
|
+
const labels = new Set(
|
|
16
|
+
'stage status name profile details information summary experience application tracking interview selection pipeline request failed'.split(
|
|
17
|
+
' '
|
|
18
|
+
)
|
|
19
|
+
);
|
|
20
|
+
|
|
21
|
+
export function detectDocumentPersonCues(text, emit) {
|
|
22
|
+
collectMatches(
|
|
23
|
+
text,
|
|
24
|
+
pattern,
|
|
25
|
+
emit,
|
|
26
|
+
{
|
|
27
|
+
type: 'PERSON',
|
|
28
|
+
rule: 'document-person-cue',
|
|
29
|
+
category: 'personal',
|
|
30
|
+
confidence: 0.85,
|
|
31
|
+
},
|
|
32
|
+
1,
|
|
33
|
+
(value) => !labels.has(value.toLowerCase())
|
|
34
|
+
);
|
|
35
|
+
}
|
|
@@ -1,71 +1,83 @@
|
|
|
1
1
|
// Authored, deliberately finite vocabulary; no network or model downloads.
|
|
2
2
|
// Each row follows localeVocabulary: locale, legal forms, school types, places.
|
|
3
|
-
export const documentVocabulary =
|
|
3
|
+
export const documentVocabulary = Object.freeze(
|
|
4
4
|
[
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
'
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
'
|
|
70
|
-
|
|
71
|
-
|
|
5
|
+
[
|
|
6
|
+
'en',
|
|
7
|
+
'Inc|LLC|Ltd|PLC|Corporation',
|
|
8
|
+
'University|College|Institute|School',
|
|
9
|
+
'Springfield|Riverton|London|New York|California',
|
|
10
|
+
],
|
|
11
|
+
[
|
|
12
|
+
'es',
|
|
13
|
+
'S.A.|S.L.|S.L.U.',
|
|
14
|
+
'Universidad|Colegio|Instituto|Escuela',
|
|
15
|
+
'Madrid|Barcelona|Andalucía',
|
|
16
|
+
],
|
|
17
|
+
[
|
|
18
|
+
'fr',
|
|
19
|
+
'SARL|SAS|SA|S.A.',
|
|
20
|
+
'Université|Collège|Institut|École',
|
|
21
|
+
'Paris|Lyon|Bretagne',
|
|
22
|
+
],
|
|
23
|
+
[
|
|
24
|
+
'de',
|
|
25
|
+
'GmbH|AG|KG|UG',
|
|
26
|
+
'Universität|Hochschule|Institut|Schule',
|
|
27
|
+
'Berlin|München|Bayern',
|
|
28
|
+
],
|
|
29
|
+
[
|
|
30
|
+
'pt',
|
|
31
|
+
'Ltda|S.A.|Unipessoal',
|
|
32
|
+
'Universidade|Faculdade|Instituto|Escola',
|
|
33
|
+
'Lisboa|São Paulo|Algarve',
|
|
34
|
+
],
|
|
35
|
+
[
|
|
36
|
+
'ru',
|
|
37
|
+
'ООО|АО|ПАО|ИП|ЗАО',
|
|
38
|
+
'Университет|университет|Институт|институт|Школа|школа',
|
|
39
|
+
'Москва|Санкт-Петербург|Казань|Татарстан|Ярославль|Кострома|Новосибирск|Екатеринбург|Нижний Новгород|Самара|Омск|Челябинск|Ростов-на-Дону|Уфа|Красноярск|Пермь|Воронеж|Волгоград|Саратов|Тюмень|Тольятти|Ижевск|Барнаул|Иркутск|Хабаровск|Владивосток|Махачкала|Томск|Оренбург|Кемерово|Рязань|Астрахань|Пенза|Липецк|Киров|Чебоксары|Тула|Калининград|Курск|Улан-Удэ|Ставрополь|Сочи|Тверь|Брянск|Белгород|Владимир|Архангельск|Калуга|Смоленск|Вологда|Саранск|Орёл',
|
|
40
|
+
],
|
|
41
|
+
['ja', '株式会社|有限会社|合同会社', '大学|学院|学校', '東京|大阪|北海道'],
|
|
42
|
+
['zh', '有限公司|股份有限公司', '大学|学院|学校', '北京|上海|广东'],
|
|
43
|
+
[
|
|
44
|
+
'zh-Hant',
|
|
45
|
+
'有限公司|股份有限公司',
|
|
46
|
+
'大學|學院|學校',
|
|
47
|
+
'臺北|台北|高雄|臺灣',
|
|
48
|
+
],
|
|
49
|
+
['ko', '주식회사|유한회사', '대학교|대학|학교', '서울|부산|경기도'],
|
|
50
|
+
['ar', 'ذ.م.م|شركة', 'جامعة|معهد|مدرسة', 'القاهرة|دبي|الرياض'],
|
|
51
|
+
[
|
|
52
|
+
'he',
|
|
53
|
+
'בע״מ|בע"מ|חברה',
|
|
54
|
+
'אוניברסיטת|אוניברסיטה|מכללת|מכון',
|
|
55
|
+
'ירושלים|תל אביב|חיפה',
|
|
56
|
+
],
|
|
57
|
+
[
|
|
58
|
+
'hi',
|
|
59
|
+
'प्राइवेट लिमिटेड|लिमिटेड',
|
|
60
|
+
'विश्वविद्यालय|महाविद्यालय|संस्थान',
|
|
61
|
+
'दिल्ली|मुंबई|महाराष्ट्र',
|
|
62
|
+
],
|
|
63
|
+
[
|
|
64
|
+
'tr',
|
|
65
|
+
'A.Ş.|Ltd. Şti.',
|
|
66
|
+
'Üniversitesi|Üniversite|Enstitüsü|Okulu',
|
|
67
|
+
'İstanbul|Ankara|İzmir',
|
|
68
|
+
],
|
|
69
|
+
['id', 'PT|CV|Tbk', 'Universitas|Institut|Sekolah', 'Jakarta|Bandung|Bali'],
|
|
70
|
+
[
|
|
71
|
+
'vi',
|
|
72
|
+
'TNHH|CTCP',
|
|
73
|
+
'Đại học|Trường|Học viện',
|
|
74
|
+
'Hà Nội|Hồ Chí Minh|Đà Nẵng',
|
|
75
|
+
],
|
|
76
|
+
[
|
|
77
|
+
'th',
|
|
78
|
+
'บริษัท|จำกัด',
|
|
79
|
+
'มหาวิทยาลัย|วิทยาลัย|โรงเรียน',
|
|
80
|
+
'กรุงเทพมหานคร|เชียงใหม่|ภูเก็ต',
|
|
81
|
+
],
|
|
82
|
+
].map((row) => Object.freeze(row))
|
|
83
|
+
);
|