@link-foundation/sensitive-data-sanitizer 0.16.1 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/docs/COVERAGE.md +69 -2
- package/package.json +18 -3
- package/src/context.js +15 -10
- package/src/credentials.js +22 -6
- package/src/detection.js +8 -5
- package/src/document-patterns.js +34 -13
- package/src/identity.js +3 -3
- package/src/personal.js +1 -1
- package/src/public.js +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.16.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Make CI security and validation gates fail on incomplete evidence, recover transient release push failures, preserve validated release dependencies, and prevent quadratic scanning of PuTTY headers, travel identifiers and postal-address whitespace.
|
|
8
|
+
|
|
9
|
+
Preserve durations, stack frames, empty document fields and local bind addresses; protect personal URL paths and additional document forms. Reduce repeated label-pattern construction and document structured JSON per-call costs.
|
|
10
|
+
|
|
3
11
|
## 0.16.1
|
|
4
12
|
|
|
5
13
|
### Patch Changes
|
package/docs/COVERAGE.md
CHANGED
|
@@ -236,7 +236,8 @@ The `documents` profile additionally recognizes:
|
|
|
236
236
|
forms and lower case, and surnames after г-жа/господин/госпожа.
|
|
237
237
|
- Birth years with д.р./г.р./год рождения/родился/родилась/Born in context;
|
|
238
238
|
Russian age expressions and Пол labels, English Sex, and gender beside age.
|
|
239
|
-
|
|
239
|
+
Bare age-in-years expressions are ambiguous and remain unchanged without
|
|
240
|
+
affirmative personal-age context; durations are preserved.
|
|
240
241
|
- Labelled citizenship, marital status, children, nationality, religion,
|
|
241
242
|
disability and criminal record, including the reported English labels.
|
|
242
243
|
- Company/employer/workplace labels, capitalized company/holding/bank names,
|
|
@@ -258,7 +259,7 @@ recognizer when these fields must be removed. Initial/name rules do not cover
|
|
|
258
259
|
all surnames, languages, cases or arbitrary lower-case names.
|
|
259
260
|
|
|
260
261
|
Operational value policy (#79): a single-slash relative URL with no query,
|
|
261
|
-
fragment, userinfo or explicit user/profile/member/home path stays unchanged
|
|
262
|
+
fragment, userinfo or explicit u/in/user/profile/member/home/people path stays unchanged
|
|
262
263
|
under `url`; other detectors and known values still apply. An ISO-style
|
|
263
264
|
`YYYY-MM-DDTHH:mm:ss[.fraction](Z|±HH:mm)` timestamp under `time` stays unchanged.
|
|
264
265
|
This exemption is syntactic, not date validation. Userinfo, token-bearing URLs
|
|
@@ -270,3 +271,69 @@ provider-specific rules and required external scanners remain authoritative.
|
|
|
270
271
|
|
|
271
272
|
The [complete requirement inventory and alternatives](case-studies/issue-84/README.md)
|
|
272
273
|
links every reported text row to its regression case.
|
|
274
|
+
|
|
275
|
+
## Document context refinements (issues 87–88)
|
|
276
|
+
|
|
277
|
+
Age requires a label (`возраст 41`, `Age 41`), personal pronoun (`мне 29 лет`),
|
|
278
|
+
birthday verb (`исполнилось 34`) or gender followed by age (`Женщина, 34 года`).
|
|
279
|
+
Bare `34 года` remains unchanged as an ambiguous limit. Work experience, tenure,
|
|
280
|
+
contract lengths and elapsed/future durations stay unchanged. `Дети: нет` and
|
|
281
|
+
`Гражданство: не указано` are treated as empty fields; negative disability and
|
|
282
|
+
criminal-record statements remain sensitive attributes.
|
|
283
|
+
|
|
284
|
+
Additional document forms include `контактный телефон` and `сот.` phone labels,
|
|
285
|
+
parenthesized `(собака)` email separators, `телега`/`Zoom` messenger labels,
|
|
286
|
+
dated `ТП УФМС`/`УФМС` passport issuers, `р/с` account numbers, `г-н` surname
|
|
287
|
+
titles, `1985 года рождения` and `г.р. 1985`, abbreviated/single-letter gender
|
|
288
|
+
values (`Пол: Ж`, `Пол: М`, `пол: жен.`, `пол муж.`), `гражданин РФ`,
|
|
289
|
+
`есть двое детей`, `проживаю в Перми`, and English `Employer`, `Company` and
|
|
290
|
+
`Education` labels. A capitalized Cyrillic word followed by two initials is
|
|
291
|
+
recognized regardless of surname ending (`Цой В. Р.`, `Ким Д. С.`,
|
|
292
|
+
`Шевченко Т. Г.`). This structural rule can flag non-name text with the same
|
|
293
|
+
shape; it does not provide a complete surname dictionary.
|
|
294
|
+
|
|
295
|
+
JSON escape boundaries cannot start an obfuscated-email local part, preserving
|
|
296
|
+
stack newlines and frames after IP:port. Relative `/u/<name>` and `/in/<name>`
|
|
297
|
+
URLs are personal paths under `url`, like the existing user/profile paths.
|
|
298
|
+
Exact `0.0.0.0` is an operational unspecified bind address, like loopback/local
|
|
299
|
+
addresses. `publicKnowledge: false` and explicit private known values still
|
|
300
|
+
redact it.
|
|
301
|
+
|
|
302
|
+
## Structured JSON cost (issue 86)
|
|
303
|
+
|
|
304
|
+
Structured sanitization scans labelled raw strings and each decoded string with
|
|
305
|
+
its own line boundaries, then verifies sanitized output. This preserves detection
|
|
306
|
+
in arrays, CV fragments and embedded JSON, but adds fixed work per string/key.
|
|
307
|
+
Costs depend on string count, enabled engines, content, runtime and hardware;
|
|
308
|
+
line length alone cannot predict synchronous main-thread delay.
|
|
309
|
+
|
|
310
|
+
A finite serial measurement on Linux x64, Node 26.13.0, using the documents
|
|
311
|
+
profile and structured JSON, produced these median milliseconds per `sanitize`
|
|
312
|
+
call (100 warm-ups, three 100-call trials):
|
|
313
|
+
|
|
314
|
+
| Fixture | 0.16.0 | 0.16.1 | This change |
|
|
315
|
+
| ----------------------------------------- | -----: | -----: | ----------: |
|
|
316
|
+
| Short JSON (89 characters) | 1.21 | 8.44 | 4.67 |
|
|
317
|
+
| Exact issue reproduction (264 characters) | 1.81 | 19.71 | 11.15 |
|
|
318
|
+
| Longer synthetic JSON (9,396 characters) | 10.04 | 37.88 | 28.32 |
|
|
319
|
+
|
|
320
|
+
The short and longer synthetic rows approximate the issue's unpublished
|
|
321
|
+
100/9,453-character inputs; only the 264-character input was provided in full.
|
|
322
|
+
These results are evidence for this container, not a universal latency guarantee
|
|
323
|
+
or an Apple Silicon comparison. The issue reporter measured 0.16.1 at
|
|
324
|
+
2.31/5.61/34.85 ms on their short/264-character/9 KiB rows and 1.10 ms for the
|
|
325
|
+
264-character text-mode control. The remaining structured overhead is **above
|
|
326
|
+
2x** of 0.16.0 here; documenting that cost fulfills the issue's alternative
|
|
327
|
+
acceptance criterion. No detection view is removed to meet a timing target.
|
|
328
|
+
|
|
329
|
+
Personal-label patterns are now constructed once instead of for every scan, and
|
|
330
|
+
redundant explicit RegExp cloning is avoided when `matchAll` supplies isolation.
|
|
331
|
+
Reproduce with `node --max-old-space-size=256 experiments/issue-89/benchmark.mjs`
|
|
332
|
+
and optionally pass a released checkout's `src/index.js`. For latency-sensitive
|
|
333
|
+
logging, measure the real workload before synchronously sanitizing every line;
|
|
334
|
+
the existing JSONL record-worker API/CLI can move record scanning off the main
|
|
335
|
+
thread (see [API.md](API.md)). External engines and more sensitive values can add
|
|
336
|
+
further work. These numbers do not describe `createSanitizer` with Secretlint.
|
|
337
|
+
|
|
338
|
+
The [complete issue 89 requirement inventory](case-studies/issue-89/README.md)
|
|
339
|
+
records every row, root cause, alternative and regression mapping.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.2",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -92,7 +92,7 @@
|
|
|
92
92
|
},
|
|
93
93
|
"devDependencies": {
|
|
94
94
|
"@changesets/cli": "^3.0.3",
|
|
95
|
-
"eslint": "^
|
|
95
|
+
"eslint": "^10.13.0",
|
|
96
96
|
"eslint-config-prettier": "^10.1.8",
|
|
97
97
|
"eslint-plugin-prettier": "^5.5.4",
|
|
98
98
|
"husky": "^9.1.7",
|
|
@@ -101,7 +101,10 @@
|
|
|
101
101
|
"mrz": "^5.0.2",
|
|
102
102
|
"prettier": "^3.6.2",
|
|
103
103
|
"secretlint": "13.0.7",
|
|
104
|
-
"test-anywhere": "^0.8.48"
|
|
104
|
+
"test-anywhere": "^0.8.48",
|
|
105
|
+
"@eslint/js": "^10.0.1",
|
|
106
|
+
"command-stream": "2.4.0",
|
|
107
|
+
"lino-arguments": "0.4.0"
|
|
105
108
|
},
|
|
106
109
|
"lint-staged": {
|
|
107
110
|
"*.{js,mjs,cjs}": [
|
|
@@ -112,6 +115,12 @@
|
|
|
112
115
|
"*.md": [
|
|
113
116
|
"prettier --write",
|
|
114
117
|
"prettier --check"
|
|
118
|
+
],
|
|
119
|
+
"*.sh": [
|
|
120
|
+
"pipx run --spec shellcheck-py==0.11.0.1 shellcheck"
|
|
121
|
+
],
|
|
122
|
+
".husky/*": [
|
|
123
|
+
"pipx run --spec shellcheck-py==0.11.0.1 shellcheck"
|
|
115
124
|
]
|
|
116
125
|
},
|
|
117
126
|
"dependencies": {
|
|
@@ -124,5 +133,11 @@
|
|
|
124
133
|
"re2-wasm": "1.0.2",
|
|
125
134
|
"re2js": "^2.9.0",
|
|
126
135
|
"yaml": "^2.9.1"
|
|
136
|
+
},
|
|
137
|
+
"allowScripts": {
|
|
138
|
+
"node-pty": false
|
|
139
|
+
},
|
|
140
|
+
"overrides": {
|
|
141
|
+
"minimatch@>=7 <11": "^11.0.4"
|
|
127
142
|
}
|
|
128
143
|
}
|
package/src/context.js
CHANGED
|
@@ -247,7 +247,7 @@ function operationalValue(value, type) {
|
|
|
247
247
|
return (
|
|
248
248
|
(type === 'URL' &&
|
|
249
249
|
/^\/(?!\/)[^\s?@#]*$/.test(value) &&
|
|
250
|
-
!/(?:^|\/)(?:users?|profiles?|people|members?|home)\/[^/]+/i.test(
|
|
250
|
+
!/(?:^|\/)(?:u|in|users?|profiles?|people|members?|home)\/[^/]+/i.test(
|
|
251
251
|
value
|
|
252
252
|
)) ||
|
|
253
253
|
(type === 'TIME' &&
|
|
@@ -393,19 +393,24 @@ function detectCredentials(text, emit) {
|
|
|
393
393
|
}
|
|
394
394
|
}
|
|
395
395
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
continue;
|
|
400
|
-
}
|
|
396
|
+
const personalPrefixes = Object.entries(labels)
|
|
397
|
+
.filter(([type]) => type !== 'credential')
|
|
398
|
+
.map(([type, words]) => {
|
|
401
399
|
const separator =
|
|
402
400
|
type === 'ID'
|
|
403
401
|
? `(?:[:=:]|(?:${verbs})(?![\\p{L}\\p{N}_]))?[ \\t]+|[:=:][ \\t]*`
|
|
404
402
|
: `(?:[:=:]|(?:${verbs})(?![\\p{L}\\p{N}_]))[ \\t]*`;
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
403
|
+
return [
|
|
404
|
+
type,
|
|
405
|
+
new RegExp(
|
|
406
|
+
`(?<![\\p{L}\\p{N}_])(?:\\\\*["'])?(?:${words.map(escapePattern).join('|')})(?![\\p{L}\\p{N}_])(?:\\\\*["'])?[ \\t]*(?:${separator})`,
|
|
407
|
+
'giu'
|
|
408
|
+
),
|
|
409
|
+
];
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
function detectPersonalLabels(text, emit) {
|
|
413
|
+
for (const [type, prefix] of personalPrefixes) {
|
|
409
414
|
for (const match of text.matchAll(prefix)) {
|
|
410
415
|
const identity = type === 'ID';
|
|
411
416
|
const suffix = identity
|
package/src/credentials.js
CHANGED
|
@@ -45,10 +45,26 @@ export function detectCredentialFormats(text, emit) {
|
|
|
45
45
|
(value) =>
|
|
46
46
|
!credentialPlaceholder(value) && value !== 'put your unique phrase here'
|
|
47
47
|
);
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
48
|
+
detectPuttyPrivateKeys(text, emit);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Scan each consumed range once. A regex combining an unbounded header line
|
|
52
|
+
// and a lazy body rescans every repeated header when the newline is missing.
|
|
53
|
+
function detectPuttyPrivateKeys(text, emit) {
|
|
54
|
+
const headers = /PuTTY-User-Key-File-[23]:/g;
|
|
55
|
+
const lineEnd = /[\r\n]/g;
|
|
56
|
+
const mac = /Private-MAC:[ \t]*[a-fA-F0-9]+/g;
|
|
57
|
+
let header;
|
|
58
|
+
while ((header = headers.exec(text))) {
|
|
59
|
+
lineEnd.lastIndex = headers.lastIndex;
|
|
60
|
+
const newline = lineEnd.exec(text);
|
|
61
|
+
if (!newline) {
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
mac.lastIndex = newline.index + 1;
|
|
65
|
+
const terminator = mac.exec(text);
|
|
66
|
+
const end = terminator ? mac.lastIndex : text.length;
|
|
67
|
+
emit({ start: header.index, end, ...details('putty-private-key') });
|
|
68
|
+
headers.lastIndex = end;
|
|
69
|
+
}
|
|
54
70
|
}
|
package/src/detection.js
CHANGED
|
@@ -58,11 +58,14 @@ export function collectMatches(
|
|
|
58
58
|
group = 0,
|
|
59
59
|
accept = () => true
|
|
60
60
|
) {
|
|
61
|
-
//
|
|
62
|
-
const pattern =
|
|
63
|
-
regex.
|
|
64
|
-
|
|
65
|
-
|
|
61
|
+
// matchAll clones global expressions and leaves their lastIndex untouched.
|
|
62
|
+
const pattern =
|
|
63
|
+
regex.global && regex.lastIndex === 0
|
|
64
|
+
? regex
|
|
65
|
+
: new RegExp(
|
|
66
|
+
regex.source,
|
|
67
|
+
regex.global ? regex.flags : `${regex.flags}g`
|
|
68
|
+
);
|
|
66
69
|
for (const match of text.matchAll(pattern)) {
|
|
67
70
|
const raw = match[group];
|
|
68
71
|
const value = details.category === 'personal' ? raw?.trim() : raw;
|
package/src/document-patterns.js
CHANGED
|
@@ -11,7 +11,7 @@ const given = [
|
|
|
11
11
|
...new Set(Object.values(firstNames).flatMap((s) => s.split(' '))),
|
|
12
12
|
].join('|');
|
|
13
13
|
const nameInitials = new RegExp(
|
|
14
|
-
`(?<!${boundary})(?:${surname}[ \\t]+${initials}|${initials}(?:${surname}|${latinSurname})|${latinSurname}[ \\t]+${initials}|(?:${given})[ \\t]+[А-ЯЁA-Z]\\.)(?!${boundary})`,
|
|
14
|
+
`(?<!${boundary})(?:${word}[ \\t]+(?:[А-ЯЁ]\\.[ \\t]*){2}|${surname}[ \\t]+${initials}|${initials}(?:${surname}|${latinSurname})|${latinSurname}[ \\t]+${initials}|(?:${given})[ \\t]+[А-ЯЁA-Z]\\.)(?!${boundary})`,
|
|
15
15
|
'gu'
|
|
16
16
|
);
|
|
17
17
|
const namePairs = new RegExp(
|
|
@@ -30,13 +30,13 @@ const rules = [
|
|
|
30
30
|
[
|
|
31
31
|
'PHONE',
|
|
32
32
|
'document-phone-label',
|
|
33
|
-
/(?<![\p{L}\p{N}_])(
|
|
33
|
+
/(?<![\p{L}\p{N}_])(?:(?:контактный[ \t]+)?телефон|тел\.?|моб\.?|сот\.?|WhatsApp)(?:[ \t]*:[ \t]*|[ \t]+)((?:[78][ () .-]*)?9\d{2}[ () .-]*\d{3}[ .-]*\d{2}[ .-]*\d{2})(?!\d)/giu,
|
|
34
34
|
1,
|
|
35
35
|
],
|
|
36
36
|
[
|
|
37
37
|
'EMAIL',
|
|
38
38
|
'obfuscated-email',
|
|
39
|
-
/(?<![\p{L}\p{N}_
|
|
39
|
+
/(?<![\p{L}\p{N}_.+\\-])[\p{L}\p{N}_+-]{1,64}(?:(?:\.|[ \t]{1,8}dot[ \t]{1,8})[\p{L}\p{N}_+-]{1,64}){0,4}(?:[ \t]{0,8}\((?:at|собака)\)[ \t]{0,8}|[ \t]{0,8}\[at\][ \t]{0,8}|[ \t]{1,8}(?:at|собака)[ \t]{1,8})[\p{L}\p{N}-]{1,63}(?:(?:\.|[ \t]{1,8}dot[ \t]{1,8})[\p{L}\p{N}-]{1,63}){1,8}(?![\p{L}\p{N}_])/giu,
|
|
40
40
|
0,
|
|
41
41
|
],
|
|
42
42
|
[
|
|
@@ -48,7 +48,7 @@ const rules = [
|
|
|
48
48
|
[
|
|
49
49
|
'USERNAME',
|
|
50
50
|
'messenger-label',
|
|
51
|
-
/(?<![\p{L}\p{N}_])(?:Telegram
|
|
51
|
+
/(?<![\p{L}\p{N}_])(?:Telegram|телеграм|телега|tg|skype|Zoom|ник)(?:[ \t]*:[ \t]*|[ \t]+)(@?[a-z][a-z\d_.-]{2,63})(?![\p{L}\p{N}_])/giu,
|
|
52
52
|
1,
|
|
53
53
|
],
|
|
54
54
|
[
|
|
@@ -60,13 +60,13 @@ const rules = [
|
|
|
60
60
|
[
|
|
61
61
|
'ID',
|
|
62
62
|
'identity-number',
|
|
63
|
-
/(?<![\p{L}\p{N}_])(?:ОГРНИП[ \t]+(\d{15})|код подразделения[ \t]+(\d{3}-\d{3})|(?:водительское удостоверение|в\/у|Driver's licen[cs]e)[ \t]+(\d{2}[ \t]?\d{2}[ \t]?\d{6})|(?:ОМС|полис)[ \t]+(\d{16})
|
|
63
|
+
/(?<![\p{L}\p{N}_])(?:ОГРНИП[ \t]+(\d{15})|код подразделения[ \t]+(\d{3}-\d{3})|(?:водительское удостоверение|в\/у|Driver's licen[cs]e)[ \t]+(\d{2}[ \t]?\d{2}[ \t]?\d{6})|(?:ОМС|полис)[ \t]+(\d{16})|(?:сч[её]т|р\/с)[ \t]+(\d{20})|диплом[ \t]+(?:№|No|#)[ \t]*(\d{6}))(?!\d)/giu,
|
|
64
64
|
-1,
|
|
65
65
|
],
|
|
66
66
|
[
|
|
67
67
|
'ID',
|
|
68
68
|
'passport-issuer',
|
|
69
|
-
/(?<![\p{L}\p{N}_])выдан[ \t]+((
|
|
69
|
+
/(?<![\p{L}\p{N}_])выдан[ \t]+((?:ОВД|УВД|МВД|(?:ТП[ \t]+)?УФМС)[ \t]+[^\r\n";|]{1,120}?\d{2}\.\d{2}\.\d{4})(?!\d)/giu,
|
|
70
70
|
1,
|
|
71
71
|
],
|
|
72
72
|
[
|
|
@@ -81,12 +81,24 @@ const rules = [
|
|
|
81
81
|
/(?<![\p{L}\p{N}_])(?:образование|ВУЗ|университет|окончил[аи]?)(?:[ \t]*:[ \t]*|[ \t]+)([А-ЯЁ]{2,12}(?:[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{2,40})?)(?![\p{L}\p{N}_])/giu,
|
|
82
82
|
1,
|
|
83
83
|
],
|
|
84
|
+
[
|
|
85
|
+
'EDUCATION',
|
|
86
|
+
'education-label',
|
|
87
|
+
/(?<![\p{L}\p{N}_])Education[ \t]*:[ \t]*([A-Z][A-Za-z.-]{1,47})(?![\p{L}\p{N}_])/gu,
|
|
88
|
+
1,
|
|
89
|
+
],
|
|
90
|
+
[
|
|
91
|
+
'PERSONAL_ATTRIBUTE',
|
|
92
|
+
'profile-prose',
|
|
93
|
+
/(?<![\p{L}\p{N}_])(?:граждан(?:ин|ка)[ \t]+([А-ЯЁ]{2,4})|есть[ \t]+((?:один|одна|двое|трое|четверо|пятеро|\d{1,2})[ \t]+(?:детей|реб[её]нок|реб[её]нка)))(?![\p{L}\p{N}_])/gu,
|
|
94
|
+
-1,
|
|
95
|
+
],
|
|
84
96
|
['PERSON', 'name-initials', nameInitials, 0],
|
|
85
97
|
[
|
|
86
98
|
'PERSON',
|
|
87
99
|
'name-title',
|
|
88
100
|
new RegExp(
|
|
89
|
-
`(?<!${boundary})(
|
|
101
|
+
`(?<!${boundary})(?:г-жа|г-н|господин|госпожа)[ \\t]+(${word})(?!${boundary})`,
|
|
90
102
|
'gu'
|
|
91
103
|
),
|
|
92
104
|
1,
|
|
@@ -94,19 +106,19 @@ const rules = [
|
|
|
94
106
|
[
|
|
95
107
|
'DATE_OF_BIRTH',
|
|
96
108
|
'birth-year',
|
|
97
|
-
/(?<![\p{L}\p{N}_])(
|
|
109
|
+
/(?<![\p{L}\p{N}_])(?:д\.р\.|г\.р\.|год рождения|родил(?:ся|ась)(?:[ \t]+в)?|Born(?:[ \t]+in)?)[ \t]*:?[ \t]*((?:19|20)\d{2})(?!\d)|(?<!\d)((?:19|20)\d{2})(?=[ \t]+(?:г\.р\.|года[ \t]+рождения)(?![\p{L}\p{N}_]))/giu,
|
|
98
110
|
-1,
|
|
99
111
|
],
|
|
100
112
|
[
|
|
101
113
|
'AGE',
|
|
102
114
|
'age-context',
|
|
103
|
-
/(?<![\p{L}\p{N}_])(
|
|
115
|
+
/(?<![\p{L}\p{N}_])(?:(?:возраст|Age)(?:[ \t]*:[ \t]*|[ \t]+)|(?:мне|исполнилось)[ \t]+)(\d{1,3})(?!\d)|(?<![\p{L}\p{N}_])(?:Женщина|Мужчина),[ \t]*(\d{1,3})(?=[ \t]+(?:года|год|лет)(?![\p{L}\p{N}_]))/giu,
|
|
104
116
|
-1,
|
|
105
117
|
],
|
|
106
118
|
[
|
|
107
119
|
'GENDER',
|
|
108
120
|
'gender-context',
|
|
109
|
-
/(?<![\p{L}\p{N}_])(?:Пол|Sex)[ \t]*:[ \t]
|
|
121
|
+
/(?<![\p{L}\p{N}_])(?:Пол|Sex)(?:[ \t]*:[ \t]*|[ \t]+)(женский|мужской|жен\.|муж\.|Ж|М|male|female)(?![\p{L}\p{N}_])|(?<![\p{L}\p{N}_])(Женщина|Мужчина)(?=,[ \t]*\d{1,3}[ \t]+(?:года|год|лет))/giu,
|
|
110
122
|
-1,
|
|
111
123
|
],
|
|
112
124
|
[
|
|
@@ -119,7 +131,7 @@ const rules = [
|
|
|
119
131
|
'ORGANIZATION',
|
|
120
132
|
'employer-label',
|
|
121
133
|
new RegExp(
|
|
122
|
-
`(?<!${boundary})(?:[Кк]омпания|[Рр]аботодатель|[Мм]есто
|
|
134
|
+
`(?<!${boundary})(?:[Кк]омпания|[Рр]аботодатель|[Мм]есто работы|[Cc]ompany|[Ee]mployer)[ \\t]*:[ \\t]*([\\p{Lu}][\\p{L}\\p{M}.-]{1,47})(?!${boundary})`,
|
|
123
135
|
'gu'
|
|
124
136
|
),
|
|
125
137
|
1,
|
|
@@ -146,7 +158,7 @@ const rules = [
|
|
|
146
158
|
'LOCATION',
|
|
147
159
|
'residence-prose',
|
|
148
160
|
new RegExp(
|
|
149
|
-
`(?<!${boundary})(
|
|
161
|
+
`(?<!${boundary})(?:(?:живу|проживаю)[ \\t]+в|из|м\\.|метро|район)[ \\t]+(${word})(?!${boundary})`,
|
|
150
162
|
'gu'
|
|
151
163
|
),
|
|
152
164
|
1,
|
|
@@ -166,7 +178,12 @@ export function detectDocumentPatterns(text, emit) {
|
|
|
166
178
|
group === -1
|
|
167
179
|
? match.slice(1).find((part) => part !== undefined)
|
|
168
180
|
: match[group].trim();
|
|
169
|
-
if (
|
|
181
|
+
if (
|
|
182
|
+
!documentValueAllowed(rule, value) ||
|
|
183
|
+
(rule === 'profile-attribute' &&
|
|
184
|
+
/^Дети[ \t]*:/iu.test(match[0]) &&
|
|
185
|
+
/^нет$/iu.test(value))
|
|
186
|
+
) {
|
|
170
187
|
continue;
|
|
171
188
|
}
|
|
172
189
|
const start =
|
|
@@ -219,6 +236,10 @@ function documentValueAllowed(rule, value) {
|
|
|
219
236
|
rule === 'school-acronym-context' &&
|
|
220
237
|
!/^[А-ЯЁ]{2,12}(?![А-ЯЁа-яё])/.test(value)
|
|
221
238
|
) &&
|
|
239
|
+
!(
|
|
240
|
+
rule === 'profile-attribute' &&
|
|
241
|
+
/^(?:не указан[оаы]?|not specified|unknown|n\/a)$/iu.test(value)
|
|
242
|
+
) &&
|
|
222
243
|
!(rule === 'messenger-label' && /^(?:недоступен|unavailable)$/i.test(value))
|
|
223
244
|
);
|
|
224
245
|
}
|
package/src/identity.js
CHANGED
|
@@ -181,11 +181,11 @@ function detectPassports(text, emit) {
|
|
|
181
181
|
});
|
|
182
182
|
}
|
|
183
183
|
const booking =
|
|
184
|
-
/(?<![\p{L}\p{N}_])(?:booking(?:[ _-]+(?:reference|code))?|reservation(?:[ _-]+code)?|confirmation(?:[ _-]+code)?|бронь|record[ _-]+locator|PNR|код брони|номер брони|бронирование)[ \t]*[:=:]
|
|
184
|
+
/(?<![\p{L}\p{N}_])(?:booking(?:[ _-]+(?:reference|code))?|reservation(?:[ _-]+code)?|confirmation(?:[ _-]+code)?|бронь|record[ _-]+locator|PNR|код брони|номер брони|бронирование)[ \t]*(?:[:=:][ \t]*)?[`"']?([A-Z0-9]{4,8})(?![\p{L}\p{N}_])/giu;
|
|
185
185
|
const ticket =
|
|
186
|
-
/(?<![\p{L}\p{N}_])(?:e[ -]?ticket|ticket[ _-]*(?:number|no)?|номер билета|электронный билет)[ \t]*[:=:]
|
|
186
|
+
/(?<![\p{L}\p{N}_])(?:e[ -]?ticket|ticket[ _-]*(?:number|no)?|номер билета|электронный билет)[ \t]*(?:[:=:][ \t]*)?[`"']?(\d{3}[ -]?\d{10})(?![\p{L}\p{N}_])/giu;
|
|
187
187
|
const visa =
|
|
188
|
-
/(?<![\p{L}\p{N}_])(?:e[ -]?visa|visa|номер визы|виза)(?:[ \t_-]+(?:registration|application|number|no|code|номер|код)){0,3}[ \t]*[:=:]
|
|
188
|
+
/(?<![\p{L}\p{N}_])(?:e[ -]?visa|visa|номер визы|виза)(?:[ \t_-]+(?:registration|application|number|no|code|номер|код)){0,3}[ \t]*(?:[:=:][ \t]*)?[`"']?([A-Z0-9][A-Z0-9-]{4,31})(?![\p{L}\p{N}_])/giu;
|
|
189
189
|
const placeholder =
|
|
190
190
|
/^(?:NONE|UNKNOWN|NULL|UNDEFINED|REDACTED|EXAMPLE|SAMPLE|PENDING|HOTEL|BOOKING|REFERENCE|REGISTRATION|APPLICATION|NUMBER|CODE|FAILED|ERROR|HTTP|OK|CANCELLED|CONFIRMED|SUCCESS|READY|ACTIVE|CLOSED)$/i;
|
|
191
191
|
function detectTravel(text, emit) {
|
package/src/personal.js
CHANGED
|
@@ -224,7 +224,7 @@ export function detectPersonal(text, emit) {
|
|
|
224
224
|
);
|
|
225
225
|
collectMatches(
|
|
226
226
|
text,
|
|
227
|
-
/(?<![\p{L}\p{N}])\d{1,6}[A-Za-z]?[ \t]+(?:[\p{Lu}][\p{L}.'-]*[ \t]+){1,5}(?:Street|St|Road|Rd|Avenue|Ave|Lane|Ln|Drive|Dr|Boulevard|Blvd|Way|Straße|Strasse|Rue|Calle|Rua)\b(?:,[ \t]+(?:[^\r\n<>]{
|
|
227
|
+
/(?<![\p{L}\p{N}])\d{1,6}[A-Za-z]?[ \t]+(?:[\p{Lu}][\p{L}.'-]*[ \t]+){1,5}(?:Street|St|Road|Rd|Avenue|Ave|Lane|Ln|Drive|Dr|Boulevard|Blvd|Way|Straße|Strasse|Rue|Calle|Rua)\b(?:,[ \t]+(?:(?:[^ \t\r\n<>][^\r\n<>]{0,79}?[ \t]+)?(?:[A-Z]{1,2}\d[A-Z\d]?[ \t]+\d[A-Z]{2}|\d{5}(?:-\d{4})?)\b|(?:[\p{Lu}][\p{Ll}.'-]+[ \t]*){1,4}))?/gu,
|
|
228
228
|
emit,
|
|
229
229
|
details('ADDRESS', 'postal-address')
|
|
230
230
|
);
|