@link-foundation/sensitive-data-sanitizer 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +17 -0
- package/bin/sensitive-data-sanitizer.js +8 -0
- package/docs/API.md +64 -4
- package/docs/COVERAGE.md +51 -5
- package/examples/prose-logs.mjs +16 -0
- package/package.json +3 -2
- package/src/context.js +9 -1
- package/src/detection.js +2 -1
- package/src/document-dates.js +15 -0
- package/src/document-entities.js +21 -14
- package/src/document-experience.js +110 -0
- package/src/document-extras.js +96 -64
- package/src/document-fields.js +3 -1
- package/src/engines.js +12 -2
- package/src/index.d.ts +6 -1
- package/src/placeholder-types.js +1 -0
- package/src/propagation.js +5 -1
- package/src/proper-names.js +222 -0
- package/src/sanitizer.js +15 -2
- package/src/structured.js +24 -16
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.18.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- f0016ea: Add opt-in structural proper-name redaction with per-string activation and
|
|
8
|
+
configurable Unicode scripts. Broaden dated document employers, group and school
|
|
9
|
+
cues, comma-separated mobiles and short share links; preserve operational paths
|
|
10
|
+
and technology controls. Normalize caller-sensitive fields like built-in names.
|
|
11
|
+
|
|
3
12
|
## 0.17.1
|
|
4
13
|
|
|
5
14
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -22,6 +22,23 @@ sensitive-data-sanitizer redact session.jsonl --stream --jsonl --two-pass --outp
|
|
|
22
22
|
|
|
23
23
|
## Quick Start
|
|
24
24
|
|
|
25
|
+
For stored prose that can tolerate extra redaction, opt into structural proper
|
|
26
|
+
names independently for each string value:
|
|
27
|
+
|
|
28
|
+
```js
|
|
29
|
+
sanitize('Работал в Ромашке менеджером по продажам.', {
|
|
30
|
+
profile: 'documents',
|
|
31
|
+
properNames: 'capitalized',
|
|
32
|
+
}).text;
|
|
33
|
+
// [REDACTED] в [REDACTED] менеджером по продажам.
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The rule defaults to Cyrillic or structural data triggers, keeps JSON keys and
|
|
37
|
+
numbers, and supports caller-selected Unicode `scripts`. It is off by default.
|
|
38
|
+
See [activation rules and limits](docs/API.md#structural-prose-redaction) and the
|
|
39
|
+
[prose-log example](examples/prose-logs.mjs). CLI:
|
|
40
|
+
`redact input.jsonl --jsonl --proper-names capitalized --profile documents`.
|
|
41
|
+
|
|
25
42
|
### Readable placeholders and document profiles
|
|
26
43
|
|
|
27
44
|
Use the opt-in `documents` profile for CVs and similar prose. It adds bounded
|
|
@@ -60,6 +60,8 @@ history rewrite SOURCE --output DIRECTORY [--apply]
|
|
|
60
60
|
--jsonl Sanitize JSONL records; combine with --stream for sessions
|
|
61
61
|
--identity-mask First/last 2 identity mask (publication profile default)
|
|
62
62
|
--profile NAME publication (redact default) or documents (CV/entity rules)
|
|
63
|
+
--proper-names MODE Structural prose rule: capitalized (off by default)
|
|
64
|
+
--scripts LIST Comma-separated Unicode Script names (default Cyrillic)
|
|
63
65
|
--placeholder Typed numbered placeholders; requires --map or --no-map
|
|
64
66
|
--map PATH Private JSON/.lino restore map; reuse it across files/runs
|
|
65
67
|
--no-map Discard placeholder mapping after full anonymization
|
|
@@ -122,6 +124,8 @@ function parseArgs(argv) {
|
|
|
122
124
|
['--worker-timeout-ms', 'workerTimeoutMs'],
|
|
123
125
|
['--filter-repo', 'filterRepo'],
|
|
124
126
|
['--profile', 'profile'],
|
|
127
|
+
['--proper-names', 'properNames'],
|
|
128
|
+
['--scripts', 'scripts'],
|
|
125
129
|
['--map', 'map'],
|
|
126
130
|
['--map-format', 'mapFormat'],
|
|
127
131
|
['--placeholder-format', 'placeholderFormat'],
|
|
@@ -341,6 +345,10 @@ async function loadOptions(config) {
|
|
|
341
345
|
|
|
342
346
|
async function featureOptions(config, options) {
|
|
343
347
|
await localCredentials(config, options);
|
|
348
|
+
options.properNames = config.properNames ?? options.properNames;
|
|
349
|
+
if (config.scripts !== undefined) {
|
|
350
|
+
options.scripts = config.scripts.split(',');
|
|
351
|
+
}
|
|
344
352
|
if (config.knownOnly) {
|
|
345
353
|
options.knownOnly = true;
|
|
346
354
|
}
|
package/docs/API.md
CHANGED
|
@@ -140,7 +140,9 @@ and a shared map; such conflicting occurrence metadata is refused. Choose
|
|
|
140
140
|
separate maps for those documents if exact independent restoration is needed.
|
|
141
141
|
|
|
142
142
|
JSON/JSONL keys and values participate equally in placeholder mode, including
|
|
143
|
-
ordinary identifier keys and declared structural fields.
|
|
143
|
+
ordinary identifier keys and declared structural fields. With `properNames`,
|
|
144
|
+
JSON keys remain unchanged by automatic personal detection in all transformation
|
|
145
|
+
modes. Explicit known values and credentials retain priority. Parseability and
|
|
144
146
|
duplicate-key checks remain enforced. Plain redact/fake policies retain their
|
|
145
147
|
existing metadata/key behavior. Use `structured` for valid structured input.
|
|
146
148
|
The normal encoding limits and credential redaction policy continue to apply.
|
|
@@ -157,6 +159,8 @@ new document recognizers; `documentEntities: false` turns them off explicitly.
|
|
|
157
159
|
| Option | Default / meaning |
|
|
158
160
|
| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
159
161
|
| `documentEntities` | Off; bounded organization, education and location rules; enabled by the documents profile |
|
|
162
|
+
| `properNames` | Off; `'capitalized'` enables structural name redaction in activated prose strings |
|
|
163
|
+
| `scripts` | `['Cyrillic']`; Unicode Script aliases that activate the prose rule; `[]` disables only script activation |
|
|
160
164
|
| `profile` | Off; `'publication'` sets a 0.3 threshold and identity masking; `'documents'` enables document entities |
|
|
161
165
|
| `knownOnly` | `false`; disable all automatic detection (including credentials, Secretlint and custom detectors); use only explicit known values and reused map identities |
|
|
162
166
|
| `knownSecrets` | `[]`; exact literal credential values, regardless of length or format |
|
|
@@ -176,7 +180,7 @@ new document recognizers; `documentEntities: false` turns them off explicitly.
|
|
|
176
180
|
| `fakeIdentity` | Off; `'specimen-and-synthetic'` retains explicitly fake identity values |
|
|
177
181
|
| `fakeValues` | `[]`; exact caller-reviewed identity values to retain, regardless of fakeIdentity |
|
|
178
182
|
| `structured` | Off; `'json'` or `'jsonl'` validates and sanitizes string keys/values separately |
|
|
179
|
-
| `sensitiveFields` | `[]`;
|
|
183
|
+
| `sensitiveFields` | `[]`; JSON/JSONL names normalized for case and `_`/`-`; complete values become a redaction string; supports synchronous calls |
|
|
180
184
|
| `structuralFields` | `[]`; declared property names suppress weak name/ID/entropy heuristics in their string values |
|
|
181
185
|
|
|
182
186
|
Unknown options and invalid limits are errors. `knownSecretsFromEnv(env, names?)` discovers credential-named variables (excluding auth configuration names) when names are omitted, or uses an explicit selection, deduplicating nonempty values. It never invokes shell commands or reads auth files. `personalVariants(entries)` generates numeric and ISO/day-first date spellings; name case variants are already handled by detection. Neither helper invents transliterations.
|
|
@@ -478,8 +482,9 @@ Authorization headers (including Proxy-Authorization) redact the scheme and one
|
|
|
478
482
|
|
|
479
483
|
## Caller-declared sensitive fields
|
|
480
484
|
|
|
481
|
-
In structured JSON/JSONL, `sensitiveFields: string[]`
|
|
482
|
-
|
|
485
|
+
In structured JSON/JSONL, `sensitiveFields: string[]` ignores case and removes
|
|
486
|
+
underscores and hyphens at any depth, like built-in document fields. For example,
|
|
487
|
+
`fooBar` also matches `foo_bar`, `FooBar` and `foo-bar`. The complete property value becomes
|
|
483
488
|
`"[REDACTED]"`, including empty strings, numbers, booleans, null, arrays and
|
|
484
489
|
objects. This deliberate schema policy can change value types. It takes
|
|
485
490
|
priority over `structuralFields` and transformations, preserves property names
|
|
@@ -505,3 +510,58 @@ sanitize('Lyutik работал', {
|
|
|
505
510
|
context. For computed synchronous detection, callers can supply validated
|
|
506
511
|
`findings` for the exact input. `detectors` remains an asynchronous-engine
|
|
507
512
|
option; `sanitize` does not invoke these callbacks.
|
|
513
|
+
|
|
514
|
+
## Structural prose redaction
|
|
515
|
+
|
|
516
|
+
`properNames: 'capitalized'` is an optional, deliberately aggressive native rule
|
|
517
|
+
for stored prose. It works with any profile; the examples use `documents` for
|
|
518
|
+
its other document rules. It is off by default.
|
|
519
|
+
|
|
520
|
+
```js
|
|
521
|
+
sanitize('Работал в Ромашке менеджером по продажам.', {
|
|
522
|
+
profile: 'documents',
|
|
523
|
+
properNames: 'capitalized',
|
|
524
|
+
}).text;
|
|
525
|
+
// [REDACTED] в [REDACTED] менеджером по продажам.
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
Each string activates independently if it contains a letter from `scripts`, a
|
|
529
|
+
parenthesized document date range, a comma followed by an 11-digit `79…`/`89…`
|
|
530
|
+
mobile shape, or an HTTP(S)/bare-domain link with a `/d/<id>` path. `scripts`
|
|
531
|
+
defaults to `['Cyrillic']`, suitable for English operational logs; choose Unicode
|
|
532
|
+
Script aliases such as `['Greek', 'Cyrillic']` for your workload. Script names
|
|
533
|
+
are case-sensitive runtime Unicode property aliases, not language codes.
|
|
534
|
+
`scripts: []` disables script activation while retaining the other triggers.
|
|
535
|
+
|
|
536
|
+
Inside activated values, the rule replaces words containing an uppercase letter
|
|
537
|
+
anywhere (`iPhone`, `A.B`, `Санкт-Петербург`), typographic quoted spans (including
|
|
538
|
+
lowercase `«ромашка лютик»`), links including `example.org/path`, and bare domestic
|
|
539
|
+
mobile shapes. Labels immediately before a colon (also `Label :`), lowercase
|
|
540
|
+
words, numbers, date ranges and path tokens stay. A path can still match existing
|
|
541
|
+
personal-path or credential rules. Numeric primitives and JSON keys stay; string
|
|
542
|
+
escapes and surrounding source formatting are retained. JSON/JSONL activation
|
|
543
|
+
also applies independently within serialized JSON strings, to the existing
|
|
544
|
+
bounded decoding depth. Findings do not seed document-wide name propagation.
|
|
545
|
+
Repeated redaction is stable. Reusing a placeholder map for the same original
|
|
546
|
+
produces stable tokens and exact restoration; the existing placeholder-namespace
|
|
547
|
+
collision checks still reject input that already contains typed placeholders.
|
|
548
|
+
|
|
549
|
+
This is a structural heuristic, so capitalized ordinary words also disappear.
|
|
550
|
+
Lowercase organizations and `Ромашка: менеджер` stay. English prose without a
|
|
551
|
+
trigger (`Worked at Romashka for three years.`), isolated neutral-key values
|
|
552
|
+
`{"note":"Romashka"}` / `{"note":"1991-03-12"}`, and a personal website root
|
|
553
|
+
without a trigger stay under this rule. The existing detectors and explicit
|
|
554
|
+
known/sensitive-field policies still run: an unactivated value is byte-for-byte
|
|
555
|
+
unchanged by this additional rule, not exempt from credential detection.
|
|
556
|
+
`credentialsOnly` and `knownOnly` suppress automatic prose detection.
|
|
557
|
+
|
|
558
|
+
The new scanner uses a fixed number of forward passes over maximal tokens,
|
|
559
|
+
including long tokens without spaces. This does not impose a linear-time bound
|
|
560
|
+
on optional external engines or every preexisting detection rule. Debugging is
|
|
561
|
+
off by default; `debug` receives value-free `proper-name-prose` and
|
|
562
|
+
`document-experience` hit counts when those scans run.
|
|
563
|
+
|
|
564
|
+
CLI equivalents: `--proper-names capitalized --scripts Cyrillic,Greek`.
|
|
565
|
+
Configuration files accept the same `properNames` and `scripts` API options;
|
|
566
|
+
use a configuration file for `scripts: []`. See the runnable
|
|
567
|
+
[prose-log example](../examples/prose-logs.mjs).
|
package/docs/COVERAGE.md
CHANGED
|
@@ -60,7 +60,7 @@ The `documents` profile also reads employers from CV experience lines anchored o
|
|
|
60
60
|
a date range: `Title, Employer (2019-05 – 2021-03)`, `Title — Employer, 2019–2021`
|
|
61
61
|
and `Title (Employer), 2019–2021`. Ranges use years, `YYYY-MM`, `MM.YYYY`, full
|
|
62
62
|
dates or `month YYYY`, with `по настоящее время`, `н.в.`, `по н.в.`, `present` and
|
|
63
|
-
similar open ends.
|
|
63
|
+
similar open ends. For the non-parenthesized layouts, titles have one to nine words. A label of one or two words
|
|
64
64
|
before the title (`Опыт: `, `Опыт работы - `, `Опыт — `) is read like a list marker,
|
|
65
65
|
and a segment after `|` is read like a line start, so
|
|
66
66
|
`Должность: Менеджер | Опыт: Аналитик, Ромашка (2018 – 2020)` also yields the
|
|
@@ -68,9 +68,9 @@ employer. The date range stays the anchor and the employer must still start with
|
|
|
68
68
|
capital letter, a digit or a quote. Confidence is 0.60, and the employer then
|
|
69
69
|
propagates through the document like other organizations. The default profile
|
|
70
70
|
ignores these lines. Lines
|
|
71
|
-
with
|
|
72
|
-
|
|
73
|
-
the same shape can be flagged.
|
|
71
|
+
with parenthesized ranges additionally support arbitrary title prefixes,
|
|
72
|
+
comma-containing employers and titles, as described below. Date-anchored prose
|
|
73
|
+
with the same shape can be flagged.
|
|
74
74
|
|
|
75
75
|
The gazetteer has only the cities/regions explicitly listed in that module,
|
|
76
76
|
including Springfield and Riverton. It is not a worldwide geocoder. Location
|
|
@@ -376,7 +376,8 @@ Employer labels consume multi-word values to a comma or line/quote boundary.
|
|
|
376
376
|
Dated CV entries additionally support compound titles, title parentheses/slashes,
|
|
377
377
|
quoted group names, legal-form/location suffixes, empty/`null` range ends,
|
|
378
378
|
multiple entries, employer-first role entries and pipe-separated entries.
|
|
379
|
-
The
|
|
379
|
+
The parenthesized date-anchor scanner accepts arbitrary forward titles and
|
|
380
|
+
lead-ins. The reversed `Company, manager` layout retains finite role cues. Undated
|
|
380
381
|
`Менеджер, Ромашка Лютик`, `- Ромашка Лютик 5 лет` and
|
|
381
382
|
`Последнее место работы Ромашка Лютик` remain explicit ambiguous limits, as
|
|
382
383
|
permitted by #94. Use known values or schema declarations for them.
|
|
@@ -402,3 +403,48 @@ redaction replaces the fragment instead of repeating an unchanged result until
|
|
|
402
403
|
|
|
403
404
|
The [requirement inventory and research](case-studies/issue-100/README.md)
|
|
404
405
|
records the complete scope and regression evidence.
|
|
406
|
+
|
|
407
|
+
## Dated entries and optional prose (issues 104–106)
|
|
408
|
+
|
|
409
|
+
Parenthesized date anchors now delimit successive entries on the same line,
|
|
410
|
+
including arbitrary title words and lead-ins (`Стаж = 5 лет:`, `= 3 года,`).
|
|
411
|
+
Employers can contain comma suffixes, nested parentheses, slash-separated names,
|
|
412
|
+
legal forms and multiple words. A pipe after the employer preserves the industry
|
|
413
|
+
suffix. `Founder, consultant, Romashka (2019–2021)` and equivalent Russian entries
|
|
414
|
+
share the parser. The employer must start with a capital letter, a quote, or a
|
|
415
|
+
company-style digit/letter combination. **Lowercase employers remain a limit**:
|
|
416
|
+
`Менеджер, romashka (2019–2021)` / `Менеджер, ромашка (2019–2021)` stay, preserving
|
|
417
|
+
ambiguous controls such as `Встреча, понедельник (2019–2021)`.
|
|
418
|
+
|
|
419
|
+
`ГК` and `группа компаний` introduce quoted or unquoted group names. Employment
|
|
420
|
+
lists continue through repeated `, потом в` and `, затем в` items. Domestic
|
|
421
|
+
`79…`/`89…` mobile shapes after a comma work in Russian and English text; plain
|
|
422
|
+
`order`/`id` numbers stay. School detection also covers `(МГУ, 2012)`,
|
|
423
|
+
`Образование (МГУ)`, `в МГТУ им. Баумана` and `Обучение: Skillbox, 2020`.
|
|
424
|
+
The acronym vocabulary remains finite.
|
|
425
|
+
|
|
426
|
+
Weak organization experience/duration cues preserve a small explicit technology
|
|
427
|
+
set: Python, Go, SQL, Excel, Power BI, Kubernetes, PostgreSQL, Telegram and Scrum.
|
|
428
|
+
This is an ambiguity policy, not general recognition of every product; a real
|
|
429
|
+
employer with one of those names needs `knownPersonal` or a declared sensitive
|
|
430
|
+
field. Explicit private values/fields and stronger legal-form cues still win.
|
|
431
|
+
|
|
432
|
+
Document share-link detection also recognizes HTTP(S) URLs containing `/d/<id>`
|
|
433
|
+
for any host and nonempty id length. Ordinary website roots remain a limit.
|
|
434
|
+
Relative operational paths such as `{"location":"apps/api"}` stay; explicit
|
|
435
|
+
personal paths and credential rules remain active. Caller-sensitive field names
|
|
436
|
+
now use the same case/underscore/hyphen normalization as built-in document names.
|
|
437
|
+
|
|
438
|
+
For no-cue prose, enable [`properNames: 'capitalized'`](API.md#structural-prose-redaction).
|
|
439
|
+
It activates per value, defaults to Cyrillic script letters or structural data
|
|
440
|
+
triggers, and supports configurable Unicode scripts. It intentionally loses
|
|
441
|
+
capitalized ordinary words. Labels, lowercase words, dates, paths, JSON keys and
|
|
442
|
+
untriggered values are outside this additional rule. English untriggered prose,
|
|
443
|
+
lowercase names and a name immediately before a colon remain documented limits.
|
|
444
|
+
The API describes the interaction with existing detectors and placeholder modes.
|
|
445
|
+
|
|
446
|
+
The date-anchor scan visits each preceding entry once; the optional prose scanner
|
|
447
|
+
classifies maximal tokens in fixed passes, avoiding repeated suffix searches on
|
|
448
|
+
long unspaced input. This is not a latency guarantee for the whole pipeline.
|
|
449
|
+
[The requirement inventory](case-studies/issue-106/README.md) records alternatives,
|
|
450
|
+
research, regression evidence and reproducible bounded performance measurements.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { sanitize } from '../src/index.js';
|
|
2
|
+
|
|
3
|
+
// Synthetic mixed records: only the prose string activates the extra rule.
|
|
4
|
+
const source = JSON.stringify({
|
|
5
|
+
note: 'Работал в Ромашке менеджером по продажам.',
|
|
6
|
+
operational: 'Connection to Postgres lost, retrying',
|
|
7
|
+
count: 42,
|
|
8
|
+
});
|
|
9
|
+
console.log(
|
|
10
|
+
sanitize(source, {
|
|
11
|
+
profile: 'documents',
|
|
12
|
+
properNames: 'capitalized',
|
|
13
|
+
scripts: ['Cyrillic'],
|
|
14
|
+
structured: 'json',
|
|
15
|
+
}).text
|
|
16
|
+
);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.0",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -24,7 +24,8 @@
|
|
|
24
24
|
"LICENSE",
|
|
25
25
|
"README.md",
|
|
26
26
|
"examples/shared-tool-logs.mjs",
|
|
27
|
-
"examples/reversible-placeholders.mjs"
|
|
27
|
+
"examples/reversible-placeholders.mjs",
|
|
28
|
+
"examples/prose-logs.mjs"
|
|
28
29
|
],
|
|
29
30
|
"exports": {
|
|
30
31
|
".": {
|
package/src/context.js
CHANGED
|
@@ -257,6 +257,11 @@ function operationalValue(value, type) {
|
|
|
257
257
|
);
|
|
258
258
|
}
|
|
259
259
|
|
|
260
|
+
function operationalLocationPath(value) {
|
|
261
|
+
return (
|
|
262
|
+
servicePath(value) || /^(?:\.?\.?\/)?[\w.-]+(?:\/[\w.-]+)+\/?$/.test(value)
|
|
263
|
+
);
|
|
264
|
+
}
|
|
260
265
|
// Shared by native labels and the vendor catalogs. Code expressions and
|
|
261
266
|
// line escapes are not personal field values, even when a key is familiar.
|
|
262
267
|
export function personalValueAllowed(text, range, type, labelStart) {
|
|
@@ -264,7 +269,10 @@ export function personalValueAllowed(text, range, type, labelStart) {
|
|
|
264
269
|
if (!plausibleLabel(value, type)) {
|
|
265
270
|
return false;
|
|
266
271
|
}
|
|
267
|
-
if (
|
|
272
|
+
if (
|
|
273
|
+
['LOCATION', 'ADDRESS'].includes(type) &&
|
|
274
|
+
operationalLocationPath(value)
|
|
275
|
+
) {
|
|
268
276
|
return false;
|
|
269
277
|
}
|
|
270
278
|
if (type === 'PHONE') {
|
package/src/detection.js
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
import { localAddress } from './public.js';
|
|
16
16
|
import { detectKnownPersonal } from './personal-variants.js';
|
|
17
17
|
import { detectDocumentEntities } from './document-entities.js';
|
|
18
|
+
import { detectProseAndDocuments } from './proper-names.js';
|
|
18
19
|
|
|
19
20
|
export const REDACTED = '[REDACTED]';
|
|
20
21
|
export const failure = (code = 'ERR_SANITIZATION', location = {}) =>
|
|
@@ -173,7 +174,7 @@ export function nativeDetect(text, options, emit) {
|
|
|
173
174
|
detectContext(text, emit, options.credentialsOnly);
|
|
174
175
|
if (!options.credentialsOnly) {
|
|
175
176
|
detectPersonal(text, emit);
|
|
176
|
-
|
|
177
|
+
detectProseAndDocuments(text, emit, options, detectDocumentEntities);
|
|
177
178
|
detectCatalog(text, emit);
|
|
178
179
|
detectIdentity(text, emit);
|
|
179
180
|
detectNational(text, emit);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { escapePattern } from './detection.js';
|
|
2
|
+
import { months } from './personal-context.js';
|
|
3
|
+
|
|
4
|
+
const boundary = '[\\p{L}\\p{M}\\p{N}_]';
|
|
5
|
+
const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
|
|
6
|
+
const month = [
|
|
7
|
+
...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
|
|
8
|
+
].join('|');
|
|
9
|
+
const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
|
|
10
|
+
const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'по н.в.']
|
|
11
|
+
.concat(['н.в.', 'по н/в', 'н/в', 'present', 'now', 'current'])
|
|
12
|
+
.flatMap((word) => [word, capitalized(word)])
|
|
13
|
+
.map(escapePattern)
|
|
14
|
+
.join('|');
|
|
15
|
+
export const documentDateRange = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen}|null|(?=\\)))(?!${boundary})`;
|
package/src/document-entities.js
CHANGED
|
@@ -2,9 +2,10 @@ import { collectMatches, escapePattern } from './detection.js';
|
|
|
2
2
|
import { firstNames } from './names.js';
|
|
3
3
|
import { organizationCore, personalRegex } from './personal-variants.js';
|
|
4
4
|
import { documentVocabulary } from './document-vocabulary.js';
|
|
5
|
-
import {
|
|
5
|
+
import { documentDateRange as range } from './document-dates.js';
|
|
6
6
|
import { detectDocumentExtras } from './document-extras.js';
|
|
7
7
|
import { detectDocumentPatterns } from './document-patterns.js';
|
|
8
|
+
import { isDocumentTechnology } from './document-experience.js';
|
|
8
9
|
|
|
9
10
|
const words = (column) =>
|
|
10
11
|
[...new Set(documentVocabulary.flatMap((row) => row[column].split('|')))]
|
|
@@ -58,17 +59,6 @@ const groupOrganization = new RegExp(
|
|
|
58
59
|
// CV experience lines: the date range anchors the employer slot, so prose with
|
|
59
60
|
// a comma is untouched. Titles are not required to be lowercase because English
|
|
60
61
|
// ones are often title case (`Senior QA Engineer`).
|
|
61
|
-
const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
|
|
62
|
-
const month = [
|
|
63
|
-
...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
|
|
64
|
-
].join('|');
|
|
65
|
-
const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
|
|
66
|
-
const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'по н.в.']
|
|
67
|
-
.concat(['н.в.', 'по н/в', 'н/в', 'present', 'now', 'current'])
|
|
68
|
-
.flatMap((word) => [word, capitalized(word)])
|
|
69
|
-
.map(escapePattern)
|
|
70
|
-
.join('|');
|
|
71
|
-
const range = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen}|null|(?=\\)))(?!${boundary})`;
|
|
72
62
|
const titleWord = '[\\p{L}\\p{N}][\\p{L}\\p{N}/+#.&-]{0,39}';
|
|
73
63
|
// A segment starts a line or follows `|`. `Опыт:` or `Опыт -` before the title
|
|
74
64
|
// reads like a list marker: the date range remains the anchor of the rule.
|
|
@@ -90,7 +80,23 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
90
80
|
return;
|
|
91
81
|
}
|
|
92
82
|
detectDocumentPatterns(text, emit);
|
|
93
|
-
detectDocumentExtras(text, emit, range);
|
|
83
|
+
detectDocumentExtras(text, emit, range, options.debug);
|
|
84
|
+
collectMatches(
|
|
85
|
+
text,
|
|
86
|
+
new RegExp(
|
|
87
|
+
`(?:^|[\\r\\n])[ \\t]*[^|\\r\\n]{1,120}\\|[ \\t]*(${name})[ \\t]*\\|[ \\t]*${range}`,
|
|
88
|
+
'gu'
|
|
89
|
+
),
|
|
90
|
+
emit,
|
|
91
|
+
{
|
|
92
|
+
type: 'ORGANIZATION',
|
|
93
|
+
rule: 'organization-experience',
|
|
94
|
+
category: 'personal',
|
|
95
|
+
confidence: 0.85,
|
|
96
|
+
},
|
|
97
|
+
1,
|
|
98
|
+
(value) => !isDocumentTechnology(value)
|
|
99
|
+
);
|
|
94
100
|
const found = [];
|
|
95
101
|
const capture = (finding) => {
|
|
96
102
|
found.push(finding);
|
|
@@ -113,7 +119,8 @@ export function detectDocumentEntities(text, emit, options) {
|
|
|
113
119
|
pattern,
|
|
114
120
|
capture,
|
|
115
121
|
{ type, rule, category: 'personal', confidence },
|
|
116
|
-
group
|
|
122
|
+
group,
|
|
123
|
+
(value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
|
|
117
124
|
);
|
|
118
125
|
}
|
|
119
126
|
for (const match of text.matchAll(ruResidence)) {
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
const technology = new Set(
|
|
2
|
+
[
|
|
3
|
+
'Python',
|
|
4
|
+
'Go',
|
|
5
|
+
'SQL',
|
|
6
|
+
'Excel',
|
|
7
|
+
'Power BI',
|
|
8
|
+
'Kubernetes',
|
|
9
|
+
'PostgreSQL',
|
|
10
|
+
'Telegram',
|
|
11
|
+
'Scrum',
|
|
12
|
+
].map((s) => s.toLowerCase())
|
|
13
|
+
);
|
|
14
|
+
export const isDocumentTechnology = (value) =>
|
|
15
|
+
technology.has(value.trim().toLowerCase());
|
|
16
|
+
|
|
17
|
+
// Used only to disambiguate the reversed `Company, manager` layout. Forward
|
|
18
|
+
// entries deliberately accept any title, without a title vocabulary.
|
|
19
|
+
const reversedRole =
|
|
20
|
+
/^(?:менеджер|директор|руководитель|инженер|аналитик|разработчик|manager|director|engineer|developer|senior|qa)(?!\p{L})/iu;
|
|
21
|
+
|
|
22
|
+
function commaParts(text) {
|
|
23
|
+
const parts = [];
|
|
24
|
+
let start = 0,
|
|
25
|
+
depth = 0,
|
|
26
|
+
quote;
|
|
27
|
+
for (let i = 0; i < text.length; i++) {
|
|
28
|
+
const char = text[i];
|
|
29
|
+
if (quote) {
|
|
30
|
+
if (char === quote) {
|
|
31
|
+
quote = undefined;
|
|
32
|
+
}
|
|
33
|
+
} else if ('«“"'.includes(char)) {
|
|
34
|
+
quote = { '«': '»', '“': '”', '"': '"' }[char];
|
|
35
|
+
} else if (char === '(') {
|
|
36
|
+
depth++;
|
|
37
|
+
} else if (char === ')') {
|
|
38
|
+
depth = Math.max(0, depth - 1);
|
|
39
|
+
} else if (char === ',' && !depth) {
|
|
40
|
+
parts.push({ start, text: text.slice(start, i).trim() });
|
|
41
|
+
start = i + 1;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
parts.push({ start, text: text.slice(start).trim() });
|
|
45
|
+
return parts;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function employerSpan(prefix) {
|
|
49
|
+
// An industry suffix belongs to the entry, but is not an employer. A pipe
|
|
50
|
+
// before the title remains a section separator, as in `Опыт | Менеджер, ...`.
|
|
51
|
+
const comma = prefix.indexOf(',');
|
|
52
|
+
const pipe = comma < 0 ? -1 : prefix.indexOf('|', comma);
|
|
53
|
+
const content = pipe < 0 ? prefix : prefix.slice(0, pipe);
|
|
54
|
+
const parts = commaParts(content);
|
|
55
|
+
let index = 1;
|
|
56
|
+
if (parts.length === 1) {
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
if (/^[=\s\d]/u.test(parts[0].text)) {
|
|
60
|
+
index++;
|
|
61
|
+
}
|
|
62
|
+
if (parts.length === 2 && reversedRole.test(parts[1].text)) {
|
|
63
|
+
index = 0;
|
|
64
|
+
} else {
|
|
65
|
+
while (index < parts.length - 1 && /^\p{Ll}/u.test(parts[index].text)) {
|
|
66
|
+
index++;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const part = parts[index];
|
|
70
|
+
if (!part || !/^(?:[\p{Lu}«“"]|\d{1,4}-?\p{L})/u.test(part.text)) {
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
const end = index === 0 ? content.indexOf(',') : content.trimEnd().length;
|
|
74
|
+
const start = part.start + content.slice(part.start).search(/\S/u);
|
|
75
|
+
return isDocumentTechnology(content.slice(start, end))
|
|
76
|
+
? undefined
|
|
77
|
+
: { start, end };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function detectDatedEmployers(text, emit, range, debug) {
|
|
81
|
+
let cursor = 0,
|
|
82
|
+
count = 0;
|
|
83
|
+
// Match only the anchor, then visit each preceding segment once. No lazy
|
|
84
|
+
// prefix search restarts at every position on a long nonmatching line.
|
|
85
|
+
for (const match of text.matchAll(new RegExp(`\\(${range}\\)`, 'gu'))) {
|
|
86
|
+
const segment = text.slice(cursor, match.index);
|
|
87
|
+
const boundary =
|
|
88
|
+
Math.max(
|
|
89
|
+
segment.lastIndexOf('\n'),
|
|
90
|
+
segment.lastIndexOf('\r'),
|
|
91
|
+
segment.lastIndexOf(';')
|
|
92
|
+
) + 1;
|
|
93
|
+
const prefix = segment.slice(boundary).replace(/^[., \t]+/u, '');
|
|
94
|
+
const span = employerSpan(prefix);
|
|
95
|
+
if (span) {
|
|
96
|
+
const offset = match.index - prefix.length;
|
|
97
|
+
emit({
|
|
98
|
+
start: offset + span.start,
|
|
99
|
+
end: offset + span.end,
|
|
100
|
+
type: 'ORGANIZATION',
|
|
101
|
+
category: 'personal',
|
|
102
|
+
rule: 'organization-experience',
|
|
103
|
+
confidence: 0.85,
|
|
104
|
+
});
|
|
105
|
+
count++;
|
|
106
|
+
}
|
|
107
|
+
cursor = match.index + match[0].length;
|
|
108
|
+
}
|
|
109
|
+
debug?.({ event: 'document-experience', count });
|
|
110
|
+
}
|
package/src/document-extras.js
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import { collectMatches } from './detection.js';
|
|
2
|
+
import {
|
|
3
|
+
detectDatedEmployers,
|
|
4
|
+
isDocumentTechnology,
|
|
5
|
+
} from './document-experience.js';
|
|
6
|
+
import { isDocumentShareLink } from './proper-names.js';
|
|
2
7
|
|
|
3
8
|
const boundary = '[\\p{L}\\p{N}_]';
|
|
4
9
|
const proper = '[\\p{Lu}][\\p{L}\\p{M}\\p{N}.-]{0,46}[\\p{L}\\p{M}\\p{N}]';
|
|
5
10
|
const name = `${proper}(?:[ \\t]+${proper}){0,5}`;
|
|
11
|
+
const employerName = `(?:«[^«»\\r\\n]{1,160}»|“[^“”\\r\\n]{1,160}”|"[^"\\r\\n]{1,160}"|${name})`;
|
|
6
12
|
const orgList = `(?:«[^«»\\r\\n]{1,160}»|${name})(?:[ \\t]*(?:,|и|and)[ \\t]+${name}){0,5}`;
|
|
7
13
|
const cues = new RegExp(
|
|
8
14
|
`(?<!${boundary})(?:компании|опыт[ \\t]+в|проекты[ \\t]+для|[Рр]оль[ \\t]+в)[ \\t]+(${orgList})(?!${boundary})`,
|
|
@@ -12,13 +18,42 @@ const duration = new RegExp(
|
|
|
12
18
|
`(?<!${boundary})(${name})(?=[ \\t]*\\(\\d{1,2}[ \\t]+(?:год|года|лет|years?)\\))`,
|
|
13
19
|
'gu'
|
|
14
20
|
);
|
|
15
|
-
|
|
16
|
-
const role =
|
|
17
|
-
/^(?:менеджер|директор|руководитель|инженер|аналитик|разработчик)(?![\p{L}])|^(?:manager|director|engineer|developer|senior|qa)\b/iu;
|
|
18
|
-
|
|
19
|
-
export function detectDocumentExtras(text, emit, range) {
|
|
21
|
+
export function detectDocumentExtras(text, emit, range, debug) {
|
|
20
22
|
for (const [type, rule, pattern, group] of [
|
|
21
23
|
['PHONE', 'document-mobile-bare', /^[78]9\d{9}$/gm, 0],
|
|
24
|
+
[
|
|
25
|
+
'PHONE',
|
|
26
|
+
'document-mobile-comma',
|
|
27
|
+
/,[ \t]*([78]9\d{9})(?![\p{L}\p{N}_])/gu,
|
|
28
|
+
1,
|
|
29
|
+
],
|
|
30
|
+
[
|
|
31
|
+
'ORGANIZATION',
|
|
32
|
+
'organization-group-cue',
|
|
33
|
+
new RegExp(
|
|
34
|
+
`(?<!${boundary})(?:ГК|[Гг]руппа компаний)[ \\t]+${employerName}`,
|
|
35
|
+
'gu'
|
|
36
|
+
),
|
|
37
|
+
0,
|
|
38
|
+
],
|
|
39
|
+
[
|
|
40
|
+
'EDUCATION',
|
|
41
|
+
'education-bracket',
|
|
42
|
+
/(?<=\()(МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ)(?=[,)])/gu,
|
|
43
|
+
1,
|
|
44
|
+
],
|
|
45
|
+
[
|
|
46
|
+
'EDUCATION',
|
|
47
|
+
'education-named-acronym',
|
|
48
|
+
/(?<![\p{L}\p{N}_])(?:МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ)[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{2,40}/gu,
|
|
49
|
+
0,
|
|
50
|
+
],
|
|
51
|
+
[
|
|
52
|
+
'EDUCATION',
|
|
53
|
+
'education-training',
|
|
54
|
+
/(?<![\p{L}\p{N}_])(?:Обучение|Training)[ \t]*:[ \t]*([\p{Lu}][\p{L}\p{M}.-]{1,47})/gu,
|
|
55
|
+
1,
|
|
56
|
+
],
|
|
22
57
|
[
|
|
23
58
|
'PHONE',
|
|
24
59
|
'document-mobile-context',
|
|
@@ -56,7 +91,8 @@ export function detectDocumentExtras(text, emit, range) {
|
|
|
56
91
|
pattern,
|
|
57
92
|
emit,
|
|
58
93
|
{ type, rule, category: 'personal', confidence: 0.85 },
|
|
59
|
-
group
|
|
94
|
+
group,
|
|
95
|
+
(value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
|
|
60
96
|
);
|
|
61
97
|
}
|
|
62
98
|
for (const match of text.matchAll(cues)) {
|
|
@@ -64,6 +100,9 @@ export function detectDocumentExtras(text, emit, range) {
|
|
|
64
100
|
for (const company of match[1].matchAll(
|
|
65
101
|
new RegExp(`«[^«»\\r\\n]{1,160}»|${name}`, 'gu')
|
|
66
102
|
)) {
|
|
103
|
+
if (isDocumentTechnology(company[0])) {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
67
106
|
emit({
|
|
68
107
|
start: offset + company.index,
|
|
69
108
|
end: offset + company.index + company[0].length,
|
|
@@ -75,7 +114,57 @@ export function detectDocumentExtras(text, emit, range) {
|
|
|
75
114
|
}
|
|
76
115
|
}
|
|
77
116
|
detectQueryContacts(text, emit);
|
|
78
|
-
|
|
117
|
+
collectMatches(
|
|
118
|
+
text,
|
|
119
|
+
/https?:\/\/[^\s"<>\\]+/gu,
|
|
120
|
+
emit,
|
|
121
|
+
{
|
|
122
|
+
type: 'URL',
|
|
123
|
+
category: 'personal',
|
|
124
|
+
rule: 'document-share-path',
|
|
125
|
+
confidence: 0.85,
|
|
126
|
+
},
|
|
127
|
+
0,
|
|
128
|
+
isDocumentShareLink
|
|
129
|
+
);
|
|
130
|
+
detectEmployerContinuation(text, emit);
|
|
131
|
+
detectDatedEmployers(text, emit, range, debug);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function detectEmployerContinuation(text, emit) {
|
|
135
|
+
const continuation = new RegExp(
|
|
136
|
+
`,[ \\t]*(?:потом|затем)[ \\t]+в[ \\t]+(${employerName})`,
|
|
137
|
+
'uy'
|
|
138
|
+
);
|
|
139
|
+
for (const match of text.matchAll(
|
|
140
|
+
new RegExp(
|
|
141
|
+
`(?<!${boundary})[Рр]аботал[аи]?[ \\t]+в[ \\t]+(${employerName})`,
|
|
142
|
+
'gu'
|
|
143
|
+
)
|
|
144
|
+
)) {
|
|
145
|
+
continuation.lastIndex = match.index + match[0].length;
|
|
146
|
+
let next = continuation.exec(text);
|
|
147
|
+
if (next) {
|
|
148
|
+
emitEmployer(match);
|
|
149
|
+
}
|
|
150
|
+
while (next) {
|
|
151
|
+
emitEmployer(next);
|
|
152
|
+
next = continuation.exec(text);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function emitEmployer(match) {
|
|
156
|
+
const end = match.index + match[0].length;
|
|
157
|
+
if (!isDocumentTechnology(match[1])) {
|
|
158
|
+
emit({
|
|
159
|
+
start: end - match[1].length,
|
|
160
|
+
end,
|
|
161
|
+
type: 'ORGANIZATION',
|
|
162
|
+
category: 'personal',
|
|
163
|
+
rule: 'organization-continuation',
|
|
164
|
+
confidence: 0.85,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
}
|
|
79
168
|
}
|
|
80
169
|
|
|
81
170
|
function detectQueryContacts(text, emit) {
|
|
@@ -111,60 +200,3 @@ function detectQueryContacts(text, emit) {
|
|
|
111
200
|
});
|
|
112
201
|
}
|
|
113
202
|
}
|
|
114
|
-
|
|
115
|
-
function detectExperience(text, emit, range) {
|
|
116
|
-
// Date anchors bound each entry; do not guess at undated employer/title fragments.
|
|
117
|
-
collectMatches(
|
|
118
|
-
text,
|
|
119
|
-
new RegExp(
|
|
120
|
-
`(?:^|[\\r\\n])[ \\t]*[^|\\r\\n]{1,120}\\|[ \\t]*(${name})[ \\t]*\\|[ \\t]*${range}`,
|
|
121
|
-
'gu'
|
|
122
|
-
),
|
|
123
|
-
emit,
|
|
124
|
-
{
|
|
125
|
-
type: 'ORGANIZATION',
|
|
126
|
-
rule: 'organization-experience',
|
|
127
|
-
category: 'personal',
|
|
128
|
-
confidence: 0.85,
|
|
129
|
-
},
|
|
130
|
-
1
|
|
131
|
-
);
|
|
132
|
-
const dated = new RegExp(`([^\\r\\n;|]{1,320}?)\\((${range})\\)`, 'gu');
|
|
133
|
-
for (const match of text.matchAll(dated)) {
|
|
134
|
-
const prefix = match[1];
|
|
135
|
-
const content = prefix
|
|
136
|
-
.replace(/^.*\(\d[^)]*\)[, \t]*/, '')
|
|
137
|
-
.replace(/^[, \t]+/, '');
|
|
138
|
-
const chunks = [...content.matchAll(/[^,]+/g)];
|
|
139
|
-
const parts = chunks.map((part) => part[0].trim());
|
|
140
|
-
let value;
|
|
141
|
-
if (parts.length > 1 && role.test(parts[0])) {
|
|
142
|
-
let index = 1;
|
|
143
|
-
while (index < parts.length - 1 && /^[\p{Ll}]/u.test(parts[index])) {
|
|
144
|
-
index++;
|
|
145
|
-
}
|
|
146
|
-
value = content.slice(chunks[index].index).trim();
|
|
147
|
-
} else if (parts.length === 2 && role.test(parts[1])) {
|
|
148
|
-
value = parts[0].trim();
|
|
149
|
-
}
|
|
150
|
-
if (
|
|
151
|
-
!value ||
|
|
152
|
-
!/^[\p{Lu}«“"]/u.test(value) ||
|
|
153
|
-
value.includes('[REDACTED]')
|
|
154
|
-
) {
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
const start = match.index + prefix.lastIndexOf(value);
|
|
158
|
-
if (start < match.index) {
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
emit({
|
|
162
|
-
start,
|
|
163
|
-
end: start + value.length,
|
|
164
|
-
type: 'ORGANIZATION',
|
|
165
|
-
category: 'personal',
|
|
166
|
-
rule: 'organization-experience',
|
|
167
|
-
confidence: 0.85,
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
}
|
package/src/document-fields.js
CHANGED
|
@@ -15,5 +15,7 @@ const types = new Map(
|
|
|
15
15
|
)
|
|
16
16
|
);
|
|
17
17
|
export function documentFieldType(field) {
|
|
18
|
-
return types.get(field
|
|
18
|
+
return types.get(normalizeFieldName(field));
|
|
19
19
|
}
|
|
20
|
+
export const normalizeFieldName = (field) =>
|
|
21
|
+
field.replace(/[_-]/g, '').toLowerCase();
|
package/src/engines.js
CHANGED
|
@@ -140,7 +140,13 @@ async function analyzeStructured(
|
|
|
140
140
|
batch.text,
|
|
141
141
|
[],
|
|
142
142
|
true,
|
|
143
|
-
|
|
143
|
+
{
|
|
144
|
+
...options,
|
|
145
|
+
properNames: undefined,
|
|
146
|
+
documentEntities: options.properNames
|
|
147
|
+
? false
|
|
148
|
+
: options.documentEntities,
|
|
149
|
+
},
|
|
144
150
|
nativeFindings === undefined ? undefined : []
|
|
145
151
|
),
|
|
146
152
|
options
|
|
@@ -152,7 +158,11 @@ async function analyzeStructured(
|
|
|
152
158
|
view.text,
|
|
153
159
|
[],
|
|
154
160
|
true,
|
|
155
|
-
{
|
|
161
|
+
{
|
|
162
|
+
...options,
|
|
163
|
+
structured: undefined,
|
|
164
|
+
properNames: view.prose ? options.properNames : undefined,
|
|
165
|
+
},
|
|
156
166
|
nativeFindings === undefined ? undefined : []
|
|
157
167
|
);
|
|
158
168
|
for (const finding of detected) {
|
package/src/index.d.ts
CHANGED
|
@@ -43,6 +43,10 @@ export interface SanitizerOptions {
|
|
|
43
43
|
profile?: 'publication' | 'documents';
|
|
44
44
|
/** Opt in to bounded employer, school, city/region heuristics. */
|
|
45
45
|
documentEntities?: boolean;
|
|
46
|
+
/** Aggressive structural proper names in prose values; opt-in, off by default. */
|
|
47
|
+
properNames?: 'capitalized';
|
|
48
|
+
/** Unicode Script aliases that activate prose detection; default ['Cyrillic']. */
|
|
49
|
+
scripts?: string[];
|
|
46
50
|
confirmedPersonal?: Array<{ value: string; type: string }>;
|
|
47
51
|
knownSecrets?: string[];
|
|
48
52
|
knownPersonal?: PersonalValue[];
|
|
@@ -63,7 +67,7 @@ export interface SanitizerOptions {
|
|
|
63
67
|
structured?: 'json' | 'jsonl';
|
|
64
68
|
/** Property names whose metadata heuristics may be suppressed. */
|
|
65
69
|
structuralFields?: string[];
|
|
66
|
-
/**
|
|
70
|
+
/** Property names normalized across case/underscores/hyphens; entire JSON/JSONL values redact. */
|
|
67
71
|
sensitiveFields?: string[];
|
|
68
72
|
/** Personal replacement threshold, default 0.5. Inspect retains lower scores. */
|
|
69
73
|
minConfidence?: number;
|
|
@@ -88,6 +92,7 @@ export interface SanitizerOptions {
|
|
|
88
92
|
/** Value-free diagnostics; enabled only when a debug callback is supplied. */
|
|
89
93
|
export type DebugEvent =
|
|
90
94
|
| { event: 'detected'; count: number }
|
|
95
|
+
| { event: 'document-experience' | 'proper-name-prose'; count: number }
|
|
91
96
|
| { event: 'residual-redaction'; round: number }
|
|
92
97
|
| { event: 'two-pass-confirmation'; type: string }
|
|
93
98
|
| { event: 'propagation-skipped'; reason: 'length'; type: string }
|
package/src/placeholder-types.js
CHANGED
package/src/propagation.js
CHANGED
|
@@ -36,7 +36,11 @@ function eligibleSeed(finding, options) {
|
|
|
36
36
|
!options.knownOnly &&
|
|
37
37
|
finding.category === 'personal' &&
|
|
38
38
|
finding.type !== 'ENCODED_SENSITIVE' &&
|
|
39
|
-
![
|
|
39
|
+
![
|
|
40
|
+
'confirmed-personal',
|
|
41
|
+
'document-propagation',
|
|
42
|
+
'proper-name-prose',
|
|
43
|
+
].includes(finding.rule) &&
|
|
40
44
|
actionable(finding, options) &&
|
|
41
45
|
((options.documentEntities && finding.type === 'ORGANIZATION') ||
|
|
42
46
|
finding.confidence >= 0.85 ||
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { failure } from './detection.js';
|
|
2
|
+
import { documentDateRange } from './document-dates.js';
|
|
3
|
+
|
|
4
|
+
const scriptPatterns = new Map();
|
|
5
|
+
const dateRange = new RegExp(`\\(${documentDateRange}\\)`, 'u');
|
|
6
|
+
const commaMobile = /,[ \t]*[78]9\d{9}(?![\p{L}\p{N}_])/u;
|
|
7
|
+
const mobile = /^[78]9\d{9}$/u;
|
|
8
|
+
const tokenPattern = /[\p{L}\p{M}\p{N}_./:\\?&#%=@+'’~+-]+/gu;
|
|
9
|
+
const upper = /[\p{Lu}\p{Lt}]/u;
|
|
10
|
+
const quotePairs = new Map([
|
|
11
|
+
['«', '»'],
|
|
12
|
+
['“', '”'],
|
|
13
|
+
['„', '“'],
|
|
14
|
+
['‘', '’'],
|
|
15
|
+
['「', '」'],
|
|
16
|
+
['『', '』'],
|
|
17
|
+
]);
|
|
18
|
+
|
|
19
|
+
function scriptPattern(scripts = ['Cyrillic']) {
|
|
20
|
+
if (
|
|
21
|
+
scripts.some(
|
|
22
|
+
(script) => typeof script !== 'string' || !/^[A-Za-z_]+$/.test(script)
|
|
23
|
+
)
|
|
24
|
+
) {
|
|
25
|
+
throw failure('ERR_CONFIG');
|
|
26
|
+
}
|
|
27
|
+
const key = scripts.join('|');
|
|
28
|
+
let pattern = scriptPatterns.get(key);
|
|
29
|
+
if (!pattern) {
|
|
30
|
+
// The alias grammar prevents regexp injection; constructing the Unicode
|
|
31
|
+
// property also validates aliases supported by this runtime.
|
|
32
|
+
try {
|
|
33
|
+
pattern = new RegExp(
|
|
34
|
+
scripts.length
|
|
35
|
+
? `(?=\\p{L})(?:${scripts.map((s) => `\\p{Script=${s}}`).join('|')})`
|
|
36
|
+
: '(?!)',
|
|
37
|
+
'u'
|
|
38
|
+
);
|
|
39
|
+
} catch {
|
|
40
|
+
throw failure('ERR_CONFIG');
|
|
41
|
+
}
|
|
42
|
+
// Bound cache growth when callers submit many different configurations.
|
|
43
|
+
if (scriptPatterns.size >= 64) {
|
|
44
|
+
scriptPatterns.clear();
|
|
45
|
+
}
|
|
46
|
+
scriptPatterns.set(key, pattern);
|
|
47
|
+
}
|
|
48
|
+
return pattern;
|
|
49
|
+
}
|
|
50
|
+
export function validateProperNames(options) {
|
|
51
|
+
if (
|
|
52
|
+
(options.properNames !== undefined &&
|
|
53
|
+
options.properNames !== 'capitalized') ||
|
|
54
|
+
(options.scripts !== undefined && !Array.isArray(options.scripts))
|
|
55
|
+
) {
|
|
56
|
+
throw failure('ERR_CONFIG');
|
|
57
|
+
}
|
|
58
|
+
scriptPattern(options.scripts);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function isProseLink(token) {
|
|
62
|
+
return (
|
|
63
|
+
/^https?:\/\/[^/]+/iu.test(token) ||
|
|
64
|
+
/^[\p{L}\p{N}](?:[\p{L}\p{N}-]*\.)+[\p{L}]{2,63}(?:[/:?#]|$)/u.test(token)
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
export function isDocumentShareLink(token) {
|
|
68
|
+
return isProseLink(token) && /\/d\/[^/?#]+/u.test(token);
|
|
69
|
+
}
|
|
70
|
+
function proseData(text, options) {
|
|
71
|
+
if (
|
|
72
|
+
scriptPattern(options.scripts).test(text) ||
|
|
73
|
+
dateRange.test(text) ||
|
|
74
|
+
commaMobile.test(text)
|
|
75
|
+
) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
for (const match of text.matchAll(tokenPattern)) {
|
|
79
|
+
if (isDocumentShareLink(match[0])) {
|
|
80
|
+
return true;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
function quotedSpans(text, emit) {
|
|
86
|
+
let start = -1,
|
|
87
|
+
closing;
|
|
88
|
+
for (let i = 0; i < text.length; i++) {
|
|
89
|
+
if (start >= 0) {
|
|
90
|
+
if (text[i] === closing) {
|
|
91
|
+
emit(start, i + 1, 'PROPER_NAME');
|
|
92
|
+
start = -1;
|
|
93
|
+
}
|
|
94
|
+
} else if (quotePairs.has(text[i])) {
|
|
95
|
+
start = i;
|
|
96
|
+
closing = quotePairs.get(text[i]);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function pathToken(value) {
|
|
101
|
+
return value.includes('/') || value.includes('\\');
|
|
102
|
+
}
|
|
103
|
+
// Keep existing document coverage outside the new replacements. Two naturally
|
|
104
|
+
// ordered runs (quotes, then tokens) provide an interval index without sorting.
|
|
105
|
+
// Overlapping broad document spans must not merge separate prose replacements.
|
|
106
|
+
export function detectProseAndDocuments(text, emit, options, detectDocuments) {
|
|
107
|
+
if (options.properNames !== 'capitalized') {
|
|
108
|
+
detectDocuments(text, emit, options);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const runs = [[]];
|
|
112
|
+
detectProperNames(
|
|
113
|
+
text,
|
|
114
|
+
(finding) => {
|
|
115
|
+
if (runs.at(-1).at(-1)?.end > finding.start) {
|
|
116
|
+
runs.push([]);
|
|
117
|
+
}
|
|
118
|
+
runs.at(-1).push(finding);
|
|
119
|
+
emit(finding);
|
|
120
|
+
},
|
|
121
|
+
options
|
|
122
|
+
);
|
|
123
|
+
detectDocuments(
|
|
124
|
+
text,
|
|
125
|
+
(finding) => {
|
|
126
|
+
if (!runs.some((run) => overlapsRun(run, finding))) {
|
|
127
|
+
emit(finding);
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
options
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
function overlapsRun(run, finding) {
|
|
134
|
+
let low = 0,
|
|
135
|
+
high = run.length;
|
|
136
|
+
while (low < high) {
|
|
137
|
+
const mid = (low + high) >>> 1;
|
|
138
|
+
if (run[mid].end <= finding.start) {
|
|
139
|
+
low = mid + 1;
|
|
140
|
+
} else {
|
|
141
|
+
high = mid;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return low < run.length && run[low].start < finding.end;
|
|
145
|
+
}
|
|
146
|
+
export function detectProperNames(text, emit, options) {
|
|
147
|
+
if (options.properNames !== 'capitalized' || !proseData(text, options)) {
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
let count = 0;
|
|
151
|
+
const found = (start, end, type) => {
|
|
152
|
+
emit({
|
|
153
|
+
start,
|
|
154
|
+
end,
|
|
155
|
+
type,
|
|
156
|
+
rule: 'proper-name-prose',
|
|
157
|
+
category: 'personal',
|
|
158
|
+
confidence: 0.85,
|
|
159
|
+
});
|
|
160
|
+
count++;
|
|
161
|
+
};
|
|
162
|
+
quotedSpans(text, found);
|
|
163
|
+
const ranges = text.matchAll(new RegExp(dateRange, 'gu'));
|
|
164
|
+
let range = ranges.next().value;
|
|
165
|
+
for (const match of text.matchAll(tokenPattern)) {
|
|
166
|
+
const raw = match[0];
|
|
167
|
+
// Leave sentence punctuation outside the replacement. Classification runs
|
|
168
|
+
// once on a maximal token, including hostile tokens without spaces.
|
|
169
|
+
let length = raw.length;
|
|
170
|
+
while (length && (raw[length - 1] === '.' || raw[length - 1] === ':')) {
|
|
171
|
+
length--;
|
|
172
|
+
}
|
|
173
|
+
const value = raw.slice(0, length);
|
|
174
|
+
const start = match.index,
|
|
175
|
+
end = start + value.length;
|
|
176
|
+
while (range && range.index + range[0].length <= start) {
|
|
177
|
+
range = ranges.next().value;
|
|
178
|
+
}
|
|
179
|
+
if (preserveToken(text, raw, value, start, end, range)) {
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const type = tokenType(value);
|
|
183
|
+
if (type) {
|
|
184
|
+
found(start, end, type);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
options.debug?.({ event: 'proper-name-prose', count });
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function preserveToken(text, raw, value, start, end, range) {
|
|
192
|
+
let next = start + raw.length;
|
|
193
|
+
while (text[next] === ' ' || text[next] === '\t') {
|
|
194
|
+
next++;
|
|
195
|
+
}
|
|
196
|
+
return (
|
|
197
|
+
!value ||
|
|
198
|
+
(range && start >= range.index && end <= range.index + range[0].length) ||
|
|
199
|
+
(value === 'REDACTED' && text[start - 1] === '[' && text[end] === ']') ||
|
|
200
|
+
raw.endsWith(':') ||
|
|
201
|
+
text[next] === ':'
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
function tokenType(value) {
|
|
205
|
+
if (isProseLink(value)) {
|
|
206
|
+
return 'URL';
|
|
207
|
+
}
|
|
208
|
+
if (pathToken(value)) {
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
if (mobile.test(value)) {
|
|
212
|
+
return 'PHONE';
|
|
213
|
+
}
|
|
214
|
+
if (
|
|
215
|
+
upper.test(value) &&
|
|
216
|
+
/\p{L}/u.test(value) &&
|
|
217
|
+
!/^\d{4}-\d{2}-\d{2}/u.test(value)
|
|
218
|
+
) {
|
|
219
|
+
return 'PROPER_NAME';
|
|
220
|
+
}
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
package/src/sanitizer.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { generatedValues } from './fake.js';
|
|
2
|
+
import { validateProperNames } from './proper-names.js';
|
|
2
3
|
import { prepareFakeText } from './fake-entities.js';
|
|
3
4
|
import { alignEscapes } from './escape-spans.js';
|
|
4
5
|
import { actionable } from './confidence.js';
|
|
@@ -78,6 +79,8 @@ const optionNames = new Set([
|
|
|
78
79
|
'threshold',
|
|
79
80
|
'profile',
|
|
80
81
|
'documentEntities',
|
|
82
|
+
'properNames',
|
|
83
|
+
'scripts',
|
|
81
84
|
]);
|
|
82
85
|
|
|
83
86
|
export function applyProfile(options = {}) {
|
|
@@ -191,6 +194,7 @@ export function validateOptions(options = {}) {
|
|
|
191
194
|
}
|
|
192
195
|
}
|
|
193
196
|
validateConfidence(options);
|
|
197
|
+
validateProperNames(options);
|
|
194
198
|
validatePolicy(options);
|
|
195
199
|
if (
|
|
196
200
|
options.verifyPublic !== undefined &&
|
|
@@ -410,11 +414,20 @@ function inspectText(text, options = {}) {
|
|
|
410
414
|
...options,
|
|
411
415
|
structured: undefined,
|
|
412
416
|
findings: [],
|
|
417
|
+
properNames: undefined,
|
|
418
|
+
documentEntities: options.properNames
|
|
419
|
+
? false
|
|
420
|
+
: options.documentEntities,
|
|
413
421
|
}),
|
|
414
422
|
options
|
|
415
423
|
),
|
|
416
|
-
...inspectStructuredValues(batch, options, (value) =>
|
|
417
|
-
inspect(value, {
|
|
424
|
+
...inspectStructuredValues(batch, options, (value, prose) =>
|
|
425
|
+
inspect(value, {
|
|
426
|
+
...options,
|
|
427
|
+
structured: undefined,
|
|
428
|
+
findings: [],
|
|
429
|
+
properNames: prose ? options.properNames : undefined,
|
|
430
|
+
})
|
|
418
431
|
),
|
|
419
432
|
]);
|
|
420
433
|
return inspectFindingsOnly(
|
package/src/structured.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { failure } from './detection.js';
|
|
2
|
-
import { documentFieldType } from './document-fields.js';
|
|
2
|
+
import { documentFieldType, normalizeFieldName } from './document-fields.js';
|
|
3
3
|
import { hasPlaceholders } from './placeholders.js';
|
|
4
4
|
|
|
5
5
|
// Session metadata is opt-in at the CLI; library callers choose their own
|
|
@@ -182,10 +182,10 @@ function weakStructuralFinding(finding, entry, batchText) {
|
|
|
182
182
|
}
|
|
183
183
|
return false;
|
|
184
184
|
}
|
|
185
|
-
function ordinaryPersonalKey(entry, finding) {
|
|
185
|
+
function ordinaryPersonalKey(entry, finding, options = {}) {
|
|
186
186
|
return (
|
|
187
187
|
entry.key &&
|
|
188
|
-
/^[A-Za-z_$][\w$]*$/.test(JSON.parse(entry.raw)) &&
|
|
188
|
+
(options.properNames || /^[A-Za-z_$][\w$]*$/.test(JSON.parse(entry.raw))) &&
|
|
189
189
|
finding.category === 'personal' &&
|
|
190
190
|
finding.rule !== 'known-personal'
|
|
191
191
|
);
|
|
@@ -193,12 +193,17 @@ function ordinaryPersonalKey(entry, finding) {
|
|
|
193
193
|
// Propagation runs again at source offsets. Apply the same key policy after it
|
|
194
194
|
// so a name confirmed in a value cannot rename an ordinary identifier key.
|
|
195
195
|
export function preserveIdentifierKeys(text, findings, options) {
|
|
196
|
-
if (
|
|
196
|
+
if (
|
|
197
|
+
!options.structured ||
|
|
198
|
+
(hasPlaceholders(options) && !options.properNames)
|
|
199
|
+
) {
|
|
197
200
|
return findings;
|
|
198
201
|
}
|
|
199
202
|
const keys = structuredBatches(text, options).flatMap((batch) =>
|
|
200
203
|
batch.entries
|
|
201
|
-
.filter((entry) =>
|
|
204
|
+
.filter((entry) =>
|
|
205
|
+
ordinaryPersonalKey(entry, { category: 'personal' }, options)
|
|
206
|
+
)
|
|
202
207
|
.map((entry) => ({
|
|
203
208
|
start: entry.start,
|
|
204
209
|
end: entry.start + entry.raw.length,
|
|
@@ -222,13 +227,16 @@ export function preserveIdentifierKeys(text, findings, options) {
|
|
|
222
227
|
});
|
|
223
228
|
}
|
|
224
229
|
function preserveStructuredEntry(entry, finding, text, options) {
|
|
230
|
+
if (options.properNames && ordinaryPersonalKey(entry, finding, options)) {
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
225
233
|
if (hasPlaceholders(options)) {
|
|
226
234
|
return false;
|
|
227
235
|
}
|
|
228
236
|
return (
|
|
229
237
|
(entry.key &&
|
|
230
238
|
['confirmed-personal', 'document-propagation'].includes(finding.rule)) ||
|
|
231
|
-
ordinaryPersonalKey(entry, finding) ||
|
|
239
|
+
ordinaryPersonalKey(entry, finding, options) ||
|
|
232
240
|
(!entry.key &&
|
|
233
241
|
options.structuralFields?.includes(entry.field) &&
|
|
234
242
|
weakStructuralFinding(finding, entry, text))
|
|
@@ -346,28 +354,28 @@ export function* structuredValueViews(
|
|
|
346
354
|
end: entry.start + raw.end,
|
|
347
355
|
});
|
|
348
356
|
};
|
|
349
|
-
|
|
350
|
-
//
|
|
357
|
+
// A serialized container is a transport envelope, not one prose value.
|
|
358
|
+
// Activate the optional scanner only in its independently decoded leaves.
|
|
359
|
+
let batches = [];
|
|
351
360
|
if (depth < 2 && /^[ \t\r\n]*[[{]/.test(value)) {
|
|
352
|
-
let batches;
|
|
353
361
|
try {
|
|
354
362
|
batches = structuredBatches(value, { ...options, structured: 'json' });
|
|
355
363
|
} catch (error) {
|
|
356
364
|
if (error.code !== 'ERR_JSON') {
|
|
357
365
|
throw error;
|
|
358
366
|
}
|
|
359
|
-
batches = [];
|
|
360
|
-
}
|
|
361
|
-
for (const nested of batches) {
|
|
362
|
-
yield* structuredValueViews(nested, options, depth + 1, map);
|
|
363
367
|
}
|
|
364
368
|
}
|
|
369
|
+
yield { text: value, map, prose: !entry.key && batches.length === 0 };
|
|
370
|
+
for (const nested of batches) {
|
|
371
|
+
yield* structuredValueViews(nested, options, depth + 1, map);
|
|
372
|
+
}
|
|
365
373
|
}
|
|
366
374
|
}
|
|
367
375
|
export function inspectStructuredValues(batch, options, inspectValue) {
|
|
368
376
|
const result = [];
|
|
369
377
|
for (const view of structuredValueViews(batch, options)) {
|
|
370
|
-
const findings = inspectValue(view.text);
|
|
378
|
+
const findings = inspectValue(view.text, view.prose);
|
|
371
379
|
options.debug?.({ event: 'structured-value', count: findings.length });
|
|
372
380
|
for (const finding of findings) {
|
|
373
381
|
const mapped = view.map(finding);
|
|
@@ -399,13 +407,13 @@ export function declaredFieldFindings(text, options) {
|
|
|
399
407
|
if (!options.sensitiveFields?.length) {
|
|
400
408
|
return [];
|
|
401
409
|
}
|
|
402
|
-
const fields = new Set(options.sensitiveFields);
|
|
410
|
+
const fields = new Set(options.sensitiveFields.map(normalizeFieldName));
|
|
403
411
|
const findings = [];
|
|
404
412
|
for (const token of lexemes(text)) {
|
|
405
413
|
if (
|
|
406
414
|
token.index < (findings.at(-1)?.end ?? 0) ||
|
|
407
415
|
!token.key ||
|
|
408
|
-
!fields.has(JSON.parse(token.raw))
|
|
416
|
+
!fields.has(normalizeFieldName(JSON.parse(token.raw)))
|
|
409
417
|
) {
|
|
410
418
|
continue;
|
|
411
419
|
}
|