@link-foundation/sensitive-data-sanitizer 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,20 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.18.0
4
+
5
+ ### Minor Changes
6
+
7
+ - f0016ea: Add opt-in structural proper-name redaction with per-string activation and
8
+ configurable Unicode scripts. Broaden dated document employers, group and school
9
+ cues, comma-separated mobiles and short share links; preserve operational paths
10
+ and technology controls. Normalize caller-sensitive fields like built-in names.
11
+
12
+ ## 0.17.1
13
+
14
+ ### Patch Changes
15
+
16
+ - 8029dd8: Refresh dependencies and supported development tooling, and repair upgrade regressions in workflow validation, formatting and reproducible example installs.
17
+
3
18
  ## 0.17.0
4
19
 
5
20
  ### Minor Changes
package/README.md CHANGED
@@ -22,6 +22,23 @@ sensitive-data-sanitizer redact session.jsonl --stream --jsonl --two-pass --outp
22
22
 
23
23
  ## Quick Start
24
24
 
25
+ For stored prose that can tolerate extra redaction, opt into structural proper
26
+ names independently for each string value:
27
+
28
+ ```js
29
+ sanitize('Работал в Ромашке менеджером по продажам.', {
30
+ profile: 'documents',
31
+ properNames: 'capitalized',
32
+ }).text;
33
+ // [REDACTED] в [REDACTED] менеджером по продажам.
34
+ ```
35
+
36
+ The rule defaults to Cyrillic or structural data triggers, keeps JSON keys and
37
+ numbers, and supports caller-selected Unicode `scripts`. It is off by default.
38
+ See [activation rules and limits](docs/API.md#structural-prose-redaction) and the
39
+ [prose-log example](examples/prose-logs.mjs). CLI:
40
+ `redact input.jsonl --jsonl --proper-names capitalized --profile documents`.
41
+
25
42
  ### Readable placeholders and document profiles
26
43
 
27
44
  Use the opt-in `documents` profile for CVs and similar prose. It adds bounded
@@ -60,6 +60,8 @@ history rewrite SOURCE --output DIRECTORY [--apply]
60
60
  --jsonl Sanitize JSONL records; combine with --stream for sessions
61
61
  --identity-mask First/last 2 identity mask (publication profile default)
62
62
  --profile NAME publication (redact default) or documents (CV/entity rules)
63
+ --proper-names MODE Structural prose rule: capitalized (off by default)
64
+ --scripts LIST Comma-separated Unicode Script names (default Cyrillic)
63
65
  --placeholder Typed numbered placeholders; requires --map or --no-map
64
66
  --map PATH Private JSON/.lino restore map; reuse it across files/runs
65
67
  --no-map Discard placeholder mapping after full anonymization
@@ -122,6 +124,8 @@ function parseArgs(argv) {
122
124
  ['--worker-timeout-ms', 'workerTimeoutMs'],
123
125
  ['--filter-repo', 'filterRepo'],
124
126
  ['--profile', 'profile'],
127
+ ['--proper-names', 'properNames'],
128
+ ['--scripts', 'scripts'],
125
129
  ['--map', 'map'],
126
130
  ['--map-format', 'mapFormat'],
127
131
  ['--placeholder-format', 'placeholderFormat'],
@@ -341,6 +345,10 @@ async function loadOptions(config) {
341
345
 
342
346
  async function featureOptions(config, options) {
343
347
  await localCredentials(config, options);
348
+ options.properNames = config.properNames ?? options.properNames;
349
+ if (config.scripts !== undefined) {
350
+ options.scripts = config.scripts.split(',');
351
+ }
344
352
  if (config.knownOnly) {
345
353
  options.knownOnly = true;
346
354
  }
package/docs/API.md CHANGED
@@ -140,7 +140,9 @@ and a shared map; such conflicting occurrence metadata is refused. Choose
140
140
  separate maps for those documents if exact independent restoration is needed.
141
141
 
142
142
  JSON/JSONL keys and values participate equally in placeholder mode, including
143
- ordinary identifier keys and declared structural fields. Parseability and
143
+ ordinary identifier keys and declared structural fields. With `properNames`,
144
+ JSON keys remain unchanged by automatic personal detection in all transformation
145
+ modes. Explicit known values and credentials retain priority. Parseability and
144
146
  duplicate-key checks remain enforced. Plain redact/fake policies retain their
145
147
  existing metadata/key behavior. Use `structured` for valid structured input.
146
148
  The normal encoding limits and credential redaction policy continue to apply.
@@ -157,6 +159,8 @@ new document recognizers; `documentEntities: false` turns them off explicitly.
157
159
  | Option | Default / meaning |
158
160
  | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
159
161
  | `documentEntities` | Off; bounded organization, education and location rules; enabled by the documents profile |
162
+ | `properNames` | Off; `'capitalized'` enables structural name redaction in activated prose strings |
163
+ | `scripts` | `['Cyrillic']`; Unicode Script aliases that activate the prose rule; `[]` disables only script activation |
160
164
  | `profile` | Off; `'publication'` sets a 0.3 threshold and identity masking; `'documents'` enables document entities |
161
165
  | `knownOnly` | `false`; disable all automatic detection (including credentials, Secretlint and custom detectors); use only explicit known values and reused map identities |
162
166
  | `knownSecrets` | `[]`; exact literal credential values, regardless of length or format |
@@ -176,7 +180,7 @@ new document recognizers; `documentEntities: false` turns them off explicitly.
176
180
  | `fakeIdentity` | Off; `'specimen-and-synthetic'` retains explicitly fake identity values |
177
181
  | `fakeValues` | `[]`; exact caller-reviewed identity values to retain, regardless of fakeIdentity |
178
182
  | `structured` | Off; `'json'` or `'jsonl'` validates and sanitizes string keys/values separately |
179
- | `sensitiveFields` | `[]`; exact JSON/JSONL property names whose complete values become a redaction string; supports synchronous calls |
183
+ | `sensitiveFields` | `[]`; JSON/JSONL names normalized for case and `_`/`-`; complete values become a redaction string; supports synchronous calls |
180
184
  | `structuralFields` | `[]`; declared property names suppress weak name/ID/entropy heuristics in their string values |
181
185
 
182
186
  Unknown options and invalid limits are errors. `knownSecretsFromEnv(env, names?)` discovers credential-named variables (excluding auth configuration names) when names are omitted, or uses an explicit selection, deduplicating nonempty values. It never invokes shell commands or reads auth files. `personalVariants(entries)` generates numeric and ISO/day-first date spellings; name case variants are already handled by detection. Neither helper invents transliterations.
@@ -478,8 +482,9 @@ Authorization headers (including Proxy-Authorization) redact the scheme and one
478
482
 
479
483
  ## Caller-declared sensitive fields
480
484
 
481
- In structured JSON/JSONL, `sensitiveFields: string[]` matches exact,
482
- case-sensitive property names at any depth. The complete property value becomes
485
+ In structured JSON/JSONL, `sensitiveFields: string[]` ignores case and removes
486
+ underscores and hyphens at any depth, like built-in document fields. For example,
487
+ `fooBar` also matches `foo_bar`, `FooBar` and `foo-bar`. The complete property value becomes
483
488
  `"[REDACTED]"`, including empty strings, numbers, booleans, null, arrays and
484
489
  objects. This deliberate schema policy can change value types. It takes
485
490
  priority over `structuralFields` and transformations, preserves property names
@@ -505,3 +510,58 @@ sanitize('Lyutik работал', {
505
510
  context. For computed synchronous detection, callers can supply validated
506
511
  `findings` for the exact input. `detectors` remains an asynchronous-engine
507
512
  option; `sanitize` does not invoke these callbacks.
513
+
514
+ ## Structural prose redaction
515
+
516
+ `properNames: 'capitalized'` is an optional, deliberately aggressive native rule
517
+ for stored prose. It works with any profile; the examples use `documents` for
518
+ its other document rules. It is off by default.
519
+
520
+ ```js
521
+ sanitize('Работал в Ромашке менеджером по продажам.', {
522
+ profile: 'documents',
523
+ properNames: 'capitalized',
524
+ }).text;
525
+ // [REDACTED] в [REDACTED] менеджером по продажам.
526
+ ```
527
+
528
+ Each string activates independently if it contains a letter from `scripts`, a
529
+ parenthesized document date range, a comma followed by an 11-digit `79…`/`89…`
530
+ mobile shape, or an HTTP(S)/bare-domain link with a `/d/<id>` path. `scripts`
531
+ defaults to `['Cyrillic']`, suitable for English operational logs; choose Unicode
532
+ Script aliases such as `['Greek', 'Cyrillic']` for your workload. Script names
533
+ are case-sensitive runtime Unicode property aliases, not language codes.
534
+ `scripts: []` disables script activation while retaining the other triggers.
535
+
536
+ Inside activated values, the rule replaces words containing an uppercase letter
537
+ anywhere (`iPhone`, `A.B`, `Санкт-Петербург`), typographic quoted spans (including
538
+ lowercase `«ромашка лютик»`), links including `example.org/path`, and bare domestic
539
+ mobile shapes. Labels immediately before a colon (also `Label :`), lowercase
540
+ words, numbers, date ranges and path tokens stay. A path can still match existing
541
+ personal-path or credential rules. Numeric primitives and JSON keys stay; string
542
+ escapes and surrounding source formatting are retained. JSON/JSONL activation
543
+ also applies independently within serialized JSON strings, to the existing
544
+ bounded decoding depth. Findings do not seed document-wide name propagation.
545
+ Repeated redaction is stable. Reusing a placeholder map for the same original
546
+ produces stable tokens and exact restoration; the existing placeholder-namespace
547
+ collision checks still reject input that already contains typed placeholders.
548
+
549
+ This is a structural heuristic, so capitalized ordinary words also disappear.
550
+ Lowercase organizations and `Ромашка: менеджер` stay. English prose without a
551
+ trigger (`Worked at Romashka for three years.`), isolated neutral-key values
552
+ `{"note":"Romashka"}` / `{"note":"1991-03-12"}`, and a personal website root
553
+ without a trigger stay under this rule. The existing detectors and explicit
554
+ known/sensitive-field policies still run: an unactivated value is byte-for-byte
555
+ unchanged by this additional rule, not exempt from credential detection.
556
+ `credentialsOnly` and `knownOnly` suppress automatic prose detection.
557
+
558
+ The new scanner uses a fixed number of forward passes over maximal tokens,
559
+ including long tokens without spaces. This does not impose a linear-time bound
560
+ on optional external engines or every preexisting detection rule. Debugging is
561
+ off by default; `debug` receives value-free `proper-name-prose` and
562
+ `document-experience` hit counts when those scans run.
563
+
564
+ CLI equivalents: `--proper-names capitalized --scripts Cyrillic,Greek`.
565
+ Configuration files accept the same `properNames` and `scripts` API options;
566
+ use a configuration file for `scripts: []`. See the runnable
567
+ [prose-log example](../examples/prose-logs.mjs).
package/docs/COVERAGE.md CHANGED
@@ -60,7 +60,7 @@ The `documents` profile also reads employers from CV experience lines anchored o
60
60
  a date range: `Title, Employer (2019-05 – 2021-03)`, `Title — Employer, 2019–2021`
61
61
  and `Title (Employer), 2019–2021`. Ranges use years, `YYYY-MM`, `MM.YYYY`, full
62
62
  dates or `month YYYY`, with `по настоящее время`, `н.в.`, `по н.в.`, `present` and
63
- similar open ends. Titles have one to nine words. A label of one or two words
63
+ similar open ends. For the non-parenthesized layouts, titles have one to nine words. A label of one or two words
64
64
  before the title (`Опыт: `, `Опыт работы - `, `Опыт — `) is read like a list marker,
65
65
  and a segment after `|` is read like a line start, so
66
66
  `Должность: Менеджер | Опыт: Аналитик, Ромашка (2018 – 2020)` also yields the
@@ -68,9 +68,9 @@ employer. The date range stays the anchor and the employer must still start with
68
68
  capital letter, a digit or a quote. Confidence is 0.60, and the employer then
69
69
  propagates through the document like other organizations. The default profile
70
70
  ignores these lines. Lines
71
- with more fields (`Title, Employer, City (dates)`) and titles containing a comma
72
- are not targeted and can leave the employer in place; date-anchored prose with
73
- the same shape can be flagged.
71
+ with parenthesized ranges additionally support arbitrary title prefixes,
72
+ comma-containing employers and titles, as described below. Date-anchored prose
73
+ with the same shape can be flagged.
74
74
 
75
75
  The gazetteer has only the cities/regions explicitly listed in that module,
76
76
  including Springfield and Riverton. It is not a worldwide geocoder. Location
@@ -376,7 +376,8 @@ Employer labels consume multi-word values to a comma or line/quote boundary.
376
376
  Dated CV entries additionally support compound titles, title parentheses/slashes,
377
377
  quoted group names, legal-form/location suffixes, empty/`null` range ends,
378
378
  multiple entries, employer-first role entries and pipe-separated entries.
379
- The supplemental grammar uses finite title cues and bounded entries. Undated
379
+ The parenthesized date-anchor scanner accepts arbitrary forward titles and
380
+ lead-ins. The reversed `Company, manager` layout retains finite role cues. Undated
380
381
  `Менеджер, Ромашка Лютик`, `- Ромашка Лютик 5 лет` and
381
382
  `Последнее место работы Ромашка Лютик` remain explicit ambiguous limits, as
382
383
  permitted by #94. Use known values or schema declarations for them.
@@ -402,3 +403,48 @@ redaction replaces the fragment instead of repeating an unchanged result until
402
403
 
403
404
  The [requirement inventory and research](case-studies/issue-100/README.md)
404
405
  records the complete scope and regression evidence.
406
+
407
+ ## Dated entries and optional prose (issues 104–106)
408
+
409
+ Parenthesized date anchors now delimit successive entries on the same line,
410
+ including arbitrary title words and lead-ins (`Стаж = 5 лет:`, `= 3 года,`).
411
+ Employers can contain comma suffixes, nested parentheses, slash-separated names,
412
+ legal forms and multiple words. A pipe after the employer preserves the industry
413
+ suffix. `Founder, consultant, Romashka (2019–2021)` and equivalent Russian entries
414
+ share the parser. The employer must start with a capital letter, a quote, or a
415
+ company-style digit/letter combination. **Lowercase employers remain a limit**:
416
+ `Менеджер, romashka (2019–2021)` / `Менеджер, ромашка (2019–2021)` stay, preserving
417
+ ambiguous controls such as `Встреча, понедельник (2019–2021)`.
418
+
419
+ `ГК` and `группа компаний` introduce quoted or unquoted group names. Employment
420
+ lists continue through repeated `, потом в` and `, затем в` items. Domestic
421
+ `79…`/`89…` mobile shapes after a comma work in Russian and English text; plain
422
+ `order`/`id` numbers stay. School detection also covers `(МГУ, 2012)`,
423
+ `Образование (МГУ)`, `в МГТУ им. Баумана` and `Обучение: Skillbox, 2020`.
424
+ The acronym vocabulary remains finite.
425
+
426
+ Weak organization experience/duration cues preserve a small explicit technology
427
+ set: Python, Go, SQL, Excel, Power BI, Kubernetes, PostgreSQL, Telegram and Scrum.
428
+ This is an ambiguity policy, not general recognition of every product; a real
429
+ employer with one of those names needs `knownPersonal` or a declared sensitive
430
+ field. Explicit private values/fields and stronger legal-form cues still win.
431
+
432
+ Document share-link detection also recognizes HTTP(S) URLs containing `/d/<id>`
433
+ for any host and nonempty id length. Ordinary website roots remain a limit.
434
+ Relative operational paths such as `{"location":"apps/api"}` stay; explicit
435
+ personal paths and credential rules remain active. Caller-sensitive field names
436
+ now use the same case/underscore/hyphen normalization as built-in document names.
437
+
438
+ For no-cue prose, enable [`properNames: 'capitalized'`](API.md#structural-prose-redaction).
439
+ It activates per value, defaults to Cyrillic script letters or structural data
440
+ triggers, and supports configurable Unicode scripts. It intentionally loses
441
+ capitalized ordinary words. Labels, lowercase words, dates, paths, JSON keys and
442
+ untriggered values are outside this additional rule. English untriggered prose,
443
+ lowercase names and a name immediately before a colon remain documented limits.
444
+ The API describes the interaction with existing detectors and placeholder modes.
445
+
446
+ The date-anchor scan visits each preceding entry once; the optional prose scanner
447
+ classifies maximal tokens in fixed passes, avoiding repeated suffix searches on
448
+ long unspaced input. This is not a latency guarantee for the whole pipeline.
449
+ [The requirement inventory](case-studies/issue-106/README.md) records alternatives,
450
+ research, regression evidence and reproducible bounded performance measurements.
@@ -0,0 +1,16 @@
1
+ import { sanitize } from '../src/index.js';
2
+
3
+ // Synthetic mixed records: only the prose string activates the extra rule.
4
+ const source = JSON.stringify({
5
+ note: 'Работал в Ромашке менеджером по продажам.',
6
+ operational: 'Connection to Postgres lost, retrying',
7
+ count: 42,
8
+ });
9
+ console.log(
10
+ sanitize(source, {
11
+ profile: 'documents',
12
+ properNames: 'capitalized',
13
+ scripts: ['Cyrillic'],
14
+ structured: 'json',
15
+ }).text
16
+ );
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@link-foundation/sensitive-data-sanitizer",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
@@ -24,7 +24,8 @@
24
24
  "LICENSE",
25
25
  "README.md",
26
26
  "examples/shared-tool-logs.mjs",
27
- "examples/reversible-placeholders.mjs"
27
+ "examples/reversible-placeholders.mjs",
28
+ "examples/prose-logs.mjs"
28
29
  ],
29
30
  "exports": {
30
31
  ".": {
@@ -77,7 +78,9 @@
77
78
  "license": "Unlicense",
78
79
  "devEngines": {
79
80
  "packageManager": {
80
- "name": "npm"
81
+ "name": "npm",
82
+ "version": ">=12.2.2",
83
+ "onFail": "error"
81
84
  }
82
85
  },
83
86
  "publishConfig": {
@@ -92,19 +95,23 @@
92
95
  },
93
96
  "devDependencies": {
94
97
  "@changesets/cli": "^3.0.3",
95
- "eslint": "^10.13.0",
98
+ "@eslint/js": "^10.0.1",
99
+ "browser-commander": "0.34.1",
100
+ "cmd-shim": "^9.0.2",
101
+ "command-stream": "2.6.0",
102
+ "eslint": "^10.14.0",
96
103
  "eslint-config-prettier": "^10.1.8",
97
- "eslint-plugin-prettier": "^5.5.4",
104
+ "eslint-plugin-prettier": "^5.5.7",
98
105
  "husky": "^9.1.7",
99
- "jscpd": "^5.4.0",
100
- "lint-staged": "^16.4.0",
106
+ "jscpd": "^5.4.2",
107
+ "lino-arguments": "0.4.0",
108
+ "lint-staged": "^17.6.0",
101
109
  "mrz": "^5.0.2",
102
- "prettier": "^3.6.2",
103
- "secretlint": "13.0.7",
104
- "test-anywhere": "^0.8.48",
105
- "@eslint/js": "^10.0.1",
106
- "command-stream": "2.4.0",
107
- "lino-arguments": "0.4.0"
110
+ "playwright": "1.65.0",
111
+ "prettier": "^3.9.10",
112
+ "secretlint": "13.1.0",
113
+ "semver": "^7.8.6",
114
+ "test-anywhere": "^0.9.1"
108
115
  },
109
116
  "lint-staged": {
110
117
  "*.{js,mjs,cjs}": [
@@ -120,9 +127,9 @@
120
127
  "pipx run --spec shellcheck-py==0.11.0.1 shellcheck"
121
128
  ],
122
129
  "*.py": [
123
- "pipx run --spec ruff==0.17.3 ruff check --fix --force-exclude",
124
- "pipx run --spec ruff==0.17.3 ruff format --force-exclude",
125
- "pipx run --spec ruff==0.17.3 ruff format --check --force-exclude"
130
+ "pipx run --spec ruff==0.17.4 ruff check --fix --force-exclude",
131
+ "pipx run --spec ruff==0.17.4 ruff format --force-exclude",
132
+ "pipx run --spec ruff==0.17.4 ruff format --check --force-exclude"
126
133
  ],
127
134
  ".husky/*": [
128
135
  "pipx run --spec shellcheck-py==0.11.0.1 shellcheck"
@@ -130,19 +137,19 @@
130
137
  },
131
138
  "dependencies": {
132
139
  "@faker-js/faker": "^10.6.0",
133
- "@secretlint/core": "13.0.7",
134
- "@secretlint/profiler": "13.0.7",
135
- "@secretlint/secretlint-rule-preset-recommend": "13.0.7",
136
- "libphonenumber-js": "1.13.14",
137
- "links-notation": "0.30.0",
140
+ "@secretlint/core": "13.1.0",
141
+ "@secretlint/profiler": "13.1.0",
142
+ "@secretlint/secretlint-rule-preset-recommend": "13.1.0",
143
+ "libphonenumber-js": "1.13.15",
144
+ "links-notation": "0.31.0",
138
145
  "re2-wasm": "1.0.2",
139
- "re2js": "^2.9.0",
146
+ "re2js": "^2.9.1",
140
147
  "yaml": "^2.9.1"
141
148
  },
142
149
  "allowScripts": {
143
150
  "node-pty": false
144
151
  },
145
152
  "overrides": {
146
- "minimatch@>=7 <11": "^11.0.4"
153
+ "minimatch@>=7 <11": "^11.0.5"
147
154
  }
148
155
  }
package/src/context.js CHANGED
@@ -257,6 +257,11 @@ function operationalValue(value, type) {
257
257
  );
258
258
  }
259
259
 
260
+ function operationalLocationPath(value) {
261
+ return (
262
+ servicePath(value) || /^(?:\.?\.?\/)?[\w.-]+(?:\/[\w.-]+)+\/?$/.test(value)
263
+ );
264
+ }
260
265
  // Shared by native labels and the vendor catalogs. Code expressions and
261
266
  // line escapes are not personal field values, even when a key is familiar.
262
267
  export function personalValueAllowed(text, range, type, labelStart) {
@@ -264,7 +269,10 @@ export function personalValueAllowed(text, range, type, labelStart) {
264
269
  if (!plausibleLabel(value, type)) {
265
270
  return false;
266
271
  }
267
- if (['LOCATION', 'ADDRESS'].includes(type) && servicePath(value)) {
272
+ if (
273
+ ['LOCATION', 'ADDRESS'].includes(type) &&
274
+ operationalLocationPath(value)
275
+ ) {
268
276
  return false;
269
277
  }
270
278
  if (type === 'PHONE') {
package/src/detection.js CHANGED
@@ -15,6 +15,7 @@ import {
15
15
  import { localAddress } from './public.js';
16
16
  import { detectKnownPersonal } from './personal-variants.js';
17
17
  import { detectDocumentEntities } from './document-entities.js';
18
+ import { detectProseAndDocuments } from './proper-names.js';
18
19
 
19
20
  export const REDACTED = '[REDACTED]';
20
21
  export const failure = (code = 'ERR_SANITIZATION', location = {}) =>
@@ -173,7 +174,7 @@ export function nativeDetect(text, options, emit) {
173
174
  detectContext(text, emit, options.credentialsOnly);
174
175
  if (!options.credentialsOnly) {
175
176
  detectPersonal(text, emit);
176
- detectDocumentEntities(text, emit, options);
177
+ detectProseAndDocuments(text, emit, options, detectDocumentEntities);
177
178
  detectCatalog(text, emit);
178
179
  detectIdentity(text, emit);
179
180
  detectNational(text, emit);
@@ -0,0 +1,15 @@
1
+ import { escapePattern } from './detection.js';
2
+ import { months } from './personal-context.js';
3
+
4
+ const boundary = '[\\p{L}\\p{M}\\p{N}_]';
5
+ const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
6
+ const month = [
7
+ ...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
8
+ ].join('|');
9
+ const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
10
+ const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'по н.в.']
11
+ .concat(['н.в.', 'по н/в', 'н/в', 'present', 'now', 'current'])
12
+ .flatMap((word) => [word, capitalized(word)])
13
+ .map(escapePattern)
14
+ .join('|');
15
+ export const documentDateRange = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen}|null|(?=\\)))(?!${boundary})`;
@@ -2,9 +2,10 @@ import { collectMatches, escapePattern } from './detection.js';
2
2
  import { firstNames } from './names.js';
3
3
  import { organizationCore, personalRegex } from './personal-variants.js';
4
4
  import { documentVocabulary } from './document-vocabulary.js';
5
- import { months } from './personal-context.js';
5
+ import { documentDateRange as range } from './document-dates.js';
6
6
  import { detectDocumentExtras } from './document-extras.js';
7
7
  import { detectDocumentPatterns } from './document-patterns.js';
8
+ import { isDocumentTechnology } from './document-experience.js';
8
9
 
9
10
  const words = (column) =>
10
11
  [...new Set(documentVocabulary.flatMap((row) => row[column].split('|')))]
@@ -58,17 +59,6 @@ const groupOrganization = new RegExp(
58
59
  // CV experience lines: the date range anchors the employer slot, so prose with
59
60
  // a comma is untouched. Titles are not required to be lowercase because English
60
61
  // ones are often title case (`Senior QA Engineer`).
61
- const capitalized = (word) => word[0].toUpperCase() + word.slice(1);
62
- const month = [
63
- ...new Set(months.split('|').flatMap((word) => [word, capitalized(word)])),
64
- ].join('|');
65
- const rangeStart = `(?:(?:19|20)\\d{2}(?:-\\d{2}){0,2}|\\d{2}[./](?:\\d{2}\\.)?\\d{4}|(?:${month})\\.?[ \\t]+(?:19|20)\\d{2})`;
66
- const rangeOpen = ['по настоящее время', 'настоящее время', 'сейчас', 'по н.в.']
67
- .concat(['н.в.', 'по н/в', 'н/в', 'present', 'now', 'current'])
68
- .flatMap((word) => [word, capitalized(word)])
69
- .map(escapePattern)
70
- .join('|');
71
- const range = `${rangeStart}(?:[ \\t]*[-–—][ \\t]*|[ \\t]+(?:по|to)[ \\t]+)(?:${rangeStart}|${rangeOpen}|null|(?=\\)))(?!${boundary})`;
72
62
  const titleWord = '[\\p{L}\\p{N}][\\p{L}\\p{N}/+#.&-]{0,39}';
73
63
  // A segment starts a line or follows `|`. `Опыт:` or `Опыт -` before the title
74
64
  // reads like a list marker: the date range remains the anchor of the rule.
@@ -90,7 +80,23 @@ export function detectDocumentEntities(text, emit, options) {
90
80
  return;
91
81
  }
92
82
  detectDocumentPatterns(text, emit);
93
- detectDocumentExtras(text, emit, range);
83
+ detectDocumentExtras(text, emit, range, options.debug);
84
+ collectMatches(
85
+ text,
86
+ new RegExp(
87
+ `(?:^|[\\r\\n])[ \\t]*[^|\\r\\n]{1,120}\\|[ \\t]*(${name})[ \\t]*\\|[ \\t]*${range}`,
88
+ 'gu'
89
+ ),
90
+ emit,
91
+ {
92
+ type: 'ORGANIZATION',
93
+ rule: 'organization-experience',
94
+ category: 'personal',
95
+ confidence: 0.85,
96
+ },
97
+ 1,
98
+ (value) => !isDocumentTechnology(value)
99
+ );
94
100
  const found = [];
95
101
  const capture = (finding) => {
96
102
  found.push(finding);
@@ -113,7 +119,8 @@ export function detectDocumentEntities(text, emit, options) {
113
119
  pattern,
114
120
  capture,
115
121
  { type, rule, category: 'personal', confidence },
116
- group
122
+ group,
123
+ (value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
117
124
  );
118
125
  }
119
126
  for (const match of text.matchAll(ruResidence)) {
@@ -0,0 +1,110 @@
1
+ const technology = new Set(
2
+ [
3
+ 'Python',
4
+ 'Go',
5
+ 'SQL',
6
+ 'Excel',
7
+ 'Power BI',
8
+ 'Kubernetes',
9
+ 'PostgreSQL',
10
+ 'Telegram',
11
+ 'Scrum',
12
+ ].map((s) => s.toLowerCase())
13
+ );
14
+ export const isDocumentTechnology = (value) =>
15
+ technology.has(value.trim().toLowerCase());
16
+
17
+ // Used only to disambiguate the reversed `Company, manager` layout. Forward
18
+ // entries deliberately accept any title, without a title vocabulary.
19
+ const reversedRole =
20
+ /^(?:менеджер|директор|руководитель|инженер|аналитик|разработчик|manager|director|engineer|developer|senior|qa)(?!\p{L})/iu;
21
+
22
+ function commaParts(text) {
23
+ const parts = [];
24
+ let start = 0,
25
+ depth = 0,
26
+ quote;
27
+ for (let i = 0; i < text.length; i++) {
28
+ const char = text[i];
29
+ if (quote) {
30
+ if (char === quote) {
31
+ quote = undefined;
32
+ }
33
+ } else if ('«“"'.includes(char)) {
34
+ quote = { '«': '»', '“': '”', '"': '"' }[char];
35
+ } else if (char === '(') {
36
+ depth++;
37
+ } else if (char === ')') {
38
+ depth = Math.max(0, depth - 1);
39
+ } else if (char === ',' && !depth) {
40
+ parts.push({ start, text: text.slice(start, i).trim() });
41
+ start = i + 1;
42
+ }
43
+ }
44
+ parts.push({ start, text: text.slice(start).trim() });
45
+ return parts;
46
+ }
47
+
48
+ function employerSpan(prefix) {
49
+ // An industry suffix belongs to the entry, but is not an employer. A pipe
50
+ // before the title remains a section separator, as in `Опыт | Менеджер, ...`.
51
+ const comma = prefix.indexOf(',');
52
+ const pipe = comma < 0 ? -1 : prefix.indexOf('|', comma);
53
+ const content = pipe < 0 ? prefix : prefix.slice(0, pipe);
54
+ const parts = commaParts(content);
55
+ let index = 1;
56
+ if (parts.length === 1) {
57
+ return undefined;
58
+ }
59
+ if (/^[=\s\d]/u.test(parts[0].text)) {
60
+ index++;
61
+ }
62
+ if (parts.length === 2 && reversedRole.test(parts[1].text)) {
63
+ index = 0;
64
+ } else {
65
+ while (index < parts.length - 1 && /^\p{Ll}/u.test(parts[index].text)) {
66
+ index++;
67
+ }
68
+ }
69
+ const part = parts[index];
70
+ if (!part || !/^(?:[\p{Lu}«“"]|\d{1,4}-?\p{L})/u.test(part.text)) {
71
+ return undefined;
72
+ }
73
+ const end = index === 0 ? content.indexOf(',') : content.trimEnd().length;
74
+ const start = part.start + content.slice(part.start).search(/\S/u);
75
+ return isDocumentTechnology(content.slice(start, end))
76
+ ? undefined
77
+ : { start, end };
78
+ }
79
+
80
+ export function detectDatedEmployers(text, emit, range, debug) {
81
+ let cursor = 0,
82
+ count = 0;
83
+ // Match only the anchor, then visit each preceding segment once. No lazy
84
+ // prefix search restarts at every position on a long nonmatching line.
85
+ for (const match of text.matchAll(new RegExp(`\\(${range}\\)`, 'gu'))) {
86
+ const segment = text.slice(cursor, match.index);
87
+ const boundary =
88
+ Math.max(
89
+ segment.lastIndexOf('\n'),
90
+ segment.lastIndexOf('\r'),
91
+ segment.lastIndexOf(';')
92
+ ) + 1;
93
+ const prefix = segment.slice(boundary).replace(/^[., \t]+/u, '');
94
+ const span = employerSpan(prefix);
95
+ if (span) {
96
+ const offset = match.index - prefix.length;
97
+ emit({
98
+ start: offset + span.start,
99
+ end: offset + span.end,
100
+ type: 'ORGANIZATION',
101
+ category: 'personal',
102
+ rule: 'organization-experience',
103
+ confidence: 0.85,
104
+ });
105
+ count++;
106
+ }
107
+ cursor = match.index + match[0].length;
108
+ }
109
+ debug?.({ event: 'document-experience', count });
110
+ }
@@ -1,8 +1,14 @@
1
1
  import { collectMatches } from './detection.js';
2
+ import {
3
+ detectDatedEmployers,
4
+ isDocumentTechnology,
5
+ } from './document-experience.js';
6
+ import { isDocumentShareLink } from './proper-names.js';
2
7
 
3
8
  const boundary = '[\\p{L}\\p{N}_]';
4
9
  const proper = '[\\p{Lu}][\\p{L}\\p{M}\\p{N}.-]{0,46}[\\p{L}\\p{M}\\p{N}]';
5
10
  const name = `${proper}(?:[ \\t]+${proper}){0,5}`;
11
+ const employerName = `(?:«[^«»\\r\\n]{1,160}»|“[^“”\\r\\n]{1,160}”|"[^"\\r\\n]{1,160}"|${name})`;
6
12
  const orgList = `(?:«[^«»\\r\\n]{1,160}»|${name})(?:[ \\t]*(?:,|и|and)[ \\t]+${name}){0,5}`;
7
13
  const cues = new RegExp(
8
14
  `(?<!${boundary})(?:компании|опыт[ \\t]+в|проекты[ \\t]+для|[Рр]оль[ \\t]+в)[ \\t]+(${orgList})(?!${boundary})`,
@@ -12,13 +18,42 @@ const duration = new RegExp(
12
18
  `(?<!${boundary})(${name})(?=[ \\t]*\\(\\d{1,2}[ \\t]+(?:год|года|лет|years?)\\))`,
13
19
  'gu'
14
20
  );
15
- // JavaScript's \b is ASCII-only; Cyrillic titles use a Unicode lookahead.
16
- const role =
17
- /^(?:менеджер|директор|руководитель|инженер|аналитик|разработчик)(?![\p{L}])|^(?:manager|director|engineer|developer|senior|qa)\b/iu;
18
-
19
- export function detectDocumentExtras(text, emit, range) {
21
+ export function detectDocumentExtras(text, emit, range, debug) {
20
22
  for (const [type, rule, pattern, group] of [
21
23
  ['PHONE', 'document-mobile-bare', /^[78]9\d{9}$/gm, 0],
24
+ [
25
+ 'PHONE',
26
+ 'document-mobile-comma',
27
+ /,[ \t]*([78]9\d{9})(?![\p{L}\p{N}_])/gu,
28
+ 1,
29
+ ],
30
+ [
31
+ 'ORGANIZATION',
32
+ 'organization-group-cue',
33
+ new RegExp(
34
+ `(?<!${boundary})(?:ГК|[Гг]руппа компаний)[ \\t]+${employerName}`,
35
+ 'gu'
36
+ ),
37
+ 0,
38
+ ],
39
+ [
40
+ 'EDUCATION',
41
+ 'education-bracket',
42
+ /(?<=\()(МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ)(?=[,)])/gu,
43
+ 1,
44
+ ],
45
+ [
46
+ 'EDUCATION',
47
+ 'education-named-acronym',
48
+ /(?<![\p{L}\p{N}_])(?:МГУ|СПбГУ|МГТУ|РГГУ|ВШЭ)[ \t]+им\.[ \t]+[А-ЯЁ][а-яё-]{2,40}/gu,
49
+ 0,
50
+ ],
51
+ [
52
+ 'EDUCATION',
53
+ 'education-training',
54
+ /(?<![\p{L}\p{N}_])(?:Обучение|Training)[ \t]*:[ \t]*([\p{Lu}][\p{L}\p{M}.-]{1,47})/gu,
55
+ 1,
56
+ ],
22
57
  [
23
58
  'PHONE',
24
59
  'document-mobile-context',
@@ -56,7 +91,8 @@ export function detectDocumentExtras(text, emit, range) {
56
91
  pattern,
57
92
  emit,
58
93
  { type, rule, category: 'personal', confidence: 0.85 },
59
- group
94
+ group,
95
+ (value) => type !== 'ORGANIZATION' || !isDocumentTechnology(value)
60
96
  );
61
97
  }
62
98
  for (const match of text.matchAll(cues)) {
@@ -64,6 +100,9 @@ export function detectDocumentExtras(text, emit, range) {
64
100
  for (const company of match[1].matchAll(
65
101
  new RegExp(`«[^«»\\r\\n]{1,160}»|${name}`, 'gu')
66
102
  )) {
103
+ if (isDocumentTechnology(company[0])) {
104
+ continue;
105
+ }
67
106
  emit({
68
107
  start: offset + company.index,
69
108
  end: offset + company.index + company[0].length,
@@ -75,7 +114,57 @@ export function detectDocumentExtras(text, emit, range) {
75
114
  }
76
115
  }
77
116
  detectQueryContacts(text, emit);
78
- detectExperience(text, emit, range);
117
+ collectMatches(
118
+ text,
119
+ /https?:\/\/[^\s"<>\\]+/gu,
120
+ emit,
121
+ {
122
+ type: 'URL',
123
+ category: 'personal',
124
+ rule: 'document-share-path',
125
+ confidence: 0.85,
126
+ },
127
+ 0,
128
+ isDocumentShareLink
129
+ );
130
+ detectEmployerContinuation(text, emit);
131
+ detectDatedEmployers(text, emit, range, debug);
132
+ }
133
+
134
+ function detectEmployerContinuation(text, emit) {
135
+ const continuation = new RegExp(
136
+ `,[ \\t]*(?:потом|затем)[ \\t]+в[ \\t]+(${employerName})`,
137
+ 'uy'
138
+ );
139
+ for (const match of text.matchAll(
140
+ new RegExp(
141
+ `(?<!${boundary})[Рр]аботал[аи]?[ \\t]+в[ \\t]+(${employerName})`,
142
+ 'gu'
143
+ )
144
+ )) {
145
+ continuation.lastIndex = match.index + match[0].length;
146
+ let next = continuation.exec(text);
147
+ if (next) {
148
+ emitEmployer(match);
149
+ }
150
+ while (next) {
151
+ emitEmployer(next);
152
+ next = continuation.exec(text);
153
+ }
154
+ }
155
+ function emitEmployer(match) {
156
+ const end = match.index + match[0].length;
157
+ if (!isDocumentTechnology(match[1])) {
158
+ emit({
159
+ start: end - match[1].length,
160
+ end,
161
+ type: 'ORGANIZATION',
162
+ category: 'personal',
163
+ rule: 'organization-continuation',
164
+ confidence: 0.85,
165
+ });
166
+ }
167
+ }
79
168
  }
80
169
 
81
170
  function detectQueryContacts(text, emit) {
@@ -111,60 +200,3 @@ function detectQueryContacts(text, emit) {
111
200
  });
112
201
  }
113
202
  }
114
-
115
- function detectExperience(text, emit, range) {
116
- // Date anchors bound each entry; do not guess at undated employer/title fragments.
117
- collectMatches(
118
- text,
119
- new RegExp(
120
- `(?:^|[\\r\\n])[ \\t]*[^|\\r\\n]{1,120}\\|[ \\t]*(${name})[ \\t]*\\|[ \\t]*${range}`,
121
- 'gu'
122
- ),
123
- emit,
124
- {
125
- type: 'ORGANIZATION',
126
- rule: 'organization-experience',
127
- category: 'personal',
128
- confidence: 0.85,
129
- },
130
- 1
131
- );
132
- const dated = new RegExp(`([^\\r\\n;|]{1,320}?)\\((${range})\\)`, 'gu');
133
- for (const match of text.matchAll(dated)) {
134
- const prefix = match[1];
135
- const content = prefix
136
- .replace(/^.*\(\d[^)]*\)[, \t]*/, '')
137
- .replace(/^[, \t]+/, '');
138
- const chunks = [...content.matchAll(/[^,]+/g)];
139
- const parts = chunks.map((part) => part[0].trim());
140
- let value;
141
- if (parts.length > 1 && role.test(parts[0])) {
142
- let index = 1;
143
- while (index < parts.length - 1 && /^[\p{Ll}]/u.test(parts[index])) {
144
- index++;
145
- }
146
- value = content.slice(chunks[index].index).trim();
147
- } else if (parts.length === 2 && role.test(parts[1])) {
148
- value = parts[0].trim();
149
- }
150
- if (
151
- !value ||
152
- !/^[\p{Lu}«“"]/u.test(value) ||
153
- value.includes('[REDACTED]')
154
- ) {
155
- continue;
156
- }
157
- const start = match.index + prefix.lastIndexOf(value);
158
- if (start < match.index) {
159
- continue;
160
- }
161
- emit({
162
- start,
163
- end: start + value.length,
164
- type: 'ORGANIZATION',
165
- category: 'personal',
166
- rule: 'organization-experience',
167
- confidence: 0.85,
168
- });
169
- }
170
- }
@@ -15,5 +15,7 @@ const types = new Map(
15
15
  )
16
16
  );
17
17
  export function documentFieldType(field) {
18
- return types.get(field.replace(/[_-]/g, '').toLowerCase());
18
+ return types.get(normalizeFieldName(field));
19
19
  }
20
+ export const normalizeFieldName = (field) =>
21
+ field.replace(/[_-]/g, '').toLowerCase();
package/src/engines.js CHANGED
@@ -140,7 +140,13 @@ async function analyzeStructured(
140
140
  batch.text,
141
141
  [],
142
142
  true,
143
- options,
143
+ {
144
+ ...options,
145
+ properNames: undefined,
146
+ documentEntities: options.properNames
147
+ ? false
148
+ : options.documentEntities,
149
+ },
144
150
  nativeFindings === undefined ? undefined : []
145
151
  ),
146
152
  options
@@ -152,7 +158,11 @@ async function analyzeStructured(
152
158
  view.text,
153
159
  [],
154
160
  true,
155
- { ...options, structured: undefined },
161
+ {
162
+ ...options,
163
+ structured: undefined,
164
+ properNames: view.prose ? options.properNames : undefined,
165
+ },
156
166
  nativeFindings === undefined ? undefined : []
157
167
  );
158
168
  for (const finding of detected) {
package/src/index.d.ts CHANGED
@@ -9,11 +9,7 @@ export interface Finding {
9
9
  /** Detector confidence, not a cross-engine probability calibration. */
10
10
  confidence?: number;
11
11
  likelihood?:
12
- | 'VERY_UNLIKELY'
13
- | 'UNLIKELY'
14
- | 'POSSIBLE'
15
- | 'LIKELY'
16
- | 'VERY_LIKELY';
12
+ 'VERY_UNLIKELY' | 'UNLIKELY' | 'POSSIBLE' | 'LIKELY' | 'VERY_LIKELY';
17
13
  /** One-based line and UTF-16 column, populated by inspect. */
18
14
  line?: number;
19
15
  column?: number;
@@ -47,6 +43,10 @@ export interface SanitizerOptions {
47
43
  profile?: 'publication' | 'documents';
48
44
  /** Opt in to bounded employer, school, city/region heuristics. */
49
45
  documentEntities?: boolean;
46
+ /** Aggressive structural proper names in prose values; opt-in, off by default. */
47
+ properNames?: 'capitalized';
48
+ /** Unicode Script aliases that activate prose detection; default ['Cyrillic']. */
49
+ scripts?: string[];
50
50
  confirmedPersonal?: Array<{ value: string; type: string }>;
51
51
  knownSecrets?: string[];
52
52
  knownPersonal?: PersonalValue[];
@@ -67,7 +67,7 @@ export interface SanitizerOptions {
67
67
  structured?: 'json' | 'jsonl';
68
68
  /** Property names whose metadata heuristics may be suppressed. */
69
69
  structuralFields?: string[];
70
- /** Exact, case-sensitive property names whose entire values are redacted in JSON/JSONL. */
70
+ /** Property names normalized across case/underscores/hyphens; entire JSON/JSONL values redact. */
71
71
  sensitiveFields?: string[];
72
72
  /** Personal replacement threshold, default 0.5. Inspect retains lower scores. */
73
73
  minConfidence?: number;
@@ -92,6 +92,7 @@ export interface SanitizerOptions {
92
92
  /** Value-free diagnostics; enabled only when a debug callback is supplied. */
93
93
  export type DebugEvent =
94
94
  | { event: 'detected'; count: number }
95
+ | { event: 'document-experience' | 'proper-name-prose'; count: number }
95
96
  | { event: 'residual-redaction'; round: number }
96
97
  | { event: 'two-pass-confirmation'; type: string }
97
98
  | { event: 'propagation-skipped'; reason: 'length'; type: string }
package/src/log-bytes.js CHANGED
@@ -13,7 +13,7 @@ export function decodeLogBytes(input) {
13
13
  } catch {
14
14
  const parts = [];
15
15
  let start = 0;
16
- for (let i = 0; i < bytes.length; ) {
16
+ for (let i = 0; i < bytes.length;) {
17
17
  const width = sequenceWidth(bytes[i]);
18
18
  if (validSequence(bytes, i, width)) {
19
19
  i += width;
@@ -22,6 +22,7 @@ export const reservedPlaceholderTypes = () => [
22
22
  'PASSPORT_MRZ',
23
23
  'PASSPORT_NUMBER',
24
24
  'PERSON',
25
+ 'PROPER_NAME',
25
26
  'PHONE',
26
27
  'PL_PESEL',
27
28
  'RU_INN',
@@ -36,7 +36,11 @@ function eligibleSeed(finding, options) {
36
36
  !options.knownOnly &&
37
37
  finding.category === 'personal' &&
38
38
  finding.type !== 'ENCODED_SENSITIVE' &&
39
- !['confirmed-personal', 'document-propagation'].includes(finding.rule) &&
39
+ ![
40
+ 'confirmed-personal',
41
+ 'document-propagation',
42
+ 'proper-name-prose',
43
+ ].includes(finding.rule) &&
40
44
  actionable(finding, options) &&
41
45
  ((options.documentEntities && finding.type === 'ORGANIZATION') ||
42
46
  finding.confidence >= 0.85 ||
@@ -0,0 +1,222 @@
1
+ import { failure } from './detection.js';
2
+ import { documentDateRange } from './document-dates.js';
3
+
4
+ const scriptPatterns = new Map();
5
+ const dateRange = new RegExp(`\\(${documentDateRange}\\)`, 'u');
6
+ const commaMobile = /,[ \t]*[78]9\d{9}(?![\p{L}\p{N}_])/u;
7
+ const mobile = /^[78]9\d{9}$/u;
8
+ const tokenPattern = /[\p{L}\p{M}\p{N}_./:\\?&#%=@+'’~+-]+/gu;
9
+ const upper = /[\p{Lu}\p{Lt}]/u;
10
+ const quotePairs = new Map([
11
+ ['«', '»'],
12
+ ['“', '”'],
13
+ ['„', '“'],
14
+ ['‘', '’'],
15
+ ['「', '」'],
16
+ ['『', '』'],
17
+ ]);
18
+
19
+ function scriptPattern(scripts = ['Cyrillic']) {
20
+ if (
21
+ scripts.some(
22
+ (script) => typeof script !== 'string' || !/^[A-Za-z_]+$/.test(script)
23
+ )
24
+ ) {
25
+ throw failure('ERR_CONFIG');
26
+ }
27
+ const key = scripts.join('|');
28
+ let pattern = scriptPatterns.get(key);
29
+ if (!pattern) {
30
+ // The alias grammar prevents regexp injection; constructing the Unicode
31
+ // property also validates aliases supported by this runtime.
32
+ try {
33
+ pattern = new RegExp(
34
+ scripts.length
35
+ ? `(?=\\p{L})(?:${scripts.map((s) => `\\p{Script=${s}}`).join('|')})`
36
+ : '(?!)',
37
+ 'u'
38
+ );
39
+ } catch {
40
+ throw failure('ERR_CONFIG');
41
+ }
42
+ // Bound cache growth when callers submit many different configurations.
43
+ if (scriptPatterns.size >= 64) {
44
+ scriptPatterns.clear();
45
+ }
46
+ scriptPatterns.set(key, pattern);
47
+ }
48
+ return pattern;
49
+ }
50
+ export function validateProperNames(options) {
51
+ if (
52
+ (options.properNames !== undefined &&
53
+ options.properNames !== 'capitalized') ||
54
+ (options.scripts !== undefined && !Array.isArray(options.scripts))
55
+ ) {
56
+ throw failure('ERR_CONFIG');
57
+ }
58
+ scriptPattern(options.scripts);
59
+ }
60
+
61
+ export function isProseLink(token) {
62
+ return (
63
+ /^https?:\/\/[^/]+/iu.test(token) ||
64
+ /^[\p{L}\p{N}](?:[\p{L}\p{N}-]*\.)+[\p{L}]{2,63}(?:[/:?#]|$)/u.test(token)
65
+ );
66
+ }
67
+ export function isDocumentShareLink(token) {
68
+ return isProseLink(token) && /\/d\/[^/?#]+/u.test(token);
69
+ }
70
+ function proseData(text, options) {
71
+ if (
72
+ scriptPattern(options.scripts).test(text) ||
73
+ dateRange.test(text) ||
74
+ commaMobile.test(text)
75
+ ) {
76
+ return true;
77
+ }
78
+ for (const match of text.matchAll(tokenPattern)) {
79
+ if (isDocumentShareLink(match[0])) {
80
+ return true;
81
+ }
82
+ }
83
+ return false;
84
+ }
85
+ function quotedSpans(text, emit) {
86
+ let start = -1,
87
+ closing;
88
+ for (let i = 0; i < text.length; i++) {
89
+ if (start >= 0) {
90
+ if (text[i] === closing) {
91
+ emit(start, i + 1, 'PROPER_NAME');
92
+ start = -1;
93
+ }
94
+ } else if (quotePairs.has(text[i])) {
95
+ start = i;
96
+ closing = quotePairs.get(text[i]);
97
+ }
98
+ }
99
+ }
100
+ function pathToken(value) {
101
+ return value.includes('/') || value.includes('\\');
102
+ }
103
+ // Keep existing document coverage outside the new replacements. Two naturally
104
+ // ordered runs (quotes, then tokens) provide an interval index without sorting.
105
+ // Overlapping broad document spans must not merge separate prose replacements.
106
+ export function detectProseAndDocuments(text, emit, options, detectDocuments) {
107
+ if (options.properNames !== 'capitalized') {
108
+ detectDocuments(text, emit, options);
109
+ return;
110
+ }
111
+ const runs = [[]];
112
+ detectProperNames(
113
+ text,
114
+ (finding) => {
115
+ if (runs.at(-1).at(-1)?.end > finding.start) {
116
+ runs.push([]);
117
+ }
118
+ runs.at(-1).push(finding);
119
+ emit(finding);
120
+ },
121
+ options
122
+ );
123
+ detectDocuments(
124
+ text,
125
+ (finding) => {
126
+ if (!runs.some((run) => overlapsRun(run, finding))) {
127
+ emit(finding);
128
+ }
129
+ },
130
+ options
131
+ );
132
+ }
133
+ function overlapsRun(run, finding) {
134
+ let low = 0,
135
+ high = run.length;
136
+ while (low < high) {
137
+ const mid = (low + high) >>> 1;
138
+ if (run[mid].end <= finding.start) {
139
+ low = mid + 1;
140
+ } else {
141
+ high = mid;
142
+ }
143
+ }
144
+ return low < run.length && run[low].start < finding.end;
145
+ }
146
+ export function detectProperNames(text, emit, options) {
147
+ if (options.properNames !== 'capitalized' || !proseData(text, options)) {
148
+ return false;
149
+ }
150
+ let count = 0;
151
+ const found = (start, end, type) => {
152
+ emit({
153
+ start,
154
+ end,
155
+ type,
156
+ rule: 'proper-name-prose',
157
+ category: 'personal',
158
+ confidence: 0.85,
159
+ });
160
+ count++;
161
+ };
162
+ quotedSpans(text, found);
163
+ const ranges = text.matchAll(new RegExp(dateRange, 'gu'));
164
+ let range = ranges.next().value;
165
+ for (const match of text.matchAll(tokenPattern)) {
166
+ const raw = match[0];
167
+ // Leave sentence punctuation outside the replacement. Classification runs
168
+ // once on a maximal token, including hostile tokens without spaces.
169
+ let length = raw.length;
170
+ while (length && (raw[length - 1] === '.' || raw[length - 1] === ':')) {
171
+ length--;
172
+ }
173
+ const value = raw.slice(0, length);
174
+ const start = match.index,
175
+ end = start + value.length;
176
+ while (range && range.index + range[0].length <= start) {
177
+ range = ranges.next().value;
178
+ }
179
+ if (preserveToken(text, raw, value, start, end, range)) {
180
+ continue;
181
+ }
182
+ const type = tokenType(value);
183
+ if (type) {
184
+ found(start, end, type);
185
+ }
186
+ }
187
+ options.debug?.({ event: 'proper-name-prose', count });
188
+ return true;
189
+ }
190
+
191
+ function preserveToken(text, raw, value, start, end, range) {
192
+ let next = start + raw.length;
193
+ while (text[next] === ' ' || text[next] === '\t') {
194
+ next++;
195
+ }
196
+ return (
197
+ !value ||
198
+ (range && start >= range.index && end <= range.index + range[0].length) ||
199
+ (value === 'REDACTED' && text[start - 1] === '[' && text[end] === ']') ||
200
+ raw.endsWith(':') ||
201
+ text[next] === ':'
202
+ );
203
+ }
204
+ function tokenType(value) {
205
+ if (isProseLink(value)) {
206
+ return 'URL';
207
+ }
208
+ if (pathToken(value)) {
209
+ return undefined;
210
+ }
211
+ if (mobile.test(value)) {
212
+ return 'PHONE';
213
+ }
214
+ if (
215
+ upper.test(value) &&
216
+ /\p{L}/u.test(value) &&
217
+ !/^\d{4}-\d{2}-\d{2}/u.test(value)
218
+ ) {
219
+ return 'PROPER_NAME';
220
+ }
221
+ return undefined;
222
+ }
package/src/sanitizer.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { generatedValues } from './fake.js';
2
+ import { validateProperNames } from './proper-names.js';
2
3
  import { prepareFakeText } from './fake-entities.js';
3
4
  import { alignEscapes } from './escape-spans.js';
4
5
  import { actionable } from './confidence.js';
@@ -78,6 +79,8 @@ const optionNames = new Set([
78
79
  'threshold',
79
80
  'profile',
80
81
  'documentEntities',
82
+ 'properNames',
83
+ 'scripts',
81
84
  ]);
82
85
 
83
86
  export function applyProfile(options = {}) {
@@ -191,6 +194,7 @@ export function validateOptions(options = {}) {
191
194
  }
192
195
  }
193
196
  validateConfidence(options);
197
+ validateProperNames(options);
194
198
  validatePolicy(options);
195
199
  if (
196
200
  options.verifyPublic !== undefined &&
@@ -410,11 +414,20 @@ function inspectText(text, options = {}) {
410
414
  ...options,
411
415
  structured: undefined,
412
416
  findings: [],
417
+ properNames: undefined,
418
+ documentEntities: options.properNames
419
+ ? false
420
+ : options.documentEntities,
413
421
  }),
414
422
  options
415
423
  ),
416
- ...inspectStructuredValues(batch, options, (value) =>
417
- inspect(value, { ...options, structured: undefined, findings: [] })
424
+ ...inspectStructuredValues(batch, options, (value, prose) =>
425
+ inspect(value, {
426
+ ...options,
427
+ structured: undefined,
428
+ findings: [],
429
+ properNames: prose ? options.properNames : undefined,
430
+ })
418
431
  ),
419
432
  ]);
420
433
  return inspectFindingsOnly(
package/src/structured.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { failure } from './detection.js';
2
- import { documentFieldType } from './document-fields.js';
2
+ import { documentFieldType, normalizeFieldName } from './document-fields.js';
3
3
  import { hasPlaceholders } from './placeholders.js';
4
4
 
5
5
  // Session metadata is opt-in at the CLI; library callers choose their own
@@ -182,10 +182,10 @@ function weakStructuralFinding(finding, entry, batchText) {
182
182
  }
183
183
  return false;
184
184
  }
185
- function ordinaryPersonalKey(entry, finding) {
185
+ function ordinaryPersonalKey(entry, finding, options = {}) {
186
186
  return (
187
187
  entry.key &&
188
- /^[A-Za-z_$][\w$]*$/.test(JSON.parse(entry.raw)) &&
188
+ (options.properNames || /^[A-Za-z_$][\w$]*$/.test(JSON.parse(entry.raw))) &&
189
189
  finding.category === 'personal' &&
190
190
  finding.rule !== 'known-personal'
191
191
  );
@@ -193,12 +193,17 @@ function ordinaryPersonalKey(entry, finding) {
193
193
  // Propagation runs again at source offsets. Apply the same key policy after it
194
194
  // so a name confirmed in a value cannot rename an ordinary identifier key.
195
195
  export function preserveIdentifierKeys(text, findings, options) {
196
- if (!options.structured || hasPlaceholders(options)) {
196
+ if (
197
+ !options.structured ||
198
+ (hasPlaceholders(options) && !options.properNames)
199
+ ) {
197
200
  return findings;
198
201
  }
199
202
  const keys = structuredBatches(text, options).flatMap((batch) =>
200
203
  batch.entries
201
- .filter((entry) => ordinaryPersonalKey(entry, { category: 'personal' }))
204
+ .filter((entry) =>
205
+ ordinaryPersonalKey(entry, { category: 'personal' }, options)
206
+ )
202
207
  .map((entry) => ({
203
208
  start: entry.start,
204
209
  end: entry.start + entry.raw.length,
@@ -222,13 +227,16 @@ export function preserveIdentifierKeys(text, findings, options) {
222
227
  });
223
228
  }
224
229
  function preserveStructuredEntry(entry, finding, text, options) {
230
+ if (options.properNames && ordinaryPersonalKey(entry, finding, options)) {
231
+ return true;
232
+ }
225
233
  if (hasPlaceholders(options)) {
226
234
  return false;
227
235
  }
228
236
  return (
229
237
  (entry.key &&
230
238
  ['confirmed-personal', 'document-propagation'].includes(finding.rule)) ||
231
- ordinaryPersonalKey(entry, finding) ||
239
+ ordinaryPersonalKey(entry, finding, options) ||
232
240
  (!entry.key &&
233
241
  options.structuralFields?.includes(entry.field) &&
234
242
  weakStructuralFinding(finding, entry, text))
@@ -346,28 +354,28 @@ export function* structuredValueViews(
346
354
  end: entry.start + raw.end,
347
355
  });
348
356
  };
349
- yield { text: value, map };
350
- // Match the encoded detector's finite depth; arbitrary nesting is a limit.
357
+ // A serialized container is a transport envelope, not one prose value.
358
+ // Activate the optional scanner only in its independently decoded leaves.
359
+ let batches = [];
351
360
  if (depth < 2 && /^[ \t\r\n]*[[{]/.test(value)) {
352
- let batches;
353
361
  try {
354
362
  batches = structuredBatches(value, { ...options, structured: 'json' });
355
363
  } catch (error) {
356
364
  if (error.code !== 'ERR_JSON') {
357
365
  throw error;
358
366
  }
359
- batches = [];
360
- }
361
- for (const nested of batches) {
362
- yield* structuredValueViews(nested, options, depth + 1, map);
363
367
  }
364
368
  }
369
+ yield { text: value, map, prose: !entry.key && batches.length === 0 };
370
+ for (const nested of batches) {
371
+ yield* structuredValueViews(nested, options, depth + 1, map);
372
+ }
365
373
  }
366
374
  }
367
375
  export function inspectStructuredValues(batch, options, inspectValue) {
368
376
  const result = [];
369
377
  for (const view of structuredValueViews(batch, options)) {
370
- const findings = inspectValue(view.text);
378
+ const findings = inspectValue(view.text, view.prose);
371
379
  options.debug?.({ event: 'structured-value', count: findings.length });
372
380
  for (const finding of findings) {
373
381
  const mapped = view.map(finding);
@@ -384,7 +392,7 @@ function stringOffsets(raw) {
384
392
  }
385
393
  const starts = [],
386
394
  ends = [];
387
- for (let i = 1; i < raw.length - 1; ) {
395
+ for (let i = 1; i < raw.length - 1;) {
388
396
  const start = i;
389
397
  i += raw[i] === '\\' ? (raw[i + 1] === 'u' ? 6 : 2) : 1;
390
398
  starts.push(start);
@@ -399,13 +407,13 @@ export function declaredFieldFindings(text, options) {
399
407
  if (!options.sensitiveFields?.length) {
400
408
  return [];
401
409
  }
402
- const fields = new Set(options.sensitiveFields);
410
+ const fields = new Set(options.sensitiveFields.map(normalizeFieldName));
403
411
  const findings = [];
404
412
  for (const token of lexemes(text)) {
405
413
  if (
406
414
  token.index < (findings.at(-1)?.end ?? 0) ||
407
415
  !token.key ||
408
- !fields.has(JSON.parse(token.raw))
416
+ !fields.has(normalizeFieldName(JSON.parse(token.raw)))
409
417
  ) {
410
418
  continue;
411
419
  }