@link-foundation/sensitive-data-sanitizer 0.20.0 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +14 -1
- package/bin/sensitive-data-sanitizer.js +1 -0
- package/docs/API.md +179 -16
- package/docs/COVERAGE.md +16 -6
- package/docs/RULES.md +1 -0
- package/examples/second-pass-placeholders.mjs +28 -0
- package/package.json +12 -3
- package/src/context.js +176 -25
- package/src/credential-spans.js +180 -0
- package/src/credentials-entry.d.ts +23 -0
- package/src/credentials-entry.js +62 -0
- package/src/credentials.js +8 -1
- package/src/detection.js +68 -37
- package/src/encoded.js +137 -51
- package/src/engines.js +33 -10
- package/src/exempt.js +72 -0
- package/src/fake-memory.js +32 -0
- package/src/fake.js +4 -28
- package/src/features.js +29 -0
- package/src/html-tags.js +8 -0
- package/src/identity-types.js +63 -0
- package/src/identity.js +3 -61
- package/src/index.d.ts +41 -2
- package/src/index.js +6 -0
- package/src/known.js +33 -15
- package/src/personal-features.js +39 -0
- package/src/personal.js +29 -5
- package/src/placeholder-types.js +164 -4
- package/src/placeholders.js +144 -45
- package/src/projection.js +1 -8
- package/src/proper-names.js +48 -5
- package/src/prose-words.js +116 -0
- package/src/public-technologies.js +32 -0
- package/src/public.js +3 -0
- package/src/record-worker.js +1 -0
- package/src/rule-catalog.js +2 -0
- package/src/rules.js +8 -0
- package/src/sanitizer.js +92 -21
- package/src/stream-worker.js +1 -0
- package/src/strict-attributes.js +60 -4
- package/src/strict.js +1 -1
- package/src/structured.js +2 -0
- package/src/transforms.js +38 -8
- package/src/vocabulary.js +39 -2
- package/src/worker-options.js +1 -0
package/src/features.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Personal-data features that the main entry registers through
|
|
2
|
+
// `personal-features.js`. The `./credentials` entry never imports that module,
|
|
3
|
+
// so it does not load the personal detectors, faker or libphonenumber
|
|
4
|
+
// (issue 137). This module must stay free of imports.
|
|
5
|
+
const registered = {};
|
|
6
|
+
|
|
7
|
+
export function registerFeatures(entries) {
|
|
8
|
+
Object.assign(registered, entries);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** Runs the registered implementation, or returns `fallback` without one. */
|
|
12
|
+
export function withFeature(name, fallback, ...args) {
|
|
13
|
+
const value = registered[name];
|
|
14
|
+
return value ? value(...args) : fallback;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** The registered implementation; fails closed when it is unavailable. */
|
|
18
|
+
export function feature(name) {
|
|
19
|
+
const value = registered[name];
|
|
20
|
+
if (!value) {
|
|
21
|
+
throw Object.assign(
|
|
22
|
+
new Error(
|
|
23
|
+
'This option needs the personal-data modules of the main entry.'
|
|
24
|
+
),
|
|
25
|
+
{ code: 'ERR_CONFIG' }
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
return value;
|
|
29
|
+
}
|
package/src/html-tags.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// HTML formatting tags split tokens (`SECRET_</span><span>REST`). Inline tags
|
|
2
|
+
// are dropped and block tags become line breaks. Only presentation attributes
|
|
3
|
+
// are allowed, so other attribute values (URLs, data-*) stay in the view.
|
|
4
|
+
export const inlineTags =
|
|
5
|
+
'abbr|b|cite|code|del|em|font|i|ins|kbd|mark|q|s|samp|small|span|strong|sub|sup|tt|u|var';
|
|
6
|
+
const blockTags =
|
|
7
|
+
'blockquote|br|dd|div|dt|h[1-6]|hr|li|ol|p|pre|table|tbody|td|th|thead|tr|ul';
|
|
8
|
+
export const htmlTag = String.raw`<\/?(?:${inlineTags}|${blockTags})(?:\s+(?:class|dir|lang|style)="[^"<>=]*")*\s*\/?>`;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Identity-document finding types, shared by the transforms and the
|
|
2
|
+
// identity detectors without loading the detectors themselves.
|
|
3
|
+
export const identityTypes = new Set([
|
|
4
|
+
'SSN',
|
|
5
|
+
'US_SSN',
|
|
6
|
+
'BRAZIL_CPF_NUMBER',
|
|
7
|
+
'INDIA_PAN_INDIVIDUAL',
|
|
8
|
+
'UK_NATIONAL_INSURANCE_NUMBER',
|
|
9
|
+
'SPAIN_DNI_NUMBER',
|
|
10
|
+
'US_HEALTHCARE_NPI',
|
|
11
|
+
'DRIVER_LICENSE',
|
|
12
|
+
'DRIVERS_LICENSE',
|
|
13
|
+
'NATIONAL_ID',
|
|
14
|
+
'CHINA_RESIDENT_ID_NUMBER',
|
|
15
|
+
'HONG_KONG_ID_NUMBER',
|
|
16
|
+
'SOUTH_AFRICA_ID_NUMBER',
|
|
17
|
+
'TAIWAN_ID_NUMBER',
|
|
18
|
+
'TURKEY_ID_NUMBER',
|
|
19
|
+
'GOVERNMENT_ID',
|
|
20
|
+
'DOD_ID_NUMBER',
|
|
21
|
+
'JAPAN_INDIVIDUAL_NUMBER',
|
|
22
|
+
'INDIA_GST_INDIVIDUAL',
|
|
23
|
+
'US_MEDICARE_BENEFICIARY_ID_NUMBER',
|
|
24
|
+
'ID',
|
|
25
|
+
'PASSPORT_NUMBER',
|
|
26
|
+
'PASSPORT_MRZ',
|
|
27
|
+
'BOOKING_REFERENCE',
|
|
28
|
+
'TICKET_NUMBER',
|
|
29
|
+
'VISA_NUMBER',
|
|
30
|
+
'UK_NHS',
|
|
31
|
+
'KR_RRN',
|
|
32
|
+
'IT_FISCAL_CODE',
|
|
33
|
+
'SG_NRIC_FIN',
|
|
34
|
+
'PL_PESEL',
|
|
35
|
+
'SE_PERSONNUMMER',
|
|
36
|
+
'ZA_ID_NUMBER',
|
|
37
|
+
'AU_TFN',
|
|
38
|
+
'CA_SIN',
|
|
39
|
+
'RU_SNILS',
|
|
40
|
+
'RU_INN',
|
|
41
|
+
'US_ITIN',
|
|
42
|
+
'US_DRIVER_LICENSE',
|
|
43
|
+
'UK_DRIVING_LICENCE',
|
|
44
|
+
'ES_NIE',
|
|
45
|
+
'IN_AADHAAR',
|
|
46
|
+
'IN_VOTER',
|
|
47
|
+
'IN_PASSPORT',
|
|
48
|
+
'FI_PERSONAL_IDENTITY_CODE',
|
|
49
|
+
'TH_TNIN',
|
|
50
|
+
'AU_MEDICARE',
|
|
51
|
+
'DE_TAX_ID',
|
|
52
|
+
'AE_EMIRATES_ID',
|
|
53
|
+
'SA_NATIONAL_ID',
|
|
54
|
+
'IL_ID_NUMBER',
|
|
55
|
+
]);
|
|
56
|
+
export function isIdentityType(type) {
|
|
57
|
+
return (
|
|
58
|
+
identityTypes.has(type) ||
|
|
59
|
+
/PASSPORT|(?:DRIVERS?|DRIVING)_?LICENSE|IDENTITY|NATIONAL_?(?:ID|IDENTIFICATION)|PERSONAL_?(?:ID|IDENTIFICATION)|UNIQUE_?IDENTIFICATION|SOCIAL_?(?:SECURITY|INSURANCE|IDENTIFICATION)|RESIDENT_?REGISTRATION|(?:TAX|TAXPAYER).*?(?:IDENTIFICATION|NUMBER|REFERENCE)|AADHAAR|FISCAL_CODE|PESEL|RRN|NHS|SNILS|INN_NUMBER/.test(
|
|
60
|
+
type
|
|
61
|
+
)
|
|
62
|
+
);
|
|
63
|
+
}
|
package/src/identity.js
CHANGED
|
@@ -1,66 +1,8 @@
|
|
|
1
1
|
import { collectMatches } from './detection.js';
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
'BRAZIL_CPF_NUMBER',
|
|
7
|
-
'INDIA_PAN_INDIVIDUAL',
|
|
8
|
-
'UK_NATIONAL_INSURANCE_NUMBER',
|
|
9
|
-
'SPAIN_DNI_NUMBER',
|
|
10
|
-
'US_HEALTHCARE_NPI',
|
|
11
|
-
'DRIVER_LICENSE',
|
|
12
|
-
'DRIVERS_LICENSE',
|
|
13
|
-
'NATIONAL_ID',
|
|
14
|
-
'CHINA_RESIDENT_ID_NUMBER',
|
|
15
|
-
'HONG_KONG_ID_NUMBER',
|
|
16
|
-
'SOUTH_AFRICA_ID_NUMBER',
|
|
17
|
-
'TAIWAN_ID_NUMBER',
|
|
18
|
-
'TURKEY_ID_NUMBER',
|
|
19
|
-
'GOVERNMENT_ID',
|
|
20
|
-
'DOD_ID_NUMBER',
|
|
21
|
-
'JAPAN_INDIVIDUAL_NUMBER',
|
|
22
|
-
'INDIA_GST_INDIVIDUAL',
|
|
23
|
-
'US_MEDICARE_BENEFICIARY_ID_NUMBER',
|
|
24
|
-
'ID',
|
|
25
|
-
'PASSPORT_NUMBER',
|
|
26
|
-
'PASSPORT_MRZ',
|
|
27
|
-
'BOOKING_REFERENCE',
|
|
28
|
-
'TICKET_NUMBER',
|
|
29
|
-
'VISA_NUMBER',
|
|
30
|
-
'UK_NHS',
|
|
31
|
-
'KR_RRN',
|
|
32
|
-
'IT_FISCAL_CODE',
|
|
33
|
-
'SG_NRIC_FIN',
|
|
34
|
-
'PL_PESEL',
|
|
35
|
-
'SE_PERSONNUMMER',
|
|
36
|
-
'ZA_ID_NUMBER',
|
|
37
|
-
'AU_TFN',
|
|
38
|
-
'CA_SIN',
|
|
39
|
-
'RU_SNILS',
|
|
40
|
-
'RU_INN',
|
|
41
|
-
'US_ITIN',
|
|
42
|
-
'US_DRIVER_LICENSE',
|
|
43
|
-
'UK_DRIVING_LICENCE',
|
|
44
|
-
'ES_NIE',
|
|
45
|
-
'IN_AADHAAR',
|
|
46
|
-
'IN_VOTER',
|
|
47
|
-
'IN_PASSPORT',
|
|
48
|
-
'FI_PERSONAL_IDENTITY_CODE',
|
|
49
|
-
'TH_TNIN',
|
|
50
|
-
'AU_MEDICARE',
|
|
51
|
-
'DE_TAX_ID',
|
|
52
|
-
'AE_EMIRATES_ID',
|
|
53
|
-
'SA_NATIONAL_ID',
|
|
54
|
-
'IL_ID_NUMBER',
|
|
55
|
-
]);
|
|
56
|
-
export function isIdentityType(type) {
|
|
57
|
-
return (
|
|
58
|
-
identityTypes.has(type) ||
|
|
59
|
-
/PASSPORT|(?:DRIVERS?|DRIVING)_?LICENSE|IDENTITY|NATIONAL_?(?:ID|IDENTIFICATION)|PERSONAL_?(?:ID|IDENTIFICATION)|UNIQUE_?IDENTIFICATION|SOCIAL_?(?:SECURITY|INSURANCE|IDENTIFICATION)|RESIDENT_?REGISTRATION|(?:TAX|TAXPAYER).*?(?:IDENTIFICATION|NUMBER|REFERENCE)|AADHAAR|FISCAL_CODE|PESEL|RRN|NHS|SNILS|INN_NUMBER/.test(
|
|
60
|
-
type
|
|
61
|
-
)
|
|
62
|
-
);
|
|
63
|
-
}
|
|
3
|
+
import { isIdentityType } from './identity-types.js';
|
|
4
|
+
|
|
5
|
+
export { identityTypes, isIdentityType } from './identity-types.js';
|
|
64
6
|
const personal = (type, rule, confidence = 0.85) => ({
|
|
65
7
|
type,
|
|
66
8
|
rule,
|
package/src/index.d.ts
CHANGED
|
@@ -17,6 +17,8 @@ export interface Finding {
|
|
|
17
17
|
kept?: 'fake';
|
|
18
18
|
/** Returned finding replaced with, or recognized as, a generated fake. */
|
|
19
19
|
faked?: true;
|
|
20
|
+
/** For `JSON_VALUE`: the field's personal type, e.g. `AGE`, when known. */
|
|
21
|
+
valueType?: string;
|
|
20
22
|
}
|
|
21
23
|
export interface PersonalValue {
|
|
22
24
|
type: string;
|
|
@@ -31,7 +33,9 @@ export type Vocabulary = Partial<
|
|
|
31
33
|
| 'organizationCues'
|
|
32
34
|
| 'educationCues'
|
|
33
35
|
| 'personalLabels'
|
|
34
|
-
| 'personalFields'
|
|
36
|
+
| 'personalFields'
|
|
37
|
+
/** Word stems for the strict special-category rule; needs `strict: true`. */
|
|
38
|
+
| 'personalAttributes',
|
|
35
39
|
VocabularyList
|
|
36
40
|
>
|
|
37
41
|
>;
|
|
@@ -114,6 +118,19 @@ export interface SanitizerOptions {
|
|
|
114
118
|
* same rules as allowList.
|
|
115
119
|
*/
|
|
116
120
|
inlineAllow?: boolean;
|
|
121
|
+
/**
|
|
122
|
+
* Keeps a finding the caller's policy knows is not sensitive. Runs after
|
|
123
|
+
* detection and before transformation, once per finding, with the text the
|
|
124
|
+
* offsets refer to; it must return a boolean (`true` keeps the span as-is).
|
|
125
|
+
* Verification of the output consults it again per span, so the same value
|
|
126
|
+
* elsewhere is still caught. Declared knownSecrets/knownPersonal/
|
|
127
|
+
* confirmedPersonal values and JSON_VALUE denials are never offered.
|
|
128
|
+
* createSyncSanitizer and sanitize() require a synchronous hook.
|
|
129
|
+
*/
|
|
130
|
+
exempt?: (
|
|
131
|
+
finding: Readonly<Finding & { value: string }>,
|
|
132
|
+
text: string
|
|
133
|
+
) => boolean | Promise<boolean>;
|
|
117
134
|
/** Explicit network/callback verification; credentials never qualify. */
|
|
118
135
|
verifyPublic?: (candidate: {
|
|
119
136
|
type: PublicEntity['type'];
|
|
@@ -243,7 +260,11 @@ export declare function fromOffsetReport(
|
|
|
243
260
|
): Finding[];
|
|
244
261
|
export declare function knownSecretsFromEnv(
|
|
245
262
|
env?: Record<string, string | undefined>,
|
|
246
|
-
names?: string[]
|
|
263
|
+
names?: string[],
|
|
264
|
+
options?: {
|
|
265
|
+
/** Skip values shorter than this many characters (default 1). */
|
|
266
|
+
minLength?: number;
|
|
267
|
+
}
|
|
247
268
|
): string[];
|
|
248
269
|
/** Reversible UTF-8 view: invalid bytes use unpaired U+DC80..U+DCFF. */
|
|
249
270
|
export declare function decodeLogBytes(bytes: Uint8Array): string;
|
|
@@ -300,6 +321,12 @@ export type Transformation =
|
|
|
300
321
|
map?: RestoreMap | false;
|
|
301
322
|
format?: string;
|
|
302
323
|
credentials?: boolean;
|
|
324
|
+
/**
|
|
325
|
+
* Tokens of the namespace already in the input: `'error'` (default)
|
|
326
|
+
* throws `ERR_PLACEHOLDER_COLLISION`; `'keep'` leaves them as they are
|
|
327
|
+
* and numbers new values after the highest kept number per type.
|
|
328
|
+
*/
|
|
329
|
+
existing?: 'error' | 'keep';
|
|
303
330
|
}
|
|
304
331
|
| {
|
|
305
332
|
mode: 'mask';
|
|
@@ -378,6 +405,18 @@ export declare function writeRestoreMap(
|
|
|
378
405
|
): Promise<void>;
|
|
379
406
|
/** Exact original-document restoration; derived text uses canonical surfaces. */
|
|
380
407
|
export declare function restore(text: string, map: RestoreMap): string;
|
|
408
|
+
/** Sorted, frozen list of the types built-in rules turn into `Type-n` tokens. */
|
|
409
|
+
export declare function reservedPlaceholderTypes(): readonly string[];
|
|
410
|
+
/** Whether `text` is one whole token of the placeholder namespace. */
|
|
411
|
+
export declare function isPlaceholderToken(
|
|
412
|
+
text: string,
|
|
413
|
+
options?: {
|
|
414
|
+
/** Placeholder format, default `'{Type}-{n}'`. */
|
|
415
|
+
format?: string;
|
|
416
|
+
/** Extra custom types, e.g. those of `createPatternDetector`. */
|
|
417
|
+
types?: readonly string[];
|
|
418
|
+
}
|
|
419
|
+
): boolean;
|
|
381
420
|
export interface StreamOptions extends SanitizerOptions {
|
|
382
421
|
/** Treat NUL as a log byte; default false. Incompatible with JSON/JSONL. */
|
|
383
422
|
allowBinary?: boolean;
|
package/src/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** Sensitive text detection and verified publication APIs. */
|
|
2
2
|
|
|
3
|
+
import './personal-features.js';
|
|
4
|
+
|
|
3
5
|
export { inspect, sanitize, redact } from './sanitizer.js';
|
|
4
6
|
export { createSanitizer } from './engines.js';
|
|
5
7
|
export { compileKnown } from './compiled-known.js';
|
|
@@ -58,3 +60,7 @@ export {
|
|
|
58
60
|
readRestoreMap,
|
|
59
61
|
writeRestoreMap,
|
|
60
62
|
} from './restore-map.js';
|
|
63
|
+
export {
|
|
64
|
+
reservedPlaceholderTypes,
|
|
65
|
+
isPlaceholderToken,
|
|
66
|
+
} from './placeholder-types.js';
|
package/src/known.js
CHANGED
|
@@ -71,28 +71,44 @@ function configTokens(config) {
|
|
|
71
71
|
return [...values];
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
// Credential keywords between `_` boundaries, plus suffixes that are written
|
|
75
|
+
// without one (`PGPASSWORD`, `GITHUBTOKEN`, `SESSIONCOOKIE`).
|
|
74
76
|
const secretEnvName =
|
|
75
|
-
/(?:^|_)(?:TOKEN|(?:API|ACCESS|PRIVATE)?KEY|SECRET|PASSWORD|PASSWD|CREDENTIALS?|AUTH|PAT)(?:_|$)
|
|
76
|
-
// Configuration suffixes describe how authentication works
|
|
77
|
-
//
|
|
78
|
-
|
|
79
|
-
|
|
77
|
+
/(?:^|_)(?:TOKEN|(?:API|ACCESS|PRIVATE)?KEY|SECRET|PASSWORD|PASSWD|CREDENTIALS?|O?AUTH|PAT|COOKIE)(?:_|$)|(?:PASSWORD|PASSWD|TOKEN|COOKIE)$/i;
|
|
78
|
+
// Configuration suffixes describe a credential (how authentication works,
|
|
79
|
+
// counters, limits, locations, names, identifiers and guard flags), not the
|
|
80
|
+
// credential itself. Explicit names bypass this filter; credential suffixes
|
|
81
|
+
// such as AUTH_MODE_TOKEN still win.
|
|
82
|
+
const configurationName =
|
|
83
|
+
/_(?:TRANSPORT|MODE|METHOD|TYPE|URL|URI|HOST|HOSTNAME|PATH|ENABLED|DISABLED|PORT|PROTOCOL|SCHEME|GUARD|COUNT|LIMIT|FILE|FILENAME|DIR|DIRECTORY|NAME|ID)$|^GIT_CONFIG_KEY_\d+$|^GOOGLE_APPLICATION_CREDENTIALS$/i;
|
|
84
|
+
// `GIT_CONFIG_KEY_<n>` names a git setting; `GIT_CONFIG_VALUE_<n>` holds the
|
|
85
|
+
// value, which is a credential for headers, passwords and tokens.
|
|
86
|
+
const gitCredentialSetting =
|
|
87
|
+
/(?:^|\.)(?:extraheader|password|token|secret|cookiefile|auth[a-z]*)$/i;
|
|
88
|
+
|
|
89
|
+
function discoveredNames(env) {
|
|
90
|
+
return Object.keys(env).filter((name) => {
|
|
91
|
+
const git = name.match(/^GIT_CONFIG_VALUE_(\d+)$/);
|
|
92
|
+
if (git) {
|
|
93
|
+
return gitCredentialSetting.test(env[`GIT_CONFIG_KEY_${git[1]}`] ?? '');
|
|
94
|
+
}
|
|
95
|
+
return secretEnvName.test(name) && !configurationName.test(name);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
80
99
|
/** Opt-in literal values from credential-named keys, or an explicit selection. */
|
|
81
|
-
export function knownSecretsFromEnv(env = process.env, names) {
|
|
100
|
+
export function knownSecretsFromEnv(env = process.env, names, options = {}) {
|
|
101
|
+
const { minLength = 1 } = options ?? {};
|
|
82
102
|
if (names === undefined) {
|
|
83
|
-
names =
|
|
84
|
-
env && typeof env === 'object'
|
|
85
|
-
? Object.keys(env).filter(
|
|
86
|
-
(name) =>
|
|
87
|
-
secretEnvName.test(name) && !authConfigurationName.test(name)
|
|
88
|
-
)
|
|
89
|
-
: [];
|
|
103
|
+
names = env && typeof env === 'object' ? discoveredNames(env) : [];
|
|
90
104
|
}
|
|
91
105
|
if (
|
|
92
106
|
!env ||
|
|
93
107
|
typeof env !== 'object' ||
|
|
94
108
|
!Array.isArray(names) ||
|
|
95
|
-
names.some((name) => typeof name !== 'string')
|
|
109
|
+
names.some((name) => typeof name !== 'string') ||
|
|
110
|
+
!Number.isSafeInteger(minLength) ||
|
|
111
|
+
minLength < 1
|
|
96
112
|
) {
|
|
97
113
|
throw failure('ERR_CONFIG');
|
|
98
114
|
}
|
|
@@ -100,7 +116,9 @@ export function knownSecretsFromEnv(env = process.env, names) {
|
|
|
100
116
|
...new Set(
|
|
101
117
|
names
|
|
102
118
|
.map((name) => env[name])
|
|
103
|
-
.filter(
|
|
119
|
+
.filter(
|
|
120
|
+
(value) => typeof value === 'string' && value.length >= minLength
|
|
121
|
+
)
|
|
104
122
|
),
|
|
105
123
|
];
|
|
106
124
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Registers the personal-data detectors and fake generators. The main entry
|
|
2
|
+
// imports this module for its side effect; the `./credentials` entry does not,
|
|
3
|
+
// which keeps faker, libphonenumber and these detectors out of its start-up.
|
|
4
|
+
import { registerFeatures } from './features.js';
|
|
5
|
+
import { detectPersonal } from './personal.js';
|
|
6
|
+
import { detectCatalog } from './entities.js';
|
|
7
|
+
import { detectIdentity, fakeIdentityValue } from './identity.js';
|
|
8
|
+
import { detectNational } from './national.js';
|
|
9
|
+
import { detectPersonalContext } from './personal-context.js';
|
|
10
|
+
import { detectPersonalEvidence } from './personal-evidence.js';
|
|
11
|
+
import { detectDocumentEntities } from './document-entities.js';
|
|
12
|
+
import {
|
|
13
|
+
detectProseAndDocuments,
|
|
14
|
+
validateProperNames,
|
|
15
|
+
} from './proper-names.js';
|
|
16
|
+
import { detectStrict } from './strict.js';
|
|
17
|
+
import { realisticFake } from './fake.js';
|
|
18
|
+
import { prepareFakeText } from './fake-entities.js';
|
|
19
|
+
import { fakeDocumentSpans } from './specimens.js';
|
|
20
|
+
|
|
21
|
+
function detectPersonalData(text, emit, options) {
|
|
22
|
+
detectPersonal(text, emit);
|
|
23
|
+
detectProseAndDocuments(text, emit, options, detectDocumentEntities);
|
|
24
|
+
detectCatalog(text, emit);
|
|
25
|
+
detectIdentity(text, emit);
|
|
26
|
+
detectNational(text, emit);
|
|
27
|
+
detectPersonalContext(text, emit);
|
|
28
|
+
detectPersonalEvidence(text, emit);
|
|
29
|
+
detectStrict(text, emit, options);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
registerFeatures({
|
|
33
|
+
detectPersonalData,
|
|
34
|
+
validateProperNames,
|
|
35
|
+
realisticFake,
|
|
36
|
+
prepareFakeText,
|
|
37
|
+
fakeDocumentSpans,
|
|
38
|
+
fakeIdentityValue,
|
|
39
|
+
});
|
package/src/personal.js
CHANGED
|
@@ -3,6 +3,7 @@ import { firstNames } from './names.js';
|
|
|
3
3
|
import { labels } from './rules.js';
|
|
4
4
|
import { transliteratedGivenNames } from './transliteration.js';
|
|
5
5
|
import { serviceAccounts } from './public.js';
|
|
6
|
+
import { isCommonProseWord } from './prose-words.js';
|
|
6
7
|
const labelWords = new Set(Object.values(labels).flat());
|
|
7
8
|
|
|
8
9
|
const details = (type, rule, confidence = 0.85) => ({
|
|
@@ -193,14 +194,39 @@ const possessiveWord =
|
|
|
193
194
|
function slavicPairEvidence(value) {
|
|
194
195
|
const words = value.split(/[ \t]+/);
|
|
195
196
|
return !words.some(
|
|
196
|
-
(word) =>
|
|
197
|
+
(word) =>
|
|
198
|
+
possessiveWord.test(word) ||
|
|
199
|
+
/^\p{Lu}{2,4}$/u.test(word) ||
|
|
200
|
+
isCommonProseWord(word)
|
|
197
201
|
);
|
|
198
202
|
}
|
|
203
|
+
// A rejected pair is retried from its second word, so `Кандидат Петров Иван`
|
|
204
|
+
// still finds `Петров Иван` after the cue word.
|
|
205
|
+
function detectSlavicPairs(text, emit) {
|
|
206
|
+
const pattern = new RegExp(slavicPairs.source, 'gu');
|
|
207
|
+
let match;
|
|
208
|
+
while ((match = pattern.exec(text))) {
|
|
209
|
+
const value = match[0];
|
|
210
|
+
if (
|
|
211
|
+
value !== 'REDACTED' &&
|
|
212
|
+
!placeWord.test(value.split(/[ \t]+/).at(-1)) &&
|
|
213
|
+
slavicPairEvidence(value)
|
|
214
|
+
) {
|
|
215
|
+
emit({
|
|
216
|
+
start: match.index,
|
|
217
|
+
end: match.index + value.length,
|
|
218
|
+
...details('PERSON', 'name-identity', 0.9),
|
|
219
|
+
});
|
|
220
|
+
} else {
|
|
221
|
+
pattern.lastIndex = match.index + value.search(/[ \t]/u) + 1;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
199
225
|
export function detectPersonal(text, emit) {
|
|
200
226
|
detectDocumentNames(text, emit);
|
|
227
|
+
detectSlavicPairs(text, emit);
|
|
201
228
|
for (const pattern of [
|
|
202
229
|
latinPairs,
|
|
203
|
-
slavicPairs,
|
|
204
230
|
russianFull,
|
|
205
231
|
/(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+)[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)[ \t]+[А-ЯЁ][а-яё]{2,30}(?![\p{L}])/gu,
|
|
206
232
|
]) {
|
|
@@ -210,9 +236,7 @@ export function detectPersonal(text, emit) {
|
|
|
210
236
|
emit,
|
|
211
237
|
details('PERSON', 'name-identity', 0.9),
|
|
212
238
|
0,
|
|
213
|
-
(value) =>
|
|
214
|
-
!placeWord.test(value.split(/[ \t]+/).at(-1)) &&
|
|
215
|
-
(pattern !== slavicPairs || slavicPairEvidence(value))
|
|
239
|
+
(value) => !placeWord.test(value.split(/[ \t]+/).at(-1))
|
|
216
240
|
);
|
|
217
241
|
}
|
|
218
242
|
for (const pattern of names) {
|
package/src/placeholder-types.js
CHANGED
|
@@ -1,34 +1,194 @@
|
|
|
1
1
|
import { catalogTypes } from './entities.js';
|
|
2
|
+
import { escapePattern, failure } from './detection.js';
|
|
3
|
+
import { jsonStringLiterals } from './json-string-runs.js';
|
|
4
|
+
import { placeholderToken, validatePlaceholderFormat } from './restore-map.js';
|
|
2
5
|
|
|
3
|
-
// Native
|
|
4
|
-
|
|
5
|
-
|
|
6
|
+
// Native types that the shared vendor-label catalog does not already list. The
|
|
7
|
+
// national identifier types are spelled out here so that this module does not
|
|
8
|
+
// load the national rules.
|
|
9
|
+
const nativeTypes = [
|
|
10
|
+
'ABA_ROUTING_NUMBER',
|
|
6
11
|
'ADDRESS',
|
|
12
|
+
'AE_EMIRATES_ID',
|
|
13
|
+
'AGE',
|
|
14
|
+
'AU_ABN',
|
|
15
|
+
'AU_ACN',
|
|
16
|
+
'AU_MEDICARE',
|
|
7
17
|
'AU_TFN',
|
|
8
18
|
'BRAZIL_CPF_NUMBER',
|
|
19
|
+
'CA_SIN',
|
|
9
20
|
'CREDENTIAL',
|
|
10
21
|
'CREDIT_CARD',
|
|
11
22
|
'DATE',
|
|
23
|
+
'DE_TAX_ID',
|
|
12
24
|
'EDUCATION',
|
|
13
25
|
'EMAIL',
|
|
14
26
|
'ENCODED_SENSITIVE',
|
|
27
|
+
'ES_NIE',
|
|
28
|
+
'EU_VAT_NUMBER',
|
|
29
|
+
'FI_PERSONAL_IDENTITY_CODE',
|
|
30
|
+
'GENDER',
|
|
15
31
|
'IBAN',
|
|
16
32
|
'ID',
|
|
33
|
+
'IL_ID_NUMBER',
|
|
34
|
+
'IMEI',
|
|
35
|
+
'IN_AADHAAR',
|
|
36
|
+
'IN_GSTIN',
|
|
37
|
+
'IN_PASSPORT',
|
|
38
|
+
'IN_VOTER',
|
|
17
39
|
'IP_ADDRESS',
|
|
40
|
+
'IT_FISCAL_CODE',
|
|
41
|
+
'IT_VAT_CODE',
|
|
18
42
|
'KR_RRN',
|
|
19
43
|
'LOCATION',
|
|
20
44
|
'MAC_ADDRESS',
|
|
45
|
+
'MEDICAL_LICENSE',
|
|
21
46
|
'ORGANIZATION',
|
|
22
47
|
'PASSPORT_MRZ',
|
|
23
48
|
'PASSPORT_NUMBER',
|
|
24
49
|
'PERSON',
|
|
25
|
-
'
|
|
50
|
+
'PERSONAL_ATTRIBUTE',
|
|
26
51
|
'PHONE',
|
|
27
52
|
'PL_PESEL',
|
|
53
|
+
'POSTAL_CODE',
|
|
54
|
+
'PROPER_NAME',
|
|
28
55
|
'RU_INN',
|
|
29
56
|
'RU_SNILS',
|
|
57
|
+
'SA_NATIONAL_ID',
|
|
58
|
+
'SENSITIVE',
|
|
59
|
+
'SE_PERSONNUMMER',
|
|
30
60
|
'SG_NRIC_FIN',
|
|
61
|
+
'SG_UEN',
|
|
31
62
|
'STATE',
|
|
63
|
+
'STEAM_ID',
|
|
64
|
+
'SWIFT_CODE',
|
|
65
|
+
'TH_TNIN',
|
|
66
|
+
'TRACKING_NUMBER',
|
|
67
|
+
'UK_DRIVING_LICENCE',
|
|
68
|
+
'UK_NATIONAL_INSURANCE_NUMBER',
|
|
69
|
+
'UK_NHS',
|
|
70
|
+
'UK_POSTCODE',
|
|
71
|
+
'URL',
|
|
32
72
|
'USERNAME',
|
|
73
|
+
'US_BANK_NUMBER',
|
|
74
|
+
'US_DRIVER_LICENSE',
|
|
33
75
|
'US_HEALTHCARE_NPI',
|
|
76
|
+
'US_ITIN',
|
|
77
|
+
'VIN',
|
|
78
|
+
'ZA_ID_NUMBER',
|
|
34
79
|
];
|
|
80
|
+
let reserved;
|
|
81
|
+
// Every type that a built-in rule can turn into a `Type-n` token.
|
|
82
|
+
export function reservedPlaceholderTypes() {
|
|
83
|
+
reserved ??= Object.freeze(
|
|
84
|
+
[...new Set([...catalogTypes(), ...nativeTypes])].sort()
|
|
85
|
+
);
|
|
86
|
+
return reserved;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const defaultFormat = '{Type}-{n}';
|
|
90
|
+
const patterns = new Map();
|
|
91
|
+
const patternCacheSize = 64;
|
|
92
|
+
// The namespace pattern depends only on the format and on the types beyond the
|
|
93
|
+
// reserved list, so it is compiled once per format and cached.
|
|
94
|
+
export function namespacePattern(format = defaultFormat, extraTypes = []) {
|
|
95
|
+
const base = new Set(reservedPlaceholderTypes());
|
|
96
|
+
const extra = [...new Set(extraTypes)].filter((type) => !base.has(type));
|
|
97
|
+
const key = `${format}\0${extra.sort().join('\0')}`;
|
|
98
|
+
const cached = patterns.get(key);
|
|
99
|
+
if (cached) {
|
|
100
|
+
return cached;
|
|
101
|
+
}
|
|
102
|
+
const [before, after] = format.split('{n}');
|
|
103
|
+
const typed = before.match(/\{(?:Type|TYPE|type)\}/) ? before : after;
|
|
104
|
+
const types = new Map();
|
|
105
|
+
for (const type of [...base, ...extra]) {
|
|
106
|
+
const part = placeholderToken(type, 1, typed);
|
|
107
|
+
types.set(part, [...(types.get(part) ?? []), type]);
|
|
108
|
+
}
|
|
109
|
+
const alternation = [...types.keys()]
|
|
110
|
+
.sort((a, b) => b.length - a.length || a.localeCompare(b))
|
|
111
|
+
.map(escapePattern)
|
|
112
|
+
.join('|');
|
|
113
|
+
const number = '([1-9][0-9]*)';
|
|
114
|
+
const source =
|
|
115
|
+
typed === before
|
|
116
|
+
? `(${alternation})${number}${escapePattern(after)}`
|
|
117
|
+
: `${escapePattern(before)}${number}(${alternation})`;
|
|
118
|
+
const pattern = {
|
|
119
|
+
source: `(?<![\\p{L}\\p{M}\\p{N}_])(?:${source})(?![\\p{L}\\p{M}\\p{N}_])`,
|
|
120
|
+
typedFirst: typed === before,
|
|
121
|
+
types,
|
|
122
|
+
};
|
|
123
|
+
if (patterns.size >= patternCacheSize) {
|
|
124
|
+
patterns.delete(patterns.keys().next().value);
|
|
125
|
+
}
|
|
126
|
+
patterns.set(key, pattern);
|
|
127
|
+
return pattern;
|
|
128
|
+
}
|
|
129
|
+
// Tokens of the namespace with the types each spelling can stand for.
|
|
130
|
+
export function placeholderTokens(text, pattern) {
|
|
131
|
+
const tokens = [];
|
|
132
|
+
if (!/[1-9]/u.test(text)) {
|
|
133
|
+
return tokens;
|
|
134
|
+
}
|
|
135
|
+
for (const match of text.matchAll(new RegExp(pattern.source, 'gu'))) {
|
|
136
|
+
const [part, number] = pattern.typedFirst
|
|
137
|
+
? [match[1], match[2]]
|
|
138
|
+
: [match[2], match[1]];
|
|
139
|
+
tokens.push({
|
|
140
|
+
start: match.index,
|
|
141
|
+
end: match.index + match[0].length,
|
|
142
|
+
number: Number(number),
|
|
143
|
+
types: pattern.types.get(part),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return tokens;
|
|
147
|
+
}
|
|
148
|
+
export function containsPlaceholder(text, pattern) {
|
|
149
|
+
return /[1-9]/u.test(text) && new RegExp(pattern.source, 'u').test(text);
|
|
150
|
+
}
|
|
151
|
+
// Tokens spelled with JSON escapes inside string literals, such as
|
|
152
|
+
// `Person-1`, decoded so that they cannot hide from the namespace check.
|
|
153
|
+
export function escapedPlaceholderTokens(text, pattern) {
|
|
154
|
+
const tokens = [];
|
|
155
|
+
// Without an escape every literal has already been checked in the raw text.
|
|
156
|
+
if (!text.includes('\\')) {
|
|
157
|
+
return tokens;
|
|
158
|
+
}
|
|
159
|
+
for (const literal of jsonStringLiterals(text)) {
|
|
160
|
+
if (!literal.includes('\\')) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
try {
|
|
164
|
+
tokens.push(...placeholderTokens(JSON.parse(literal), pattern));
|
|
165
|
+
} catch {
|
|
166
|
+
// Ordinary free text may include strings that are not JSON literals.
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return tokens;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Public check for a whole `Type-n` token of the library namespace.
|
|
173
|
+
export function isPlaceholderToken(text, options = {}) {
|
|
174
|
+
if (
|
|
175
|
+
options === null ||
|
|
176
|
+
typeof options !== 'object' ||
|
|
177
|
+
(options.types !== undefined &&
|
|
178
|
+
(!Array.isArray(options.types) ||
|
|
179
|
+
options.types.some(
|
|
180
|
+
(type) =>
|
|
181
|
+
typeof type !== 'string' || !/^[A-Z][A-Z0-9_]{0,63}$/.test(type)
|
|
182
|
+
)))
|
|
183
|
+
) {
|
|
184
|
+
throw failure('ERR_CONFIG');
|
|
185
|
+
}
|
|
186
|
+
if (typeof text !== 'string') {
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
const pattern = namespacePattern(
|
|
190
|
+
validatePlaceholderFormat(options.format ?? defaultFormat),
|
|
191
|
+
options.types ?? []
|
|
192
|
+
);
|
|
193
|
+
return new RegExp(`^(?:${pattern.source})$`, 'u').test(text);
|
|
194
|
+
}
|