@link-foundation/sensitive-data-sanitizer 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/personal.js CHANGED
@@ -29,6 +29,9 @@ const ibanLengths = Object.fromEntries(
29
29
  .map((entry) => [entry.slice(0, 2), Number(entry.slice(2))])
30
30
  );
31
31
 
32
+ // `@handle/jane@example.com` is a mention and an address, not a package path.
33
+ const mentionedEmail =
34
+ /^\/[\p{L}\p{N}_.!#$%&'*+?^`{|}~-]+@[\p{L}\p{N}][\p{L}\p{N}.-]{0,252}\.\p{L}{2,63}(?![\p{L}\p{N}_-])/u;
32
35
  export function iban(value) {
33
36
  const compact = value.replace(/\s/g, '').toUpperCase();
34
37
  if (
@@ -179,7 +182,12 @@ const russianFull =
179
182
  /(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)|(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}(?:оглы|кызы)(?![\p{L}])/gu;
180
183
  export function detectPersonal(text, emit) {
181
184
  detectDocumentNames(text, emit);
182
- for (const pattern of [latinPairs, slavicPairs, russianFull]) {
185
+ for (const pattern of [
186
+ latinPairs,
187
+ slavicPairs,
188
+ russianFull,
189
+ /(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+)[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)[ \t]+[А-ЯЁ][а-яё]{2,30}(?![\p{L}])/gu,
190
+ ]) {
183
191
  collectMatches(
184
192
  text,
185
193
  pattern,
@@ -202,10 +210,11 @@ export function detectPersonal(text, emit) {
202
210
  }
203
211
  collectMatches(
204
212
  text,
205
- /(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&'-]{1,80})/gu,
213
+ /(?:компании|организации|company|organization|société|empresa|Firma)[ \t]+([\p{Lu}][\p{L}\p{M}&']{0,80}(?:-[\p{L}\p{M}\p{N}&']{1,40}){0,4})/gu,
206
214
  emit,
207
215
  details('ORGANIZATION', 'organization-context', 0.7),
208
- 1
216
+ 1,
217
+ (value) => value.length >= 2
209
218
  );
210
219
  collectMatches(
211
220
  text,
@@ -231,10 +240,12 @@ export function detectPersonal(text, emit) {
231
240
  emit,
232
241
  details('USERNAME', 'messenger-handle', 0.65),
233
242
  0,
234
- (_value, match) =>
235
- !/^\/[a-z0-9][a-z0-9_.-]*/i.test(
236
- text.slice(match.index + match[0].length)
237
- )
243
+ (_value, match) => {
244
+ const rest = text.slice(match.index + match[0].length);
245
+ return (
246
+ !/^\/[a-z0-9][a-z0-9_.-]*/i.test(rest) || mentionedEmail.test(rest)
247
+ );
248
+ }
238
249
  );
239
250
  collectMatches(
240
251
  text,
@@ -0,0 +1,33 @@
1
+ import { catalogTypes } from './entities.js';
2
+
3
+ // Native unlabelled types supplement the shared vendor-label catalog.
4
+ export const reservedPlaceholderTypes = () => [
5
+ ...catalogTypes(),
6
+ 'ADDRESS',
7
+ 'AU_TFN',
8
+ 'BRAZIL_CPF_NUMBER',
9
+ 'CREDENTIAL',
10
+ 'CREDIT_CARD',
11
+ 'DATE',
12
+ 'EDUCATION',
13
+ 'EMAIL',
14
+ 'ENCODED_SENSITIVE',
15
+ 'IBAN',
16
+ 'ID',
17
+ 'IP_ADDRESS',
18
+ 'KR_RRN',
19
+ 'LOCATION',
20
+ 'MAC_ADDRESS',
21
+ 'ORGANIZATION',
22
+ 'PASSPORT_MRZ',
23
+ 'PASSPORT_NUMBER',
24
+ 'PERSON',
25
+ 'PHONE',
26
+ 'PL_PESEL',
27
+ 'RU_INN',
28
+ 'RU_SNILS',
29
+ 'SG_NRIC_FIN',
30
+ 'STATE',
31
+ 'USERNAME',
32
+ 'US_HEALTHCARE_NPI',
33
+ ];
@@ -1,4 +1,6 @@
1
1
  import { failure, REDACTED, escapePattern } from './detection.js';
2
+ import { reservedPlaceholderTypes } from './placeholder-types.js';
3
+ import { samePersonal } from './personal-variants.js';
2
4
  import { nameAliases } from './transliteration.js';
3
5
  import { createHash } from 'node:crypto';
4
6
  import { jsonStringLiterals } from './json-string-runs.js';
@@ -105,11 +107,29 @@ function aliases(value, type) {
105
107
  ),
106
108
  ];
107
109
  }
110
+ // A bare name is the same entity as its numbered form, including maps saved
111
+ // when the span stopped at the hyphen of `Вектор-99`.
112
+ function numberedOrganization(value, entry) {
113
+ return (entry.values ?? []).some((original) => {
114
+ const base = original.replace(/-\d*$/u, '');
115
+ return base !== original && samePersonal(value, base, entry.type);
116
+ });
117
+ }
108
118
  function related(value, entry) {
119
+ if (
120
+ (entry.values ?? []).some((original) =>
121
+ samePersonal(value, original, entry.type)
122
+ )
123
+ ) {
124
+ return true;
125
+ }
109
126
  const candidates = aliases(value, entry.type);
110
127
  if (candidates.some((alias) => entry.aliases.includes(alias))) {
111
128
  return true;
112
129
  }
130
+ if (entry.type === 'ORGANIZATION') {
131
+ return numberedOrganization(value, entry);
132
+ }
113
133
  if (entry.type !== 'PERSON') {
114
134
  return false;
115
135
  }
@@ -139,7 +159,7 @@ function related(value, entry) {
139
159
  return false;
140
160
  }
141
161
  export function placeholderNameFindings(text, findings, options) {
142
- if (!hasPlaceholders(options)) {
162
+ if (!hasPlaceholders(options) || options.knownOnly) {
143
163
  return findings;
144
164
  }
145
165
  const people = [
@@ -199,12 +219,23 @@ export function preparePlaceholderText(text, findings, options) {
199
219
  if (!state || !hasPlaceholders(options)) {
200
220
  return;
201
221
  }
202
- // Reserve the entire namespace, not only the numbers about to be allocated.
203
- const template = state.map.format;
222
+ const types = new Set([
223
+ ...reservedPlaceholderTypes(),
224
+ ...findings.map((f) =>
225
+ f.category === 'credential' ? 'CREDENTIAL' : f.type
226
+ ),
227
+ ...(options.knownPersonal ?? []).map((e) => e.type),
228
+ ...state.map.entries.map((e) => e.type),
229
+ ]);
230
+ const [before, after] = state.map.format.split('{n}');
231
+ const tokens = [...types].map(
232
+ (type) =>
233
+ `${escapePattern(
234
+ placeholderToken(type, 1, before)
235
+ )}[1-9][0-9]*${escapePattern(placeholderToken(type, 1, after))}`
236
+ );
204
237
  const pattern = new RegExp(
205
- escapePattern(template)
206
- .replace(/\\\{(?:Type|TYPE|type)\\\}/, '[A-Za-z][A-Za-z0-9_]*')
207
- .replace('\\{n\\}', '[1-9][0-9]*'),
238
+ `(?<![\\p{L}\\p{M}\\p{N}_])(?:${tokens.join('|')})(?![\\p{L}\\p{M}\\p{N}_])`,
208
239
  'u'
209
240
  );
210
241
  if (namespaceCollision(text, pattern)) {
@@ -237,7 +268,7 @@ function namespaceCollision(text, pattern) {
237
268
  return false;
238
269
  }
239
270
 
240
- function canonicalPerson(value, people) {
271
+ export function canonicalPerson(value, people) {
241
272
  // Resolve components only when there is a unique compatible full name.
242
273
  const matches = people.filter((person) => {
243
274
  const words = person.match(/\p{L}+/gu) ?? [];
@@ -255,6 +286,18 @@ function canonicalPerson(value, people) {
255
286
  );
256
287
  return distinct.size === 1 ? [...distinct.values()][0] : value;
257
288
  }
289
+ function canonicalKnown(value, type, options, state) {
290
+ const known = (options.knownPersonal ?? []).filter(
291
+ (e) => e.type === type && samePersonal(value, e.value, type)
292
+ );
293
+ // `Петров Данил` and `Петров Данил Сергеевич` are one person: the fullest wins.
294
+ const longest = Math.max(0, ...known.map((e) => e.value.length));
295
+ const fullest = known.filter((e) => e.value.length === longest);
296
+ if (fullest.length === 1) {
297
+ return fullest[0].value;
298
+ }
299
+ return type === 'PERSON' ? canonicalPerson(value, state.people ?? []) : value;
300
+ }
258
301
  export function placeholderReplacement(value, finding, transform, options) {
259
302
  if (finding.category === 'credential' && !transform.credentials) {
260
303
  return REDACTED;
@@ -270,12 +313,16 @@ export function placeholderReplacement(value, finding, transform, options) {
270
313
  : finding.type === 'DATE_OF_BIRTH'
271
314
  ? 'DATE'
272
315
  : finding.type;
273
- const canonical =
274
- type === 'PERSON' ? canonicalPerson(value, state.people ?? []) : value;
316
+ const canonical = canonicalKnown(value, type, options, state);
275
317
  const candidates = map.entries.filter(
276
318
  (entry) => entry.type === type && related(canonical, entry)
277
319
  );
278
- let entry = candidates.length === 1 ? candidates[0] : undefined;
320
+ // Several related entities are ambiguous, unless one holds this exact value.
321
+ const exact = candidates.filter((entry) =>
322
+ entry.values.some((original) => samePersonal(canonical, original, type))
323
+ );
324
+ const matches = candidates.length === 1 ? candidates : exact;
325
+ let entry = matches.length === 1 ? matches[0] : undefined;
279
326
  if (!entry) {
280
327
  if (map.entries.length >= (options.maxFindings ?? 100000)) {
281
328
  throw failure('ERR_LIMIT');
package/src/projection.js CHANGED
@@ -27,8 +27,12 @@ export function projectText(input) {
27
27
  !/\x1b\[|\\u[\da-fA-F]{4}|[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]/u.test(
28
28
  input
29
29
  ) &&
30
- input.normalize('NFKC') === input &&
31
- !/[^0-9]/.test(input.replace(/[^\p{Nd}]/gu, ''))
30
+ // ASCII already has normalized letters and digits. Avoid constructing a
31
+ // second multi-megabyte string just to check its decimal characters.
32
+ // eslint-disable-next-line no-control-regex -- Check the ASCII range.
33
+ (!/[^\x00-\x7f]/.test(input) ||
34
+ (input.normalize('NFKC') === input &&
35
+ !/[^0-9]/.test(input.replace(/[^\p{Nd}]/gu, ''))))
32
36
  ) {
33
37
  return { text: input, changed: false };
34
38
  }
@@ -1,5 +1,6 @@
1
1
  import { personalPattern, maxPropagationLength, failure } from './detection.js';
2
2
  import { actionable } from './confidence.js';
3
+ import { organizationCore, organizationFamily } from './personal-variants.js';
3
4
  import { nameAliases } from './transliteration.js';
4
5
 
5
6
  function mrzSeeds(value, finding, seeds) {
@@ -32,11 +33,13 @@ function decodedValue(value) {
32
33
 
33
34
  function eligibleSeed(finding, options) {
34
35
  return (
36
+ !options.knownOnly &&
35
37
  finding.category === 'personal' &&
36
38
  finding.type !== 'ENCODED_SENSITIVE' &&
37
39
  !['confirmed-personal', 'document-propagation'].includes(finding.rule) &&
38
40
  actionable(finding, options) &&
39
- (finding.confidence >= 0.85 ||
41
+ ((options.documentEntities && finding.type === 'ORGANIZATION') ||
42
+ finding.confidence >= 0.85 ||
40
43
  [
41
44
  'PERSON',
42
45
  'PASSPORT_NUMBER',
@@ -47,6 +50,18 @@ function eligibleSeed(finding, options) {
47
50
  ].includes(finding.type))
48
51
  );
49
52
  }
53
+ function organizationSeed(text, value, finding, seeds, options) {
54
+ if (options.documentEntities && finding.type === 'ORGANIZATION') {
55
+ const core = organizationCore(value);
56
+ if (core.length >= 3) {
57
+ seeds.set(core, finding);
58
+ }
59
+ const family = organizationFamily(core);
60
+ for (const match of family ? text.matchAll(family.pattern) : []) {
61
+ seeds.set(match[0], finding);
62
+ }
63
+ }
64
+ }
50
65
  // Only confirmed values seed propagation. A low-score numeric coincidence
51
66
  // must never turn every log occurrence into a high-score identity.
52
67
  export function confirmedSeeds(text, findings, options) {
@@ -71,6 +86,7 @@ export function confirmedSeeds(text, findings, options) {
71
86
  mrzSeeds(value, finding, seeds);
72
87
  } else if (value.length >= 2) {
73
88
  seeds.set(value, finding);
89
+ organizationSeed(text, value, finding, seeds, options);
74
90
  if (finding.type === 'PERSON') {
75
91
  for (const name of value.match(/\p{L}{2,}/gu) ?? []) {
76
92
  seeds.set(name, finding);
@@ -0,0 +1,167 @@
1
+ // Finite Russian declension shared by fake names and fake entities. Case order:
2
+ // nominative, genitive, dative, accusative, instrumental, prepositional.
3
+ // `|` lists a rarer spelling, accepted when reading a form, never generated.
4
+ const fleeting = { павел: 'павл', лев: 'льв', ветер: 'ветр' };
5
+ const hush = /[жчшщц]$/u;
6
+ const a = (stem) => [
7
+ 'а',
8
+ /[гкхжчшщ]$/u.test(stem) ? 'и' : 'ы',
9
+ 'е',
10
+ 'у',
11
+ /[жчшщц]$/u.test(stem) ? 'ей|ею' : 'ой|ою',
12
+ 'е',
13
+ ];
14
+ const adjectives = [
15
+ [
16
+ /[ый]й$|ой$/u,
17
+ 2,
18
+ (w) => [w.slice(-2), 'ого', 'ому', w.slice(-2), 'ым', 'ом'],
19
+ ],
20
+ [/[кгхжчшщ]ий$/u, 2, () => ['ий', 'ого', 'ому', 'ий', 'им', 'ом']],
21
+ [/ий$/u, 2, () => ['ий', 'его', 'ему', 'ий', 'им', 'ем']],
22
+ [/ая$/u, 2, () => ['ая', 'ой', 'ой', 'ую', 'ой|ою', 'ой']],
23
+ [/яя$/u, 2, () => ['яя', 'ей', 'ей', 'юю', 'ей|ею', 'ей']],
24
+ [/ое$/u, 2, () => ['ое', 'ого', 'ому', 'ое', 'ым', 'ом']],
25
+ [/ее$/u, 2, () => ['ее', 'его', 'ему', 'ее', 'им', 'ем']],
26
+ ];
27
+ function noun(word, sex, animate) {
28
+ const accusative = (genitive, nominative = '') =>
29
+ animate ? genitive : nominative;
30
+ if (fleeting[word]) {
31
+ const hard = ['а', 'у', accusative('а'), 'ом', 'е'];
32
+ return [word, ...hard.map((ending) => fleeting[word] + ending)];
33
+ }
34
+ const table = [
35
+ [/ия$/u, 2, ['ия', 'ии', 'ии', 'ию', 'ией|иею', 'ии']],
36
+ [/ье$/u, 2, ['ье', 'ья', 'ью', 'ье', 'ьем', 'ье']],
37
+ [/ие$/u, 2, ['ие', 'ия', 'ию', 'ие', 'ием', 'ии']],
38
+ [/ий$/u, 2, ['ий', 'ия', 'ию', accusative('ия', 'ий'), 'ием', 'ии']],
39
+ [/й$/u, 1, ['й', 'я', 'ю', accusative('я', 'й'), 'ем', 'е']],
40
+ [/я$/u, 1, ['я', 'и', 'е', 'ю', 'ей|ею', 'е']],
41
+ [/а$/u, 1, a(word.slice(0, -1))],
42
+ [/о$/u, 1, ['о', 'а', 'у', 'о', 'ом', 'е']],
43
+ ];
44
+ for (const [ending, remove, cases] of table) {
45
+ if (ending.test(word)) {
46
+ return cases.map((suffix) => attach(word.slice(0, -remove), suffix));
47
+ }
48
+ }
49
+ if (word.endsWith('ь')) {
50
+ const stem = word.slice(0, -1);
51
+ const feminine = ['ь', 'и', 'и', 'ь', 'ью', 'и'];
52
+ const masculine = ['ь', 'я', 'ю', accusative('я', 'ь'), 'ем', 'е'];
53
+ const cases =
54
+ sex === 'female'
55
+ ? feminine
56
+ : sex === 'male'
57
+ ? masculine
58
+ : masculine.map((suffix, i) =>
59
+ suffix === feminine[i] ? suffix : `${suffix}|${feminine[i]}`
60
+ );
61
+ return cases.map((suffix) => attach(stem, suffix));
62
+ }
63
+ if (!/[бвгджзклмнпрстфхцчшщ]$/u.test(word)) {
64
+ return Array(6).fill(word); // Indeclinable: Шевченко, Сочи.
65
+ }
66
+ return [
67
+ '',
68
+ 'а',
69
+ 'у',
70
+ accusative('а'),
71
+ hush.test(word) ? 'ем' : 'ом',
72
+ 'е',
73
+ ].map((suffix) => attach(word, suffix));
74
+ }
75
+ const attach = (stem, suffix) =>
76
+ suffix
77
+ .split('|')
78
+ .map((ending) => stem + ending)
79
+ .join('|');
80
+ function adjective(word) {
81
+ for (const [ending, remove, cases] of adjectives) {
82
+ if (ending.test(word)) {
83
+ return cases(word).map((suffix) =>
84
+ attach(word.slice(0, -remove), suffix)
85
+ );
86
+ }
87
+ }
88
+ return undefined;
89
+ }
90
+ function surname(word, sex) {
91
+ if (/(?:ов|ев|ёв|ин|ын)$/u.test(word)) {
92
+ return ['', 'а', 'у', 'а', 'ым|ом', 'е'].map((s) => attach(word, s));
93
+ }
94
+ if (/(?:ова|ева|ёва|ина|ына)$/u.test(word)) {
95
+ const stem = word.slice(0, -1);
96
+ return ['а', 'ой', 'ой', 'у', 'ой|ою', 'ой'].map((s) => attach(stem, s));
97
+ }
98
+ const forms = adjective(word);
99
+ if (forms) {
100
+ return forms;
101
+ }
102
+ return sex === 'female' && !/[ая]$/u.test(word)
103
+ ? Array(6).fill(word)
104
+ : noun(word, sex, true);
105
+ }
106
+ export const russianNoun = (word, options) =>
107
+ options.adjective && adjective(word)
108
+ ? adjective(word)
109
+ : noun(word, options.sex, options.animate);
110
+ // `kind` is the name slot: lastName, firstName or middleName.
111
+ export function russianNameForms(word, kind, sex) {
112
+ const lower = word.toLowerCase();
113
+ if (kind === 'lastName') {
114
+ return surname(lower, sex);
115
+ }
116
+ if (kind === 'middleName' && /ич$/u.test(lower)) {
117
+ return ['', 'а', 'у', 'а', 'ем', 'е'].map((s) => attach(lower, s));
118
+ }
119
+ return noun(lower, sex, true);
120
+ }
121
+ const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
122
+ // Every case a surface can stand for, in case order.
123
+ export function casesOf(surface, forms) {
124
+ const wanted = fold(surface);
125
+ return forms.flatMap((form, index) =>
126
+ fold(form).split('|').includes(wanted) ? [index] : []
127
+ );
128
+ }
129
+ export const formOf = (forms, index) => forms[index].split('|')[0];
130
+ // Cases governed by a preposition, in order of preference. Bare forms like
131
+ // `Ромашке` (dative or prepositional) cannot be told apart without it.
132
+ const governed = [
133
+ [/^(?:в|во|на|о|об|обо|при)$/u, [5, 3]],
134
+ [/^(?:к|ко|по)$/u, [2]],
135
+ [/^(?:с|со|над|под|перед|за)$/u, [4, 1, 3]],
136
+ [/^(?:у|от|до|из|без|для|около|после|кроме|вокруг|против)$/u, [1]],
137
+ ];
138
+ const lastWord = /(?<![\p{L}\p{M}\p{N}_-])(\p{L}{1,7})[ \t\u00a0]+([«“"]?)$/u;
139
+ // `в Лютике и Ромашке`: a conjunct takes the case of the word before it.
140
+ const conjunct =
141
+ /[\p{L}\p{M}-]{1,32}[»”"]?(?:,|[ \t\u00a0]+(?:и|или))[ \t\u00a0]+[«“"]?$/u;
142
+ export function governedCases(text, start) {
143
+ const before = text.slice(Math.max(0, start - 64), start);
144
+ const conjunction = conjunct.exec(before);
145
+ const source = conjunction ? before.slice(0, conjunction.index) : before;
146
+ const match = lastWord.exec(source.slice(-12));
147
+ if (!match) {
148
+ return { cases: [] };
149
+ }
150
+ const word = match[1].toLowerCase();
151
+ const rule = governed.find(([pattern]) => pattern.test(word));
152
+ return { cases: rule?.[1] ?? [], quoted: Boolean(match[2]) };
153
+ }
154
+ // Intersects the candidates of each word; a word that fits no case is ignored.
155
+ // A quoted name in the nominative stays there: в «Вектор».
156
+ export function commonCase(candidates, context = { cases: [] }) {
157
+ const present = candidates.filter((cases) => cases.length);
158
+ const common = [0, 1, 2, 3, 4, 5].filter((index) =>
159
+ present.every((cases) => cases.includes(index))
160
+ );
161
+ if (context.quoted && common.includes(0)) {
162
+ return 0;
163
+ }
164
+ return (
165
+ context.cases.find((index) => common.includes(index)) ?? common[0] ?? 0
166
+ );
167
+ }
@@ -0,0 +1,36 @@
1
+ import { firstNames } from './names.js';
2
+
3
+ // Finite Russian name grammar shared by known-value matching and fake mode.
4
+ const patronymic = /(?:ич|овна|евна|ична|vich|ovna|evna)$/iu;
5
+ const surname = /(?:ов|ев|ин|ын|ский|цкий|ова|ева|ина|ына|ская|цкая)$/iu;
6
+ const fold = (word) => word.toLowerCase().replaceAll('ё', 'е');
7
+ const givenNames = new Set(firstNames.ru.split(' ').map(fold));
8
+
9
+ export const isPatronymic = (word) => patronymic.test(word);
10
+ export const isGivenName = (word) => givenNames.has(fold(word));
11
+
12
+ // Returns word indexes by role, or undefined when the roles are not evident.
13
+ // `Имя Отчество Фамилия` is checked first so that surnames in -ич, as in
14
+ // `Шостакович Дмитрий Дмитриевич`, are not mistaken for patronymics.
15
+ export function russianNameRoles(words, known = isGivenName) {
16
+ if (words.length === 3) {
17
+ if (isPatronymic(words[1])) {
18
+ return { given: 0, patronymic: 1, surname: 2 };
19
+ }
20
+ return isPatronymic(words[2])
21
+ ? { surname: 0, given: 1, patronymic: 2 }
22
+ : undefined;
23
+ }
24
+ if (words.length !== 2) {
25
+ return undefined;
26
+ }
27
+ if (isPatronymic(words[1])) {
28
+ return { given: 0, patronymic: 1 };
29
+ }
30
+ const candidates = [0, 1].filter(
31
+ (i) => surname.test(words[i]) && !known(words[i])
32
+ );
33
+ return candidates.length === 1
34
+ ? { surname: candidates[0], given: 1 - candidates[0] }
35
+ : undefined;
36
+ }
package/src/sanitizer.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { generatedValues } from './fake.js';
2
+ import { prepareFakeText } from './fake-entities.js';
2
3
  import { alignEscapes } from './escape-spans.js';
3
4
  import { actionable } from './confidence.js';
4
5
  import { propagate, confirmationOptions } from './propagation.js';
@@ -45,6 +46,7 @@ const publicTypes = new Set([
45
46
  ]);
46
47
  const optionNames = new Set([
47
48
  'credentialsOnly',
49
+ 'knownOnly',
48
50
  'knownSecrets',
49
51
  'knownPersonal',
50
52
  'confirmedPersonal',
@@ -166,6 +168,7 @@ export function validateOptions(options = {}) {
166
168
  for (const key of [
167
169
  'paranoid',
168
170
  'credentialsOnly',
171
+ 'knownOnly',
169
172
  'decode',
170
173
  'secretlint',
171
174
  'publicKnowledge',
@@ -690,11 +693,12 @@ export function redactResult(
690
693
  ) {
691
694
  options = applyProfile(options);
692
695
  const spans = validatedSpans(text, findings, options);
693
- options = initializePlaceholders(options);
696
+ options = prepareFakeText(text, findings, initializePlaceholders(options));
694
697
  preparePlaceholderText(text, findings, options);
695
698
  const parts = [];
696
699
  const faked = [];
697
700
  const occurrences = [];
701
+ let irreversibleRedactions = 0;
698
702
  let outputLength = 0;
699
703
  let cursor = 0;
700
704
  for (const span of spans) {
@@ -716,6 +720,11 @@ export function redactResult(
716
720
  const transform =
717
721
  options.transformations?.[span.type] ?? options.transformation;
718
722
  if (retainedReplacement(transform, 'placeholder', span, replacement)) {
723
+ // `Organization-1` followed by `99` would read as a token that does not
724
+ // exist, so a span that ends there fails the run instead.
725
+ if (/\d$/.test(replacement) && /\d/.test(text[span.end] ?? '')) {
726
+ throw failure('ERR_RESIDUAL');
727
+ }
719
728
  occurrences.push({
720
729
  start: outputLength,
721
730
  end: outputLength + replacement.length,
@@ -723,6 +732,12 @@ export function redactResult(
723
732
  raw: text.slice(span.start, span.end),
724
733
  });
725
734
  }
735
+ if (
736
+ hasPlaceholders(options) &&
737
+ !retainedReplacement(transform, 'placeholder', span, replacement)
738
+ ) {
739
+ irreversibleRedactions++;
740
+ }
726
741
  outputLength += replacement.length;
727
742
  if (retainedReplacement(transform, 'fake', span, replacement)) {
728
743
  faked.push(span);
@@ -731,7 +746,11 @@ export function redactResult(
731
746
  }
732
747
  parts.push(text.slice(cursor));
733
748
  const result = finishPlaceholders(
734
- { text: parts.join(''), redactions: spans.length },
749
+ {
750
+ text: parts.join(''),
751
+ redactions: spans.length,
752
+ ...(hasPlaceholders(options) ? { irreversibleRedactions } : {}),
753
+ },
735
754
  occurrences,
736
755
  options
737
756
  );
@@ -861,6 +880,13 @@ export function redactAgain(result, findings, options, round, rerun) {
861
880
  const merge = (next) => ({
862
881
  ...next,
863
882
  redactions: result.redactions + next.redactions,
883
+ ...(hasPlaceholders(options)
884
+ ? {
885
+ irreversibleRedactions:
886
+ (result.irreversibleRedactions ?? 0) +
887
+ (next.irreversibleRedactions ?? 0),
888
+ }
889
+ : {}),
864
890
  findings: auditFaked(findings, options, result),
865
891
  });
866
892
  const next = rerun({ ...options, findings: [] }, round + 1);
package/src/transforms.js CHANGED
@@ -41,6 +41,22 @@ export function validateTransforms(options) {
41
41
  function positiveInteger(value) {
42
42
  return Number.isSafeInteger(value) && value > 0;
43
43
  }
44
+ const poolTypes = new Set(['ORGANIZATION', 'LOCATION', 'EDUCATION']);
45
+ function validPools(pools) {
46
+ return (
47
+ pools === undefined ||
48
+ (Boolean(pools) &&
49
+ typeof pools === 'object' &&
50
+ !Array.isArray(pools) &&
51
+ Object.entries(pools).every(
52
+ ([type, names]) =>
53
+ poolTypes.has(type) &&
54
+ Array.isArray(names) &&
55
+ names.length > 0 &&
56
+ names.every((name) => typeof name === 'string' && name.trim())
57
+ ))
58
+ );
59
+ }
44
60
  function validateTransform(transform) {
45
61
  if (!transform || !modes.has(transform.mode)) {
46
62
  throw failure('ERR_CONFIG');
@@ -54,7 +70,8 @@ function validateTransform(transform) {
54
70
  Buffer.byteLength(transform.key) >= 16 &&
55
71
  (transform.credentials === undefined ||
56
72
  typeof transform.credentials === 'boolean') &&
57
- (transform.mrzCountry === undefined || transform.mrzCountry === 'UTO'),
73
+ (transform.mrzCountry === undefined || transform.mrzCountry === 'UTO') &&
74
+ validPools(transform.pools),
58
75
  pseudonym: () =>
59
76
  typeof transform.key === 'string' &&
60
77
  Buffer.byteLength(transform.key) >= 16,
@@ -130,7 +147,7 @@ function masked(value, finding, transform) {
130
147
  ? REDACTED
131
148
  : `${chars.slice(0, start).join('')}${transform.marker ?? '*'.repeat(chars.length - start - end)}${end ? chars.slice(-end).join('') : ''}`;
132
149
  }
133
- function transformed(value, finding, transform) {
150
+ function transformed(value, finding, transform, options) {
134
151
  if (!transform || transform.mode === 'redact') {
135
152
  return REDACTED;
136
153
  }
@@ -141,7 +158,7 @@ function transformed(value, finding, transform) {
141
158
  : REDACTED;
142
159
  }
143
160
  if (transform.mode === 'fake') {
144
- return realisticFake(value, finding, transform);
161
+ return realisticFake(value, finding, transform, options);
145
162
  }
146
163
  if (finding.category === 'credential') {
147
164
  return REDACTED;
@@ -198,6 +215,6 @@ function renderValue(text, finding, transform, options) {
198
215
  const result =
199
216
  transform?.mode === 'placeholder'
200
217
  ? placeholderReplacement(raw, finding, transform, options)
201
- : transformed(raw, finding, transform);
218
+ : transformed(raw, finding, transform, options);
202
219
  return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
203
220
  }