@crewhaus/tool-secure 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +273 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +1006 -0
- package/dist/lib/entropy.d.ts +65 -0
- package/dist/lib/entropy.js +99 -0
- package/dist/lib/evidence.d.ts +55 -0
- package/dist/lib/evidence.js +134 -0
- package/dist/lib/files.d.ts +71 -0
- package/dist/lib/files.js +227 -0
- package/dist/lib/injection.d.ts +38 -0
- package/dist/lib/injection.js +396 -0
- package/dist/lib/pii.d.ts +91 -0
- package/dist/lib/pii.js +757 -0
- package/dist/lib/policy.d.ts +42 -0
- package/dist/lib/policy.js +138 -0
- package/dist/lib/redact.d.ts +41 -0
- package/dist/lib/redact.js +121 -0
- package/dist/lib/secrets.d.ts +111 -0
- package/dist/lib/secrets.js +329 -0
- package/dist/lib/text.d.ts +129 -0
- package/dist/lib/text.js +191 -0
- package/dist/lib/unicode.d.ts +106 -0
- package/dist/lib/unicode.js +431 -0
- package/dist/lib/url.d.ts +35 -0
- package/dist/lib/url.js +363 -0
- package/dist/paths.d.ts +42 -0
- package/dist/paths.js +158 -0
- package/package.json +45 -0
package/dist/lib/pii.js
ADDED
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Personal-data detectors.
|
|
3
|
+
*
|
|
4
|
+
* ## Read this before you trust a result
|
|
5
|
+
*
|
|
6
|
+
* Every rule here is a pattern over bytes. A pattern can prove that a string
|
|
7
|
+
* has the SHAPE of an identifier, and for two formats it can prove a check
|
|
8
|
+
* digit is consistent. It can never prove that a string IS somebody's
|
|
9
|
+
* personal data, and — far more dangerous — an empty result can never prove
|
|
10
|
+
* a document has none. A name, a free-text medical detail, an employee
|
|
11
|
+
* number with a local format, an address that does not end in "Street": all
|
|
12
|
+
* invisible here.
|
|
13
|
+
*
|
|
14
|
+
* So `PiiScan` reports what it LOOKED for, not what a document contains, and
|
|
15
|
+
* nothing downstream should read "0 findings" as "safe to publish".
|
|
16
|
+
*
|
|
17
|
+
* ## The rules, and exactly what each one proves
|
|
18
|
+
*
|
|
19
|
+
* | rule | proves |
|
|
20
|
+
* |---|---|
|
|
21
|
+
* | `email.addr-spec-subset` | dot-atom local part, dotted domain, TLD of 2+ letters. Not quoted local parts, not IP-literal domains, not internationalized addresses. |
|
|
22
|
+
* | `phone.e164` | `+` then 8–15 digits with common separators, taken as the longest whole-group run within that range. Where the next number is grouped the same way the two cannot be told apart, and the span runs long; `dedupeOverlaps` lets a verified card win that overlap. |
|
|
23
|
+
* | `phone.national.XX` | a national dialling shape for one of the seven supported countries. |
|
|
24
|
+
* | `ssn.us-format` | `NNN-NN-NNNN` with a group the SSA has never issued (000/666/900+ area, 00 group, 0000 serial) excluded. NOT validity: there is no public checksum for an SSN, and this package does not pretend otherwise. |
|
|
25
|
+
* | `iban.iso7064-mod97` | the ISO 7064 mod-97-10 check digits are correct, and where the country is known, the length matches. A real check, verifiable offline. |
|
|
26
|
+
* | `card.luhn` | the Luhn check digit is correct over 12–19 digits taken as whole digit groups. A real check. It does not mean the card exists or is live. |
|
|
27
|
+
* | `ip.v4-dotted` / `ip.v6-parsed` | a parseable address literal. Private, loopback and documentation ranges are labelled, because those are usually not personal data. |
|
|
28
|
+
* | `dob.labelled` / `date.calendar` | a calendar date. Only a nearby "date of birth"-style label, or a caller-supplied `referenceDate` that makes the age plausible, raises it above `possible`. |
|
|
29
|
+
* | `address.us-street-suffix` / `address.us-city-state-zip` | a US-style street line or city/state/ZIP tail. US conventions only. |
|
|
30
|
+
*/
|
|
31
|
+
import { matchAll, withPositions } from "./text";
|
|
32
|
+
export const PII_TYPES = [
|
|
33
|
+
"credit_card",
|
|
34
|
+
"date_of_birth",
|
|
35
|
+
"email",
|
|
36
|
+
"iban",
|
|
37
|
+
"ip_address",
|
|
38
|
+
"phone",
|
|
39
|
+
"postal_address",
|
|
40
|
+
"us_ssn",
|
|
41
|
+
];
|
|
42
|
+
/** Countries whose national (non-`+`) dialling shapes are implemented. */
|
|
43
|
+
export const SUPPORTED_PHONE_COUNTRIES = ["AU", "CA", "DE", "FR", "GB", "IN", "US"];
|
|
44
|
+
export class PiiOptionError extends Error {
|
|
45
|
+
name = "PiiOptionError";
|
|
46
|
+
}
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
// check digits — the only two things in this file that PROVE anything
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
/** Luhn (ISO/IEC 7812-1) over a digits-only string. */
|
|
51
|
+
export function luhnValid(digits) {
|
|
52
|
+
if (digits.length < 12 || digits.length > 19 || !/^\d+$/.test(digits))
|
|
53
|
+
return false;
|
|
54
|
+
let sum = 0;
|
|
55
|
+
let double = false;
|
|
56
|
+
for (let i = digits.length - 1; i >= 0; i--) {
|
|
57
|
+
let d = digits.charCodeAt(i) - 48;
|
|
58
|
+
if (double) {
|
|
59
|
+
d *= 2;
|
|
60
|
+
if (d > 9)
|
|
61
|
+
d -= 9;
|
|
62
|
+
}
|
|
63
|
+
sum += d;
|
|
64
|
+
double = !double;
|
|
65
|
+
}
|
|
66
|
+
return sum % 10 === 0;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* ISO 7064 mod-97-10, as IBANs use it: rotate the first four characters to
|
|
70
|
+
* the end, map letters to two-digit numbers, and take the remainder modulo
|
|
71
|
+
* 97 in chunks, because the integer does not fit in a double.
|
|
72
|
+
*/
|
|
73
|
+
export function ibanMod97(iban) {
|
|
74
|
+
const rearranged = iban.slice(4) + iban.slice(0, 4);
|
|
75
|
+
let remainder = 0;
|
|
76
|
+
for (const ch of rearranged) {
|
|
77
|
+
const code = ch.charCodeAt(0);
|
|
78
|
+
const piece = code >= 48 && code <= 57
|
|
79
|
+
? String(code - 48)
|
|
80
|
+
: code >= 65 && code <= 90
|
|
81
|
+
? String(code - 55)
|
|
82
|
+
: undefined;
|
|
83
|
+
if (piece === undefined)
|
|
84
|
+
return -1;
|
|
85
|
+
for (const digit of piece) {
|
|
86
|
+
remainder = (remainder * 10 + (digit.charCodeAt(0) - 48)) % 97;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return remainder;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Registered IBAN lengths. A country not listed is still check-digit tested,
|
|
93
|
+
* but reported at lower confidence and flagged `unknownCountry`, because the
|
|
94
|
+
* length rule is half of what makes an IBAN verifiable.
|
|
95
|
+
*/
|
|
96
|
+
export const IBAN_LENGTHS = new Map([
|
|
97
|
+
["AD", 24],
|
|
98
|
+
["AE", 23],
|
|
99
|
+
["AL", 28],
|
|
100
|
+
["AT", 20],
|
|
101
|
+
["AZ", 28],
|
|
102
|
+
["BA", 20],
|
|
103
|
+
["BE", 16],
|
|
104
|
+
["BG", 22],
|
|
105
|
+
["BH", 22],
|
|
106
|
+
["BR", 29],
|
|
107
|
+
["BY", 28],
|
|
108
|
+
["CH", 21],
|
|
109
|
+
["CR", 22],
|
|
110
|
+
["CY", 28],
|
|
111
|
+
["CZ", 24],
|
|
112
|
+
["DE", 22],
|
|
113
|
+
["DK", 18],
|
|
114
|
+
["DO", 28],
|
|
115
|
+
["EE", 20],
|
|
116
|
+
["EG", 29],
|
|
117
|
+
["ES", 24],
|
|
118
|
+
["FI", 18],
|
|
119
|
+
["FO", 18],
|
|
120
|
+
["FR", 27],
|
|
121
|
+
["GB", 22],
|
|
122
|
+
["GE", 22],
|
|
123
|
+
["GI", 23],
|
|
124
|
+
["GL", 18],
|
|
125
|
+
["GR", 27],
|
|
126
|
+
["GT", 28],
|
|
127
|
+
["HR", 21],
|
|
128
|
+
["HU", 28],
|
|
129
|
+
["IE", 22],
|
|
130
|
+
["IL", 23],
|
|
131
|
+
["IS", 26],
|
|
132
|
+
["IT", 27],
|
|
133
|
+
["JO", 30],
|
|
134
|
+
["KW", 30],
|
|
135
|
+
["KZ", 20],
|
|
136
|
+
["LB", 28],
|
|
137
|
+
["LC", 32],
|
|
138
|
+
["LI", 21],
|
|
139
|
+
["LT", 20],
|
|
140
|
+
["LU", 20],
|
|
141
|
+
["LV", 21],
|
|
142
|
+
["LY", 25],
|
|
143
|
+
["MC", 27],
|
|
144
|
+
["MD", 24],
|
|
145
|
+
["ME", 22],
|
|
146
|
+
["MK", 19],
|
|
147
|
+
["MR", 27],
|
|
148
|
+
["MT", 31],
|
|
149
|
+
["MU", 30],
|
|
150
|
+
["NL", 18],
|
|
151
|
+
["NO", 15],
|
|
152
|
+
["PK", 24],
|
|
153
|
+
["PL", 28],
|
|
154
|
+
["PS", 29],
|
|
155
|
+
["PT", 25],
|
|
156
|
+
["QA", 29],
|
|
157
|
+
["RO", 24],
|
|
158
|
+
["RS", 22],
|
|
159
|
+
["SA", 24],
|
|
160
|
+
["SC", 31],
|
|
161
|
+
["SE", 24],
|
|
162
|
+
["SI", 19],
|
|
163
|
+
["SK", 24],
|
|
164
|
+
["SM", 27],
|
|
165
|
+
["ST", 25],
|
|
166
|
+
["SV", 28],
|
|
167
|
+
["TL", 23],
|
|
168
|
+
["TN", 24],
|
|
169
|
+
["TR", 26],
|
|
170
|
+
["UA", 29],
|
|
171
|
+
["VA", 22],
|
|
172
|
+
["VG", 24],
|
|
173
|
+
["XK", 20],
|
|
174
|
+
]);
|
|
175
|
+
/** Issuer, by the IIN prefix ranges that are stable and public. */
|
|
176
|
+
export function cardBrand(digits) {
|
|
177
|
+
const n = digits.length;
|
|
178
|
+
if (/^4/.test(digits) && (n === 13 || n === 16 || n === 19))
|
|
179
|
+
return "visa";
|
|
180
|
+
if (/^(5[1-5]|2[2-7])/.test(digits) && n === 16)
|
|
181
|
+
return "mastercard";
|
|
182
|
+
if (/^3[47]/.test(digits) && n === 15)
|
|
183
|
+
return "amex";
|
|
184
|
+
if (/^(6011|65|64[4-9])/.test(digits) && (n === 16 || n === 19))
|
|
185
|
+
return "discover";
|
|
186
|
+
if (/^35(2[89]|[3-8]\d)/.test(digits) && (n === 16 || n === 19))
|
|
187
|
+
return "jcb";
|
|
188
|
+
if (/^3(0[0-5]|[68])/.test(digits) && (n === 14 || n === 16))
|
|
189
|
+
return "diners";
|
|
190
|
+
if (/^62/.test(digits) && n >= 16)
|
|
191
|
+
return "unionpay";
|
|
192
|
+
return "unknown";
|
|
193
|
+
}
|
|
194
|
+
// ---------------------------------------------------------------------------
|
|
195
|
+
// IP literals
|
|
196
|
+
// ---------------------------------------------------------------------------
|
|
197
|
+
/** Dotted-quad parse. Rejects leading zeros, which are an obfuscation trick. */
|
|
198
|
+
export function parseIpv4(value) {
|
|
199
|
+
const parts = value.split(".");
|
|
200
|
+
if (parts.length !== 4)
|
|
201
|
+
return undefined;
|
|
202
|
+
const octets = [];
|
|
203
|
+
for (const part of parts) {
|
|
204
|
+
if (!/^\d{1,3}$/.test(part))
|
|
205
|
+
return undefined;
|
|
206
|
+
if (part.length > 1 && part.startsWith("0"))
|
|
207
|
+
return undefined;
|
|
208
|
+
const n = Number(part);
|
|
209
|
+
if (n > 255)
|
|
210
|
+
return undefined;
|
|
211
|
+
octets.push(n);
|
|
212
|
+
}
|
|
213
|
+
return octets;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* IPv6 as RFC 4291 §2.2 forms 1–3: eight hextets, `::` compression at most
|
|
217
|
+
* once, and a trailing dotted-quad. Zone identifiers (`%eth0`) and CIDR
|
|
218
|
+
* suffixes are NOT accepted — they are addresses plus something else, and
|
|
219
|
+
* guessing which is a good way to be confidently wrong.
|
|
220
|
+
*/
|
|
221
|
+
export function parseIpv6(value) {
|
|
222
|
+
if (value.includes("%") || value.includes("/"))
|
|
223
|
+
return undefined;
|
|
224
|
+
// Three or more colons in a row is not a compression, it is malformed.
|
|
225
|
+
// `":::"` splits as one `"::"` plus a stray colon and would otherwise be
|
|
226
|
+
// accepted as `"::"` — a wrong answer stated with full confidence.
|
|
227
|
+
if (value.includes(":::"))
|
|
228
|
+
return undefined;
|
|
229
|
+
const doubleColons = value.split("::").length - 1;
|
|
230
|
+
if (doubleColons > 1)
|
|
231
|
+
return undefined;
|
|
232
|
+
let head = value;
|
|
233
|
+
let tailGroups = [];
|
|
234
|
+
const lastColon = value.lastIndexOf(":");
|
|
235
|
+
if (lastColon >= 0 && value.slice(lastColon + 1).includes(".")) {
|
|
236
|
+
const v4 = parseIpv4(value.slice(lastColon + 1));
|
|
237
|
+
if (!v4)
|
|
238
|
+
return undefined;
|
|
239
|
+
const [a, b, c, d] = v4;
|
|
240
|
+
tailGroups = [(a << 8) | b, (c << 8) | d];
|
|
241
|
+
head = value.slice(0, lastColon + 1);
|
|
242
|
+
if (!head.endsWith("::"))
|
|
243
|
+
head = head.slice(0, -1);
|
|
244
|
+
}
|
|
245
|
+
const expectedGroups = 8 - tailGroups.length;
|
|
246
|
+
const parseSide = (side) => {
|
|
247
|
+
if (side === "")
|
|
248
|
+
return [];
|
|
249
|
+
const groups = [];
|
|
250
|
+
for (const piece of side.split(":")) {
|
|
251
|
+
if (!/^[0-9a-fA-F]{1,4}$/.test(piece))
|
|
252
|
+
return undefined;
|
|
253
|
+
groups.push(Number.parseInt(piece, 16));
|
|
254
|
+
}
|
|
255
|
+
return groups;
|
|
256
|
+
};
|
|
257
|
+
if (doubleColons === 1) {
|
|
258
|
+
const [beforeRaw = "", afterRaw = ""] = head.split("::");
|
|
259
|
+
const before = parseSide(beforeRaw.replace(/:$/, ""));
|
|
260
|
+
const after = parseSide(afterRaw.replace(/^:/, ""));
|
|
261
|
+
if (!before || !after)
|
|
262
|
+
return undefined;
|
|
263
|
+
const missing = expectedGroups - before.length - after.length;
|
|
264
|
+
if (missing < 1)
|
|
265
|
+
return undefined;
|
|
266
|
+
return [...before, ...new Array(missing).fill(0), ...after, ...tailGroups];
|
|
267
|
+
}
|
|
268
|
+
const groups = parseSide(head.replace(/:$/, ""));
|
|
269
|
+
if (!groups || groups.length !== expectedGroups)
|
|
270
|
+
return undefined;
|
|
271
|
+
return [...groups, ...tailGroups];
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Label an IPv4 address that is almost certainly not personal data, by the
|
|
275
|
+
* IANA special-purpose registry. The prefixes are named exactly as the RFCs
|
|
276
|
+
* define them — `198.51.100.0/24` is documentation, `198.18.0.0/15` is
|
|
277
|
+
* benchmarking, and calling the second one "documentation" would put a wrong
|
|
278
|
+
* label on a finding a reviewer is meant to be able to check.
|
|
279
|
+
*/
|
|
280
|
+
function ipv4Scope(octets) {
|
|
281
|
+
const [a = 0, b = 0, c = 0] = octets;
|
|
282
|
+
if (a === 10)
|
|
283
|
+
return "private"; // 10/8, RFC 1918
|
|
284
|
+
if (a === 172 && b >= 16 && b <= 31)
|
|
285
|
+
return "private"; // 172.16/12, RFC 1918
|
|
286
|
+
if (a === 192 && b === 168)
|
|
287
|
+
return "private"; // 192.168/16, RFC 1918
|
|
288
|
+
if (a === 127)
|
|
289
|
+
return "loopback"; // 127/8, RFC 1122
|
|
290
|
+
if (a === 169 && b === 254)
|
|
291
|
+
return "link-local"; // 169.254/16, RFC 3927
|
|
292
|
+
if (a === 100 && b >= 64 && b <= 127)
|
|
293
|
+
return "shared-address-space"; // 100.64/10, RFC 6598
|
|
294
|
+
if (a === 192 && b === 0 && c === 2)
|
|
295
|
+
return "documentation"; // TEST-NET-1, RFC 5737
|
|
296
|
+
if (a === 198 && b === 51 && c === 100)
|
|
297
|
+
return "documentation"; // TEST-NET-2, RFC 5737
|
|
298
|
+
if (a === 203 && b === 0 && c === 113)
|
|
299
|
+
return "documentation"; // TEST-NET-3, RFC 5737
|
|
300
|
+
if (a === 198 && (b === 18 || b === 19))
|
|
301
|
+
return "benchmarking"; // 198.18/15, RFC 2544
|
|
302
|
+
if (a === 0 || a >= 224)
|
|
303
|
+
return "reserved"; // 0/8, multicast and 240/4
|
|
304
|
+
return "public";
|
|
305
|
+
}
|
|
306
|
+
// ---------------------------------------------------------------------------
|
|
307
|
+
// patterns
|
|
308
|
+
// ---------------------------------------------------------------------------
|
|
309
|
+
const EMAIL = /[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+/g;
|
|
310
|
+
const SSN = /\b(\d{3})-(\d{2})-(\d{4})\b/g;
|
|
311
|
+
const IBAN_CANDIDATE = /\b[A-Z]{2}\d{2}[ ]?(?:[A-Z0-9]{4}[ ]?){2,7}[A-Z0-9]{1,4}\b/g;
|
|
312
|
+
const CARD_RUN = /\b\d(?:[ -]?\d){11,}/g;
|
|
313
|
+
const IPV4 = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g;
|
|
314
|
+
const IPV6_CANDIDATE = /\b(?=[0-9A-Fa-f:]*::|(?:[0-9A-Fa-f]{1,4}:){7})[0-9A-Fa-f:]{2,39}(?:\.\d{1,3}){0,3}/g;
|
|
315
|
+
const PHONE_E164_RUN = /\+\d(?:[\s().-]?\d)+/g;
|
|
316
|
+
const DATE_ISO = /\b(19|20)\d{2}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])\b/g;
|
|
317
|
+
const DATE_US = /\b(0?[1-9]|1[0-2])\/(0?[1-9]|[12]\d|3[01])\/((?:19|20)\d{2})\b/g;
|
|
318
|
+
const DATE_DOTTED = /\b(0?[1-9]|[12]\d|3[01])\.(0?[1-9]|1[0-2])\.((?:19|20)\d{2})\b/g;
|
|
319
|
+
const DATE_LONG = /\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),?\s+((?:19|20)\d{2})\b/gi;
|
|
320
|
+
const DOB_LABEL = /(date of birth|d\.?o\.?b\.?|birth ?date|birthday|born(?:\s+on)?)\s*[:\-]?\s*$/i;
|
|
321
|
+
const STREET_SUFFIX = "(?:Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Lane|Ln|Drive|Dr|Court|Ct|Way|Terrace|Ter|Place|Pl|Circle|Cir|Parkway|Pkwy|Highway|Hwy|Square|Sq|Trail|Trl)";
|
|
322
|
+
const ADDRESS_STREET = new RegExp(`\\b\\d{1,6}\\s+(?:[A-Za-z0-9.'#-]+\\s+){0,4}${STREET_SUFFIX}\\b\\.?(?:\\s+(?:Apt|Suite|Ste|Unit|#)\\s*[A-Za-z0-9-]+)?`, "g");
|
|
323
|
+
const ADDRESS_CITY_STATE_ZIP = /\b[A-Z][A-Za-z.'-]+(?:\s+[A-Z][A-Za-z.'-]+){0,3},\s*(A[KLRZ]|C[AOT]|D[CE]|FL|GA|HI|I[ADLN]|K[SY]|LA|M[ADEINOST]|N[CDEHJMVY]|O[HKR]|P[AR]|RI|S[CD]|T[NX]|UT|V[AIT]|W[AIVY])\s+\d{5}(?:-\d{4})?\b/g;
|
|
324
|
+
const NATIONAL_PHONE = {
|
|
325
|
+
// NANP: area and exchange codes both start 2-9.
|
|
326
|
+
US: /(?:\(\d{3}\)|\b[2-9]\d{2})[\s.-]?[2-9]\d{2}[\s.-]?\d{4}\b/g,
|
|
327
|
+
CA: /(?:\(\d{3}\)|\b[2-9]\d{2})[\s.-]?[2-9]\d{2}[\s.-]?\d{4}\b/g,
|
|
328
|
+
GB: /\b0\d{2,4}[\s-]?\d{3,4}[\s-]?\d{3,4}\b/g,
|
|
329
|
+
DE: /\b0\d{2,4}[\s/-]?\d{3,8}\b/g,
|
|
330
|
+
FR: /\b0[1-9](?:[\s.-]?\d{2}){4}\b/g,
|
|
331
|
+
AU: /\b0[2-478][\s-]?\d{4}[\s-]?\d{4}\b/g,
|
|
332
|
+
IN: /\b[6-9]\d{4}[\s-]?\d{5}\b/g,
|
|
333
|
+
};
|
|
334
|
+
// ---------------------------------------------------------------------------
|
|
335
|
+
function selected(options, type) {
|
|
336
|
+
return options.types === undefined || options.types.includes(type);
|
|
337
|
+
}
|
|
338
|
+
/** Split a matched run into its digit groups, keeping absolute offsets. */
|
|
339
|
+
function digitGroups(run, base) {
|
|
340
|
+
const groups = [];
|
|
341
|
+
let i = 0;
|
|
342
|
+
while (i < run.length) {
|
|
343
|
+
if (run.charCodeAt(i) < 48 || run.charCodeAt(i) > 57) {
|
|
344
|
+
i += 1;
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
const from = i;
|
|
348
|
+
while (i < run.length && run.charCodeAt(i) >= 48 && run.charCodeAt(i) <= 57)
|
|
349
|
+
i += 1;
|
|
350
|
+
groups.push({ digits: run.slice(from, i), start: base + from, end: base + i });
|
|
351
|
+
}
|
|
352
|
+
return groups;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* The longest run of WHOLE digit groups that satisfies `accept`, searched
|
|
356
|
+
* from `firstGroup` onward, or undefined.
|
|
357
|
+
*
|
|
358
|
+
* Why whole groups: a card is written `4111 1111 1111 1111`, so every real
|
|
359
|
+
* boundary is already a separator in the text. Anchoring candidates at those
|
|
360
|
+
* boundaries is what makes this safe to run — sliding a 12-to-19-digit window
|
|
361
|
+
* over an arbitrary digit run instead would find a Luhn-valid slice by chance
|
|
362
|
+
* roughly one time in ten and turn the detector into noise.
|
|
363
|
+
*
|
|
364
|
+
* Why it exists at all: the old patterns matched one greedy run and gave up
|
|
365
|
+
* if it failed the check. `4111111111111111 123-45-6789` extended the card
|
|
366
|
+
* candidate across the space into the SSN, failed Luhn on the 19 digits that
|
|
367
|
+
* produced, and reported NOTHING — so `PiiRedact` handed back a document with
|
|
368
|
+
* a valid card number still in it.
|
|
369
|
+
*/
|
|
370
|
+
function longestAcceptedRange(groups, firstGroup, minDigits, maxDigits, accept) {
|
|
371
|
+
let best;
|
|
372
|
+
for (let from = firstGroup; from < groups.length; from++) {
|
|
373
|
+
let digits = "";
|
|
374
|
+
for (let to = from; to < groups.length; to++) {
|
|
375
|
+
const group = groups[to];
|
|
376
|
+
if (!group)
|
|
377
|
+
break;
|
|
378
|
+
digits += group.digits;
|
|
379
|
+
if (digits.length > maxDigits)
|
|
380
|
+
break;
|
|
381
|
+
if (digits.length < minDigits)
|
|
382
|
+
continue;
|
|
383
|
+
if (!accept(digits))
|
|
384
|
+
continue;
|
|
385
|
+
const head = groups[from];
|
|
386
|
+
if (!head)
|
|
387
|
+
break;
|
|
388
|
+
if (best === undefined || digits.length > best.digits.length) {
|
|
389
|
+
best = { start: head.start, end: group.end, digits, lastGroup: to };
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
// Anchored at the first group that can start a match: a later start would
|
|
393
|
+
// report the tail of the same number as a second, shorter finding.
|
|
394
|
+
if (best !== undefined)
|
|
395
|
+
return best;
|
|
396
|
+
}
|
|
397
|
+
return best;
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* Days since 1970-01-01 for a valid proleptic Gregorian Y-M-D, or undefined.
|
|
401
|
+
*
|
|
402
|
+
* `Date.UTC` maps years 0–99 to 1900–1999, so `calendarDay(50, 1, 1)` would
|
|
403
|
+
* silently answer for 1950; the year is set explicitly to keep a four-digit
|
|
404
|
+
* `referenceDate` of `"0050-01-01"` meaning the year 50 rather than 1950.
|
|
405
|
+
*/
|
|
406
|
+
export function calendarDay(year, month, day) {
|
|
407
|
+
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
|
|
408
|
+
return undefined;
|
|
409
|
+
}
|
|
410
|
+
if (month < 1 || month > 12 || day < 1)
|
|
411
|
+
return undefined;
|
|
412
|
+
const leap = (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0;
|
|
413
|
+
const lengths = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
414
|
+
if (day > (lengths[month - 1] ?? 0))
|
|
415
|
+
return undefined;
|
|
416
|
+
const at = new Date(0);
|
|
417
|
+
at.setUTCFullYear(year, month - 1, day);
|
|
418
|
+
at.setUTCHours(0, 0, 0, 0);
|
|
419
|
+
const ms = at.getTime();
|
|
420
|
+
if (!Number.isFinite(ms))
|
|
421
|
+
return undefined;
|
|
422
|
+
return Math.floor(ms / 86_400_000);
|
|
423
|
+
}
|
|
424
|
+
/** Whole years between two calendar dates. Used only for DOB plausibility. */
|
|
425
|
+
function yearsBetween(from, to) {
|
|
426
|
+
let age = to.y - from.y;
|
|
427
|
+
if (to.m < from.m || (to.m === from.m && to.d < from.d))
|
|
428
|
+
age -= 1;
|
|
429
|
+
return age;
|
|
430
|
+
}
|
|
431
|
+
const MONTH_NAMES = [
|
|
432
|
+
"january",
|
|
433
|
+
"february",
|
|
434
|
+
"march",
|
|
435
|
+
"april",
|
|
436
|
+
"may",
|
|
437
|
+
"june",
|
|
438
|
+
"july",
|
|
439
|
+
"august",
|
|
440
|
+
"september",
|
|
441
|
+
"october",
|
|
442
|
+
"november",
|
|
443
|
+
"december",
|
|
444
|
+
];
|
|
445
|
+
function collectDates(text) {
|
|
446
|
+
const out = [];
|
|
447
|
+
for (const { index, match } of matchAll(text, DATE_ISO)) {
|
|
448
|
+
const [raw] = match;
|
|
449
|
+
const [y, m, d] = raw.split("-").map(Number);
|
|
450
|
+
out.push({ start: index, end: index + raw.length, raw, date: { y, m, d } });
|
|
451
|
+
}
|
|
452
|
+
for (const { index, match } of matchAll(text, DATE_US)) {
|
|
453
|
+
const raw = match[0];
|
|
454
|
+
const y = Number(match[3]);
|
|
455
|
+
const m = Number(match[1]);
|
|
456
|
+
const d = Number(match[2]);
|
|
457
|
+
out.push({ start: index, end: index + raw.length, raw, date: { y, m, d } });
|
|
458
|
+
}
|
|
459
|
+
for (const { index, match } of matchAll(text, DATE_DOTTED)) {
|
|
460
|
+
const raw = match[0];
|
|
461
|
+
out.push({
|
|
462
|
+
start: index,
|
|
463
|
+
end: index + raw.length,
|
|
464
|
+
raw,
|
|
465
|
+
date: { y: Number(match[3]), m: Number(match[2]), d: Number(match[1]) },
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
for (const { index, match } of matchAll(text, DATE_LONG)) {
|
|
469
|
+
const raw = match[0];
|
|
470
|
+
const month = MONTH_NAMES.indexOf((match[1] ?? "").toLowerCase()) + 1;
|
|
471
|
+
out.push({
|
|
472
|
+
start: index,
|
|
473
|
+
end: index + raw.length,
|
|
474
|
+
raw,
|
|
475
|
+
date: { y: Number(match[3]), m: month, d: Number(match[2]) },
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
return out;
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Run the selected detectors. Overlaps are NOT resolved here — the caller
|
|
482
|
+
* runs `dedupeOverlaps`, so a caller that wants every raw hit can have it.
|
|
483
|
+
*/
|
|
484
|
+
export function scanPii(text, options = {}) {
|
|
485
|
+
const raw = [];
|
|
486
|
+
if (options.country !== undefined) {
|
|
487
|
+
const upper = options.country.toUpperCase();
|
|
488
|
+
if (!SUPPORTED_PHONE_COUNTRIES.includes(upper)) {
|
|
489
|
+
throw new PiiOptionError(`country "${options.country}" has no national phone rule here; supported: ${SUPPORTED_PHONE_COUNTRIES.join(", ")}. Leave it unset to scan E.164 (+…) numbers only.`);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
let reference;
|
|
493
|
+
if (options.referenceDate !== undefined) {
|
|
494
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(options.referenceDate);
|
|
495
|
+
const parsed = m
|
|
496
|
+
? calendarDay(Number(m[1]), Number(m[2]), Number(m[3])) !== undefined
|
|
497
|
+
? { y: Number(m[1]), m: Number(m[2]), d: Number(m[3]) }
|
|
498
|
+
: undefined
|
|
499
|
+
: undefined;
|
|
500
|
+
if (!parsed) {
|
|
501
|
+
throw new PiiOptionError(`referenceDate "${options.referenceDate}" is not a valid YYYY-MM-DD calendar date`);
|
|
502
|
+
}
|
|
503
|
+
reference = parsed;
|
|
504
|
+
}
|
|
505
|
+
if (selected(options, "email")) {
|
|
506
|
+
for (const { index, match } of matchAll(text, EMAIL)) {
|
|
507
|
+
raw.push({
|
|
508
|
+
type: "email",
|
|
509
|
+
rule: "email.addr-spec-subset",
|
|
510
|
+
confidence: "likely",
|
|
511
|
+
start: index,
|
|
512
|
+
end: index + match[0].length,
|
|
513
|
+
value: match[0],
|
|
514
|
+
detail: { domain: (match[0].split("@")[1] ?? "").toLowerCase() },
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
if (selected(options, "us_ssn")) {
|
|
519
|
+
for (const { index, match } of matchAll(text, SSN)) {
|
|
520
|
+
const area = match[1] ?? "";
|
|
521
|
+
const group = match[2] ?? "";
|
|
522
|
+
const serial = match[3] ?? "";
|
|
523
|
+
// Ranges the SSA has never issued. Excluding them cuts false positives
|
|
524
|
+
// without ever claiming the remainder was issued.
|
|
525
|
+
if (area === "000" || area === "666" || area.startsWith("9"))
|
|
526
|
+
continue;
|
|
527
|
+
if (group === "00" || serial === "0000")
|
|
528
|
+
continue;
|
|
529
|
+
raw.push({
|
|
530
|
+
type: "us_ssn",
|
|
531
|
+
rule: "ssn.us-format",
|
|
532
|
+
confidence: "possible",
|
|
533
|
+
start: index,
|
|
534
|
+
end: index + match[0].length,
|
|
535
|
+
value: match[0],
|
|
536
|
+
detail: { note: "format only; an SSN has no public check digit" },
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
if (selected(options, "iban")) {
|
|
541
|
+
for (const { index, match } of matchAll(text, IBAN_CANDIDATE)) {
|
|
542
|
+
const compact = match[0].replace(/\s/g, "");
|
|
543
|
+
if (compact.length < 15 || compact.length > 34)
|
|
544
|
+
continue;
|
|
545
|
+
if (ibanMod97(compact) !== 1)
|
|
546
|
+
continue;
|
|
547
|
+
const country = compact.slice(0, 2);
|
|
548
|
+
const expected = IBAN_LENGTHS.get(country);
|
|
549
|
+
if (expected !== undefined && expected !== compact.length)
|
|
550
|
+
continue;
|
|
551
|
+
raw.push({
|
|
552
|
+
type: "iban",
|
|
553
|
+
rule: expected === undefined ? "iban.iso7064-mod97-unknown-country" : "iban.iso7064-mod97",
|
|
554
|
+
confidence: expected === undefined ? "likely" : "verified",
|
|
555
|
+
start: index,
|
|
556
|
+
end: index + match[0].length,
|
|
557
|
+
value: match[0],
|
|
558
|
+
detail: { country, length: compact.length, unknownCountry: expected === undefined },
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
if (selected(options, "credit_card")) {
|
|
563
|
+
for (const { index, match } of matchAll(text, CARD_RUN)) {
|
|
564
|
+
const groups = digitGroups(match[0], index);
|
|
565
|
+
// Only a passing Luhn check is reported. A digit run that fails it is
|
|
566
|
+
// an order number far more often than it is a mistyped card. But the
|
|
567
|
+
// run may hold a card AND something else — a card followed by an order
|
|
568
|
+
// number, an SSN, a year — so every whole-group range is considered,
|
|
569
|
+
// not just the greedy whole run, and every card in the run is reported.
|
|
570
|
+
let from = 0;
|
|
571
|
+
while (from < groups.length) {
|
|
572
|
+
const found = longestAcceptedRange(groups, from, 12, 19, luhnValid);
|
|
573
|
+
if (!found)
|
|
574
|
+
break;
|
|
575
|
+
raw.push({
|
|
576
|
+
type: "credit_card",
|
|
577
|
+
rule: "card.luhn",
|
|
578
|
+
confidence: "verified",
|
|
579
|
+
start: found.start,
|
|
580
|
+
end: found.end,
|
|
581
|
+
value: text.slice(found.start, found.end),
|
|
582
|
+
detail: { brand: cardBrand(found.digits), digits: found.digits.length },
|
|
583
|
+
});
|
|
584
|
+
from = found.lastGroup + 1;
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
if (selected(options, "ip_address")) {
|
|
589
|
+
for (const { index, match } of matchAll(text, IPV4)) {
|
|
590
|
+
const octets = parseIpv4(match[0]);
|
|
591
|
+
if (!octets)
|
|
592
|
+
continue;
|
|
593
|
+
const scope = ipv4Scope(octets);
|
|
594
|
+
raw.push({
|
|
595
|
+
type: "ip_address",
|
|
596
|
+
rule: "ip.v4-dotted",
|
|
597
|
+
confidence: scope === "public" ? "likely" : "possible",
|
|
598
|
+
start: index,
|
|
599
|
+
end: index + match[0].length,
|
|
600
|
+
value: match[0],
|
|
601
|
+
detail: { version: 4, scope },
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
for (const { index, match } of matchAll(text, IPV6_CANDIDATE)) {
|
|
605
|
+
const candidate = match[0].replace(/[:.]+$/, "");
|
|
606
|
+
if (!candidate.includes(":"))
|
|
607
|
+
continue;
|
|
608
|
+
const groups = parseIpv6(candidate);
|
|
609
|
+
if (!groups)
|
|
610
|
+
continue;
|
|
611
|
+
const first = groups[0] ?? 0;
|
|
612
|
+
const loopback = groups.every((g, i) => (i === 7 ? g === 1 : g === 0));
|
|
613
|
+
const scope = loopback
|
|
614
|
+
? "loopback"
|
|
615
|
+
: (first & 0xfe00) === 0xfc00
|
|
616
|
+
? "unique-local"
|
|
617
|
+
: (first & 0xffc0) === 0xfe80
|
|
618
|
+
? "link-local"
|
|
619
|
+
: first === 0x2001 && (groups[1] ?? 0) === 0x0db8
|
|
620
|
+
? "documentation"
|
|
621
|
+
: "public";
|
|
622
|
+
raw.push({
|
|
623
|
+
type: "ip_address",
|
|
624
|
+
rule: "ip.v6-parsed",
|
|
625
|
+
confidence: scope === "public" ? "likely" : "possible",
|
|
626
|
+
start: index,
|
|
627
|
+
end: index + candidate.length,
|
|
628
|
+
value: candidate,
|
|
629
|
+
detail: { version: 6, scope },
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
if (selected(options, "phone")) {
|
|
634
|
+
for (const { index, match } of matchAll(text, PHONE_E164_RUN)) {
|
|
635
|
+
// The run starts at the `+`, so the number does too. It may run on into
|
|
636
|
+
// whatever follows — `+14155550132 4111111111111111` is one run of 28
|
|
637
|
+
// digits — and dropping the whole run for being too long is how a phone
|
|
638
|
+
// number next to a card number went unreported and unredacted.
|
|
639
|
+
const groups = digitGroups(match[0], index);
|
|
640
|
+
const found = longestAcceptedRange(groups, 0, 8, 15, () => true);
|
|
641
|
+
if (!found || found.start !== index + 1)
|
|
642
|
+
continue;
|
|
643
|
+
raw.push({
|
|
644
|
+
type: "phone",
|
|
645
|
+
rule: "phone.e164",
|
|
646
|
+
confidence: "likely",
|
|
647
|
+
// The `+` is part of the number.
|
|
648
|
+
start: index,
|
|
649
|
+
end: found.end,
|
|
650
|
+
value: text.slice(index, found.end),
|
|
651
|
+
detail: { digits: found.digits.length, form: "e164" },
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
if (options.country !== undefined) {
|
|
655
|
+
const country = options.country.toUpperCase();
|
|
656
|
+
for (const { index, match } of matchAll(text, NATIONAL_PHONE[country])) {
|
|
657
|
+
const digits = match[0].replace(/\D/g, "");
|
|
658
|
+
if (digits.length < 9 || digits.length > 12)
|
|
659
|
+
continue;
|
|
660
|
+
raw.push({
|
|
661
|
+
type: "phone",
|
|
662
|
+
rule: `phone.national.${country}`,
|
|
663
|
+
confidence: "possible",
|
|
664
|
+
start: index,
|
|
665
|
+
end: index + match[0].length,
|
|
666
|
+
value: match[0],
|
|
667
|
+
detail: { digits: digits.length, country, form: "national" },
|
|
668
|
+
});
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
if (selected(options, "date_of_birth")) {
|
|
673
|
+
for (const hit of collectDates(text)) {
|
|
674
|
+
const { y, m, d } = hit.date;
|
|
675
|
+
if (calendarDay(y, m, d) === undefined)
|
|
676
|
+
continue;
|
|
677
|
+
const before = text.slice(Math.max(0, hit.start - 40), hit.start);
|
|
678
|
+
const labelled = DOB_LABEL.test(before);
|
|
679
|
+
let confidence = "possible";
|
|
680
|
+
let rule = "date.calendar";
|
|
681
|
+
const detail = {
|
|
682
|
+
iso: `${y}-${String(m).padStart(2, "0")}-${String(d).padStart(2, "0")}`,
|
|
683
|
+
};
|
|
684
|
+
if (reference) {
|
|
685
|
+
const age = yearsBetween(hit.date, reference);
|
|
686
|
+
detail["ageAtReference"] = age;
|
|
687
|
+
if (age < 0 || age > 120)
|
|
688
|
+
continue; // not a plausible birth date
|
|
689
|
+
if (labelled) {
|
|
690
|
+
confidence = "likely";
|
|
691
|
+
rule = "dob.labelled";
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
else if (labelled) {
|
|
695
|
+
confidence = "likely";
|
|
696
|
+
rule = "dob.labelled";
|
|
697
|
+
}
|
|
698
|
+
if (!labelled && !reference)
|
|
699
|
+
detail["note"] = "no label and no referenceDate: any calendar date matches";
|
|
700
|
+
raw.push({
|
|
701
|
+
type: "date_of_birth",
|
|
702
|
+
rule,
|
|
703
|
+
confidence,
|
|
704
|
+
start: hit.start,
|
|
705
|
+
end: hit.end,
|
|
706
|
+
value: hit.raw,
|
|
707
|
+
detail,
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
if (selected(options, "postal_address")) {
|
|
712
|
+
for (const { index, match } of matchAll(text, ADDRESS_STREET)) {
|
|
713
|
+
raw.push({
|
|
714
|
+
type: "postal_address",
|
|
715
|
+
rule: "address.us-street-suffix",
|
|
716
|
+
confidence: "possible",
|
|
717
|
+
start: index,
|
|
718
|
+
end: index + match[0].length,
|
|
719
|
+
value: match[0],
|
|
720
|
+
detail: { convention: "US" },
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
for (const { index, match } of matchAll(text, ADDRESS_CITY_STATE_ZIP)) {
|
|
724
|
+
raw.push({
|
|
725
|
+
type: "postal_address",
|
|
726
|
+
rule: "address.us-city-state-zip",
|
|
727
|
+
confidence: "possible",
|
|
728
|
+
start: index,
|
|
729
|
+
end: index + match[0].length,
|
|
730
|
+
value: match[0],
|
|
731
|
+
detail: { convention: "US", state: match[1] ?? "" },
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return withPositions(text, raw);
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* The value a pseudonym is derived from, so the same identifier produces the
|
|
739
|
+
* same token no matter how it was written. Case and separators only — no
|
|
740
|
+
* semantic normalization, because that would need judgement.
|
|
741
|
+
*/
|
|
742
|
+
export function canonicalPiiValue(type, value) {
|
|
743
|
+
switch (type) {
|
|
744
|
+
case "email":
|
|
745
|
+
return value.trim().toLowerCase();
|
|
746
|
+
case "credit_card":
|
|
747
|
+
case "phone":
|
|
748
|
+
case "us_ssn":
|
|
749
|
+
return value.replace(/[^\d+]/g, "");
|
|
750
|
+
case "iban":
|
|
751
|
+
return value.replace(/\s/g, "").toUpperCase();
|
|
752
|
+
case "ip_address":
|
|
753
|
+
return value.trim().toLowerCase();
|
|
754
|
+
default:
|
|
755
|
+
return value.trim().replace(/\s+/g, " ");
|
|
756
|
+
}
|
|
757
|
+
}
|