@crewhaus/tool-secure 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +18 -1
- package/dist/index.d.ts +16 -0
- package/dist/index.js +113 -37
- package/dist/lib/injection.d.ts +31 -0
- package/dist/lib/injection.js +176 -6
- package/dist/lib/ipv6-scan.d.ts +16 -0
- package/dist/lib/ipv6-scan.js +73 -0
- package/dist/lib/literal.d.ts +55 -0
- package/dist/lib/literal.js +146 -0
- package/dist/lib/pii.d.ts +19 -30
- package/dist/lib/pii.js +71 -8
- package/dist/lib/policy.d.ts +24 -2
- package/dist/lib/policy.js +176 -79
- package/dist/lib/secrets.d.ts +8 -0
- package/dist/lib/secrets.js +43 -5
- package/dist/lib/text.d.ts +61 -2
- package/dist/lib/text.js +174 -5
- package/dist/paths.d.ts +21 -2
- package/dist/paths.js +72 -13
- package/package.json +12 -6
package/dist/lib/injection.js
CHANGED
|
@@ -28,8 +28,117 @@
|
|
|
28
28
|
* attacker-controlled text. Show it to a person; do not paste it into a
|
|
29
29
|
* prompt as if it were instructions.
|
|
30
30
|
*/
|
|
31
|
-
import { compareStrings, matchAll, withPositions } from "./text";
|
|
31
|
+
import { compareStrings, matchAll, matchAllSkipping, withPositions } from "./text";
|
|
32
32
|
import { scanInvisible } from "./unicode";
|
|
33
|
+
/** A code unit `\s` matches: WhiteSpace or LineTerminator. */
|
|
34
|
+
function isRegExpSpace(c) {
|
|
35
|
+
return ((c >= 0x09 && c <= 0x0d) ||
|
|
36
|
+
c === 0x20 ||
|
|
37
|
+
c === 0xa0 ||
|
|
38
|
+
c === 0x1680 ||
|
|
39
|
+
(c >= 0x2000 && c <= 0x200a) ||
|
|
40
|
+
c === 0x2028 ||
|
|
41
|
+
c === 0x2029 ||
|
|
42
|
+
c === 0x202f ||
|
|
43
|
+
c === 0x205f ||
|
|
44
|
+
c === 0x3000 ||
|
|
45
|
+
c === 0xfeff);
|
|
46
|
+
}
|
|
47
|
+
/** Where `^` matches under the `m` flag. */
|
|
48
|
+
function isLineStart(text, at) {
|
|
49
|
+
if (at === 0)
|
|
50
|
+
return true;
|
|
51
|
+
const c = text.charCodeAt(at - 1);
|
|
52
|
+
return c === 0x0a || c === 0x0d || c === 0x2028 || c === 0x2029;
|
|
53
|
+
}
|
|
54
|
+
const CHAT_MARKUP = /<\/?(?:system|assistant|human|user)>|\[\/?INST\]|<\|im_(?:start|end)\|>|^\s*###\s*(?:instruction|system)/gim;
|
|
55
|
+
/**
|
|
56
|
+
* CHAT_MARKUP's matches. Its `^\s*###` branch starts at every line and runs
|
|
57
|
+
* its `\s*` on across blank lines, so a run of blank lines was rescanned from
|
|
58
|
+
* each line start in it: quadratic in the run's length. The other branches
|
|
59
|
+
* start at `<` or `[`, which no whitespace is, so a try at a line start that
|
|
60
|
+
* fails rules out every start up to the end of its whitespace.
|
|
61
|
+
*/
|
|
62
|
+
function chatMarkupMatches(text) {
|
|
63
|
+
return matchAllSkipping(text, CHAT_MARKUP, (t, at) => {
|
|
64
|
+
const c = t.charCodeAt(at);
|
|
65
|
+
return c === 0x3c || c === 0x5b || isLineStart(t, at);
|
|
66
|
+
}, (t, at) => {
|
|
67
|
+
if (!isLineStart(t, at))
|
|
68
|
+
return at + 1;
|
|
69
|
+
let i = at;
|
|
70
|
+
while (i < t.length && isRegExpSpace(t.charCodeAt(i)))
|
|
71
|
+
i++;
|
|
72
|
+
return i;
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
const PIPE_TO_SHELL = /\b(?:curl|wget)\s+[^\n|]{1,200}\|\s*(?:sudo\s+)?(?:ba|z|d|fi)?sh\b/gi;
|
|
76
|
+
/** What PIPE_TO_SHELL reads after its `|`, tried at one place. */
|
|
77
|
+
const PIPE_TO_SHELL_TAIL = /\s*(?:sudo\s+)?(?:ba|z|d|fi)?sh\b/iy;
|
|
78
|
+
/** A code unit `\w` matches. */
|
|
79
|
+
function isWordUnit(c) {
|
|
80
|
+
return ((c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) || c === 0x5f);
|
|
81
|
+
}
|
|
82
|
+
/** `word` (lowercase ASCII) at `at`, in either case, as the `i` flag reads it. */
|
|
83
|
+
function asciiWordAt(text, at, word) {
|
|
84
|
+
for (let i = 0; i < word.length; i++) {
|
|
85
|
+
const c = text.charCodeAt(at + i);
|
|
86
|
+
if (c !== word.charCodeAt(i) && c !== word.charCodeAt(i) - 0x20)
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* PIPE_TO_SHELL's matches. Its blanks after `curl` and the up to 200
|
|
93
|
+
* characters before `|` overlap on blanks, so a try that fails gave the
|
|
94
|
+
* blanks back one at a time and read up to 200 characters again after each,
|
|
95
|
+
* and every `curl` in a stretch did it again: hundreds of steps a character
|
|
96
|
+
* over text of up to 2M characters. A try is made only where it matches:
|
|
97
|
+
* after `curl` or `wget` and its blanks, the first `|` or `\n` must be a `|`
|
|
98
|
+
* that some split of the blanks leaves 1 to 200 characters before, with no
|
|
99
|
+
* `\n` among them, and the shell must follow it. Those ends only move
|
|
100
|
+
* forward, so each is found once.
|
|
101
|
+
*/
|
|
102
|
+
function pipeToShellMatches(text) {
|
|
103
|
+
let stop = -1;
|
|
104
|
+
let tailAt = -1;
|
|
105
|
+
let tailMatches = false;
|
|
106
|
+
return matchAllSkipping(text, PIPE_TO_SHELL, (t, at) => {
|
|
107
|
+
if (!(asciiWordAt(t, at, "curl") || asciiWordAt(t, at, "wget")))
|
|
108
|
+
return false;
|
|
109
|
+
if (at > 0 && isWordUnit(t.charCodeAt(at - 1)))
|
|
110
|
+
return false;
|
|
111
|
+
const from = at + 4;
|
|
112
|
+
let blanksEnd = from;
|
|
113
|
+
let lastBreak = -1;
|
|
114
|
+
while (blanksEnd < t.length && isRegExpSpace(t.charCodeAt(blanksEnd))) {
|
|
115
|
+
if (t.charCodeAt(blanksEnd) === 0x0a)
|
|
116
|
+
lastBreak = blanksEnd;
|
|
117
|
+
blanksEnd++;
|
|
118
|
+
}
|
|
119
|
+
if (blanksEnd === from)
|
|
120
|
+
return false;
|
|
121
|
+
if (stop < blanksEnd) {
|
|
122
|
+
stop = blanksEnd;
|
|
123
|
+
while (stop < t.length && t.charCodeAt(stop) !== 0x7c && t.charCodeAt(stop) !== 0x0a)
|
|
124
|
+
stop++;
|
|
125
|
+
}
|
|
126
|
+
if (t.charCodeAt(stop) !== 0x7c)
|
|
127
|
+
return false;
|
|
128
|
+
// The blanks the `\s+` keeps, `w`: at least one, no `\n` left after
|
|
129
|
+
// them, and 1 to 200 characters from them to the `|`.
|
|
130
|
+
const lo = Math.max(1, stop - from - 200, lastBreak - from + 1);
|
|
131
|
+
const hi = Math.min(blanksEnd - from, stop - from - 1);
|
|
132
|
+
if (lo > hi)
|
|
133
|
+
return false;
|
|
134
|
+
if (tailAt !== stop + 1) {
|
|
135
|
+
tailAt = stop + 1;
|
|
136
|
+
PIPE_TO_SHELL_TAIL.lastIndex = tailAt;
|
|
137
|
+
tailMatches = PIPE_TO_SHELL_TAIL.test(t);
|
|
138
|
+
}
|
|
139
|
+
return tailMatches;
|
|
140
|
+
}, (_, at) => at + 1);
|
|
141
|
+
}
|
|
33
142
|
/** The phrase rules. Also run against decoded base64, once, no recursion. */
|
|
34
143
|
export const INJECTION_RULES = [
|
|
35
144
|
{
|
|
@@ -65,7 +174,8 @@ export const INJECTION_RULES = [
|
|
|
65
174
|
category: "override",
|
|
66
175
|
weight: 30,
|
|
67
176
|
description: "embeds chat-template or role markup, trying to close the current turn",
|
|
68
|
-
pattern:
|
|
177
|
+
pattern: CHAT_MARKUP,
|
|
178
|
+
matches: chatMarkupMatches,
|
|
69
179
|
},
|
|
70
180
|
{
|
|
71
181
|
id: "role.you-are-now",
|
|
@@ -121,7 +231,8 @@ export const INJECTION_RULES = [
|
|
|
121
231
|
category: "tool-coercion",
|
|
122
232
|
weight: 35,
|
|
123
233
|
description: "a download-and-execute one-liner",
|
|
124
|
-
pattern:
|
|
234
|
+
pattern: PIPE_TO_SHELL,
|
|
235
|
+
matches: pipeToShellMatches,
|
|
125
236
|
},
|
|
126
237
|
{
|
|
127
238
|
id: "tool.credential-request",
|
|
@@ -171,6 +282,65 @@ const HIDDEN_STYLE = /style\s*=\s*["'][^"']{0,200}(?:display\s*:\s*none|visibili
|
|
|
171
282
|
const MARKDOWN_LINK = /\[([^\]\n]{1,200})\]\(([^)\s]{1,500})\)/g;
|
|
172
283
|
const BASE64_BLOB = /[A-Za-z0-9+/]{40,}={0,2}/g;
|
|
173
284
|
const DOMAIN_IN_TEXT = /\b(?:https?:\/\/)?([a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+)/i;
|
|
285
|
+
/**
|
|
286
|
+
* HTML_COMMENT's matches. From each `<!--` it reads up to 4,000 characters
|
|
287
|
+
* for a `-->`, so a text of `<!--` with none read 4,000 characters again
|
|
288
|
+
* from every one: thousands of steps a character, seconds over 2M. A try is
|
|
289
|
+
* made only where the first `-->` after the opener lies within reach; that
|
|
290
|
+
* `-->` only moves forward, so it is searched for once.
|
|
291
|
+
*/
|
|
292
|
+
export function htmlCommentMatches(text) {
|
|
293
|
+
let close = -2;
|
|
294
|
+
return matchAllSkipping(text, HTML_COMMENT, (t, at) => {
|
|
295
|
+
if (!t.startsWith("<!--", at))
|
|
296
|
+
return false;
|
|
297
|
+
const from = at + 4;
|
|
298
|
+
if (close !== -1 && close < from)
|
|
299
|
+
close = t.indexOf("-->", from);
|
|
300
|
+
return close !== -1 && close - from <= 4_000;
|
|
301
|
+
}, (_, at) => at + 1);
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* MARKDOWN_LINK's matches. From each `[` it reads up to 200 label and 500
|
|
305
|
+
* target characters, so a run of `[` read 200 again from every one. A try
|
|
306
|
+
* is made only where it matches: the label runs from the `[` to the first
|
|
307
|
+
* `]` or `\n` (1 to 200 characters, ending at `]`), `(` follows, and the
|
|
308
|
+
* target runs to the first `)` or blank (1 to 500, ending at `)`). Both ends
|
|
309
|
+
* only move forward, so each is found once.
|
|
310
|
+
*/
|
|
311
|
+
export function markdownLinkMatches(text) {
|
|
312
|
+
let labelEnd = -1;
|
|
313
|
+
let targetEnd = -1;
|
|
314
|
+
return matchAllSkipping(text, MARKDOWN_LINK, (t, at) => {
|
|
315
|
+
if (t.charCodeAt(at) !== 0x5b)
|
|
316
|
+
return false;
|
|
317
|
+
const label = at + 1;
|
|
318
|
+
if (labelEnd < label) {
|
|
319
|
+
labelEnd = label;
|
|
320
|
+
while (labelEnd < t.length &&
|
|
321
|
+
t.charCodeAt(labelEnd) !== 0x5d &&
|
|
322
|
+
t.charCodeAt(labelEnd) !== 0x0a) {
|
|
323
|
+
labelEnd++;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
const labelLength = labelEnd - label;
|
|
327
|
+
if (labelLength < 1 || labelLength > 200 || t.charCodeAt(labelEnd) !== 0x5d)
|
|
328
|
+
return false;
|
|
329
|
+
if (t.charCodeAt(labelEnd + 1) !== 0x28)
|
|
330
|
+
return false;
|
|
331
|
+
const target = labelEnd + 2;
|
|
332
|
+
if (targetEnd < target) {
|
|
333
|
+
targetEnd = target;
|
|
334
|
+
while (targetEnd < t.length &&
|
|
335
|
+
t.charCodeAt(targetEnd) !== 0x29 &&
|
|
336
|
+
!isRegExpSpace(t.charCodeAt(targetEnd))) {
|
|
337
|
+
targetEnd++;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
const targetLength = targetEnd - target;
|
|
341
|
+
return targetLength >= 1 && targetLength <= 500 && t.charCodeAt(targetEnd) === 0x29;
|
|
342
|
+
}, (_, at) => at + 1);
|
|
343
|
+
}
|
|
174
344
|
/** Decoded blobs are bounded so one base64 wall cannot blow up memory. */
|
|
175
345
|
export const MAX_BASE64_CANDIDATE_CHARS = 100_000;
|
|
176
346
|
/** Decoding happens once. Content inside a decoded blob is never decoded again. */
|
|
@@ -220,7 +390,7 @@ export function decodeBase64Text(candidate) {
|
|
|
220
390
|
function phraseHits(text, source) {
|
|
221
391
|
const raw = [];
|
|
222
392
|
for (const rule of INJECTION_RULES) {
|
|
223
|
-
for (const { index, match } of matchAll(text, rule.pattern)) {
|
|
393
|
+
for (const { index, match } of rule.matches?.(text) ?? matchAll(text, rule.pattern)) {
|
|
224
394
|
raw.push({
|
|
225
395
|
type: rule.category,
|
|
226
396
|
rule: rule.id,
|
|
@@ -295,7 +465,7 @@ export function scanInjection(text) {
|
|
|
295
465
|
});
|
|
296
466
|
}
|
|
297
467
|
// Concealment: instruction text inside an HTML comment.
|
|
298
|
-
for (const { index, match } of
|
|
468
|
+
for (const { index, match } of htmlCommentMatches(text)) {
|
|
299
469
|
const inner = match[1] ?? "";
|
|
300
470
|
if (phraseHits(inner, "text").length === 0)
|
|
301
471
|
continue;
|
|
@@ -321,7 +491,7 @@ export function scanInjection(text) {
|
|
|
321
491
|
});
|
|
322
492
|
}
|
|
323
493
|
// Concealment: a link whose label names a different host than its target.
|
|
324
|
-
for (const { index, match } of
|
|
494
|
+
for (const { index, match } of markdownLinkMatches(text)) {
|
|
325
495
|
const label = match[1] ?? "";
|
|
326
496
|
const href = match[2] ?? "";
|
|
327
497
|
const labelDomain = DOMAIN_IN_TEXT.exec(label)?.[1]?.toLowerCase();
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The matches of the IPv6 candidate pattern
|
|
3
|
+
* `/\b(?=[0-9A-Fa-f:]*::|(?:[0-9A-Fa-f]{1,4}:){7})[0-9A-Fa-f:]{2,39}(?:\.\d{1,3}){0,3}/g`,
|
|
4
|
+
* in order, start for start.
|
|
5
|
+
*
|
|
6
|
+
* The lookahead's `[0-9A-Fa-f:]*::` read on to the end of a run of hex
|
|
7
|
+
* digits and colons from every word boundary in it, and a run like
|
|
8
|
+
* `a:a:a:…` has one every second character and matches every 39: quadratic
|
|
9
|
+
* in the run, minutes over the 2M characters `PiiScan` accepts. Here the run
|
|
10
|
+
* is read once: a start sees `::` ahead exactly when the run's last `::`
|
|
11
|
+
* begins at or after it.
|
|
12
|
+
*/
|
|
13
|
+
export declare function ipv6Candidates(text: string): Array<{
|
|
14
|
+
index: number;
|
|
15
|
+
value: string;
|
|
16
|
+
}>;
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
const isHexDigit = (c) => (c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x46) || (c >= 0x61 && c <= 0x66);
|
|
2
|
+
const isHexOrColon = (c) => isHexDigit(c) || c === 0x3a;
|
|
3
|
+
const isDigit = (c) => c >= 0x30 && c <= 0x39;
|
|
4
|
+
/** A code unit `\w` matches, which is what `\b` looks at. */
|
|
5
|
+
const isWordUnit = (c) => isDigit(c) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) || c === 0x5f;
|
|
6
|
+
/** `(?:[0-9A-Fa-f]{1,4}:){7}` at `at`. */
|
|
7
|
+
function sevenGroups(text, at) {
|
|
8
|
+
let p = at;
|
|
9
|
+
for (let g = 0; g < 7; g++) {
|
|
10
|
+
let d = p;
|
|
11
|
+
while (d < p + 4 && isHexDigit(text.charCodeAt(d)))
|
|
12
|
+
d++;
|
|
13
|
+
if (d === p || text.charCodeAt(d) !== 0x3a)
|
|
14
|
+
return false;
|
|
15
|
+
p = d + 1;
|
|
16
|
+
}
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The matches of the IPv6 candidate pattern
|
|
21
|
+
* `/\b(?=[0-9A-Fa-f:]*::|(?:[0-9A-Fa-f]{1,4}:){7})[0-9A-Fa-f:]{2,39}(?:\.\d{1,3}){0,3}/g`,
|
|
22
|
+
* in order, start for start.
|
|
23
|
+
*
|
|
24
|
+
* The lookahead's `[0-9A-Fa-f:]*::` read on to the end of a run of hex
|
|
25
|
+
* digits and colons from every word boundary in it, and a run like
|
|
26
|
+
* `a:a:a:…` has one every second character and matches every 39: quadratic
|
|
27
|
+
* in the run, minutes over the 2M characters `PiiScan` accepts. Here the run
|
|
28
|
+
* is read once: a start sees `::` ahead exactly when the run's last `::`
|
|
29
|
+
* begins at or after it.
|
|
30
|
+
*/
|
|
31
|
+
export function ipv6Candidates(text) {
|
|
32
|
+
const out = [];
|
|
33
|
+
// The run of hex digits and colons the current start is in, and where its
|
|
34
|
+
// last `::` begins (-1 for none).
|
|
35
|
+
let runEnd = -1;
|
|
36
|
+
let lastPair = -1;
|
|
37
|
+
let at = 0;
|
|
38
|
+
while (at < text.length) {
|
|
39
|
+
if (!isHexOrColon(text.charCodeAt(at))) {
|
|
40
|
+
at++;
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if (at >= runEnd) {
|
|
44
|
+
runEnd = at;
|
|
45
|
+
lastPair = -1;
|
|
46
|
+
while (runEnd < text.length && isHexOrColon(text.charCodeAt(runEnd))) {
|
|
47
|
+
if (runEnd > at &&
|
|
48
|
+
text.charCodeAt(runEnd) === 0x3a &&
|
|
49
|
+
text.charCodeAt(runEnd - 1) === 0x3a) {
|
|
50
|
+
lastPair = runEnd - 1;
|
|
51
|
+
}
|
|
52
|
+
runEnd++;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const boundary = (at > 0 && isWordUnit(text.charCodeAt(at - 1))) !== isWordUnit(text.charCodeAt(at));
|
|
56
|
+
if (boundary && runEnd - at >= 2 && (lastPair >= at || sevenGroups(text, at))) {
|
|
57
|
+
let end = at + Math.min(39, runEnd - at);
|
|
58
|
+
for (let g = 0; g < 3; g++) {
|
|
59
|
+
if (text.charCodeAt(end) !== 0x2e || !isDigit(text.charCodeAt(end + 1)))
|
|
60
|
+
break;
|
|
61
|
+
let d = end + 1;
|
|
62
|
+
while (d < end + 4 && isDigit(text.charCodeAt(d)))
|
|
63
|
+
d++;
|
|
64
|
+
end = d;
|
|
65
|
+
}
|
|
66
|
+
out.push({ index: at, value: text.slice(at, end) });
|
|
67
|
+
at = end;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
at++;
|
|
71
|
+
}
|
|
72
|
+
return out;
|
|
73
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A literal phrase, found in linear time, exactly where
|
|
3
|
+
* `new RegExp(escaped, "g")` or `"gi"` finds it.
|
|
4
|
+
*
|
|
5
|
+
* ContentPolicyCheck's `*_phrase` rules ran as an escaped regex on the
|
|
6
|
+
* caller's thread. Case-sensitive, JavaScriptCore finds a literal quickly;
|
|
7
|
+
* case-insensitive (the default) it tries the whole phrase at every position,
|
|
8
|
+
* so the scan costs the text's length times the phrase's. Neither is capped:
|
|
9
|
+
* four rules of `a`×50 000 then `b` over 2 000 000 `a`s held the process for
|
|
10
|
+
* 16.6 s without one event-loop turn (C073's residual, bounds review).
|
|
11
|
+
*
|
|
12
|
+
* Here the text and the phrase are folded to one code unit per character the
|
|
13
|
+
* way the regex `i` flag compares them, and searched with Knuth-Morris-Pratt:
|
|
14
|
+
* each text position is passed once, whatever the phrase. Matches are what a
|
|
15
|
+
* global regex reports: leftmost, not overlapping, at UTF-16 code-unit
|
|
16
|
+
* offsets.
|
|
17
|
+
*/
|
|
18
|
+
/** Where one match starts, and the text it covers (the text's own spelling). */
|
|
19
|
+
export type LiteralMatch = {
|
|
20
|
+
readonly index: number;
|
|
21
|
+
readonly match: string;
|
|
22
|
+
};
|
|
23
|
+
/** Every match counted, the first few located. */
|
|
24
|
+
export type LiteralFound = {
|
|
25
|
+
readonly matches: ReadonlyArray<LiteralMatch>;
|
|
26
|
+
readonly count: number;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* The code unit the regex `i` flag (without `u`) compares `c` as: the
|
|
30
|
+
* ECMAScript Canonicalize — its upper case when that is one code unit, and not
|
|
31
|
+
* ASCII for a non-ASCII `c` — as this engine's own tables have it. The
|
|
32
|
+
* engine's case tables can be older than `toUpperCase`'s (Bun 1.3's upper-cases
|
|
33
|
+
* U+019B to U+A7DC, which its regex does not match), so a pairing the regex
|
|
34
|
+
* does not make is not made here either.
|
|
35
|
+
*/
|
|
36
|
+
export declare function canonicalUnit(c: number): number;
|
|
37
|
+
/**
|
|
38
|
+
* Searches one text for many phrases. The case-folded text is prepared once,
|
|
39
|
+
* on first need, and shared by every case-insensitive phrase.
|
|
40
|
+
*/
|
|
41
|
+
export declare class LiteralSearch {
|
|
42
|
+
#private;
|
|
43
|
+
private readonly text;
|
|
44
|
+
constructor(text: string);
|
|
45
|
+
/**
|
|
46
|
+
* Every match of `phrase`, counted in full, with the first `keep` located.
|
|
47
|
+
* An empty phrase matches at every position, as an empty regex does.
|
|
48
|
+
*
|
|
49
|
+
* Knuth-Morris-Pratt, except that from a standing start it jumps to the
|
|
50
|
+
* next place the phrase's first two code units occur: no match can start
|
|
51
|
+
* anywhere else, and a two-unit search passes each position once, so the
|
|
52
|
+
* whole search still does. On prose that jump is most of the text.
|
|
53
|
+
*/
|
|
54
|
+
find(phrase: string, caseSensitive: boolean, keep: number): LiteralFound;
|
|
55
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A literal phrase, found in linear time, exactly where
|
|
3
|
+
* `new RegExp(escaped, "g")` or `"gi"` finds it.
|
|
4
|
+
*
|
|
5
|
+
* ContentPolicyCheck's `*_phrase` rules ran as an escaped regex on the
|
|
6
|
+
* caller's thread. Case-sensitive, JavaScriptCore finds a literal quickly;
|
|
7
|
+
* case-insensitive (the default) it tries the whole phrase at every position,
|
|
8
|
+
* so the scan costs the text's length times the phrase's. Neither is capped:
|
|
9
|
+
* four rules of `a`×50 000 then `b` over 2 000 000 `a`s held the process for
|
|
10
|
+
* 16.6 s without one event-loop turn (C073's residual, bounds review).
|
|
11
|
+
*
|
|
12
|
+
* Here the text and the phrase are folded to one code unit per character the
|
|
13
|
+
* way the regex `i` flag compares them, and searched with Knuth-Morris-Pratt:
|
|
14
|
+
* each text position is passed once, whatever the phrase. Matches are what a
|
|
15
|
+
* global regex reports: leftmost, not overlapping, at UTF-16 code-unit
|
|
16
|
+
* offsets.
|
|
17
|
+
*/
|
|
18
|
+
/** Canonical code unit per code unit, filled on first use; -1 is "not yet". */
|
|
19
|
+
const CANONICAL = new Int32Array(0x10000).fill(-1);
|
|
20
|
+
/**
|
|
21
|
+
* The code unit the regex `i` flag (without `u`) compares `c` as: the
|
|
22
|
+
* ECMAScript Canonicalize — its upper case when that is one code unit, and not
|
|
23
|
+
* ASCII for a non-ASCII `c` — as this engine's own tables have it. The
|
|
24
|
+
* engine's case tables can be older than `toUpperCase`'s (Bun 1.3's upper-cases
|
|
25
|
+
* U+019B to U+A7DC, which its regex does not match), so a pairing the regex
|
|
26
|
+
* does not make is not made here either.
|
|
27
|
+
*/
|
|
28
|
+
export function canonicalUnit(c) {
|
|
29
|
+
const known = CANONICAL[c];
|
|
30
|
+
if (known !== -1)
|
|
31
|
+
return known;
|
|
32
|
+
let out = c;
|
|
33
|
+
if (c < 0x80) {
|
|
34
|
+
if (c >= 0x61 && c <= 0x7a)
|
|
35
|
+
out = c - 0x20;
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
const upper = String.fromCharCode(c).toUpperCase();
|
|
39
|
+
if (upper.length === 1) {
|
|
40
|
+
const u = upper.charCodeAt(0);
|
|
41
|
+
if (u >= 0x80 && u !== c && regexFolds(c, u))
|
|
42
|
+
out = u;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
CANONICAL[c] = out;
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
function regexFolds(a, b) {
|
|
49
|
+
return new RegExp(`^\\u${a.toString(16).padStart(4, "0")}$`, "i").test(String.fromCharCode(b));
|
|
50
|
+
}
|
|
51
|
+
/** `text`'s code units, each folded as the regex `i` flag compares it. */
|
|
52
|
+
function foldUnits(text) {
|
|
53
|
+
const out = new Uint16Array(text.length);
|
|
54
|
+
for (let i = 0; i < text.length; i++)
|
|
55
|
+
out[i] = canonicalUnit(text.charCodeAt(i));
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
function rawUnits(text) {
|
|
59
|
+
const out = new Uint16Array(text.length);
|
|
60
|
+
for (let i = 0; i < text.length; i++)
|
|
61
|
+
out[i] = text.charCodeAt(i);
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
/** KMP's failure function: for each prefix, its longest proper border. */
|
|
65
|
+
function borders(p) {
|
|
66
|
+
const f = new Int32Array(p.length);
|
|
67
|
+
let k = 0;
|
|
68
|
+
for (let i = 1; i < p.length; i++) {
|
|
69
|
+
while (k > 0 && p[i] !== p[k])
|
|
70
|
+
k = f[k - 1];
|
|
71
|
+
if (p[i] === p[k])
|
|
72
|
+
k += 1;
|
|
73
|
+
f[i] = k;
|
|
74
|
+
}
|
|
75
|
+
return f;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Searches one text for many phrases. The case-folded text is prepared once,
|
|
79
|
+
* on first need, and shared by every case-insensitive phrase.
|
|
80
|
+
*/
|
|
81
|
+
export class LiteralSearch {
|
|
82
|
+
text;
|
|
83
|
+
#folded;
|
|
84
|
+
constructor(text) {
|
|
85
|
+
this.text = text;
|
|
86
|
+
}
|
|
87
|
+
#hay(caseSensitive) {
|
|
88
|
+
if (caseSensitive)
|
|
89
|
+
return this.text;
|
|
90
|
+
if (this.#folded === undefined) {
|
|
91
|
+
this.#folded = Buffer.from(foldUnits(this.text).buffer).toString("utf16le");
|
|
92
|
+
}
|
|
93
|
+
return this.#folded;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Every match of `phrase`, counted in full, with the first `keep` located.
|
|
97
|
+
* An empty phrase matches at every position, as an empty regex does.
|
|
98
|
+
*
|
|
99
|
+
* Knuth-Morris-Pratt, except that from a standing start it jumps to the
|
|
100
|
+
* next place the phrase's first two code units occur: no match can start
|
|
101
|
+
* anywhere else, and a two-unit search passes each position once, so the
|
|
102
|
+
* whole search still does. On prose that jump is most of the text.
|
|
103
|
+
*/
|
|
104
|
+
find(phrase, caseSensitive, keep) {
|
|
105
|
+
const text = this.text;
|
|
106
|
+
const length = phrase.length;
|
|
107
|
+
const matches = [];
|
|
108
|
+
if (length === 0) {
|
|
109
|
+
for (let i = 0; i <= text.length && matches.length < keep; i++) {
|
|
110
|
+
matches.push({ index: i, match: "" });
|
|
111
|
+
}
|
|
112
|
+
return { matches, count: text.length + 1 };
|
|
113
|
+
}
|
|
114
|
+
if (length > text.length)
|
|
115
|
+
return { matches, count: 0 };
|
|
116
|
+
const hay = this.#hay(caseSensitive);
|
|
117
|
+
const p = caseSensitive ? rawUnits(phrase) : foldUnits(phrase);
|
|
118
|
+
const head = String.fromCharCode(...p.subarray(0, 2));
|
|
119
|
+
const f = borders(p);
|
|
120
|
+
let k = 0;
|
|
121
|
+
let count = 0;
|
|
122
|
+
for (let i = 0; i < hay.length; i++) {
|
|
123
|
+
if (k === 0) {
|
|
124
|
+
const next = hay.indexOf(head, i);
|
|
125
|
+
if (next === -1)
|
|
126
|
+
break;
|
|
127
|
+
i = next;
|
|
128
|
+
}
|
|
129
|
+
const ch = hay.charCodeAt(i);
|
|
130
|
+
while (k > 0 && ch !== p[k])
|
|
131
|
+
k = f[k - 1];
|
|
132
|
+
if (ch === p[k])
|
|
133
|
+
k += 1;
|
|
134
|
+
if (k === length) {
|
|
135
|
+
count += 1;
|
|
136
|
+
if (matches.length < keep) {
|
|
137
|
+
const at = i + 1 - length;
|
|
138
|
+
matches.push({ index: at, match: text.slice(at, i + 1) });
|
|
139
|
+
}
|
|
140
|
+
// Not overlapping: the next match starts after this one ends.
|
|
141
|
+
k = 0;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return { matches, count };
|
|
145
|
+
}
|
|
146
|
+
}
|
package/dist/lib/pii.d.ts
CHANGED
|
@@ -1,33 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Personal-data detectors.
|
|
3
|
-
*
|
|
4
|
-
* ## Read this before you trust a result
|
|
5
|
-
*
|
|
6
|
-
* Every rule here is a pattern over bytes. A pattern can prove that a string
|
|
7
|
-
* has the SHAPE of an identifier, and for two formats it can prove a check
|
|
8
|
-
* digit is consistent. It can never prove that a string IS somebody's
|
|
9
|
-
* personal data, and — far more dangerous — an empty result can never prove
|
|
10
|
-
* a document has none. A name, a free-text medical detail, an employee
|
|
11
|
-
* number with a local format, an address that does not end in "Street": all
|
|
12
|
-
* invisible here.
|
|
13
|
-
*
|
|
14
|
-
* So `PiiScan` reports what it LOOKED for, not what a document contains, and
|
|
15
|
-
* nothing downstream should read "0 findings" as "safe to publish".
|
|
16
|
-
*
|
|
17
|
-
* ## The rules, and exactly what each one proves
|
|
18
|
-
*
|
|
19
|
-
* | rule | proves |
|
|
20
|
-
* |---|---|
|
|
21
|
-
* | `email.addr-spec-subset` | dot-atom local part, dotted domain, TLD of 2+ letters. Not quoted local parts, not IP-literal domains, not internationalized addresses. |
|
|
22
|
-
* | `phone.e164` | `+` then 8–15 digits with common separators, taken as the longest whole-group run within that range. Where the next number is grouped the same way the two cannot be told apart, and the span runs long; `dedupeOverlaps` lets a verified card win that overlap. |
|
|
23
|
-
* | `phone.national.XX` | a national dialling shape for one of the seven supported countries. |
|
|
24
|
-
* | `ssn.us-format` | `NNN-NN-NNNN` with a group the SSA has never issued (000/666/900+ area, 00 group, 0000 serial) excluded. NOT validity: there is no public checksum for an SSN, and this package does not pretend otherwise. |
|
|
25
|
-
* | `iban.iso7064-mod97` | the ISO 7064 mod-97-10 check digits are correct, and where the country is known, the length matches. A real check, verifiable offline. |
|
|
26
|
-
* | `card.luhn` | the Luhn check digit is correct over 12–19 digits taken as whole digit groups. A real check. It does not mean the card exists or is live. |
|
|
27
|
-
* | `ip.v4-dotted` / `ip.v6-parsed` | a parseable address literal. Private, loopback and documentation ranges are labelled, because those are usually not personal data. |
|
|
28
|
-
* | `dob.labelled` / `date.calendar` | a calendar date. Only a nearby "date of birth"-style label, or a caller-supplied `referenceDate` that makes the age plausible, raises it above `possible`. |
|
|
29
|
-
* | `address.us-street-suffix` / `address.us-city-state-zip` | a US-style street line or city/state/ZIP tail. US conventions only. |
|
|
30
|
-
*/
|
|
31
1
|
import { type Finding } from "./text";
|
|
32
2
|
export declare const PII_TYPES: readonly ["credit_card", "date_of_birth", "email", "iban", "ip_address", "phone", "postal_address", "us_ssn"];
|
|
33
3
|
export type PiiType = (typeof PII_TYPES)[number];
|
|
@@ -70,6 +40,25 @@ export declare function parseIpv4(value: string): number[] | undefined;
|
|
|
70
40
|
* guessing which is a good way to be confidently wrong.
|
|
71
41
|
*/
|
|
72
42
|
export declare function parseIpv6(value: string): number[] | undefined;
|
|
43
|
+
export declare const EMAIL: RegExp;
|
|
44
|
+
/**
|
|
45
|
+
* EMAIL's matches. It opens with an unbounded local-part run, so it is tried
|
|
46
|
+
* only where one starts, and a failed try skips the rest of that run (see
|
|
47
|
+
* matchAllSkipping): 100K of base64 cost seconds as a global search.
|
|
48
|
+
*/
|
|
49
|
+
export declare function emailMatches(text: string): Iterable<{
|
|
50
|
+
index: number;
|
|
51
|
+
match: RegExpExecArray;
|
|
52
|
+
}>;
|
|
53
|
+
/**
|
|
54
|
+
* ADDRESS_CITY_STATE_ZIP's matches, tried where a capitalised word starts; a
|
|
55
|
+
* failed try skips the rest of its `[A-Za-z.'-]` run.
|
|
56
|
+
*/
|
|
57
|
+
export declare function cityStateZipMatches(text: string): Iterable<{
|
|
58
|
+
index: number;
|
|
59
|
+
match: RegExpExecArray;
|
|
60
|
+
}>;
|
|
61
|
+
export declare const ADDRESS_CITY_STATE_ZIP: RegExp;
|
|
73
62
|
/**
|
|
74
63
|
* Days since 1970-01-01 for a valid proleptic Gregorian Y-M-D, or undefined.
|
|
75
64
|
*
|