@conarium-ai/core 0.2.54 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/LIMITATIONS.md +94 -1
- package/LIMITATIONS.tr.md +96 -1
- package/dist/custom_patterns.d.ts +13 -12
- package/dist/custom_patterns.d.ts.map +1 -1
- package/dist/custom_patterns.js +576 -14
- package/dist/custom_patterns.js.map +1 -1
- package/dist/governance.d.ts.map +1 -1
- package/dist/governance.js +37 -139
- package/dist/governance.js.map +1 -1
- package/dist/mask/belt.d.ts +36 -0
- package/dist/mask/belt.d.ts.map +1 -0
- package/dist/mask/belt.js +771 -0
- package/dist/mask/belt.js.map +1 -0
- package/dist/mask/core.d.ts +36 -0
- package/dist/mask/core.d.ts.map +1 -0
- package/dist/mask/core.js +119 -0
- package/dist/mask/core.js.map +1 -0
- package/dist/mask/detectors/bearer.d.ts +6 -0
- package/dist/mask/detectors/bearer.d.ts.map +1 -0
- package/dist/mask/detectors/bearer.js +30 -0
- package/dist/mask/detectors/bearer.js.map +1 -0
- package/dist/mask/detectors/connection.d.ts +7 -0
- package/dist/mask/detectors/connection.d.ts.map +1 -0
- package/dist/mask/detectors/connection.js +20 -0
- package/dist/mask/detectors/connection.js.map +1 -0
- package/dist/mask/detectors/custom.d.ts +12 -0
- package/dist/mask/detectors/custom.d.ts.map +1 -0
- package/dist/mask/detectors/custom.js +44 -0
- package/dist/mask/detectors/custom.js.map +1 -0
- package/dist/mask/detectors/email.d.ts +3 -0
- package/dist/mask/detectors/email.d.ts.map +1 -0
- package/dist/mask/detectors/email.js +41 -0
- package/dist/mask/detectors/email.js.map +1 -0
- package/dist/mask/detectors/embedded.d.ts +3 -0
- package/dist/mask/detectors/embedded.d.ts.map +1 -0
- package/dist/mask/detectors/embedded.js +73 -0
- package/dist/mask/detectors/embedded.js.map +1 -0
- package/dist/mask/detectors/iban.d.ts +3 -0
- package/dist/mask/detectors/iban.d.ts.map +1 -0
- package/dist/mask/detectors/iban.js +92 -0
- package/dist/mask/detectors/iban.js.map +1 -0
- package/dist/mask/detectors/ip.d.ts +3 -0
- package/dist/mask/detectors/ip.d.ts.map +1 -0
- package/dist/mask/detectors/ip.js +42 -0
- package/dist/mask/detectors/ip.js.map +1 -0
- package/dist/mask/detectors/labelled_secret.d.ts +6 -0
- package/dist/mask/detectors/labelled_secret.d.ts.map +1 -0
- package/dist/mask/detectors/labelled_secret.js +31 -0
- package/dist/mask/detectors/labelled_secret.js.map +1 -0
- package/dist/mask/detectors/mrz.d.ts +3 -0
- package/dist/mask/detectors/mrz.d.ts.map +1 -0
- package/dist/mask/detectors/mrz.js +42 -0
- package/dist/mask/detectors/mrz.js.map +1 -0
- package/dist/mask/detectors/name.d.ts +6 -0
- package/dist/mask/detectors/name.d.ts.map +1 -0
- package/dist/mask/detectors/name.js +41 -0
- package/dist/mask/detectors/name.js.map +1 -0
- package/dist/mask/detectors/numeric.d.ts +3 -0
- package/dist/mask/detectors/numeric.d.ts.map +1 -0
- package/dist/mask/detectors/numeric.js +240 -0
- package/dist/mask/detectors/numeric.js.map +1 -0
- package/dist/mask/detectors/secret.d.ts +6 -0
- package/dist/mask/detectors/secret.d.ts.map +1 -0
- package/dist/mask/detectors/secret.js +31 -0
- package/dist/mask/detectors/secret.js.map +1 -0
- package/dist/mask/detectors/whole_base64.d.ts +3 -0
- package/dist/mask/detectors/whole_base64.d.ts.map +1 -0
- package/dist/mask/detectors/whole_base64.js +28 -0
- package/dist/mask/detectors/whole_base64.js.map +1 -0
- package/dist/mask/legacy_0254/conn_linear.d.ts +26 -0
- package/dist/mask/legacy_0254/conn_linear.d.ts.map +1 -0
- package/dist/mask/legacy_0254/conn_linear.js +97 -0
- package/dist/mask/legacy_0254/conn_linear.js.map +1 -0
- package/dist/mask/legacy_0254/custom_patterns.d.ts +42 -0
- package/dist/mask/legacy_0254/custom_patterns.d.ts.map +1 -0
- package/dist/mask/legacy_0254/custom_patterns.js +164 -0
- package/dist/mask/legacy_0254/custom_patterns.js.map +1 -0
- package/dist/mask/legacy_0254/digit_pii.d.ts +46 -0
- package/dist/mask/legacy_0254/digit_pii.d.ts.map +1 -0
- package/dist/mask/legacy_0254/digit_pii.js +230 -0
- package/dist/mask/legacy_0254/digit_pii.js.map +1 -0
- package/dist/mask/legacy_0254/iban.d.ts +56 -0
- package/dist/mask/legacy_0254/iban.d.ts.map +1 -0
- package/dist/mask/legacy_0254/iban.js +171 -0
- package/dist/mask/legacy_0254/iban.js.map +1 -0
- package/dist/mask/legacy_0254/ip_detect.d.ts +28 -0
- package/dist/mask/legacy_0254/ip_detect.d.ts.map +1 -0
- package/dist/mask/legacy_0254/ip_detect.js +92 -0
- package/dist/mask/legacy_0254/ip_detect.js.map +1 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.d.ts +20 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.d.ts.map +1 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.js +171 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.js.map +1 -0
- package/dist/mask/legacy_0254/legacy_mask_text.d.ts +11 -0
- package/dist/mask/legacy_0254/legacy_mask_text.d.ts.map +1 -0
- package/dist/mask/legacy_0254/legacy_mask_text.js +46 -0
- package/dist/mask/legacy_0254/legacy_mask_text.js.map +1 -0
- package/dist/mask/legacy_0254/mrz.d.ts +7 -0
- package/dist/mask/legacy_0254/mrz.d.ts.map +1 -0
- package/dist/mask/legacy_0254/mrz.js +74 -0
- package/dist/mask/legacy_0254/mrz.js.map +1 -0
- package/dist/mask/legacy_0254/pii_normalize.d.ts +38 -0
- package/dist/mask/legacy_0254/pii_normalize.d.ts.map +1 -0
- package/dist/mask/legacy_0254/pii_normalize.js +96 -0
- package/dist/mask/legacy_0254/pii_normalize.js.map +1 -0
- package/dist/mask/legacy_0254/safe-object.d.ts +14 -0
- package/dist/mask/legacy_0254/safe-object.d.ts.map +1 -0
- package/dist/mask/legacy_0254/safe-object.js +27 -0
- package/dist/mask/legacy_0254/safe-object.js.map +1 -0
- package/dist/mask/legacy_0254/tckn.d.ts +10 -0
- package/dist/mask/legacy_0254/tckn.d.ts.map +1 -0
- package/dist/mask/legacy_0254/tckn.js +70 -0
- package/dist/mask/legacy_0254/tckn.js.map +1 -0
- package/dist/mask/legacy_0254/types.d.ts +207 -0
- package/dist/mask/legacy_0254/types.d.ts.map +1 -0
- package/dist/mask/legacy_0254/types.js +2 -0
- package/dist/mask/legacy_0254/types.js.map +1 -0
- package/dist/mask-text.d.ts +11 -1
- package/dist/mask-text.d.ts.map +1 -1
- package/dist/mask-text.js +189 -30
- package/dist/mask-text.js.map +1 -1
- package/docs/claims/reviews/0.3.0.md +72 -0
- package/package.json +3 -1
package/dist/custom_patterns.js
CHANGED
|
@@ -1,3 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* User-defined PII patterns from config.
|
|
3
|
+
*
|
|
4
|
+
* Same response-path scanner as the built-in detectors — not a second masker.
|
|
5
|
+
* The pattern text is treated as sensitive: errors, logs and receipts carry
|
|
6
|
+
* the rule NAME only.
|
|
7
|
+
*
|
|
8
|
+
* User regex is ReDoS-risky. Compile keeps the 0.2.54 reject list (lookaround,
|
|
9
|
+
* backreferences, `+` / `*`, open `{n,}`, source and quantifier caps, the two
|
|
10
|
+
* old group rules). Accepted patterns are parsed once and emitted as re2js
|
|
11
|
+
* source, which matches in linear time.
|
|
12
|
+
* Nested repeats whose written maxima divide a budget of 1000 down to zero
|
|
13
|
+
* are rejected, because re2js refuses that product while parsing. re2js checks
|
|
14
|
+
* each repeat on its own from 1000; a zero-count repeat only cuts the count
|
|
15
|
+
* coming from outside, and the nest inside is still checked.
|
|
16
|
+
* A broken rule fails the config (silent drop = leak).
|
|
17
|
+
*/
|
|
18
|
+
import { RegExpParser } from '@eslint-community/regexpp';
|
|
19
|
+
import { RE2JS } from 're2js';
|
|
1
20
|
export const CUSTOM_PATTERN_MAX_SOURCE = 200;
|
|
2
21
|
export const CUSTOM_PATTERN_MAX_QUANTIFIER = 64;
|
|
3
22
|
export const CUSTOM_PATTERN_MAX_MATCHES = 256;
|
|
@@ -9,6 +28,10 @@ const REJECT_MESSAGE = {
|
|
|
9
28
|
'too-long': (name) => `customPatterns: rule "${name}" exceeds the pattern length cap`,
|
|
10
29
|
'bad-name': (name) => `customPatterns: rule name "${name}" is not allowed`,
|
|
11
30
|
'bad-label': (name) => `customPatterns: rule "${name}" has an invalid mask label`,
|
|
31
|
+
'unsupported-flag': (name) => `customPatterns: rule "${name}" was rejected (multiline flag m)`,
|
|
32
|
+
'unsupported-case-word': (name) => `customPatterns: rule "${name}" was rejected (case-insensitive word characters)`,
|
|
33
|
+
'unsupported-empty-choice': (name) => `customPatterns: rule "${name}" was rejected (empty choice inside a repeated group)`,
|
|
34
|
+
'unsupported-surrogate': (name) => `customPatterns: rule "${name}" was rejected (lone surrogate escape)`,
|
|
12
35
|
};
|
|
13
36
|
export class CustomPatternError extends Error {
|
|
14
37
|
ruleName;
|
|
@@ -23,6 +46,43 @@ export class CustomPatternError extends Error {
|
|
|
23
46
|
}
|
|
24
47
|
const NAME_RE = /^[A-Za-z][A-Za-z0-9._-]{0,63}$/;
|
|
25
48
|
const LABEL_RE = /^\[[A-Za-z][A-Za-z0-9._-]{0,61}\]$/;
|
|
49
|
+
/**
|
|
50
|
+
* JS `u` whitespace, one span list. `\s` and `\S` (inside and outside a class)
|
|
51
|
+
* are both written from this list.
|
|
52
|
+
*/
|
|
53
|
+
const JS_SPACE_SPANS = [
|
|
54
|
+
[0x0009, 0x000d],
|
|
55
|
+
[0x0020, 0x0020],
|
|
56
|
+
[0x00a0, 0x00a0],
|
|
57
|
+
[0x1680, 0x1680],
|
|
58
|
+
[0x2000, 0x200a],
|
|
59
|
+
[0x2028, 0x2029],
|
|
60
|
+
[0x202f, 0x202f],
|
|
61
|
+
[0x205f, 0x205f],
|
|
62
|
+
[0x3000, 0x3000],
|
|
63
|
+
[0xfeff, 0xfeff],
|
|
64
|
+
];
|
|
65
|
+
const DIGIT_SPANS = [[0x30, 0x39]];
|
|
66
|
+
/** ASCII `\w`: digits, `A–Z`, `_`, `a–z`. Case-insensitive `\w` is refused. */
|
|
67
|
+
const WORD_SPANS = [
|
|
68
|
+
[0x30, 0x39],
|
|
69
|
+
[0x41, 0x5a],
|
|
70
|
+
[0x5f, 0x5f],
|
|
71
|
+
[0x61, 0x7a],
|
|
72
|
+
];
|
|
73
|
+
/** JS `u` dot: any code point except LF, CR, LS, PS. */
|
|
74
|
+
const JS_DOT = '[^\\n\\r\\x{2028}\\x{2029}]';
|
|
75
|
+
/** `[]` matches nothing. `[^]` matches every code point, line breaks included. */
|
|
76
|
+
const JS_NEVER = '[^\\x{0}-\\x{10ffff}]';
|
|
77
|
+
const JS_ANY = '[\\x{0}-\\x{10ffff}]';
|
|
78
|
+
/**
|
|
79
|
+
* re2js checks each repeat on its own from this budget. Each written
|
|
80
|
+
* quantifier integer-divides it by its maximum (`?` is 1). A zero-count
|
|
81
|
+
* repeat (`{0}`, `{0,0}`) does not divide: it only cuts the count coming
|
|
82
|
+
* from outside, and the body is checked again from 1000. A budget of zero
|
|
83
|
+
* is a refusal.
|
|
84
|
+
*/
|
|
85
|
+
const NESTED_REPEAT_BUDGET = 1000;
|
|
26
86
|
export function compileCustomPatterns(list) {
|
|
27
87
|
if (!list || list.length === 0)
|
|
28
88
|
return [];
|
|
@@ -60,16 +120,32 @@ function compileOne(item) {
|
|
|
60
120
|
// SyntaxError text includes the pattern. Drop it.
|
|
61
121
|
throw new CustomPatternError(item.name, 'invalid');
|
|
62
122
|
}
|
|
123
|
+
let engine;
|
|
124
|
+
try {
|
|
125
|
+
engine = RE2JS.compile(toEngineSource(source, item.name));
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
if (err instanceof CustomPatternError)
|
|
129
|
+
throw err;
|
|
130
|
+
throw new CustomPatternError(item.name, 'invalid');
|
|
131
|
+
}
|
|
63
132
|
return {
|
|
64
133
|
name: item.name,
|
|
65
134
|
columns: (item.columns ?? []).filter(Boolean),
|
|
66
135
|
label,
|
|
67
136
|
re,
|
|
137
|
+
matches(text) {
|
|
138
|
+
return scanEngine(engine, text);
|
|
139
|
+
},
|
|
68
140
|
};
|
|
69
141
|
}
|
|
70
142
|
/**
|
|
71
143
|
* Conservative static reject. Linear class quantifiers (`[0-9]{8}`) pass.
|
|
72
144
|
* Anything that can explode backtracking — or is simply unbounded — does not.
|
|
145
|
+
*
|
|
146
|
+
* This is the 0.2.54 rule. Nested quantifiers, alternation inside a quantified
|
|
147
|
+
* group, and adjacent variable quantifiers are not rejected here: those shapes
|
|
148
|
+
* run on the linear engine.
|
|
73
149
|
*/
|
|
74
150
|
export function looksUnsafe(source) {
|
|
75
151
|
if (/\(\?[=!<]/.test(source))
|
|
@@ -100,6 +176,471 @@ export function looksUnsafe(source) {
|
|
|
100
176
|
}
|
|
101
177
|
return false;
|
|
102
178
|
}
|
|
179
|
+
/**
|
|
180
|
+
* re2js source for a pattern JavaScript already accepted with the `u` flag.
|
|
181
|
+
*
|
|
182
|
+
* One parse and one walk. The walk carries an `i` / `s` stack (both start
|
|
183
|
+
* off; the compile flags are `gu`) and, on every node, the empty-choice
|
|
184
|
+
* attributes and the re2js text. `s` only chooses which set `.` becomes.
|
|
185
|
+
* `s` and `m` are never written out. The walk starts with a budget of 1000;
|
|
186
|
+
* a quantifier that is written passes its child the budget it received,
|
|
187
|
+
* integer-divided by that quantifier's maximum (its minimum
|
|
188
|
+
* when the repeat is open). A zero-count repeat (`{0}`, `{0,0}`) only cuts
|
|
189
|
+
* the count coming from outside: its body starts again at 1000, and a nested
|
|
190
|
+
* repeat found there is still carried upward. A zero budget is rejected
|
|
191
|
+
* because re2js will not parse the nest.
|
|
192
|
+
* Refusals are thrown in order:
|
|
193
|
+
* case-insensitive word characters, an empty choice in a repeated group,
|
|
194
|
+
* a lone surrogate, the multiline flag, nested repeats, then `invalid`.
|
|
195
|
+
* A node that matches nothing is a single never-set and is never left under
|
|
196
|
+
* a quantifier. Characters are `\x{…}`. `\s` / `\S` / `\p` / `\P` become
|
|
197
|
+
* the ranges JavaScript uses; a property is cached by its raw brace text.
|
|
198
|
+
* An assertion (`^` `$` `\b` `\B`) is not an empty choice.
|
|
199
|
+
*/
|
|
200
|
+
function toEngineSource(source, ruleName = '') {
|
|
201
|
+
let pattern;
|
|
202
|
+
try {
|
|
203
|
+
pattern = new RegExpParser({ ecmaVersion: 2025 }).parsePattern(source, 0, source.length, {
|
|
204
|
+
unicode: true,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
// The engine accepted it and the parser did not. Fail closed; the message
|
|
209
|
+
// stays the rule name.
|
|
210
|
+
throw new CustomPatternError(ruleName, 'invalid');
|
|
211
|
+
}
|
|
212
|
+
const ctx = {
|
|
213
|
+
ruleName,
|
|
214
|
+
refusals: {
|
|
215
|
+
caseIgnore: false,
|
|
216
|
+
word: false,
|
|
217
|
+
emptyChoice: false,
|
|
218
|
+
surrogate: false,
|
|
219
|
+
multiline: false,
|
|
220
|
+
nestedRepeat: false,
|
|
221
|
+
invalid: false,
|
|
222
|
+
},
|
|
223
|
+
};
|
|
224
|
+
const walked = walkNode(pattern, { ignoreCase: false, dotAll: false }, ctx, NESTED_REPEAT_BUDGET);
|
|
225
|
+
if (walked.repeatOver)
|
|
226
|
+
ctx.refusals.nestedRepeat = true;
|
|
227
|
+
throwCollected(ruleName, ctx.refusals);
|
|
228
|
+
// `[]|` drops the nothing branch and leaves only an empty alternative.
|
|
229
|
+
return walked.text === '' ? '(?:)' : walked.text;
|
|
230
|
+
}
|
|
231
|
+
function charNode() {
|
|
232
|
+
return { el: true, canEmpty: false, canNonEmpty: true };
|
|
233
|
+
}
|
|
234
|
+
function emptyNode() {
|
|
235
|
+
return { el: true, canEmpty: true, canNonEmpty: false };
|
|
236
|
+
}
|
|
237
|
+
function nothingNode() {
|
|
238
|
+
return { el: true, canEmpty: false, canNonEmpty: false };
|
|
239
|
+
}
|
|
240
|
+
function isNothing(attr) {
|
|
241
|
+
return !attr.canEmpty && !attr.canNonEmpty;
|
|
242
|
+
}
|
|
243
|
+
function isSurrogateValue(value) {
|
|
244
|
+
return value >= 0xd800 && value <= 0xdfff;
|
|
245
|
+
}
|
|
246
|
+
function throwCollected(ruleName, refusals) {
|
|
247
|
+
if (refusals.caseIgnore && refusals.word) {
|
|
248
|
+
throw new CustomPatternError(ruleName, 'unsupported-case-word');
|
|
249
|
+
}
|
|
250
|
+
if (refusals.emptyChoice)
|
|
251
|
+
throw new CustomPatternError(ruleName, 'unsupported-empty-choice');
|
|
252
|
+
if (refusals.surrogate)
|
|
253
|
+
throw new CustomPatternError(ruleName, 'unsupported-surrogate');
|
|
254
|
+
if (refusals.multiline)
|
|
255
|
+
throw new CustomPatternError(ruleName, 'unsupported-flag');
|
|
256
|
+
if (refusals.nestedRepeat)
|
|
257
|
+
throw new CustomPatternError(ruleName, 'unsafe');
|
|
258
|
+
if (refusals.invalid)
|
|
259
|
+
throw new CustomPatternError(ruleName, 'invalid');
|
|
260
|
+
}
|
|
261
|
+
function quantify(atom, min, max, lazy, ctx) {
|
|
262
|
+
if (max > min && atom.canEmpty && !atom.el)
|
|
263
|
+
ctx.refusals.emptyChoice = true;
|
|
264
|
+
const nothing = !atom.canEmpty && !atom.canNonEmpty;
|
|
265
|
+
const canEmpty = nothing ? min === 0 : min === 0 || atom.canEmpty;
|
|
266
|
+
const canNonEmpty = !nothing && atom.canNonEmpty && max >= 1;
|
|
267
|
+
const variable = max > min;
|
|
268
|
+
let el;
|
|
269
|
+
if (!canEmpty)
|
|
270
|
+
el = true;
|
|
271
|
+
else if (lazy && variable && atom.canNonEmpty)
|
|
272
|
+
el = false;
|
|
273
|
+
else
|
|
274
|
+
el = atom.el;
|
|
275
|
+
return { el, canEmpty, canNonEmpty };
|
|
276
|
+
}
|
|
277
|
+
/** Concatenation. A leading assertion does not clear `el`. */
|
|
278
|
+
function concatNode(items) {
|
|
279
|
+
if (items.length === 0)
|
|
280
|
+
return emptyNode();
|
|
281
|
+
const blocked = items.some((p) => !p.canEmpty && !p.canNonEmpty);
|
|
282
|
+
const canEmpty = !blocked && items.every((p) => p.canEmpty);
|
|
283
|
+
const canNonEmpty = !blocked && items.some((p) => p.canNonEmpty);
|
|
284
|
+
return {
|
|
285
|
+
el: !canEmpty || items.every((p) => p.el),
|
|
286
|
+
canEmpty,
|
|
287
|
+
canNonEmpty,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
/** Alternation. An empty branch before a later non-empty one clears `el`. */
|
|
291
|
+
function altNode(branches) {
|
|
292
|
+
let emptyThenFull = false;
|
|
293
|
+
for (let b = 0; b < branches.length && !emptyThenFull; b++) {
|
|
294
|
+
if (!branches[b].canEmpty)
|
|
295
|
+
continue;
|
|
296
|
+
for (let a = b + 1; a < branches.length; a++) {
|
|
297
|
+
if (branches[a].canNonEmpty) {
|
|
298
|
+
emptyThenFull = true;
|
|
299
|
+
break;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
const canEmpty = branches.some((b) => b.canEmpty);
|
|
304
|
+
const canNonEmpty = branches.some((b) => b.canNonEmpty);
|
|
305
|
+
return {
|
|
306
|
+
el: !canEmpty || (branches.every((b) => b.el) && !emptyThenFull),
|
|
307
|
+
canEmpty,
|
|
308
|
+
canNonEmpty,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
function repeatOver(parts) {
|
|
312
|
+
return parts.some((p) => p.repeatOver === true);
|
|
313
|
+
}
|
|
314
|
+
function concatAll(parts) {
|
|
315
|
+
const attr = concatNode(parts.map((p) => p.attr));
|
|
316
|
+
if (isNothing(attr))
|
|
317
|
+
return { attr, text: JS_NEVER };
|
|
318
|
+
return { attr, text: parts.map((p) => p.text).join(''), repeatOver: repeatOver(parts) };
|
|
319
|
+
}
|
|
320
|
+
function emitAlt(branches) {
|
|
321
|
+
const attr = altNode(branches.map((b) => b.attr));
|
|
322
|
+
if (isNothing(attr))
|
|
323
|
+
return { attr, text: JS_NEVER };
|
|
324
|
+
const kept = branches.filter((b) => !isNothing(b.attr));
|
|
325
|
+
if (kept.length === 0)
|
|
326
|
+
return { attr, text: JS_NEVER };
|
|
327
|
+
if (kept.length === 1)
|
|
328
|
+
return { attr, text: kept[0].text, repeatOver: kept[0].repeatOver };
|
|
329
|
+
return {
|
|
330
|
+
attr,
|
|
331
|
+
text: kept.map((b) => (b.text.length === 0 ? '(?:)' : b.text)).join('|'),
|
|
332
|
+
repeatOver: repeatOver(kept),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
function unionSpans(parts) {
|
|
336
|
+
const flat = [];
|
|
337
|
+
for (const part of parts) {
|
|
338
|
+
for (const [lo, hi] of part)
|
|
339
|
+
flat.push([lo, hi]);
|
|
340
|
+
}
|
|
341
|
+
flat.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
342
|
+
const out = [];
|
|
343
|
+
for (const span of flat) {
|
|
344
|
+
const last = out[out.length - 1];
|
|
345
|
+
if (!last || span[0] > last[1] + 1)
|
|
346
|
+
out.push(span);
|
|
347
|
+
else if (span[1] > last[1])
|
|
348
|
+
out[out.length - 1] = [last[0], span[1]];
|
|
349
|
+
}
|
|
350
|
+
return out;
|
|
351
|
+
}
|
|
352
|
+
function closed(ctx) {
|
|
353
|
+
ctx.refusals.invalid = true;
|
|
354
|
+
return { attr: nothingNode(), text: JS_NEVER };
|
|
355
|
+
}
|
|
356
|
+
function unexpectedNode(_node, ctx) {
|
|
357
|
+
return closed(ctx);
|
|
358
|
+
}
|
|
359
|
+
function noteSurrogate(value, ctx) {
|
|
360
|
+
if (isSurrogateValue(value))
|
|
361
|
+
ctx.refusals.surrogate = true;
|
|
362
|
+
}
|
|
363
|
+
function propertySpans(raw, negate, ctx) {
|
|
364
|
+
try {
|
|
365
|
+
const ranges = jsPropertyRanges(raw.slice(3, -1), ctx.ruleName);
|
|
366
|
+
return negate ? complementSpans(ranges) : ranges;
|
|
367
|
+
}
|
|
368
|
+
catch (err) {
|
|
369
|
+
if (err instanceof CustomPatternError) {
|
|
370
|
+
ctx.refusals.invalid = true;
|
|
371
|
+
return [];
|
|
372
|
+
}
|
|
373
|
+
throw err;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
function spansOfShorthand(kind, negate) {
|
|
377
|
+
const base = kind === 'digit' ? DIGIT_SPANS : kind === 'word' ? WORD_SPANS : JS_SPACE_SPANS;
|
|
378
|
+
return negate ? complementSpans(base) : base;
|
|
379
|
+
}
|
|
380
|
+
function emitShorthand(kind, negate) {
|
|
381
|
+
if (kind === 'digit')
|
|
382
|
+
return negate ? '[^0-9]' : '[0-9]';
|
|
383
|
+
const body = formatSpans(kind === 'word' ? WORD_SPANS : JS_SPACE_SPANS);
|
|
384
|
+
return negate ? `[^${body}]` : `[${body}]`;
|
|
385
|
+
}
|
|
386
|
+
function spansOfClassSet(node, ctx) {
|
|
387
|
+
if (node.kind === 'any') {
|
|
388
|
+
ctx.refusals.invalid = true;
|
|
389
|
+
return [];
|
|
390
|
+
}
|
|
391
|
+
if (node.kind === 'property') {
|
|
392
|
+
if (node.strings) {
|
|
393
|
+
ctx.refusals.invalid = true;
|
|
394
|
+
return [];
|
|
395
|
+
}
|
|
396
|
+
return propertySpans(node.raw, node.negate, ctx);
|
|
397
|
+
}
|
|
398
|
+
if (node.kind === 'word')
|
|
399
|
+
ctx.refusals.word = true;
|
|
400
|
+
return spansOfShorthand(node.kind, node.negate);
|
|
401
|
+
}
|
|
402
|
+
function walkClass(node, ctx) {
|
|
403
|
+
const parts = [];
|
|
404
|
+
for (const el of node.elements) {
|
|
405
|
+
switch (el.type) {
|
|
406
|
+
case 'Character':
|
|
407
|
+
noteSurrogate(el.value, ctx);
|
|
408
|
+
parts.push([[el.value, el.value]]);
|
|
409
|
+
break;
|
|
410
|
+
case 'CharacterClassRange':
|
|
411
|
+
noteSurrogate(el.min.value, ctx);
|
|
412
|
+
noteSurrogate(el.max.value, ctx);
|
|
413
|
+
if (el.min.value > el.max.value) {
|
|
414
|
+
ctx.refusals.invalid = true;
|
|
415
|
+
break;
|
|
416
|
+
}
|
|
417
|
+
parts.push([[el.min.value, el.max.value]]);
|
|
418
|
+
break;
|
|
419
|
+
case 'CharacterSet':
|
|
420
|
+
parts.push(spansOfClassSet(el, ctx));
|
|
421
|
+
break;
|
|
422
|
+
default:
|
|
423
|
+
return unexpectedNode(el, ctx);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
const spans = unionSpans(parts);
|
|
427
|
+
if (!node.negate) {
|
|
428
|
+
if (spans.length === 0)
|
|
429
|
+
return { attr: nothingNode(), text: JS_NEVER };
|
|
430
|
+
return { attr: charNode(), text: `[${formatSpans(spans)}]` };
|
|
431
|
+
}
|
|
432
|
+
if (spans.length === 0)
|
|
433
|
+
return { attr: charNode(), text: JS_ANY };
|
|
434
|
+
if (complementSpans(spans).length === 0)
|
|
435
|
+
return { attr: nothingNode(), text: JS_NEVER };
|
|
436
|
+
return { attr: charNode(), text: `[^${formatSpans(spans)}]` };
|
|
437
|
+
}
|
|
438
|
+
function flagsAfterGroup(node, flags, ctx) {
|
|
439
|
+
const mods = node.modifiers;
|
|
440
|
+
if (!mods)
|
|
441
|
+
return flags;
|
|
442
|
+
let ignoreCase = flags.ignoreCase;
|
|
443
|
+
let dotAll = flags.dotAll;
|
|
444
|
+
if (mods.add.ignoreCase) {
|
|
445
|
+
ignoreCase = true;
|
|
446
|
+
ctx.refusals.caseIgnore = true;
|
|
447
|
+
}
|
|
448
|
+
if (mods.add.dotAll)
|
|
449
|
+
dotAll = true;
|
|
450
|
+
if (mods.add.multiline)
|
|
451
|
+
ctx.refusals.multiline = true;
|
|
452
|
+
if (mods.remove?.ignoreCase)
|
|
453
|
+
ignoreCase = false;
|
|
454
|
+
if (mods.remove?.dotAll)
|
|
455
|
+
dotAll = false;
|
|
456
|
+
return { ignoreCase, dotAll };
|
|
457
|
+
}
|
|
458
|
+
function wrapGroup(text, parent, child) {
|
|
459
|
+
if (child.ignoreCase === parent.ignoreCase)
|
|
460
|
+
return `(?:${text})`;
|
|
461
|
+
const body = text.length === 0 ? '(?:)' : text;
|
|
462
|
+
return child.ignoreCase ? `(?i:${body})` : `(?-i:${body})`;
|
|
463
|
+
}
|
|
464
|
+
function walkAlts(alts, flags, ctx, budget) {
|
|
465
|
+
return emitAlt(alts.map((alt) => walkNode(alt, flags, ctx, budget)));
|
|
466
|
+
}
|
|
467
|
+
/** Largest count of a repeat: its max, or its min when the repeat is open. `{0}` is 0. */
|
|
468
|
+
function repeatMax(min, max) {
|
|
469
|
+
const m = Number.isFinite(max) ? max : min;
|
|
470
|
+
return m > 0 ? m : 0;
|
|
471
|
+
}
|
|
472
|
+
function walkNode(node, flags, ctx, budget) {
|
|
473
|
+
switch (node.type) {
|
|
474
|
+
case 'Pattern':
|
|
475
|
+
case 'Group': {
|
|
476
|
+
if (node.type === 'Group') {
|
|
477
|
+
const inner = flagsAfterGroup(node, flags, ctx);
|
|
478
|
+
const body = walkAlts(node.alternatives, inner, ctx, budget);
|
|
479
|
+
if (isNothing(body.attr))
|
|
480
|
+
return { attr: body.attr, text: JS_NEVER };
|
|
481
|
+
return { attr: body.attr, text: wrapGroup(body.text, flags, inner), repeatOver: body.repeatOver };
|
|
482
|
+
}
|
|
483
|
+
return walkAlts(node.alternatives, flags, ctx, budget);
|
|
484
|
+
}
|
|
485
|
+
case 'CapturingGroup': {
|
|
486
|
+
const body = walkAlts(node.alternatives, flags, ctx, budget);
|
|
487
|
+
if (isNothing(body.attr))
|
|
488
|
+
return { attr: body.attr, text: JS_NEVER };
|
|
489
|
+
return { attr: body.attr, text: `(?:${body.text})`, repeatOver: body.repeatOver };
|
|
490
|
+
}
|
|
491
|
+
case 'Alternative':
|
|
492
|
+
return concatAll(node.elements.map((el) => walkNode(el, flags, ctx, budget)));
|
|
493
|
+
case 'Quantifier': {
|
|
494
|
+
if (!Number.isFinite(node.min) || !Number.isFinite(node.max) || node.max < node.min) {
|
|
495
|
+
ctx.refusals.invalid = true;
|
|
496
|
+
}
|
|
497
|
+
const m = repeatMax(node.min, node.max);
|
|
498
|
+
// re2js checks every repeat from 1000 on its own. A zero-count repeat
|
|
499
|
+
// only cuts the count coming from outside, so the body starts again at
|
|
500
|
+
// 1000; the nest that walk finds is carried upward as usual.
|
|
501
|
+
const childBudget = m > 0 ? Math.floor(budget / m) : NESTED_REPEAT_BUDGET;
|
|
502
|
+
const body = walkNode(node.element, flags, ctx, childBudget);
|
|
503
|
+
const attr = quantify(body.attr, node.min, node.max, !node.greedy, ctx);
|
|
504
|
+
// An unsatisfiable body under `{0,n}` (n ≥ 2) makes re2js throw
|
|
505
|
+
// `unexpected InstFail`. Nothing is rewritten, never quantified.
|
|
506
|
+
if (isNothing(body.attr))
|
|
507
|
+
return { attr, text: node.min === 0 ? '(?:)' : JS_NEVER };
|
|
508
|
+
const suffix = node.min === 0 && node.max === 1 ? '?' : `{${node.min},${node.max}}`;
|
|
509
|
+
return {
|
|
510
|
+
attr,
|
|
511
|
+
text: body.text + suffix + (node.greedy ? '' : '?'),
|
|
512
|
+
// A zero-count repeat does not hide body.repeatOver: it only cuts
|
|
513
|
+
// the outside count.
|
|
514
|
+
repeatOver: (m > 0 && childBudget === 0) || body.repeatOver === true,
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
case 'Character':
|
|
518
|
+
noteSurrogate(node.value, ctx);
|
|
519
|
+
return { attr: charNode(), text: `\\x{${node.value.toString(16)}}` };
|
|
520
|
+
case 'CharacterSet': {
|
|
521
|
+
if (node.kind === 'any') {
|
|
522
|
+
return { attr: charNode(), text: flags.dotAll ? JS_ANY : JS_DOT };
|
|
523
|
+
}
|
|
524
|
+
if (node.kind === 'property') {
|
|
525
|
+
if (node.strings)
|
|
526
|
+
return closed(ctx);
|
|
527
|
+
const spans = propertySpans(node.raw, node.negate, ctx);
|
|
528
|
+
if (spans.length === 0)
|
|
529
|
+
return { attr: nothingNode(), text: JS_NEVER };
|
|
530
|
+
return { attr: charNode(), text: `[${formatSpans(spans)}]` };
|
|
531
|
+
}
|
|
532
|
+
if (node.kind === 'word')
|
|
533
|
+
ctx.refusals.word = true;
|
|
534
|
+
return { attr: charNode(), text: emitShorthand(node.kind, node.negate) };
|
|
535
|
+
}
|
|
536
|
+
case 'CharacterClass':
|
|
537
|
+
if (node.unicodeSets)
|
|
538
|
+
return closed(ctx);
|
|
539
|
+
return walkClass(node, ctx);
|
|
540
|
+
case 'Assertion': {
|
|
541
|
+
if (node.kind === 'lookahead' || node.kind === 'lookbehind')
|
|
542
|
+
return closed(ctx);
|
|
543
|
+
if (node.kind === 'word') {
|
|
544
|
+
ctx.refusals.word = true;
|
|
545
|
+
return { attr: emptyNode(), text: node.negate ? '\\B' : '\\b' };
|
|
546
|
+
}
|
|
547
|
+
return { attr: emptyNode(), text: node.kind === 'start' ? '^' : '$' };
|
|
548
|
+
}
|
|
549
|
+
case 'Backreference':
|
|
550
|
+
case 'CharacterClassRange':
|
|
551
|
+
case 'ClassIntersection':
|
|
552
|
+
case 'ClassStringDisjunction':
|
|
553
|
+
case 'ClassSubtraction':
|
|
554
|
+
case 'ExpressionCharacterClass':
|
|
555
|
+
case 'Flags':
|
|
556
|
+
case 'ModifierFlags':
|
|
557
|
+
case 'Modifiers':
|
|
558
|
+
case 'RegExpLiteral':
|
|
559
|
+
return closed(ctx);
|
|
560
|
+
case 'StringAlternative':
|
|
561
|
+
for (const ch of node.elements)
|
|
562
|
+
noteSurrogate(ch.value, ctx);
|
|
563
|
+
return closed(ctx);
|
|
564
|
+
default:
|
|
565
|
+
return unexpectedNode(node, ctx);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
const PROPERTY_BODY_RE = /^[A-Za-z0-9_=]+$/;
|
|
569
|
+
const UNICODE_LAST = 0x10ffff;
|
|
570
|
+
/** One entry per raw property body, for the process. */
|
|
571
|
+
const propertyRangeCache = new Map();
|
|
572
|
+
/**
|
|
573
|
+
* Code points JS's `u` flag accepts for `\p{body}`, including the surrogate
|
|
574
|
+
* range when JS itself matches them. The second compile of the same body
|
|
575
|
+
* reads the map and does not scan again.
|
|
576
|
+
*/
|
|
577
|
+
function jsPropertyRanges(body, ruleName) {
|
|
578
|
+
if (!PROPERTY_BODY_RE.test(body))
|
|
579
|
+
throw new CustomPatternError(ruleName, 'invalid');
|
|
580
|
+
const cached = propertyRangeCache.get(body);
|
|
581
|
+
if (cached)
|
|
582
|
+
return cached;
|
|
583
|
+
let probe;
|
|
584
|
+
try {
|
|
585
|
+
probe = new RegExp('^\\p{' + body + '}$', 'u');
|
|
586
|
+
}
|
|
587
|
+
catch {
|
|
588
|
+
throw new CustomPatternError(ruleName, 'invalid');
|
|
589
|
+
}
|
|
590
|
+
const ranges = [];
|
|
591
|
+
let start = -1;
|
|
592
|
+
for (let cp = 0; cp <= UNICODE_LAST; cp++) {
|
|
593
|
+
if (probe.test(String.fromCodePoint(cp))) {
|
|
594
|
+
if (start < 0)
|
|
595
|
+
start = cp;
|
|
596
|
+
}
|
|
597
|
+
else if (start >= 0) {
|
|
598
|
+
ranges.push([start, cp - 1]);
|
|
599
|
+
start = -1;
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
if (start >= 0)
|
|
603
|
+
ranges.push([start, UNICODE_LAST]);
|
|
604
|
+
propertyRangeCache.set(body, ranges);
|
|
605
|
+
return ranges;
|
|
606
|
+
}
|
|
607
|
+
function complementSpans(ranges) {
|
|
608
|
+
const out = [];
|
|
609
|
+
let next = 0;
|
|
610
|
+
for (const [lo, hi] of ranges) {
|
|
611
|
+
if (lo > next)
|
|
612
|
+
out.push([next, lo - 1]);
|
|
613
|
+
next = hi + 1;
|
|
614
|
+
}
|
|
615
|
+
if (next <= UNICODE_LAST)
|
|
616
|
+
out.push([next, UNICODE_LAST]);
|
|
617
|
+
return out;
|
|
618
|
+
}
|
|
619
|
+
function formatSpans(ranges) {
|
|
620
|
+
let out = '';
|
|
621
|
+
for (const [lo, hi] of ranges) {
|
|
622
|
+
out += `\\x{${lo.toString(16)}}`;
|
|
623
|
+
if (hi !== lo)
|
|
624
|
+
out += `-\\x{${hi.toString(16)}}`;
|
|
625
|
+
}
|
|
626
|
+
return out;
|
|
627
|
+
}
|
|
628
|
+
/** Global search: after a zero-width hit, re2js steps one code point. */
|
|
629
|
+
function scanEngine(engine, text) {
|
|
630
|
+
const matcher = engine.matcher(text);
|
|
631
|
+
const out = [];
|
|
632
|
+
const limit = text.length + 1;
|
|
633
|
+
while (matcher.find()) {
|
|
634
|
+
out.push({ start: matcher.start(), end: matcher.end() });
|
|
635
|
+
if (out.length > limit)
|
|
636
|
+
break;
|
|
637
|
+
}
|
|
638
|
+
return out;
|
|
639
|
+
}
|
|
640
|
+
/** Same scan as {@link CompiledCustomPattern.matches}, for a source string. */
|
|
641
|
+
export function matchCustomPatternSource(source, text) {
|
|
642
|
+
return scanEngine(RE2JS.compile(toEngineSource(source)), text);
|
|
643
|
+
}
|
|
103
644
|
function matchGlob(pattern, value) {
|
|
104
645
|
const p = pattern.trim().toLowerCase();
|
|
105
646
|
const v = value.trim().toLowerCase();
|
|
@@ -123,15 +664,41 @@ export function columnMatches(globs, column) {
|
|
|
123
664
|
return p.slice(p.lastIndexOf('.') + 1) === keyLower || p.slice(p.lastIndexOf('.') + 1) === bare;
|
|
124
665
|
});
|
|
125
666
|
}
|
|
667
|
+
function ownCount(bag, key) {
|
|
668
|
+
if (!Object.prototype.hasOwnProperty.call(bag, key))
|
|
669
|
+
return 0;
|
|
670
|
+
const n = bag[key];
|
|
671
|
+
return typeof n === 'number' ? n : 0;
|
|
672
|
+
}
|
|
673
|
+
function addOwnCount(bag, key, n) {
|
|
674
|
+
Object.defineProperty(bag, key, {
|
|
675
|
+
value: ownCount(bag, key) + n,
|
|
676
|
+
writable: true,
|
|
677
|
+
enumerable: true,
|
|
678
|
+
configurable: true,
|
|
679
|
+
});
|
|
680
|
+
}
|
|
126
681
|
export function mergeByClass(into, add) {
|
|
127
682
|
if (!add)
|
|
128
683
|
return into;
|
|
129
|
-
for (const
|
|
684
|
+
for (const key of Object.keys(add)) {
|
|
685
|
+
const n = ownCount(add, key);
|
|
130
686
|
if (n > 0)
|
|
131
|
-
|
|
687
|
+
addOwnCount(into, key, n);
|
|
132
688
|
}
|
|
133
689
|
return into;
|
|
134
690
|
}
|
|
691
|
+
function replaceWithLabel(text, matches, label) {
|
|
692
|
+
let out = '';
|
|
693
|
+
let cursor = 0;
|
|
694
|
+
for (const match of matches) {
|
|
695
|
+
out += text.slice(cursor, match.start);
|
|
696
|
+
out += label;
|
|
697
|
+
cursor = match.end;
|
|
698
|
+
}
|
|
699
|
+
out += text.slice(cursor);
|
|
700
|
+
return out;
|
|
701
|
+
}
|
|
135
702
|
export function applyCustomPatterns(text, compiled, column) {
|
|
136
703
|
if (!compiled.length)
|
|
137
704
|
return { text, count: 0, byClass: {} };
|
|
@@ -143,21 +710,16 @@ export function applyCustomPatterns(text, compiled, column) {
|
|
|
143
710
|
if (!column || !columnMatches(rule.columns, column))
|
|
144
711
|
continue;
|
|
145
712
|
}
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
const next = out.replace(re, () => {
|
|
149
|
-
hits++;
|
|
150
|
-
return rule.label;
|
|
151
|
-
});
|
|
152
|
-
if (hits === 0)
|
|
713
|
+
const hits = rule.matches(out);
|
|
714
|
+
if (hits.length === 0)
|
|
153
715
|
continue;
|
|
154
|
-
if (hits > CUSTOM_PATTERN_MAX_MATCHES) {
|
|
155
|
-
byClass
|
|
716
|
+
if (hits.length > CUSTOM_PATTERN_MAX_MATCHES) {
|
|
717
|
+
addOwnCount(byClass, rule.name, 1);
|
|
156
718
|
return { text: rule.label, count: count + 1, byClass };
|
|
157
719
|
}
|
|
158
|
-
out =
|
|
159
|
-
count += hits;
|
|
160
|
-
byClass
|
|
720
|
+
out = replaceWithLabel(out, hits, rule.label);
|
|
721
|
+
count += hits.length;
|
|
722
|
+
addOwnCount(byClass, rule.name, hits.length);
|
|
161
723
|
}
|
|
162
724
|
return { text: out, count, byClass };
|
|
163
725
|
}
|