@conarium-ai/core 0.2.54 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/LIMITATIONS.md +94 -1
- package/LIMITATIONS.tr.md +96 -1
- package/dist/custom_patterns.d.ts +13 -12
- package/dist/custom_patterns.d.ts.map +1 -1
- package/dist/custom_patterns.js +576 -14
- package/dist/custom_patterns.js.map +1 -1
- package/dist/governance.d.ts.map +1 -1
- package/dist/governance.js +37 -139
- package/dist/governance.js.map +1 -1
- package/dist/mask/belt.d.ts +36 -0
- package/dist/mask/belt.d.ts.map +1 -0
- package/dist/mask/belt.js +771 -0
- package/dist/mask/belt.js.map +1 -0
- package/dist/mask/core.d.ts +36 -0
- package/dist/mask/core.d.ts.map +1 -0
- package/dist/mask/core.js +119 -0
- package/dist/mask/core.js.map +1 -0
- package/dist/mask/detectors/bearer.d.ts +6 -0
- package/dist/mask/detectors/bearer.d.ts.map +1 -0
- package/dist/mask/detectors/bearer.js +30 -0
- package/dist/mask/detectors/bearer.js.map +1 -0
- package/dist/mask/detectors/connection.d.ts +7 -0
- package/dist/mask/detectors/connection.d.ts.map +1 -0
- package/dist/mask/detectors/connection.js +20 -0
- package/dist/mask/detectors/connection.js.map +1 -0
- package/dist/mask/detectors/custom.d.ts +12 -0
- package/dist/mask/detectors/custom.d.ts.map +1 -0
- package/dist/mask/detectors/custom.js +44 -0
- package/dist/mask/detectors/custom.js.map +1 -0
- package/dist/mask/detectors/email.d.ts +3 -0
- package/dist/mask/detectors/email.d.ts.map +1 -0
- package/dist/mask/detectors/email.js +41 -0
- package/dist/mask/detectors/email.js.map +1 -0
- package/dist/mask/detectors/embedded.d.ts +3 -0
- package/dist/mask/detectors/embedded.d.ts.map +1 -0
- package/dist/mask/detectors/embedded.js +73 -0
- package/dist/mask/detectors/embedded.js.map +1 -0
- package/dist/mask/detectors/iban.d.ts +3 -0
- package/dist/mask/detectors/iban.d.ts.map +1 -0
- package/dist/mask/detectors/iban.js +92 -0
- package/dist/mask/detectors/iban.js.map +1 -0
- package/dist/mask/detectors/ip.d.ts +3 -0
- package/dist/mask/detectors/ip.d.ts.map +1 -0
- package/dist/mask/detectors/ip.js +42 -0
- package/dist/mask/detectors/ip.js.map +1 -0
- package/dist/mask/detectors/labelled_secret.d.ts +6 -0
- package/dist/mask/detectors/labelled_secret.d.ts.map +1 -0
- package/dist/mask/detectors/labelled_secret.js +31 -0
- package/dist/mask/detectors/labelled_secret.js.map +1 -0
- package/dist/mask/detectors/mrz.d.ts +3 -0
- package/dist/mask/detectors/mrz.d.ts.map +1 -0
- package/dist/mask/detectors/mrz.js +42 -0
- package/dist/mask/detectors/mrz.js.map +1 -0
- package/dist/mask/detectors/name.d.ts +6 -0
- package/dist/mask/detectors/name.d.ts.map +1 -0
- package/dist/mask/detectors/name.js +41 -0
- package/dist/mask/detectors/name.js.map +1 -0
- package/dist/mask/detectors/numeric.d.ts +3 -0
- package/dist/mask/detectors/numeric.d.ts.map +1 -0
- package/dist/mask/detectors/numeric.js +240 -0
- package/dist/mask/detectors/numeric.js.map +1 -0
- package/dist/mask/detectors/secret.d.ts +6 -0
- package/dist/mask/detectors/secret.d.ts.map +1 -0
- package/dist/mask/detectors/secret.js +31 -0
- package/dist/mask/detectors/secret.js.map +1 -0
- package/dist/mask/detectors/whole_base64.d.ts +3 -0
- package/dist/mask/detectors/whole_base64.d.ts.map +1 -0
- package/dist/mask/detectors/whole_base64.js +28 -0
- package/dist/mask/detectors/whole_base64.js.map +1 -0
- package/dist/mask/legacy_0254/conn_linear.d.ts +26 -0
- package/dist/mask/legacy_0254/conn_linear.d.ts.map +1 -0
- package/dist/mask/legacy_0254/conn_linear.js +97 -0
- package/dist/mask/legacy_0254/conn_linear.js.map +1 -0
- package/dist/mask/legacy_0254/custom_patterns.d.ts +42 -0
- package/dist/mask/legacy_0254/custom_patterns.d.ts.map +1 -0
- package/dist/mask/legacy_0254/custom_patterns.js +164 -0
- package/dist/mask/legacy_0254/custom_patterns.js.map +1 -0
- package/dist/mask/legacy_0254/digit_pii.d.ts +46 -0
- package/dist/mask/legacy_0254/digit_pii.d.ts.map +1 -0
- package/dist/mask/legacy_0254/digit_pii.js +230 -0
- package/dist/mask/legacy_0254/digit_pii.js.map +1 -0
- package/dist/mask/legacy_0254/iban.d.ts +56 -0
- package/dist/mask/legacy_0254/iban.d.ts.map +1 -0
- package/dist/mask/legacy_0254/iban.js +171 -0
- package/dist/mask/legacy_0254/iban.js.map +1 -0
- package/dist/mask/legacy_0254/ip_detect.d.ts +28 -0
- package/dist/mask/legacy_0254/ip_detect.d.ts.map +1 -0
- package/dist/mask/legacy_0254/ip_detect.js +92 -0
- package/dist/mask/legacy_0254/ip_detect.js.map +1 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.d.ts +20 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.d.ts.map +1 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.js +171 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.js.map +1 -0
- package/dist/mask/legacy_0254/legacy_mask_text.d.ts +11 -0
- package/dist/mask/legacy_0254/legacy_mask_text.d.ts.map +1 -0
- package/dist/mask/legacy_0254/legacy_mask_text.js +46 -0
- package/dist/mask/legacy_0254/legacy_mask_text.js.map +1 -0
- package/dist/mask/legacy_0254/mrz.d.ts +7 -0
- package/dist/mask/legacy_0254/mrz.d.ts.map +1 -0
- package/dist/mask/legacy_0254/mrz.js +74 -0
- package/dist/mask/legacy_0254/mrz.js.map +1 -0
- package/dist/mask/legacy_0254/pii_normalize.d.ts +38 -0
- package/dist/mask/legacy_0254/pii_normalize.d.ts.map +1 -0
- package/dist/mask/legacy_0254/pii_normalize.js +96 -0
- package/dist/mask/legacy_0254/pii_normalize.js.map +1 -0
- package/dist/mask/legacy_0254/safe-object.d.ts +14 -0
- package/dist/mask/legacy_0254/safe-object.d.ts.map +1 -0
- package/dist/mask/legacy_0254/safe-object.js +27 -0
- package/dist/mask/legacy_0254/safe-object.js.map +1 -0
- package/dist/mask/legacy_0254/tckn.d.ts +10 -0
- package/dist/mask/legacy_0254/tckn.d.ts.map +1 -0
- package/dist/mask/legacy_0254/tckn.js +70 -0
- package/dist/mask/legacy_0254/tckn.js.map +1 -0
- package/dist/mask/legacy_0254/types.d.ts +207 -0
- package/dist/mask/legacy_0254/types.d.ts.map +1 -0
- package/dist/mask/legacy_0254/types.js +2 -0
- package/dist/mask/legacy_0254/types.js.map +1 -0
- package/dist/mask-text.d.ts +11 -1
- package/dist/mask-text.d.ts.map +1 -1
- package/dist/mask-text.js +189 -30
- package/dist/mask-text.js.map +1 -1
- package/docs/claims/reviews/0.3.0.md +72 -0
- package/package.json +3 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,74 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0 - masking rewritten: each detector reads the original text, and the 0.2.54 steps run alongside
|
|
4
|
+
|
|
5
|
+
The masking that runs over audit arguments and error text (`maskSensitiveText`)
|
|
6
|
+
and over what the model reads (`Governance.maskPII`) was two copies of one
|
|
7
|
+
sequence of steps, each step reading the output of the step before it. A step
|
|
8
|
+
could hide a value from the next one. Both paths now run one core: every
|
|
9
|
+
detector reads the same normalised text, the spans they find are merged, and
|
|
10
|
+
the text is replaced once.
|
|
11
|
+
|
|
12
|
+
Values 0.2.54 left readable and 0.3.0 masks:
|
|
13
|
+
|
|
14
|
+
- a Bearer token, or a value behind `api_key=` or `password:`, that is shaped
|
|
15
|
+
like an IBAN (two letters, two digits, then letters and digits);
|
|
16
|
+
- a word shaped like an IBAN hiding the words after it
|
|
17
|
+
(`ref AB12cdef 05321234567`);
|
|
18
|
+
- a passport MRZ whose document number starts with two letters and two digits,
|
|
19
|
+
or whose personal-number field holds a digit run of its own;
|
|
20
|
+
- a number joined to the word before it by one separator
|
|
21
|
+
(`siparis 2024 10000000146`, `oda 5 4111 1111 1111 1111`), when the joined
|
|
22
|
+
part passes its checksum or is a Turkish mobile number;
|
|
23
|
+
- a Turkish mobile number written `+90 532 123 45 67`;
|
|
24
|
+
- the tail of an IBAN after an IBAN-shaped run that passed the checksum by
|
|
25
|
+
chance and ran into it.
|
|
26
|
+
|
|
27
|
+
Each field is also run through a frozen copy of the 0.2.54 steps, and every
|
|
28
|
+
span that copy masked is masked again. When that copy's output cannot be lined
|
|
29
|
+
up with the text, the field is masked whole. A position-by-position comparison
|
|
30
|
+
with 0.2.54 over 8 000 generated cases found no character that 0.2.54 masked and
|
|
31
|
+
0.3.0 shows.
|
|
32
|
+
|
|
33
|
+
Changes an operator can see:
|
|
34
|
+
|
|
35
|
+
- `maskedCount` counts masked regions. A value that a label and a detector both
|
|
36
|
+
cover is one region: `token: a@b.com` was 2 and is 1.
|
|
37
|
+
- `byClass` counts the custom rules the core matched on the original text. A
|
|
38
|
+
rule that matched only text 0.2.54 had already masked is no longer counted.
|
|
39
|
+
- The audit path now also masks `sk_test_` keys, names after a title or a field
|
|
40
|
+
label, and a field that is entirely base64 of a personal value. The model path
|
|
41
|
+
already did.
|
|
42
|
+
- Custom patterns run on a linear-time regular expression engine (`re2js`, a new
|
|
43
|
+
dependency with no dependencies of its own). In 0.2.54 an accepted pattern
|
|
44
|
+
such as `(?:a?){30}z` could hold a field's masking for minutes. Each pattern
|
|
45
|
+
is parsed once with `@eslint-community/regexpp` (also new, also without
|
|
46
|
+
dependencies of its own) and written out for the engine from that parse. The
|
|
47
|
+
refusal rules are 0.2.54's, plus four forms where the engine would match
|
|
48
|
+
differently from JavaScript: a multiline modifier group `(?m:…)`, a
|
|
49
|
+
case-insensitive modifier group in a pattern that uses `\w`, `\W`, `\b` or
|
|
50
|
+
`\B`, an optional or repeated group that lists an empty choice before a
|
|
51
|
+
non-empty one (`(?:|a)?`), and a lone surrogate, escaped or typed. Repeats
|
|
52
|
+
nested so that their counts multiply past 1000 (`(?:(?:a{32})){32}`), which
|
|
53
|
+
0.2.54 accepted, are refused as nested quantifiers: the engine cannot compile
|
|
54
|
+
them. Unicode property classes
|
|
55
|
+
take their code points from the running JavaScript engine; every property
|
|
56
|
+
name JavaScript accepts on Node 24 was compared and is accepted (see
|
|
57
|
+
LIMITATIONS, "Custom patterns: what is refused, and the engine they run on").
|
|
58
|
+
- If masking throws, the field is masked whole and counted once.
|
|
59
|
+
- Between a title or label and a masked name, the space is kept as written:
|
|
60
|
+
`Yetkili:` followed by a tab and a name gives `Yetkili:` a tab and
|
|
61
|
+
`[MASKED_PII]`. 0.2.54 wrote one space there on the model path.
|
|
62
|
+
- On the model path, the masks over the same title without a dot starting line
|
|
63
|
+
after line (`Sayin Ahmet`, `Sayin Mehmet`, …) can merge into one. 0.2.54
|
|
64
|
+
wrote one mask for every one or two of those lines.
|
|
65
|
+
- The connection-string step no longer takes quadratic time on a long run of
|
|
66
|
+
scheme characters before `://`; it is one linear pass with the same matches.
|
|
67
|
+
|
|
68
|
+
Time: each field also runs the 0.2.54 copy. On the comparison inputs masking
|
|
69
|
+
took up to about four times as long as 0.2.54 (a one-megabyte field of dense
|
|
70
|
+
e-mail addresses, about 170 ms) and up to about three times on the others.
|
|
71
|
+
|
|
3
72
|
## 0.2.54 - two operator scripts leave the tarball
|
|
4
73
|
|
|
5
74
|
Nothing in the tool's behaviour changed. `scripts/conarium_sabah_brief.mjs` and
|
package/LIMITATIONS.md
CHANGED
|
@@ -13,7 +13,7 @@ on the roadmap.
|
|
|
13
13
|
## Not 1.0
|
|
14
14
|
<!-- s: not-1.0 -->
|
|
15
15
|
|
|
16
|
-
Version is 0.
|
|
16
|
+
Version is 0.3.x. The API can break.
|
|
17
17
|
|
|
18
18
|
## Node 20 is the floor, and Node 20 is past end-of-life
|
|
19
19
|
<!-- s: node-20-floor -->
|
|
@@ -55,6 +55,99 @@ Structured columns: deterministic (`maskColumns`).
|
|
|
55
55
|
Free text: best effort (carry-over of values this policy already masks; labelled names).
|
|
56
56
|
A bare name in running prose is not detected.
|
|
57
57
|
|
|
58
|
+
## A labelled secret is masked to its first delimiter
|
|
59
|
+
<!-- s: labelled-secret-first-delimiter -->
|
|
60
|
+
|
|
61
|
+
After a label such as `password:` or `pwd:` the value is masked up to the first
|
|
62
|
+
space, comma, semicolon, quote or brace. The rest is masked only if another
|
|
63
|
+
detector recognises it. `pwd: 4111 1111 1111 1112` leaves `1111 1111 1112`
|
|
64
|
+
readable, because that card number fails its checksum. 0.2.54 did the same.
|
|
65
|
+
|
|
66
|
+
## A number joined to its neighbours is found only when it proves itself
|
|
67
|
+
<!-- s: joined-number-windows -->
|
|
68
|
+
|
|
69
|
+
When a number is joined to the words or numbers around it by single separators,
|
|
70
|
+
a part of it is masked only when that part passes a checksum (card, Turkish ID
|
|
71
|
+
number) or is a Turkish mobile number. A joined landline (`siparis 2024
|
|
72
|
+
02321234567`) and a number with no separator before it
|
|
73
|
+
(`siparis 202410000000146`) stay readable. A long
|
|
74
|
+
run of groups can be masked although it is not one number, when some of its
|
|
75
|
+
groups happen to pass a checksum.
|
|
76
|
+
|
|
77
|
+
## Each field also runs the 0.2.54 masking steps
|
|
78
|
+
<!-- s: legacy-belt -->
|
|
79
|
+
|
|
80
|
+
Every field is also masked by a frozen copy of the 0.2.54 steps, and every span
|
|
81
|
+
that copy masked is masked again. That costs time: on the comparison inputs,
|
|
82
|
+
masking took up to about four times as long as 0.2.54 (a one-megabyte field of
|
|
83
|
+
dense e-mail addresses, about 170 ms), and up to about three times on the
|
|
84
|
+
others. Where the copy's output cannot be lined up with the text, the whole
|
|
85
|
+
field is masked. Some text next to a masked value can then be hidden although it
|
|
86
|
+
is not personal: a title without a dot (`Bay`, `Bayan`) next to a masked number,
|
|
87
|
+
or a custom rule's label written literally in the source text. On the model
|
|
88
|
+
path, when the same title without a dot starts line after line (`Sayin Ahmet`,
|
|
89
|
+
`Sayin Mehmet`, …), the copy's output can fit the text in more than one way,
|
|
90
|
+
and the masks over those lines can then merge into one.
|
|
91
|
+
|
|
92
|
+
## Custom patterns: what is refused, and the engine they run on
|
|
93
|
+
<!-- s: custom-pattern-shapes -->
|
|
94
|
+
|
|
95
|
+
Custom patterns run on a linear-time regular expression engine (re2js), so no
|
|
96
|
+
accepted pattern can hold a field's masking for minutes; in 0.2.54 some could.
|
|
97
|
+
The patterns refused when the policy loads are the ones 0.2.54 refused
|
|
98
|
+
(lookaround, backreferences, `+`, `*`, an open-ended `{n,}`, and the older group
|
|
99
|
+
rules), plus four forms where the engine would match differently from
|
|
100
|
+
JavaScript:
|
|
101
|
+
|
|
102
|
+
- a modifier group that turns on the multiline flag (`(?m:…)`; JavaScript's
|
|
103
|
+
multiline `^` and `$` also stop at a carriage return, U+2028 and U+2029);
|
|
104
|
+
- a pattern with a case-insensitive modifier group (`(?i:…)`) that also uses
|
|
105
|
+
`\w`, `\W`, `\b` or `\B` anywhere (on Node 24 such a group makes `\w` take the
|
|
106
|
+
Kelvin sign and the long s, outside the group as well);
|
|
107
|
+
- an optional or repeated group that lists an empty choice before a non-empty
|
|
108
|
+
one, such as `(?:|a)?` or `(?:a??)?` (JavaScript skips the empty choice in
|
|
109
|
+
that position, the engine takes it). The rule is structural, so it also
|
|
110
|
+
refuses some groups both engines would match the same way, such as
|
|
111
|
+
`(?:$|a)?`;
|
|
112
|
+
- a surrogate escape that is not half of a `\uHHHH\uLLLL` pair, such as
|
|
113
|
+
`\u{D83D}`, or a lone surrogate typed as a raw character.
|
|
114
|
+
|
|
115
|
+
The 0.2.54 quantifier cap also reads a braced escape written with decimal
|
|
116
|
+
digits only as a quantifier: `\u{100}`, `\u{65}` and `\u{2028}` are refused as
|
|
117
|
+
unbounded or nested quantifiers, in 0.2.54 as now; `\u0100`, `\u0065` and
|
|
118
|
+
`\u2028` are accepted.
|
|
119
|
+
|
|
120
|
+
One more refusal comes from the engine itself: repeats nested so that their
|
|
121
|
+
largest counts multiply past 1000, such as `(?:(?:a{32})){32}`, are refused as
|
|
122
|
+
unbounded or nested quantifiers. 0.2.54 accepted them. Up to 1000, such as
|
|
123
|
+
`(?:(?:a{25})){40}`, they are accepted.
|
|
124
|
+
|
|
125
|
+
Modifier groups compile only on Node versions that support them, in 0.2.54 as
|
|
126
|
+
now. Unicode property classes (`\p{…}`, `\P{…}`) take their code points from
|
|
127
|
+
the JavaScript engine Conarium runs on, so they match what 0.2.54 matched on
|
|
128
|
+
the same Node; the first use of each property costs about 25 to 45 ms when the
|
|
129
|
+
policy loads, once per process. Matches are meant to equal JavaScript's with the
|
|
130
|
+
`u` flag. That was tested on generated patterns over letters, digits, classes
|
|
131
|
+
and negated classes (with `\s`, `\S` and `\b` inside them), `[^]`, `.`, `\s`,
|
|
132
|
+
`\d`, `\w`, `\b`, `^`, `$`, `\u` escapes, bounded and lazy quantifiers,
|
|
133
|
+
alternation with empty choices and `i`/`s` modifier groups, with line breaks,
|
|
134
|
+
no-break and other Unicode spaces, the Kelvin sign and the long s in the text;
|
|
135
|
+
on listed surrogate-pair and class corners; and on every Unicode property name
|
|
136
|
+
JavaScript accepts on Node 24: all general categories by short and long name,
|
|
137
|
+
every binary property in the ECMAScript table with its alias, and 175 scripts
|
|
138
|
+
with 169 four-letter aliases under `Script=`, `sc=`, `Script_Extensions=` and
|
|
139
|
+
`scx=`. Two differences stay:
|
|
140
|
+
|
|
141
|
+
- a match of nothing (for example `\B`) between the two halves of an emoji,
|
|
142
|
+
which JavaScript reports and the engine does not; it hides no text either way;
|
|
143
|
+
- inside a case-insensitive modifier group and outside a class, seven letters
|
|
144
|
+
beyond Latin-1, typed or escaped (Ÿ, ſ, Μ, μ, ẞ, the Kelvin sign `\u212A` and
|
|
145
|
+
the Angstrom sign `\u212B`), match their Latin-1 case partners (ÿ; s and S;
|
|
146
|
+
µ; µ; ß; k and K; å and Å) in the engine on every text. On Node 24,
|
|
147
|
+
JavaScript matches them only when the text also holds a character beyond
|
|
148
|
+
Latin-1. The engine masks more there; it shows nothing JavaScript would hide.
|
|
149
|
+
Inside a class (`(?i:[\u212A])`) both match the partners.
|
|
150
|
+
|
|
58
151
|
## One production install
|
|
59
152
|
<!-- s: one-prod-install -->
|
|
60
153
|
|
package/LIMITATIONS.tr.md
CHANGED
|
@@ -12,7 +12,7 @@ güvence. Bağımsız sızma testi de henüz yok — o yol haritasında.
|
|
|
12
12
|
## 1.0 değil
|
|
13
13
|
<!-- s: not-1.0 -->
|
|
14
14
|
|
|
15
|
-
Sürüm 0.
|
|
15
|
+
Sürüm 0.3.x. API kırılabilir.
|
|
16
16
|
|
|
17
17
|
## Taban Node 20, ve Node 20 artık yama almıyor
|
|
18
18
|
<!-- s: node-20-floor -->
|
|
@@ -52,6 +52,101 @@ Yapılandırılmış kolonlar: deterministik (`maskColumns`).
|
|
|
52
52
|
Serbest metin: en iyi çaba (bu politikanın zaten maskelediği değerlerin taşınması; etiketli isimler).
|
|
53
53
|
Düz cümledeki çıplak isim yakalanmaz.
|
|
54
54
|
|
|
55
|
+
## Etiketli bir sır ilk ayırıcıya kadar maskelenir
|
|
56
|
+
<!-- s: labelled-secret-first-delimiter -->
|
|
57
|
+
|
|
58
|
+
`password:` ya da `pwd:` gibi bir etiketten sonra değer ilk boşluğa, virgüle,
|
|
59
|
+
noktalı virgüle, tırnağa ya da süslü paranteze kadar maskelenir. Kalanı ancak
|
|
60
|
+
başka bir dedektör tanırsa maskelenir. `pwd: 4111 1111 1111 1112` girdisinde
|
|
61
|
+
`1111 1111 1112` okunur kalır, çünkü o kart numarası sağlamadan geçmez. 0.2.54
|
|
62
|
+
de aynısını yapıyordu.
|
|
63
|
+
|
|
64
|
+
## Komşularına bitişik bir numara ancak kendini kanıtlarsa bulunur
|
|
65
|
+
<!-- s: joined-number-windows -->
|
|
66
|
+
|
|
67
|
+
Bir numara çevresindeki kelimelere ya da numaralara tek ayırıcıyla bitişikse,
|
|
68
|
+
içindeki bir parça ancak sağlamadan geçerse (kart, T.C. kimlik numarası) ya da
|
|
69
|
+
Türk cep telefonu numarasıysa maskelenir. Bitişik bir sabit hat (`siparis 2024
|
|
70
|
+
02321234567`) ve önünde ayırıcı olmayan bir numara (`siparis 202410000000146`)
|
|
71
|
+
okunur kalır. Bazı
|
|
72
|
+
grupları tesadüfen sağlamadan geçen uzun bir grup dizisi, tek bir numara olmadığı
|
|
73
|
+
hâlde maskelenebilir.
|
|
74
|
+
|
|
75
|
+
## Her alan 0.2.54'ün maskeleme adımlarından da geçer
|
|
76
|
+
<!-- s: legacy-belt -->
|
|
77
|
+
|
|
78
|
+
Her alan 0.2.54 adımlarının dondurulmuş bir kopyasından da geçirilir ve o
|
|
79
|
+
kopyanın maskelediği her aralık yeniden maskelenir. Bunun bir süre bedeli var:
|
|
80
|
+
karşılaştırma girdilerinde maskeleme 0.2.54'ün en çok yaklaşık dört katı sürdü
|
|
81
|
+
(yoğun e-posta adresleriyle dolu bir megabaytlık alan, yaklaşık 170 ms), diğerlerinde
|
|
82
|
+
en çok yaklaşık üç katı. Kopyanın çıktısı metinle hizalanamazsa alanın tamamı
|
|
83
|
+
maskelenir. O zaman maskeli bir değerin yanındaki, kişisel olmayan bir metin de
|
|
84
|
+
gizlenebilir: maskeli bir numaranın yanındaki noktasız bir unvan (`Bay`, `Bayan`)
|
|
85
|
+
ya da kaynak metinde aynen geçen bir özel kural etiketi. Model yolunda aynı
|
|
86
|
+
noktasız unvan satır satır tekrarlanırsa (`Sayın Ahmet`, `Sayın Mehmet`, …)
|
|
87
|
+
kopyanın çıktısı metne birden çok biçimde oturabilir; o zaman o satırlardaki
|
|
88
|
+
maskeler tek maskede birleşebilir.
|
|
89
|
+
|
|
90
|
+
## Özel desenler: neler reddedilir ve hangi motorda çalışırlar
|
|
91
|
+
<!-- s: custom-pattern-shapes -->
|
|
92
|
+
|
|
93
|
+
Özel desenler doğrusal zamanlı bir düzenli ifade motorunda (re2js) çalışır; bu
|
|
94
|
+
yüzden kabul edilen hiçbir desen bir alanın maskelenmesini dakikalarca
|
|
95
|
+
kilitleyemez. 0.2.54'te bazıları kilitleyebiliyordu. Politika yüklenirken
|
|
96
|
+
reddedilen desenler 0.2.54'ün reddettikleridir (ileri/geri bakış, geri başvuru,
|
|
97
|
+
`+`, `*`, üst sınırı olmayan `{n,}` ve eski grup kuralları); buna, motorun
|
|
98
|
+
JavaScript'ten farklı eşleşeceği dört biçim eklendi:
|
|
99
|
+
|
|
100
|
+
- çok satır bayrağını açan bir değiştirici grup (`(?m:…)`; JavaScript'in çok
|
|
101
|
+
satırlı `^` ve `$`'ı satır başı karakterinde, U+2028 ve U+2029'da da durur);
|
|
102
|
+
- büyük/küçük harf duyarsız bir değiştirici grup (`(?i:…)`) içeren ve herhangi
|
|
103
|
+
bir yerinde `\w`, `\W`, `\b` ya da `\B` kullanan bir desen (Node 24'te böyle bir
|
|
104
|
+
grup, `\w`'nin grubun dışında da Kelvin işaretini ve uzun s'yi almasına yol
|
|
105
|
+
açar);
|
|
106
|
+
- boş bir seçeneği boş olmayan bir seçenekten önce sayan isteğe bağlı ya da
|
|
107
|
+
yinelenen bir grup, örneğin `(?:|a)?` veya `(?:a??)?` (JavaScript o konumda
|
|
108
|
+
boş seçeneği atlar, motor alır). Kural yapısaldır; bu yüzden iki motorun aynı
|
|
109
|
+
eşleyeceği bazı grupları da reddeder, örneğin `(?:$|a)?`;
|
|
110
|
+
- bir `\uHHHH\uLLLL` çiftinin yarısı olmayan bir vekil kaçışı, örneğin
|
|
111
|
+
`\u{D83D}`, ya da ham karakter olarak yazılmış tek bir vekil.
|
|
112
|
+
|
|
113
|
+
0.2.54'ün niceleyici sınırı, yalnız ondalık rakamlarla yazılmış süslü
|
|
114
|
+
parantezli bir kaçışı da niceleyici sayar: `\u{100}`, `\u{65}` ve `\u{2028}`,
|
|
115
|
+
0.2.54'te olduğu gibi, sınırsız ya da iç içe niceleyici diye reddedilir;
|
|
116
|
+
`\u0100`, `\u0065` ve `\u2028` kabul edilir.
|
|
117
|
+
|
|
118
|
+
Bir ret de motorun kendisinden gelir: en büyük sayılarının çarpımı 1000'i
|
|
119
|
+
aşacak biçimde iç içe geçmiş tekrarlar, örneğin `(?:(?:a{32})){32}`, sınırsız ya
|
|
120
|
+
da iç içe niceleyici diye reddedilir. 0.2.54 bunları kabul ediyordu. 1000'e
|
|
121
|
+
kadar olanlar, örneğin `(?:(?:a{25})){40}`, kabul edilir.
|
|
122
|
+
|
|
123
|
+
Değiştirici gruplar, 0.2.54'te olduğu gibi, yalnız bunları destekleyen Node
|
|
124
|
+
sürümlerinde derlenir. Unicode özellik sınıfları (`\p{…}`, `\P{…}`) kod
|
|
125
|
+
noktalarını Conarium'un çalıştığı JavaScript motorundan alır; bu yüzden aynı
|
|
126
|
+
Node'da 0.2.54'ün eşlediğini eşler. Her özelliğin ilk kullanımı politika
|
|
127
|
+
yüklenirken süreç başına bir kez yaklaşık 25–45 ms tutar. Eşleşmelerin
|
|
128
|
+
JavaScript'in `u` bayraklı eşleşmeleriyle aynı olması amaçlanır. Bu, harf, rakam,
|
|
129
|
+
sınıf ve olumsuz sınıf (içlerinde `\s`, `\S` ve `\b` ile), `[^]`, `.`, `\s`, `\d`,
|
|
130
|
+
`\w`, `\b`, `^`, `$`, `\u` kaçışları, sınırlı ve tembel niceleyici, boş seçenekli
|
|
131
|
+
alternasyon ve `i`/`s` değiştirici grupları içeren üretilmiş desenlerde, metinde
|
|
132
|
+
satır sonu, bölünmez boşluk ve diğer Unicode boşlukları, Kelvin işareti ve uzun s
|
|
133
|
+
ile; listelenmiş vekil çifti ve sınıf kenar durumlarında; ve JavaScript'in Node
|
|
134
|
+
24'te kabul ettiği her Unicode özellik adında sınandı: kısa ve uzun adıyla bütün
|
|
135
|
+
genel kategoriler, ECMAScript tablosundaki her ikili özellik ve kısaltması, ve
|
|
136
|
+
`Script=`, `sc=`, `Script_Extensions=` ve `scx=` altında 175 betik ile 169 dört
|
|
137
|
+
harfli kısaltması. İki fark kalır:
|
|
138
|
+
|
|
139
|
+
- bir emojinin iki yarısı arasındaki boş eşleşme (örneğin `\B`); JavaScript bunu
|
|
140
|
+
bildirir, motor bildirmez. İki durumda da hiçbir metin gizlenmez;
|
|
141
|
+
- büyük/küçük harf duyarsız bir değiştirici grubun içinde ve bir sınıfın
|
|
142
|
+
dışında, Latin-1 dışındaki yedi harf, ister harf ister kaçış olarak yazılsın
|
|
143
|
+
(Ÿ, ſ, Μ, μ, ẞ, Kelvin işareti `\u212A` ve Ångström işareti `\u212B`), motorda
|
|
144
|
+
her metinde Latin-1'deki büyük/küçük eşleriyle (ÿ; s ve S; µ; µ; ß; k ve K; å
|
|
145
|
+
ve Å) eşleşir. Node 24'te JavaScript bunları yalnız metinde Latin-1 dışında
|
|
146
|
+
bir karakter de varsa eşler. Motor orada daha çok maskeler; JavaScript'in
|
|
147
|
+
gizleyeceği hiçbir şeyi açık bırakmaz. Bir sınıfın içinde (`(?i:[\u212A])`)
|
|
148
|
+
ikisi de eşleri eşler.
|
|
149
|
+
|
|
55
150
|
## Tek üretim kurulumu
|
|
56
151
|
<!-- s: one-prod-install -->
|
|
57
152
|
|
|
@@ -1,37 +1,38 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* User-defined PII patterns from config.
|
|
3
|
-
*
|
|
4
|
-
* Same response-path scanner as the built-in detectors — not a second masker.
|
|
5
|
-
* The pattern text is treated as sensitive: errors, logs and receipts carry
|
|
6
|
-
* the rule NAME only.
|
|
7
|
-
*
|
|
8
|
-
* User regex is ReDoS-risky. This module never runs an unbounded pattern:
|
|
9
|
-
* nested / overlapping quantifiers, `+` / `*`, open `{n,}` and lookaround
|
|
10
|
-
* are rejected at compile. A broken rule fails the config (silent drop = leak).
|
|
11
|
-
*/
|
|
12
1
|
import type { CustomPiiPattern } from './types.js';
|
|
13
2
|
export declare const CUSTOM_PATTERN_MAX_SOURCE = 200;
|
|
14
3
|
export declare const CUSTOM_PATTERN_MAX_QUANTIFIER = 64;
|
|
15
4
|
export declare const CUSTOM_PATTERN_MAX_MATCHES = 256;
|
|
16
|
-
export type CustomPatternReject = 'invalid' | 'unsafe' | 'duplicate' | 'empty' | 'too-long' | 'bad-name' | 'bad-label';
|
|
5
|
+
export type CustomPatternReject = 'invalid' | 'unsafe' | 'duplicate' | 'empty' | 'too-long' | 'bad-name' | 'bad-label' | 'unsupported-flag' | 'unsupported-case-word' | 'unsupported-empty-choice' | 'unsupported-surrogate';
|
|
17
6
|
export declare class CustomPatternError extends Error {
|
|
18
7
|
readonly ruleName: string;
|
|
19
8
|
readonly reason: CustomPatternReject;
|
|
20
9
|
constructor(ruleName: string, reason: CustomPatternReject);
|
|
21
10
|
}
|
|
11
|
+
export interface CustomPatternMatch {
|
|
12
|
+
start: number;
|
|
13
|
+
end: number;
|
|
14
|
+
}
|
|
22
15
|
export interface CompiledCustomPattern {
|
|
23
16
|
name: string;
|
|
24
17
|
columns: string[];
|
|
25
18
|
label: string;
|
|
26
19
|
/** Source kept only in memory for lastIndex-safe clones. Never logged. */
|
|
27
20
|
re: RegExp;
|
|
21
|
+
/** Leftmost-first matches, `String.prototype.replace` + `gu` semantics. */
|
|
22
|
+
matches(text: string): CustomPatternMatch[];
|
|
28
23
|
}
|
|
29
24
|
export declare function compileCustomPatterns(list: CustomPiiPattern[] | undefined): CompiledCustomPattern[];
|
|
30
25
|
/**
|
|
31
26
|
* Conservative static reject. Linear class quantifiers (`[0-9]{8}`) pass.
|
|
32
27
|
* Anything that can explode backtracking — or is simply unbounded — does not.
|
|
28
|
+
*
|
|
29
|
+
* This is the 0.2.54 rule. Nested quantifiers, alternation inside a quantified
|
|
30
|
+
* group, and adjacent variable quantifiers are not rejected here: those shapes
|
|
31
|
+
* run on the linear engine.
|
|
33
32
|
*/
|
|
34
33
|
export declare function looksUnsafe(source: string): boolean;
|
|
34
|
+
/** Same scan as {@link CompiledCustomPattern.matches}, for a source string. */
|
|
35
|
+
export declare function matchCustomPatternSource(source: string, text: string): CustomPatternMatch[];
|
|
35
36
|
export declare function columnMatches(globs: string[], column: string): boolean;
|
|
36
37
|
export declare function mergeByClass(into: Record<string, number>, add?: Record<string, number>): Record<string, number>;
|
|
37
38
|
export declare function applyCustomPatterns(text: string, compiled: CompiledCustomPattern[], column?: string): {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom_patterns.d.ts","sourceRoot":"","sources":["../src/custom_patterns.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"custom_patterns.d.ts","sourceRoot":"","sources":["../src/custom_patterns.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElD,eAAO,MAAM,yBAAyB,MAAM,CAAA;AAC5C,eAAO,MAAM,6BAA6B,KAAK,CAAA;AAC/C,eAAO,MAAM,0BAA0B,MAAM,CAAA;AAE7C,MAAM,MAAM,mBAAmB,GAC3B,SAAS,GACT,QAAQ,GACR,WAAW,GACX,OAAO,GACP,UAAU,GACV,UAAU,GACV,WAAW,GACX,kBAAkB,GAClB,uBAAuB,GACvB,0BAA0B,GAC1B,uBAAuB,CAAA;AAoB3B,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAA;IAEpC,YAAY,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,mBAAmB,EAMxD;CACF;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,0EAA0E;IAC1E,EAAE,EAAE,MAAM,CAAA;IACV,2EAA2E;IAC3E,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAAA;CAC5C;AA8CD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,gBAAgB,EAAE,GAAG,SAAS,GACnC,qBAAqB,EAAE,CAUzB;AA8CD;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAkBnD;AAqdD,+EAA+E;AAC/E,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAE3F;AAYD,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAQtE;AAiBD,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC5B,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOxB;AAcD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,qBAAqB,EAAE,EACjC,MAAM,CAAC,EAAE,MAAM,GACd;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAwBlE"}
|