@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +8 -1
- package/bin/sensitive-data-sanitizer.js +49 -28
- package/docs/API.md +192 -1
- package/docs/COVERAGE.md +3 -3
- package/docs/RULES.md +217 -0
- package/package.json +1 -1
- package/src/allow-list.js +93 -0
- package/src/confidence.js +24 -9
- package/src/context.js +121 -25
- package/src/credentials.js +44 -0
- package/src/crypto-addresses.js +74 -0
- package/src/detection.js +21 -6
- package/src/document-person-cues.js +1 -1
- package/src/encrypt.js +95 -0
- package/src/entities.js +4 -3
- package/src/fake.js +29 -1
- package/src/files.js +57 -12
- package/src/gitleaks.js +82 -27
- package/src/identifier-detector.js +36 -0
- package/src/identity.js +14 -0
- package/src/index.d.ts +116 -1
- package/src/index.js +7 -0
- package/src/national-checks.js +209 -0
- package/src/national.js +310 -92
- package/src/pattern-detector.js +219 -0
- package/src/personal-context.js +3 -2
- package/src/personal-evidence.js +233 -0
- package/src/personal.js +42 -10
- package/src/projection.js +85 -26
- package/src/rule-catalog.js +424 -0
- package/src/sanitizer.js +48 -3
- package/src/sarif.js +127 -0
- package/src/token-checksum.js +49 -0
- package/src/transforms.js +108 -25
- package/src/vendor/betterleaks/LICENSE +21 -0
- package/src/vendor/betterleaks/provenance.json +11 -0
- package/src/vendor/betterleaks/rules-0.json +463 -0
- package/src/vendor/betterleaks/rules-1.json +454 -0
- package/src/vendor/betterleaks/rules-2.json +461 -0
- package/src/vendor/betterleaks/rules-3.json +455 -0
- package/src/vendor/betterleaks/rules-4.json +483 -0
- package/src/vendor/betterleaks/rules-5.json +35 -0
- package/src/vendor/entities/aws.json +1 -1
- package/src/vendor/entities/azure.json +6 -2
- package/src/vendor/entities/google.json +2 -1
package/docs/RULES.md
CHANGED
|
@@ -12,16 +12,221 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
12
12
|
- `anthropic`
|
|
13
13
|
- `authorization`
|
|
14
14
|
- `aws`
|
|
15
|
+
- `betterleaks/abuseipdb-api-key.1`
|
|
16
|
+
- `betterleaks/aikido-ci-token`
|
|
17
|
+
- `betterleaks/aikido-client-secret`
|
|
18
|
+
- `betterleaks/airtable-oauth-token`
|
|
19
|
+
- `betterleaks/aiven-auth-token`
|
|
20
|
+
- `betterleaks/alibaba-sts-access-key-secret`
|
|
21
|
+
- `betterleaks/amplitude-secret-key`
|
|
22
|
+
- `betterleaks/apify-api-token`
|
|
23
|
+
- `betterleaks/apollo-api-key.1`
|
|
24
|
+
- `betterleaks/asaas-api-token`
|
|
25
|
+
- `betterleaks/assemblyai-api-key`
|
|
26
|
+
- `betterleaks/auth0-client-secret.1`
|
|
27
|
+
- `betterleaks/azure-app-configuration-connection-string`
|
|
28
|
+
- `betterleaks/azure-servicebus-connection-string`
|
|
29
|
+
- `betterleaks/azure-storage-account-key`
|
|
30
|
+
- `betterleaks/bitbucket-data-center-token`
|
|
31
|
+
- `betterleaks/bitly-access-token`
|
|
32
|
+
- `betterleaks/bitrise-access-token`
|
|
33
|
+
- `betterleaks/box-api-access-token`
|
|
34
|
+
- `betterleaks/brave-search-api-key`
|
|
35
|
+
- `betterleaks/browserstack-access-key.1`
|
|
36
|
+
- `betterleaks/buildkite-service-token`
|
|
37
|
+
- `betterleaks/buildkite-user-access-token`
|
|
38
|
+
- `betterleaks/canadian-digital-service-notify-api-key`
|
|
39
|
+
- `betterleaks/canva-client-secret`
|
|
40
|
+
- `betterleaks/cartesia-api-key.1`
|
|
41
|
+
- `betterleaks/cerebras-api-key`
|
|
42
|
+
- `betterleaks/checkout-secret-key`
|
|
43
|
+
- `betterleaks/circleci-personal-token`
|
|
44
|
+
- `betterleaks/circleci-project-token`
|
|
45
|
+
- `betterleaks/civo-api-key`
|
|
46
|
+
- `betterleaks/clerk-secret-key`
|
|
47
|
+
- `betterleaks/clickup-personal-api-token`
|
|
48
|
+
- `betterleaks/cloudflare-api-key.2`
|
|
49
|
+
- `betterleaks/cloudinary-api-secret`
|
|
50
|
+
- `betterleaks/cloudsmith-api-key`
|
|
51
|
+
- `betterleaks/cockroachlabs-cloud-api-key`
|
|
52
|
+
- `betterleaks/configcat-sdk-key`
|
|
53
|
+
- `betterleaks/configcat-sdk-key-extended`
|
|
54
|
+
- `betterleaks/couchbase-capella-api-key`
|
|
55
|
+
- `betterleaks/coveralls-personal-api-token`
|
|
56
|
+
- `betterleaks/crates-io-api-key`
|
|
57
|
+
- `betterleaks/cursor-api-key`
|
|
58
|
+
- `betterleaks/databento-api-key`
|
|
59
|
+
- `betterleaks/datadog-api-key`
|
|
60
|
+
- `betterleaks/datadog-application-key`
|
|
61
|
+
- `betterleaks/datagov-api-key`
|
|
62
|
+
- `betterleaks/datastax-astra-application-token`
|
|
63
|
+
- `betterleaks/deepgram-api-key`
|
|
64
|
+
- `betterleaks/deepseek-api-key`
|
|
65
|
+
- `betterleaks/deno-account-token`
|
|
66
|
+
- `betterleaks/devcycle-client-sdk-key`
|
|
67
|
+
- `betterleaks/devcycle-mobile-sdk-key`
|
|
68
|
+
- `betterleaks/devcycle-server-sdk-key`
|
|
69
|
+
- `betterleaks/devin-personal-api-key`
|
|
70
|
+
- `betterleaks/devin-service-api-key`
|
|
71
|
+
- `betterleaks/devin-service-user-token`
|
|
72
|
+
- `betterleaks/disqus-api-key`
|
|
73
|
+
- `betterleaks/docker-swarm-join-token`
|
|
74
|
+
- `betterleaks/docker-swarm-unlock-key`
|
|
75
|
+
- `betterleaks/dockerhub-organization-access-token`
|
|
76
|
+
- `betterleaks/dockerhub-personal-access-token`
|
|
77
|
+
- `betterleaks/ebay-client-secret`
|
|
78
|
+
- `betterleaks/elastic-cloud-api-key`
|
|
79
|
+
- `betterleaks/elevenlabs-api-key`
|
|
80
|
+
- `betterleaks/endorlabs-api-key`
|
|
81
|
+
- `betterleaks/endorlabs-api-secret`
|
|
82
|
+
- `betterleaks/etsy-open-api-key`
|
|
83
|
+
- `betterleaks/exoscale-api-key`
|
|
84
|
+
- `betterleaks/fal-api-key.1`
|
|
85
|
+
- `betterleaks/figma-personal-access-header-token`
|
|
86
|
+
- `betterleaks/figma-personal-access-token`
|
|
87
|
+
- `betterleaks/fullstory-api-key`
|
|
88
|
+
- `betterleaks/gcp-application-default-credentials`
|
|
89
|
+
- `betterleaks/gcp-gemini-api`
|
|
90
|
+
- `betterleaks/gcp-service-account`
|
|
91
|
+
- `betterleaks/gitea-access-token`
|
|
92
|
+
- `betterleaks/gitlab-incoming-mail-address-token`
|
|
93
|
+
- `betterleaks/gitlab-pat-routable-versioned`
|
|
94
|
+
- `betterleaks/greptile-api-key`
|
|
95
|
+
- `betterleaks/groq-api-key`
|
|
96
|
+
- `betterleaks/gumroad-access-token`
|
|
97
|
+
- `betterleaks/highnote-secret-live-key`
|
|
98
|
+
- `betterleaks/honeycomb-api-key`
|
|
99
|
+
- `betterleaks/hunter-api-key.1`
|
|
100
|
+
- `betterleaks/ibm-cloud-user-api-key`
|
|
101
|
+
- `betterleaks/influxdb-api-token`
|
|
102
|
+
- `betterleaks/infomaniak-api-token`
|
|
103
|
+
- `betterleaks/instantly-api-key.1`
|
|
104
|
+
- `betterleaks/ionic-personal-access-token`
|
|
105
|
+
- `betterleaks/jumpcloud-api-key`
|
|
106
|
+
- `betterleaks/kagi-api-key`
|
|
107
|
+
- `betterleaks/kimi-api-key`
|
|
108
|
+
- `betterleaks/klaviyo-api-key`
|
|
109
|
+
- `betterleaks/langchain-langsmith-personal-access-token`
|
|
110
|
+
- `betterleaks/langchain-langsmith-service-key`
|
|
111
|
+
- `betterleaks/langfuse-secret-key.1`
|
|
112
|
+
- `betterleaks/lark-app-secret`
|
|
113
|
+
- `betterleaks/lichess-personal-access-token`
|
|
114
|
+
- `betterleaks/lighton-paradigm-api-key`
|
|
115
|
+
- `betterleaks/llama-cloud-api-key.1`
|
|
116
|
+
- `betterleaks/mailersend-api-token`
|
|
117
|
+
- `betterleaks/mem0-api-key.1`
|
|
118
|
+
- `betterleaks/mercury-production-api-token`
|
|
119
|
+
- `betterleaks/mergify-application-key`
|
|
120
|
+
- `betterleaks/midtrans-production-server-client-key`
|
|
121
|
+
- `betterleaks/minimax-api-key`
|
|
122
|
+
- `betterleaks/miro-access-token`
|
|
123
|
+
- `betterleaks/miro-client-secret`
|
|
124
|
+
- `betterleaks/mistral-api-key`
|
|
125
|
+
- `betterleaks/monday-api-token.1`
|
|
126
|
+
- `betterleaks/mongodb-atlas-service-account-secret`
|
|
127
|
+
- `betterleaks/mongodb-connection-string`
|
|
128
|
+
- `betterleaks/mux-access-token-secret.1`
|
|
129
|
+
- `betterleaks/neon-api-key`
|
|
130
|
+
- `betterleaks/ngrok-api-key.1`
|
|
131
|
+
- `betterleaks/nuget-api-key`
|
|
132
|
+
- `betterleaks/nvidia-api-key`
|
|
133
|
+
- `betterleaks/nylas-api-key.1`
|
|
134
|
+
- `betterleaks/ollama-api-key`
|
|
135
|
+
- `betterleaks/onesignal-rich-authentication-token`
|
|
136
|
+
- `betterleaks/onfido-live-api-token-ca`
|
|
137
|
+
- `betterleaks/onfido-live-api-token-eu`
|
|
138
|
+
- `betterleaks/onfido-live-api-token-us`
|
|
139
|
+
- `betterleaks/openrouter-api-key`
|
|
140
|
+
- `betterleaks/openweather-api-key`
|
|
141
|
+
- `betterleaks/opsgenie-api-key.1`
|
|
142
|
+
- `betterleaks/ovh-application-secret`
|
|
143
|
+
- `betterleaks/paddle-live-api-key`
|
|
144
|
+
- `betterleaks/pagerduty-authorization-token.1`
|
|
145
|
+
- `betterleaks/paypal-client-secret.1`
|
|
146
|
+
- `betterleaks/persona-production-api-key`
|
|
147
|
+
- `betterleaks/pinecone-api-key.1`
|
|
148
|
+
- `betterleaks/pinecone-api-key.2`
|
|
149
|
+
- `betterleaks/pinterest-access-token`
|
|
150
|
+
- `betterleaks/plivo-auth-token`
|
|
151
|
+
- `betterleaks/polar-oauth-access-token`
|
|
152
|
+
- `betterleaks/polar-organization-access-token`
|
|
153
|
+
- `betterleaks/polar-personal-access-token`
|
|
154
|
+
- `betterleaks/polymarket-api-key`
|
|
155
|
+
- `betterleaks/polymarket-private-key`
|
|
156
|
+
- `betterleaks/posthog-personal-api-key`
|
|
157
|
+
- `betterleaks/posthog-project-api-key`
|
|
158
|
+
- `betterleaks/postmark-api-token.1`
|
|
159
|
+
- `betterleaks/proof-full-access-api-key`
|
|
160
|
+
- `betterleaks/rainforest-pay-production-api-key`
|
|
161
|
+
- `betterleaks/ramp-client-secret`
|
|
162
|
+
- `betterleaks/razorpay-key-secret.1`
|
|
163
|
+
- `betterleaks/redirect-pizza-api-token.1`
|
|
164
|
+
- `betterleaks/render-api-key`
|
|
165
|
+
- `betterleaks/replicate-api-token`
|
|
166
|
+
- `betterleaks/resend-api-key.1`
|
|
167
|
+
- `betterleaks/retell-api-key.1`
|
|
168
|
+
- `betterleaks/rootly-api-key.1`
|
|
169
|
+
- `betterleaks/runpod-api-key.1`
|
|
170
|
+
- `betterleaks/salesforce-access-token.1`
|
|
171
|
+
- `betterleaks/samsara-api-token.1`
|
|
172
|
+
- `betterleaks/scaleway-secret-key`
|
|
173
|
+
- `betterleaks/scalr-api-access-token.1`
|
|
174
|
+
- `betterleaks/segment-public-api-token.1`
|
|
175
|
+
- `betterleaks/slack-session-cookie`
|
|
176
|
+
- `betterleaks/slack-session-token`
|
|
177
|
+
- `betterleaks/snowflake-programmatic-access-token.1`
|
|
178
|
+
- `betterleaks/sslmate-api-key.1`
|
|
179
|
+
- `betterleaks/stability-ai-api-key`
|
|
180
|
+
- `betterleaks/supabase-management-token`
|
|
181
|
+
- `betterleaks/supabase-project-api-key`
|
|
182
|
+
- `betterleaks/tableau-personal-access-token.1`
|
|
183
|
+
- `betterleaks/tailscale-api-key.1`
|
|
184
|
+
- `betterleaks/telnyx-api-v2-key.1`
|
|
185
|
+
- `betterleaks/temporal-cloud-api-key.1`
|
|
186
|
+
- `betterleaks/thunderstore-api-token.1`
|
|
187
|
+
- `betterleaks/togetherai-api-key`
|
|
188
|
+
- `betterleaks/unkey-root-key.1`
|
|
189
|
+
- `betterleaks/upcloud-api-token`
|
|
190
|
+
- `betterleaks/upstage-api-key`
|
|
191
|
+
- `betterleaks/upstash-redis-rest-token.1`
|
|
192
|
+
- `betterleaks/val-town-api-token.1`
|
|
193
|
+
- `betterleaks/vercel-ai-gateway-key`
|
|
194
|
+
- `betterleaks/vercel-api-token`
|
|
195
|
+
- `betterleaks/vercel-app-access-token`
|
|
196
|
+
- `betterleaks/vercel-app-refresh-token`
|
|
197
|
+
- `betterleaks/vercel-integration-token`
|
|
198
|
+
- `betterleaks/vercel-personal-access-token`
|
|
199
|
+
- `betterleaks/virustotal-api-key.1`
|
|
200
|
+
- `betterleaks/voyageai-api-key`
|
|
201
|
+
- `betterleaks/vultr-api-key.1`
|
|
202
|
+
- `betterleaks/wakatime-api-key.1`
|
|
203
|
+
- `betterleaks/wakatime-api-key.2`
|
|
204
|
+
- `betterleaks/weatherstack-api-key.1`
|
|
205
|
+
- `betterleaks/weights-and-biases-api-key`
|
|
206
|
+
- `betterleaks/weights-and-biases-api-key-v1`
|
|
207
|
+
- `betterleaks/wiz-client-secret.1`
|
|
208
|
+
- `betterleaks/woocommerce-consumer-secret.1`
|
|
209
|
+
- `betterleaks/workato-developer-api-token.1`
|
|
210
|
+
- `betterleaks/workos-production-api-key.1`
|
|
211
|
+
- `betterleaks/xai-api-key`
|
|
212
|
+
- `betterleaks/xendit-production-api-key.1`
|
|
213
|
+
- `betterleaks/zai-api-key`
|
|
214
|
+
- `betterleaks/zoho-client-secret.1`
|
|
215
|
+
- `betterleaks/zoho-oauth-token.1`
|
|
216
|
+
- `betterleaks/zoho-zapi-key.1`
|
|
217
|
+
- `betterleaks/zuplo-consumer-api-key.1`
|
|
15
218
|
- `birth-context`
|
|
16
219
|
- `birth-year`
|
|
17
220
|
- `booking-context`
|
|
18
221
|
- `booking-filename`
|
|
222
|
+
- `card-context`
|
|
19
223
|
- `catalog/aws`
|
|
20
224
|
- `catalog/azure`
|
|
21
225
|
- `catalog/google`
|
|
22
226
|
- `cli-password`
|
|
23
227
|
- `context`
|
|
24
228
|
- `cookie`
|
|
229
|
+
- `credential-pair`
|
|
25
230
|
- `databricks`
|
|
26
231
|
- `detect-secrets`
|
|
27
232
|
- `digitalocean`
|
|
@@ -315,6 +520,7 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
315
520
|
- `google`
|
|
316
521
|
- `grafana`
|
|
317
522
|
- `home-path`
|
|
523
|
+
- `honorific-name`
|
|
318
524
|
- `huggingface`
|
|
319
525
|
- `iban-checksum`
|
|
320
526
|
- `identity-number`
|
|
@@ -329,8 +535,10 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
329
535
|
- `location-gazetteer`
|
|
330
536
|
- `location-label`
|
|
331
537
|
- `location-residence`
|
|
538
|
+
- `login-call`
|
|
332
539
|
- `luhn`
|
|
333
540
|
- `mac`
|
|
541
|
+
- `mail-display-name`
|
|
334
542
|
- `mailchimp`
|
|
335
543
|
- `messenger-handle`
|
|
336
544
|
- `messenger-label`
|
|
@@ -342,6 +550,7 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
342
550
|
- `name-initials`
|
|
343
551
|
- `name-title`
|
|
344
552
|
- `name-variant`
|
|
553
|
+
- `named-value`
|
|
345
554
|
- `national-format`
|
|
346
555
|
- `national-phone`
|
|
347
556
|
- `ner/offline`
|
|
@@ -365,10 +574,13 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
365
574
|
- `passport-format`
|
|
366
575
|
- `passport-issuer`
|
|
367
576
|
- `passport-series`
|
|
577
|
+
- `password-hash`
|
|
368
578
|
- `personal-profile-url`
|
|
369
579
|
- `phone-context`
|
|
580
|
+
- `po-box`
|
|
370
581
|
- `postal-address`
|
|
371
582
|
- `postal-code`
|
|
583
|
+
- `postal-label`
|
|
372
584
|
- `presidio`
|
|
373
585
|
- `private-key`
|
|
374
586
|
- `profile-attribute`
|
|
@@ -378,12 +590,17 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
|
|
|
378
590
|
- `pypi`
|
|
379
591
|
- `quoted-entropy`
|
|
380
592
|
- `residence-prose`
|
|
593
|
+
- `salutation`
|
|
381
594
|
- `school-acronym-context`
|
|
382
595
|
- `secretlint`
|
|
596
|
+
- `self-introduction`
|
|
383
597
|
- `sendgrid`
|
|
384
598
|
- `shopify`
|
|
599
|
+
- `sign-off`
|
|
385
600
|
- `slack`
|
|
386
601
|
- `square`
|
|
602
|
+
- `ssn-label`
|
|
603
|
+
- `state-zip`
|
|
387
604
|
- `strict-age`
|
|
388
605
|
- `strict-attribute`
|
|
389
606
|
- `strict-date`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-foundation/sensitive-data-sanitizer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.20.0",
|
|
4
4
|
"description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { failure } from './detection.js';
|
|
2
|
+
|
|
3
|
+
// Caller-declared values are explicit private policy; an allow list must not
|
|
4
|
+
// silently cancel them. Schema denials (JSON_VALUE) always win as well.
|
|
5
|
+
const declaredRules = new Set([
|
|
6
|
+
'known-personal',
|
|
7
|
+
'confirmed-personal',
|
|
8
|
+
'known-secret',
|
|
9
|
+
]);
|
|
10
|
+
const anchored = new WeakMap();
|
|
11
|
+
|
|
12
|
+
function fullMatcher(pattern) {
|
|
13
|
+
let compiled = anchored.get(pattern);
|
|
14
|
+
if (!compiled) {
|
|
15
|
+
compiled = new RegExp(
|
|
16
|
+
`^(?:${pattern.source})$`,
|
|
17
|
+
pattern.flags.replace(/[gy]/g, '')
|
|
18
|
+
);
|
|
19
|
+
anchored.set(pattern, compiled);
|
|
20
|
+
}
|
|
21
|
+
return compiled;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Inline suppression comments understood by Gitleaks/Betterleaks
|
|
25
|
+
// (gitleaks:allow), detect-secrets and ripsecrets (pragma: allowlist secret),
|
|
26
|
+
// Kingfisher, TruffleHog and SonarQube. Honoured only with inlineAllow: true,
|
|
27
|
+
// because anyone who writes to a log could otherwise suppress redaction.
|
|
28
|
+
const sameLine =
|
|
29
|
+
/(?:gitleaks|betterleaks|kingfisher|trufflehog|sanitizer):(?:allow|ignore)\b|pragma:\s*allowlist(?:\s+secret)?\b(?!\s+nextline)|\bNOSONAR\b/i;
|
|
30
|
+
const nextLine = /pragma:\s*allowlist\s+nextline(?:\s+secret)?\b/i;
|
|
31
|
+
|
|
32
|
+
function lineAt(text, index) {
|
|
33
|
+
const start = text.lastIndexOf('\n', index - 1) + 1;
|
|
34
|
+
const end = text.indexOf('\n', index);
|
|
35
|
+
return { start, end: end < 0 ? text.length : end };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function inlineSuppressed(text, finding) {
|
|
39
|
+
const line = lineAt(text, finding.start);
|
|
40
|
+
if (sameLine.test(text.slice(line.start, line.end))) {
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
if (line.start === 0) {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
const previous = lineAt(text, line.start - 1);
|
|
47
|
+
return nextLine.test(text.slice(previous.start, previous.end));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function validateAllowList(options) {
|
|
51
|
+
if (
|
|
52
|
+
options.inlineAllow !== undefined &&
|
|
53
|
+
typeof options.inlineAllow !== 'boolean'
|
|
54
|
+
) {
|
|
55
|
+
throw failure('ERR_CONFIG');
|
|
56
|
+
}
|
|
57
|
+
const list = options.allowList;
|
|
58
|
+
if (list === undefined) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
if (
|
|
62
|
+
!Array.isArray(list) ||
|
|
63
|
+
list.length > 10000 ||
|
|
64
|
+
!list.every(
|
|
65
|
+
(entry) =>
|
|
66
|
+
(typeof entry === 'string' && entry.length > 0) ||
|
|
67
|
+
entry instanceof RegExp
|
|
68
|
+
)
|
|
69
|
+
) {
|
|
70
|
+
throw failure('ERR_CONFIG');
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Presidio allow_list, Google exclusion rules (MATCHING_TYPE_FULL_MATCH) and
|
|
75
|
+
// DataFog allowlist_patterns: a listed string or pattern must cover the whole
|
|
76
|
+
// detected value. Partial matches never shorten a finding.
|
|
77
|
+
export function allowListed(text, finding, options) {
|
|
78
|
+
if (
|
|
79
|
+
!(options.allowList?.length || options.inlineAllow) ||
|
|
80
|
+
declaredRules.has(finding.rule) ||
|
|
81
|
+
finding.type === 'JSON_VALUE' ||
|
|
82
|
+
(finding.category === 'credential' && !options.allowCredentialDisable)
|
|
83
|
+
) {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
if (options.inlineAllow && inlineSuppressed(text, finding)) {
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
const value = text.slice(finding.start, finding.end);
|
|
90
|
+
return (options.allowList ?? []).some((entry) =>
|
|
91
|
+
typeof entry === 'string' ? entry === value : fullMatcher(entry).test(value)
|
|
92
|
+
);
|
|
93
|
+
}
|
package/src/confidence.js
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
// A valid check digit is common in arbitrary log numbers. Context decides
|
|
2
2
|
// whether that evidence merits a replacement; inspect retains low scores.
|
|
3
|
-
export function threshold(options) {
|
|
4
|
-
return
|
|
3
|
+
export function threshold(options, type) {
|
|
4
|
+
return (
|
|
5
|
+
options.minConfidenceByType?.[type] ??
|
|
6
|
+
options.minConfidence ??
|
|
7
|
+
options.threshold ??
|
|
8
|
+
0.5
|
|
9
|
+
);
|
|
5
10
|
}
|
|
6
11
|
export function actionable(finding, options) {
|
|
7
12
|
return (
|
|
8
13
|
finding.kept !== 'fake' &&
|
|
9
14
|
(finding.category === 'credential' ||
|
|
10
|
-
finding.confidence >= threshold(options))
|
|
15
|
+
finding.confidence >= threshold(options, finding.type))
|
|
11
16
|
);
|
|
12
17
|
}
|
|
13
18
|
export function numericNoise(text, start) {
|
|
@@ -29,11 +34,10 @@ export function epochNumber(value) {
|
|
|
29
34
|
);
|
|
30
35
|
}
|
|
31
36
|
export function cardEvidence(text, start, value) {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
return identityContext || (!numericNoise(text, start) && !epochNumber(value));
|
|
37
|
+
return (
|
|
38
|
+
cardContext(text, start, value) ||
|
|
39
|
+
(!numericNoise(text, start) && !epochNumber(value))
|
|
40
|
+
);
|
|
37
41
|
}
|
|
38
42
|
export function cardNetwork(value) {
|
|
39
43
|
const d = value.replace(/\D/g, ''),
|
|
@@ -60,6 +64,17 @@ function otherCard(d, n) {
|
|
|
60
64
|
n <= 19) ||
|
|
61
65
|
(/^62/.test(d) && n >= 16 && n <= 19) ||
|
|
62
66
|
(/^220[0-4]/.test(d) && n >= 16 && n <= 19) ||
|
|
63
|
-
(/^3(?:0[0-5]|[68])/.test(d) && n === 14)
|
|
67
|
+
(/^3(?:0[0-5]|[68])/.test(d) && n === 14) ||
|
|
68
|
+
// Maestro and Dankort IINs listed by Presidio and the card networks.
|
|
69
|
+
(/^(?:50(?:1[89]|20|38)|5893|6304|6759|676[1-3])/.test(d) &&
|
|
70
|
+
n >= 12 &&
|
|
71
|
+
n <= 19)
|
|
64
72
|
);
|
|
65
73
|
}
|
|
74
|
+
// UATP cards (prefix 1, 15 digits) collide with numeric IDs, so they need a
|
|
75
|
+
// card word nearby.
|
|
76
|
+
export const cardContext = (text, start, value) =>
|
|
77
|
+
/(?<![\p{L}\p{N}_])(?:card|visa|mastercard|amex|discover|jcb|unionpay|mir|diners|maestro|dankort|uatp|карта)(?![\p{L}\p{N}_])/iu.test(
|
|
78
|
+
text.slice(Math.max(0, start - 64), start + value.length + 32)
|
|
79
|
+
);
|
|
80
|
+
export const uatpCard = (value) => /^1\d{14}$/.test(value.replace(/\D/g, ''));
|
package/src/context.js
CHANGED
|
@@ -50,7 +50,11 @@ const label = labels.credential
|
|
|
50
50
|
.filter((word) => !['auth', 'authorization'].includes(word))
|
|
51
51
|
.map(escapePattern)
|
|
52
52
|
.join('|');
|
|
53
|
-
|
|
53
|
+
// Glued service prefixes (`dbpass`, `rootpassword`, `bindpw`, `tlspassword`)
|
|
54
|
+
// are fixed so that ordinary words ending in "pass" stay prose.
|
|
55
|
+
const glued =
|
|
56
|
+
'(?:db|user|root|admin|smtp|mail|ftp|sftp|ssh|redis|ldap|bind|proxy|tls|ssl|keystore|truststore|store|master|sys|app|login|account|acct|wifi|wpa)(?:password|passwd|pass|pwd|pw|secret|token)';
|
|
57
|
+
const key = `(?:[\\p{L}\\p{N}]+[_.-])*(?:auth|authorization|PGPASSWORD|${glued}|${label}|pass|pw)(?:[_.-][\\p{L}\\p{N}]+)*`;
|
|
54
58
|
// A qualifier such as "for staging" stays on one logical line: it never
|
|
55
59
|
// crosses an escaped line break, an escape sequence or an enclosing quote.
|
|
56
60
|
// Verbs are whole words, so "request" does not end in the verb "est".
|
|
@@ -58,6 +62,15 @@ const credentialPrefix = new RegExp(
|
|
|
58
62
|
`(?<![\\p{L}\\p{N}_])(?:\\\\*["'])?(${key})(?![\\p{L}\\p{N}_])(?:\\\\*["'])?(?:[ \\t]+(?:от|для|к|for|de|du|del)[ \\t]+[^:=\\r\\n"\\\\]{1,64})?[ \\t]*([:=:]|(?<![\\p{L}\\p{N}_])(?:${verbs})(?![\\p{L}\\p{N}_]))[ \\t]*`,
|
|
59
63
|
'giu'
|
|
60
64
|
);
|
|
65
|
+
// camelCase and PascalCase keys (`clientSecret`, `secretKey`, `dbPassword`,
|
|
66
|
+
// `ApiSecret`) are case-sensitive: the credential word starts a new word, so
|
|
67
|
+
// `tokenType`, `secretName` and `maxTokens` stay configuration metadata.
|
|
68
|
+
const camelKey =
|
|
69
|
+
'[\\p{L}][\\p{Ll}\\p{N}]*(?:\\p{Lu}[\\p{Ll}\\p{N}]*)*?(?:Secret|Password|Passwd|Passphrase|Pass|Pwd|Token|Credentials?|(?:Api|Private|Access|Secret|Signing|Encryption|Master|Client|App|Subscription|Account|Storage)Key)(?:Key|Value)?|(?:[Ss]ecret|[Pp]assword|[Pp]asswd|[Tt]oken|[Pp]rivate|[Aa]ccess|[Ss]igning|[Ee]ncryption)(?:Key|Value|Secret|Token)';
|
|
70
|
+
const camelPrefix = new RegExp(
|
|
71
|
+
`(?<![\\p{L}\\p{N}_$])(?:\\\\*["'])?(${camelKey})(?![\\p{L}\\p{N}_])(?:\\\\*["'])?[ \\t]*([:=])[ \\t]*`,
|
|
72
|
+
'gu'
|
|
73
|
+
);
|
|
61
74
|
// Bare generic labels are common English nouns. After a verb they need a
|
|
62
75
|
// value that looks generated; "the token is valid" is prose, not a secret.
|
|
63
76
|
function generatedWord(text, range) {
|
|
@@ -359,30 +372,54 @@ function sourceReference(label, tail) {
|
|
|
359
372
|
);
|
|
360
373
|
}
|
|
361
374
|
|
|
375
|
+
// A camelCase key is usually source code; an unquoted member chain, call or
|
|
376
|
+
// type annotation after it (`config.clientSecret`, `apiToken: string;`) is
|
|
377
|
+
// a reference, not the secret itself.
|
|
378
|
+
function codeValue(text, start) {
|
|
379
|
+
return /^(?:[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*)+(?:\([^()\r\n]*\))?|[A-Za-z_$][\w$]*\([^()\r\n]*\)|(?:string|number|boolean|any|unknown|str|bytes|String|SecretStr|Optional\[str\])(?![\w$]))[ \t]*(?:[,;)}\]]|\r?\n|$)/u.test(
|
|
380
|
+
text.slice(start, start + 256)
|
|
381
|
+
);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// Picks how far a credential value runs: to the end of a query parameter, a
|
|
385
|
+
// token, a line or a prose word.
|
|
386
|
+
function credentialMode(text, match) {
|
|
387
|
+
const prefix = text.slice(Math.max(0, match.index - 512), match.index);
|
|
388
|
+
if (
|
|
389
|
+
/(?:https?:\/\/[^\s]*[?&]|^[?&])[^\s]*$/i.test(prefix) ||
|
|
390
|
+
/[?&]$/.test(prefix)
|
|
391
|
+
) {
|
|
392
|
+
return 'query';
|
|
393
|
+
}
|
|
394
|
+
if (match[2] === '=' || /["']\s*[::]/.test(match[0])) {
|
|
395
|
+
return 'token';
|
|
396
|
+
}
|
|
397
|
+
return /[::]/.test(match[2]) ? 'line' : 'prose';
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function credentialRange(text, match, camel) {
|
|
401
|
+
const start = match.index + match[0].length;
|
|
402
|
+
if (
|
|
403
|
+
sourceReference(match[1], text.slice(start)) ||
|
|
404
|
+
(camel && codeValue(text, start))
|
|
405
|
+
) {
|
|
406
|
+
return undefined;
|
|
407
|
+
}
|
|
408
|
+
const mode = credentialMode(text, match);
|
|
409
|
+
const range = /^(?:proxy-)?authorization$/i.test(match[1])
|
|
410
|
+
? authorizationRange(text, start)
|
|
411
|
+
: valueRange(text, start, mode);
|
|
412
|
+
return range && mode === 'prose' ? generatedWord(text, range) : range;
|
|
413
|
+
}
|
|
414
|
+
|
|
362
415
|
function detectCredentials(text, emit) {
|
|
363
416
|
const details = { category: 'credential', type: 'SECRET', rule: 'context' };
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
const mode = query
|
|
371
|
-
? 'query'
|
|
372
|
-
: match[2] === '=' || /["']\s*[::]/.test(match[0])
|
|
373
|
-
? 'token'
|
|
374
|
-
: /[::]/.test(match[2])
|
|
375
|
-
? 'line'
|
|
376
|
-
: 'prose';
|
|
377
|
-
let range = /^(?:proxy-)?authorization$/i.test(match[1])
|
|
378
|
-
? authorizationRange(text, start)
|
|
379
|
-
: valueRange(text, start, mode);
|
|
380
|
-
if (range && mode === 'prose') {
|
|
381
|
-
range = generatedWord(text, range);
|
|
382
|
-
}
|
|
383
|
-
if (sourceReference(match[1], text.slice(start))) {
|
|
384
|
-
continue;
|
|
385
|
-
}
|
|
417
|
+
const matches = [
|
|
418
|
+
...[...text.matchAll(credentialPrefix)].map((match) => [match, false]),
|
|
419
|
+
...[...text.matchAll(camelPrefix)].map((match) => [match, true]),
|
|
420
|
+
];
|
|
421
|
+
for (const [match, camel] of matches) {
|
|
422
|
+
const range = credentialRange(text, match, camel);
|
|
386
423
|
if (range) {
|
|
387
424
|
emitValue(text, range.start, range.end, emit, details, match[1]);
|
|
388
425
|
}
|
|
@@ -417,12 +454,19 @@ const personalPrefixes = Object.entries(labels)
|
|
|
417
454
|
];
|
|
418
455
|
});
|
|
419
456
|
|
|
457
|
+
// "passport number is 912803456": the number word and a following linking
|
|
458
|
+
// verb belong to the label, never to the identity value.
|
|
459
|
+
const identitySuffix = new RegExp(
|
|
460
|
+
`^(?:(?:number|no\\.?|номер)[ \\t]*(?:[:=:]|(?:${verbs})(?![\\p{L}\\p{N}_]))?[ \\t]*|[№#][ \\t]*)`,
|
|
461
|
+
'iu'
|
|
462
|
+
);
|
|
463
|
+
|
|
420
464
|
function detectPersonalLabels(text, emit) {
|
|
421
465
|
for (const [type, prefix] of personalPrefixes) {
|
|
422
466
|
for (const match of text.matchAll(prefix)) {
|
|
423
467
|
const identity = type === 'ID';
|
|
424
468
|
const suffix = identity
|
|
425
|
-
? (
|
|
469
|
+
? (identitySuffix.exec(
|
|
426
470
|
text.slice(match.index + match[0].length)
|
|
427
471
|
)?.[0] ?? '')
|
|
428
472
|
: '';
|
|
@@ -488,7 +532,7 @@ function detectCommandPasswords(text, emit) {
|
|
|
488
532
|
}
|
|
489
533
|
if (/(?:^|[\s/])curl(?:\s|$)/.test(command)) {
|
|
490
534
|
for (const match of command.matchAll(
|
|
491
|
-
/(?:^|\s)(?:-u|--user)(?:[ \t]+|=)/g
|
|
535
|
+
/(?:^|\s)(?:-u|--user|-U|--proxy-user)(?:[ \t]+|=)/g
|
|
492
536
|
)) {
|
|
493
537
|
const range = valueRange(
|
|
494
538
|
text,
|
|
@@ -502,10 +546,62 @@ function detectCommandPasswords(text, emit) {
|
|
|
502
546
|
}
|
|
503
547
|
}
|
|
504
548
|
|
|
549
|
+
// Kubernetes/compose environment entries name the secret in one field and
|
|
550
|
+
// carry it in the next: `{"name":"DB_PASSWORD","value":"hunter2"}` or
|
|
551
|
+
// `- name: DB_PASSWORD` followed by `value: hunter2`.
|
|
552
|
+
const namedValue = new RegExp(
|
|
553
|
+
`(["']?)name\\1[ \\t]*:[ \\t]*(["']?)(${key})\\2[ \\t]*,?[ \\t]*(?:\\r?\\n[ \\t]*)?(["']?)value\\4[ \\t]*:[ \\t]*`,
|
|
554
|
+
'giu'
|
|
555
|
+
);
|
|
556
|
+
// `smtp.login(user, "hunter2")`, `ftp.login('anna', 'pw')`: a literal second
|
|
557
|
+
// argument of a login call is the password.
|
|
558
|
+
const loginCall =
|
|
559
|
+
/(?<![\p{L}\p{N}_])(?:login|log_in|logIn|signin|sign_in|signIn|authenticate|auth_login|bind_s|simple_bind_s|connect_with_password)[ \t]*\([ \t]*[^,()\r\n]{1,64},[ \t]*(["'])([^"'\\\r\n]{4,128})\1[ \t]*[,)]/gu;
|
|
560
|
+
// Modular crypt and Argon2 password hashes (`$2y$10$…`, `$apr1$…`, `$6$…`)
|
|
561
|
+
// from htpasswd, shadow files and user tables crack offline.
|
|
562
|
+
const passwordHash =
|
|
563
|
+
/(?<![\p{L}\p{N}_$])\$(?:2[abxy]?\$\d{2}\$[./A-Za-z0-9]{53}|apr1\$[./A-Za-z0-9]{1,8}\$[./A-Za-z0-9]{22}|1\$[./A-Za-z0-9]{1,8}\$[./A-Za-z0-9]{22}|[56]\$(?:rounds=\d{1,9}\$)?[./A-Za-z0-9]{1,16}\$[./A-Za-z0-9]{43,86}|y\$[./A-Za-z0-9]+\$[./A-Za-z0-9]{1,64}\$[./A-Za-z0-9]{43}|argon2(?:id|i|d)\$v=\d+\$m=\d+,t=\d+,p=\d+\$[A-Za-z0-9+/]{11,}\$[A-Za-z0-9+/]{16,})(?![./A-Za-z0-9$])/gu;
|
|
564
|
+
function detectCredentialForms(text, emit) {
|
|
565
|
+
const details = { category: 'credential', type: 'SECRET' };
|
|
566
|
+
if (/name/i.test(text)) {
|
|
567
|
+
for (const match of text.matchAll(namedValue)) {
|
|
568
|
+
const range = valueRange(text, match.index + match[0].length);
|
|
569
|
+
if (range) {
|
|
570
|
+
emitValue(
|
|
571
|
+
text,
|
|
572
|
+
range.start,
|
|
573
|
+
range.end,
|
|
574
|
+
emit,
|
|
575
|
+
{ ...details, rule: 'named-value' },
|
|
576
|
+
match[3]
|
|
577
|
+
);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
for (const match of text.matchAll(loginCall)) {
|
|
582
|
+
const start = match.index + match[0].lastIndexOf(match[2]);
|
|
583
|
+
emitValue(text, start, start + match[2].length, emit, {
|
|
584
|
+
...details,
|
|
585
|
+
rule: 'login-call',
|
|
586
|
+
confidence: 0.85,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
if (text.includes('$')) {
|
|
590
|
+
for (const match of text.matchAll(passwordHash)) {
|
|
591
|
+
emitValue(text, match.index, match.index + match[0].length, emit, {
|
|
592
|
+
...details,
|
|
593
|
+
rule: 'password-hash',
|
|
594
|
+
confidence: 0.95,
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
505
600
|
export function detectContext(text, emit, credentialsOnly = false) {
|
|
506
601
|
if (credentialHint.test(text)) {
|
|
507
602
|
detectCredentials(text, emit);
|
|
508
603
|
}
|
|
604
|
+
detectCredentialForms(text, emit);
|
|
509
605
|
if (!credentialsOnly) {
|
|
510
606
|
detectPersonalLabels(text, emit);
|
|
511
607
|
}
|
package/src/credentials.js
CHANGED
|
@@ -7,6 +7,50 @@ export function credentialPlaceholder(value) {
|
|
|
7
7
|
) || referencePrefix(value)?.[0] === value
|
|
8
8
|
);
|
|
9
9
|
}
|
|
10
|
+
// One template or mask token: ${NAME}, $${NAME}, $NAME, $1, <name>, [NAME],
|
|
11
|
+
// ****, XXXX or an ellipsis. Values made only of these (optionally split by
|
|
12
|
+
// a user:password colon) carry no credential material.
|
|
13
|
+
const templateToken =
|
|
14
|
+
/^(?:\$?\$\{[A-Za-z_][\w.-]*\}|\$[A-Za-z_]\w*|\$\d|<[^<>\r\n]{1,64}>|\[[A-Z][A-Z0-9_ ]{0,63}\]|\*{3,}|[xX]{4,}|\.{3}|…)$/;
|
|
15
|
+
const unquote = (part) => part.replace(/^["']|["']$/g, '');
|
|
16
|
+
|
|
17
|
+
function templatedValue(value) {
|
|
18
|
+
const body = value.replace(/^(?:token|Bearer|Basic)\s+/i, '');
|
|
19
|
+
const colon = body.indexOf(':');
|
|
20
|
+
const parts = (
|
|
21
|
+
colon < 0 ? [body] : [body.slice(0, colon), body.slice(colon + 1)]
|
|
22
|
+
).map((part) => unquote(part.trim()));
|
|
23
|
+
return (
|
|
24
|
+
parts.some(Boolean) &&
|
|
25
|
+
parts.every((part) => !part || templateToken.test(part))
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Detectors stop at whitespace or brackets, so `<API token>` or `[PASSWORD]`
|
|
30
|
+
// can arrive truncated; complete them from the following text first.
|
|
31
|
+
export function templatedCredential(text, finding) {
|
|
32
|
+
if (finding.category !== 'credential' || finding.rule === 'known-secret') {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
let value = text.slice(finding.start, finding.end);
|
|
36
|
+
const closer =
|
|
37
|
+
value.lastIndexOf('<') > value.lastIndexOf('>')
|
|
38
|
+
? '>'
|
|
39
|
+
: value.lastIndexOf('[') > value.lastIndexOf(']')
|
|
40
|
+
? ']'
|
|
41
|
+
: '';
|
|
42
|
+
if (closer) {
|
|
43
|
+
const rest = text.slice(finding.end, finding.end + 64);
|
|
44
|
+
const index = rest.indexOf(closer);
|
|
45
|
+
if (index < 0 || /[\r\n]/.test(rest.slice(0, index))) {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
value += rest.slice(0, index + 1);
|
|
49
|
+
}
|
|
50
|
+
// A template glued to literal characters (`<x>abc`, `[A]x9`) is not one.
|
|
51
|
+
const after = text[finding.start + value.length] ?? '';
|
|
52
|
+
return !/[\w$%+/=.-]/.test(after) && templatedValue(value);
|
|
53
|
+
}
|
|
10
54
|
// Only variable references and the documented gh lookup are exempt. Arbitrary
|
|
11
55
|
// shell commands and Actions expressions can contain literal credentials.
|
|
12
56
|
export function referencePrefix(value) {
|