pii_scrubber 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +236 -0
- data/Rakefile +8 -0
- data/exe/pii_scrubber +48 -0
- data/lib/pii_scrubber/configuration.rb +76 -0
- data/lib/pii_scrubber/detectors/api_key.rb +48 -0
- data/lib/pii_scrubber/detectors/base.rb +56 -0
- data/lib/pii_scrubber/detectors/canadian_sin.rb +58 -0
- data/lib/pii_scrubber/detectors/credit_card.rb +58 -0
- data/lib/pii_scrubber/detectors/database_url.rb +40 -0
- data/lib/pii_scrubber/detectors/email.rb +33 -0
- data/lib/pii_scrubber/detectors/iban.rb +60 -0
- data/lib/pii_scrubber/detectors/indian_aadhaar.rb +77 -0
- data/lib/pii_scrubber/detectors/indian_pan.rb +42 -0
- data/lib/pii_scrubber/detectors/ip_address.rb +36 -0
- data/lib/pii_scrubber/detectors/phone.rb +26 -0
- data/lib/pii_scrubber/detectors/ssn.rb +29 -0
- data/lib/pii_scrubber/detectors/uk_nino.rb +49 -0
- data/lib/pii_scrubber/integrations/faraday_middleware.rb +48 -0
- data/lib/pii_scrubber/integrations/logger_formatter.rb +41 -0
- data/lib/pii_scrubber/integrations/rack_middleware.rb +37 -0
- data/lib/pii_scrubber/scrubber.rb +141 -0
- data/lib/pii_scrubber/vault/session.rb +73 -0
- data/lib/pii_scrubber/vault.rb +70 -0
- data/lib/pii_scrubber/version.rb +5 -0
- data/lib/pii_scrubber.rb +50 -0
- metadata +142 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 1d3b640a3c54112489c787e96db29e53c56bf4780c9183eb3e3045a626e759a1
|
|
4
|
+
data.tar.gz: 91be1a34d87cf3d9e883c1110490fb4569cb9eb4fdbbec162dc1c862ce94763f
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 12752d059f4cd33bcfcae68f2b571fe76dfd80cd1697b0f386066b3ee02020f0c5988ff9646c628be71f0484fad3ada6488224c3a3417bd33bbd641588b3fb76
|
|
7
|
+
data.tar.gz: 52c5f6e8e595a95f68327156ba8947df9b6ff316369167c6c1eaf1cd43a09b53fe1342e8d5a2fa9b91b6c77e3686ae9f890a7dd92682eeacc8fe4928d663291e
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 PiiScrubber Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
# 🛡️ PiiScrubber
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/rb/pii_scrubber)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
**PiiScrubber** is a high-performance, pluggable Ruby gem for detecting and sanitizing Personally Identifiable Information (PII) and API secrets from strings, nested data structures, Rails logs, Faraday HTTP requests, and LLM prompt contexts.
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## ✨ Features
|
|
11
|
+
|
|
12
|
+
- ✉️ **Email Addresses** (`user@example.com` → `[REDACTED:EMAIL]` or `j***@e***.com`)
|
|
13
|
+
- 📱 **Phone Numbers** (`+1-800-555-0199` → `[REDACTED:PHONE]`)
|
|
14
|
+
- 💳 **Credit Cards** (Supports Luhn algorithm checksum validation to eliminate false positives)
|
|
15
|
+
- 🪪 **US Social Security Numbers (SSN)** (`123-45-6789`)
|
|
16
|
+
- 🏦 **International Bank Account Numbers (IBAN)** (ISO 7064 Modulo-97 checksum validation for 80+ countries)
|
|
17
|
+
- 🇬🇧 **UK National Insurance Numbers (NINO)** (`QQ 12 34 56 A`)
|
|
18
|
+
- 🇨🇦 **Canadian Social Insurance Numbers (SIN)** (Luhn algorithm validated)
|
|
19
|
+
- 🇮🇳 **Indian Identifiers**:
|
|
20
|
+
- **PAN Cards** (`ABCPE1234F` with taxpayer status validation)
|
|
21
|
+
- **Aadhaar Numbers** (`3675 9834 5017` with Verhoeff algorithm checksum)
|
|
22
|
+
- 🗄️ **Database Connection Strings** (`postgres://user:pass@host:5432/db`, `mysql2://`, `mongodb://`, `redis://`)
|
|
23
|
+
- 🔑 **API Keys & Cloud Secrets** (AWS Access Keys, OpenAI `sk-`, Anthropic `sk-ant-`, Hugging Face `hf_`, SendGrid `SG.`, Twilio `AC`/`SK`, Mailgun `key-`, GitLab `glpat-`, GitHub `ghp_`, Stripe `sk_live_`, Bearer Tokens, RSA Private Keys)
|
|
24
|
+
- 🌐 **IP Addresses** (IPv4 & IPv6)
|
|
25
|
+
- 🌲 **Data Structure Support**: Recursively scrubs `String`, `Hash`, `Array`, and embedded `JSON`.
|
|
26
|
+
- 🔑 **Sensitive Key Redaction**: Automatically redacts values for hash keys matching sensitive terms (`password`, `auth_token`, `secret`, `cvv`, `pin`).
|
|
27
|
+
- 🌍 **Regional Configuration Presets**: Quick switch with `:us`, `:uk`, `:eu`, `:ca`, `:in`, `:finance`, `:secrets`, or `:all`.
|
|
28
|
+
- 🛠️ **Integrations**: Drop-in wrapper for `Logger` / Rails `ActiveSupport::Logger`, `Faraday` HTTP middleware, and `Rack` middleware.
|
|
29
|
+
- 💻 **CLI Utility**: Scrub log files directly from the command line (`cat app.log | pii_scrubber`).
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## 📦 Installation
|
|
34
|
+
|
|
35
|
+
Add this line to your application's `Gemfile`:
|
|
36
|
+
|
|
37
|
+
```ruby
|
|
38
|
+
gem "pii_scrubber"
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
And then execute:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
bundle install
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## 🚀 Quick Start
|
|
50
|
+
|
|
51
|
+
### 1. Basic Usage
|
|
52
|
+
|
|
53
|
+
```ruby
|
|
54
|
+
require "pii_scrubber"
|
|
55
|
+
|
|
56
|
+
# Simple String Scrubbing
|
|
57
|
+
PiiScrubber.scrub("Please contact Jane Doe at jane@example.com or call 555-123-4567.")
|
|
58
|
+
# => "Please contact Jane Doe at [REDACTED:EMAIL] or call [REDACTED:PHONE]."
|
|
59
|
+
|
|
60
|
+
# Nested Hash & Sensitive Key Redaction
|
|
61
|
+
payload = {
|
|
62
|
+
user: {
|
|
63
|
+
username: "john_smith",
|
|
64
|
+
password: "mysecretpassword123", # Key matches sensitive key pattern
|
|
65
|
+
email: "john.smith@gmail.com",
|
|
66
|
+
api_key: "sk-proj-1234567890abcdef1234567890abcdef12345678"
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
PiiScrubber.scrub(payload)
|
|
71
|
+
# => {
|
|
72
|
+
# user: {
|
|
73
|
+
# username: "john_smith",
|
|
74
|
+
# password: "[REDACTED:PASSWORD]",
|
|
75
|
+
# email: "[REDACTED:EMAIL]",
|
|
76
|
+
# api_key: "[REDACTED:API_KEY]"
|
|
77
|
+
# }
|
|
78
|
+
# }
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
### 2. Regional Presets & International PII
|
|
84
|
+
|
|
85
|
+
Easily configure detection for specific geographic regions or security profiles:
|
|
86
|
+
|
|
87
|
+
```ruby
|
|
88
|
+
PiiScrubber.configure do |config|
|
|
89
|
+
# Use regional preset (:us, :uk, :eu, :ca, :in, :finance, :secrets, or :all)
|
|
90
|
+
config.use_preset(:eu)
|
|
91
|
+
|
|
92
|
+
# Or selectively enable/disable detectors
|
|
93
|
+
config.enable_detector(:iban, :uk_nino)
|
|
94
|
+
config.disable_detector(:ip_address)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Multi-country detection with checksum validation
|
|
98
|
+
text = "IBAN: GB82WEST12345698765432, SIN: 130-692-544, DB: postgres://user:secret@localhost/db"
|
|
99
|
+
PiiScrubber.scrub(text)
|
|
100
|
+
# => "IBAN: [REDACTED:IBAN], SIN: [REDACTED:CANADIAN_SIN], DB: [REDACTED:DATABASE_URL]"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
### 3. Custom Rules & Proc Redaction Strategy
|
|
106
|
+
|
|
107
|
+
```ruby
|
|
108
|
+
PiiScrubber.configure do |config|
|
|
109
|
+
# Add custom rule via DSL
|
|
110
|
+
config.add_rule(:employee_code, /EMP-\d{4}/)
|
|
111
|
+
|
|
112
|
+
# Custom Proc strategy
|
|
113
|
+
config.strategy = ->(match, detector_name) { "<HIDDEN_#{detector_name.to_s.upcase}>" }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
PiiScrubber.scrub("Employee EMP-1234 email is test@company.com")
|
|
117
|
+
# => "Employee <HIDDEN_EMPLOYEE_CODE> email is <HIDDEN_EMAIL>"
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
### 4. 🤖 Reversible Anonymization Vault (For LLMs & APIs)
|
|
123
|
+
|
|
124
|
+
When sending user input to external LLMs (OpenAI, Anthropic, Gemini) or third-party APIs, anonymize PII before sending the prompt and **restore the real PII values** when rendering the response back to the user:
|
|
125
|
+
|
|
126
|
+
```ruby
|
|
127
|
+
# 1. Anonymize user prompt before sending to LLM
|
|
128
|
+
session = PiiScrubber.anonymize("Draft an email to Alice at alice@company.com or call 555-123-4567.")
|
|
129
|
+
|
|
130
|
+
session.text
|
|
131
|
+
# => "Draft an email to Alice at [PII_VAULT_EMAIL_a8f91234] or call [PII_VAULT_PHONE_b4c56789]."
|
|
132
|
+
|
|
133
|
+
# 2. Pass session.text to OpenAI / LLM
|
|
134
|
+
llm_response = "Confirmed! Email drafted for [PII_VAULT_EMAIL_a8f91234]."
|
|
135
|
+
|
|
136
|
+
# 3. Restore original PII into LLM output
|
|
137
|
+
restored_response = session.restore(llm_response)
|
|
138
|
+
# => "Confirmed! Email drafted for alice@company.com."
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
#### Redis / Session Storage Serialization
|
|
142
|
+
|
|
143
|
+
Vault sessions can be serialized and stored across asynchronous HTTP requests or background jobs:
|
|
144
|
+
|
|
145
|
+
```ruby
|
|
146
|
+
# Save to Redis / Cache
|
|
147
|
+
redis.set("vault:#{session.id}", session.to_h.to_json)
|
|
148
|
+
|
|
149
|
+
# Restore in background worker
|
|
150
|
+
saved_hash = JSON.parse(redis.get("vault:#{session.id}"))
|
|
151
|
+
session = PiiScrubber::Vault::Session.from_h(saved_hash)
|
|
152
|
+
final_output = session.restore(async_llm_response)
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
`pii_scrubber` supports three redaction strategies:
|
|
156
|
+
1. `:placeholder` (Default): `[REDACTED:EMAIL]`
|
|
157
|
+
2. `:mask`: Partial masking (`j***@g***.com` or `************0366`)
|
|
158
|
+
3. `:hash`: HMAC-SHA256 deterministic hash (`[ANON:a3b1f9e8]`)
|
|
159
|
+
|
|
160
|
+
```ruby
|
|
161
|
+
PiiScrubber.configure do |config|
|
|
162
|
+
config.strategy = :mask
|
|
163
|
+
config.mask_char = "*"
|
|
164
|
+
config.detectors = [:email, :credit_card, :api_key]
|
|
165
|
+
config.ignored_keys = [:company_email] # Skip scrubbing specific hash keys
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
PiiScrubber.scrub("Email: john@gmail.com")
|
|
169
|
+
# => "Email: j***@g***.com"
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
#### HMAC Anonymization Strategy
|
|
173
|
+
|
|
174
|
+
Useful when you need consistent anonymized tokens for tracking without revealing PII:
|
|
175
|
+
|
|
176
|
+
```ruby
|
|
177
|
+
PiiScrubber.scrub("User email: john@gmail.com", strategy: :hash, hmac_salt: "my_secret_salt")
|
|
178
|
+
# => "User email: [ANON:c74d0e2b]"
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
---
|
|
182
|
+
|
|
183
|
+
### 3. Rails Logger Integration
|
|
184
|
+
|
|
185
|
+
Sanitize all Rails production logs automatically by updating `config/environments/production.rb`:
|
|
186
|
+
|
|
187
|
+
```ruby
|
|
188
|
+
# config/environments/production.rb
|
|
189
|
+
Rails.application.configure do
|
|
190
|
+
config.logger = ActiveSupport::Logger.new(STDOUT)
|
|
191
|
+
config.logger.formatter = PiiScrubber::Integrations::LoggerFormatter.new
|
|
192
|
+
end
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
### 4. Faraday HTTP Client Integration
|
|
198
|
+
|
|
199
|
+
Prevent sensitive headers (`Authorization`, `Cookie`) and PII in request/response bodies from leaking into third-party services:
|
|
200
|
+
|
|
201
|
+
```ruby
|
|
202
|
+
conn = Faraday.new(url: "https://api.example.com") do |faraday|
|
|
203
|
+
faraday.use PiiScrubber::Integrations::FaradayMiddleware, strategy: :placeholder
|
|
204
|
+
faraday.adapter Faraday.default_adapter
|
|
205
|
+
end
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
### 5. CLI Executable
|
|
211
|
+
|
|
212
|
+
PiiScrubber comes with a command-line tool to clean log files on standard input or file paths:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
# Pipe stdin
|
|
216
|
+
echo "User email is test@example.com and IP is 192.168.1.100" | pii_scrubber
|
|
217
|
+
|
|
218
|
+
# Process file with masking strategy
|
|
219
|
+
pii_scrubber -s mask production.log
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
---
|
|
223
|
+
|
|
224
|
+
## 🧪 Running Tests
|
|
225
|
+
|
|
226
|
+
To run the test suite:
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
bundle exec rspec
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
---
|
|
233
|
+
|
|
234
|
+
## 📄 License
|
|
235
|
+
|
|
236
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/exe/pii_scrubber
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "optparse"
|
|
5
|
+
require_relative "../lib/pii_scrubber"
|
|
6
|
+
|
|
7
|
+
options = {
|
|
8
|
+
strategy: :placeholder,
|
|
9
|
+
mask_char: "*"
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
OptionParser.new do |opts|
|
|
13
|
+
opts.banner = "Usage: pii_scrubber [options] [file...]"
|
|
14
|
+
|
|
15
|
+
opts.on("-s", "--strategy STRATEGY", %i[placeholder mask hash], "Redaction strategy: placeholder, mask, or hash") do |s|
|
|
16
|
+
options[:strategy] = s
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
opts.on("-m", "--mask-char CHAR", "Mask character (default: '*')") do |c|
|
|
20
|
+
options[:mask_char] = c
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
opts.on("-v", "--version", "Show version") do
|
|
24
|
+
puts "pii_scrubber v#{PiiScrubber::VERSION}"
|
|
25
|
+
exit
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
opts.on("-h", "--help", "Prints this help") do
|
|
29
|
+
puts opts
|
|
30
|
+
exit
|
|
31
|
+
end
|
|
32
|
+
end.parse!
|
|
33
|
+
|
|
34
|
+
def process_stream(stream, options)
|
|
35
|
+
stream.each_line do |line|
|
|
36
|
+
print PiiScrubber.scrub(line, **options)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
if ARGV.empty?
|
|
41
|
+
process_stream($stdin, options)
|
|
42
|
+
else
|
|
43
|
+
ARGV.each do |filename|
|
|
44
|
+
File.open(filename, "r") do |file|
|
|
45
|
+
process_stream(file, options)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PiiScrubber
|
|
4
|
+
class Configuration
|
|
5
|
+
attr_accessor :detectors, :strategy, :mask_char, :hmac_salt, :ignored_keys, :custom_rules, :sensitive_key_patterns, :redact_sensitive_hash_keys
|
|
6
|
+
|
|
7
|
+
ALL_DETECTORS = %i[
|
|
8
|
+
database_url
|
|
9
|
+
api_key
|
|
10
|
+
iban
|
|
11
|
+
credit_card
|
|
12
|
+
canadian_sin
|
|
13
|
+
ssn
|
|
14
|
+
uk_nino
|
|
15
|
+
indian_pan
|
|
16
|
+
indian_aadhaar
|
|
17
|
+
email
|
|
18
|
+
phone
|
|
19
|
+
ip_address
|
|
20
|
+
].freeze
|
|
21
|
+
|
|
22
|
+
PRESETS = {
|
|
23
|
+
all: ALL_DETECTORS,
|
|
24
|
+
us: %i[email phone ssn credit_card api_key ip_address database_url],
|
|
25
|
+
uk: %i[email phone uk_nino iban credit_card api_key ip_address database_url],
|
|
26
|
+
eu: %i[email phone iban credit_card api_key ip_address database_url],
|
|
27
|
+
ca: %i[email phone canadian_sin credit_card api_key ip_address database_url],
|
|
28
|
+
in: %i[email phone indian_pan indian_aadhaar credit_card api_key ip_address database_url],
|
|
29
|
+
finance: %i[credit_card iban],
|
|
30
|
+
secrets: %i[api_key database_url]
|
|
31
|
+
}.freeze
|
|
32
|
+
|
|
33
|
+
def initialize
|
|
34
|
+
@detectors = ALL_DETECTORS.dup
|
|
35
|
+
@strategy = :placeholder
|
|
36
|
+
@mask_char = "*"
|
|
37
|
+
@hmac_salt = nil
|
|
38
|
+
@ignored_keys = []
|
|
39
|
+
@custom_rules = []
|
|
40
|
+
@redact_sensitive_hash_keys = true
|
|
41
|
+
@sensitive_key_patterns = [
|
|
42
|
+
/password/i,
|
|
43
|
+
/secret/i,
|
|
44
|
+
/auth_token/i,
|
|
45
|
+
/access_token/i,
|
|
46
|
+
/bearer_token/i,
|
|
47
|
+
/private_key/i,
|
|
48
|
+
/cvv/i,
|
|
49
|
+
/card_number/i
|
|
50
|
+
]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def use_preset(preset_name)
|
|
54
|
+
preset = PRESETS[preset_name.to_sym]
|
|
55
|
+
raise ArgumentError, "Unknown preset: #{preset_name}. Valid presets: #{PRESETS.keys.join(', ')}" unless preset
|
|
56
|
+
|
|
57
|
+
@detectors = preset.dup
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def enable_detector(*names)
|
|
61
|
+
names.each do |name|
|
|
62
|
+
sym = name.to_sym
|
|
63
|
+
@detectors << sym unless @detectors.include?(sym)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def disable_detector(*names)
|
|
68
|
+
syms = names.map(&:to_sym)
|
|
69
|
+
@detectors.reject! { |d| syms.include?(d) }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def add_rule(name, pattern, &validator)
|
|
73
|
+
@custom_rules << { name: name.to_sym, pattern: pattern, validator: validator }
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module PiiScrubber
|
|
6
|
+
module Detectors
|
|
7
|
+
class ApiKey < Base
|
|
8
|
+
PATTERNS = [
|
|
9
|
+
# AWS Access Key ID
|
|
10
|
+
/\b(AKIA[0-9A-Z]{16})\b/,
|
|
11
|
+
# OpenAI API Key
|
|
12
|
+
/\b(sk-[a-zA-Z0-9]{32,}|sk-proj-[a-zA-Z0-9\-_]{32,})\b/,
|
|
13
|
+
# Anthropic API Key
|
|
14
|
+
/\b(sk-ant-api[0-9a-zA-Z\-_]{32,})\b/,
|
|
15
|
+
# Google API Key
|
|
16
|
+
/\b(AIzaSy[0-9A-Za-z\-_]{35})\b/,
|
|
17
|
+
# GitHub Tokens
|
|
18
|
+
/\b(gh[pousr]_[a-zA-Z0-9]{36,40})\b/,
|
|
19
|
+
# Stripe Keys
|
|
20
|
+
/\b([sr]k_live_[0-9a-zA-Z]{24,34})\b/,
|
|
21
|
+
# JWT Bearer Token
|
|
22
|
+
/\b(Bearer\s+eyJ[a-zA-Z0-9\-_]+\.eyJ[a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_]+)\b/,
|
|
23
|
+
# Generic Slack / Slack Bot Token
|
|
24
|
+
/\b(xox[baprs]-[0-9a-zA-Z]{10,48})\b/,
|
|
25
|
+
# Hugging Face Access Token
|
|
26
|
+
/\b(hf_[a-zA-Z0-9]{34,})\b/,
|
|
27
|
+
# SendGrid API Key
|
|
28
|
+
/\b(SG\.[a-zA-Z0-9_\-]{16,32}\.[a-zA-Z0-9_\-]{32,64})\b/,
|
|
29
|
+
# Twilio API Key / SID
|
|
30
|
+
/\b((?:AC|SK)[a-f0-9]{32})\b/,
|
|
31
|
+
# Mailgun API Key
|
|
32
|
+
/\b(key-[a-zA-Z0-9]{32})\b/,
|
|
33
|
+
# GitLab Personal Access Token
|
|
34
|
+
/\b(glpat-[0-9a-zA-Z\-_]{20,})\b/,
|
|
35
|
+
# RSA / PEM Private Keys
|
|
36
|
+
/-----BEGIN (?:RSA|EC|DSA|OPENSSH) PRIVATE KEY-----[[\s\S]]*?-----END (?:RSA|EC|DSA|OPENSSH) PRIVATE KEY-----/
|
|
37
|
+
].freeze
|
|
38
|
+
|
|
39
|
+
def initialize
|
|
40
|
+
super(name: :api_key)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def patterns
|
|
44
|
+
PATTERNS
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module PiiScrubber
|
|
6
|
+
module Detectors
|
|
7
|
+
class Base
|
|
8
|
+
attr_reader :name
|
|
9
|
+
|
|
10
|
+
def initialize(name:)
|
|
11
|
+
@name = name.to_sym
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Returns array of Regexp objects for matching
|
|
15
|
+
def patterns
|
|
16
|
+
[]
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Validates if the regex match is genuinely a PII instance (prevents false positives)
|
|
20
|
+
def valid?(_match)
|
|
21
|
+
true
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Generates replacement text for the match based on strategy
|
|
25
|
+
def replace(match, strategy: :placeholder, mask_char: "*", hmac_salt: nil)
|
|
26
|
+
if strategy.is_a?(Proc)
|
|
27
|
+
return strategy.call(match, name)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
case strategy
|
|
31
|
+
when :placeholder
|
|
32
|
+
"[REDACTED:#{name.to_s.upcase}]"
|
|
33
|
+
when :mask
|
|
34
|
+
mask_value(match, mask_char: mask_char)
|
|
35
|
+
when :hash
|
|
36
|
+
digest = hmac_salt ? Digest::SHA256.hexdigest("#{hmac_salt}:#{match}")[0..7] : Digest::SHA256.hexdigest(match)[0..7]
|
|
37
|
+
"[ANON:#{digest}]"
|
|
38
|
+
else
|
|
39
|
+
"[REDACTED:#{name.to_s.upcase}]"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
protected
|
|
44
|
+
|
|
45
|
+
def mask_value(val, mask_char: "*")
|
|
46
|
+
return mask_char * val.length if val.length <= 4
|
|
47
|
+
|
|
48
|
+
# Keep first and last character visible, mask middle
|
|
49
|
+
first = val[0]
|
|
50
|
+
last = val[-1]
|
|
51
|
+
masked_length = [val.length - 2, 1].max
|
|
52
|
+
"#{first}#{mask_char * masked_length}#{last}"
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module PiiScrubber
|
|
6
|
+
module Detectors
|
|
7
|
+
class CanadianSin < Base
|
|
8
|
+
# Matches 9-digit Canadian Social Insurance Numbers (e.g. 046-454-286, 046 454 286, 046454286)
|
|
9
|
+
SIN_REGEX = /\b[0-9]{3}[ -]?[0-9]{3}[ -]?[0-9]{3}\b/
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
super(name: :canadian_sin)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def patterns
|
|
16
|
+
[SIN_REGEX]
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def valid?(match)
|
|
20
|
+
digits = match.scan(/\d/).map(&:to_i)
|
|
21
|
+
return false unless digits.size == 9
|
|
22
|
+
|
|
23
|
+
# First digit cannot be 0 or 8 in valid Canadian SINs
|
|
24
|
+
return false if [0, 8].include?(digits.first)
|
|
25
|
+
|
|
26
|
+
luhn_valid?(digits)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def replace(match, strategy: :placeholder, mask_char: "*", hmac_salt: nil)
|
|
30
|
+
return super unless strategy == :mask
|
|
31
|
+
|
|
32
|
+
digits = match.scan(/\d/)
|
|
33
|
+
return super if digits.size != 9
|
|
34
|
+
|
|
35
|
+
last_three = digits.last(3).join
|
|
36
|
+
"#{mask_char * 3}-#{mask_char * 3}-#{last_three}"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def luhn_valid?(digits)
|
|
42
|
+
checksum = 0
|
|
43
|
+
reverse_digits = digits.reverse
|
|
44
|
+
|
|
45
|
+
reverse_digits.each_with_index do |digit, index|
|
|
46
|
+
if index.odd?
|
|
47
|
+
doubled = digit * 2
|
|
48
|
+
checksum += doubled > 9 ? doubled - 9 : doubled
|
|
49
|
+
else
|
|
50
|
+
checksum += digit
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
(checksum % 10).zero?
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module PiiScrubber
|
|
6
|
+
module Detectors
|
|
7
|
+
class CreditCard < Base
|
|
8
|
+
# Matches 13-19 digit card numbers separated by optional dashes or spaces
|
|
9
|
+
CREDIT_CARD_REGEX = /\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|6(?:011|5[0-9]{2})[0-9]{12}|(?:2131|1800|35\d{3})\d{11})\b/
|
|
10
|
+
GENERIC_CARD_REGEX = /\b(?:\d[ -]*?){13,19}\b/
|
|
11
|
+
|
|
12
|
+
def initialize
|
|
13
|
+
super(name: :credit_card)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def patterns
|
|
17
|
+
[CREDIT_CARD_REGEX, GENERIC_CARD_REGEX]
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def valid?(match)
|
|
21
|
+
digits = match.scan(/\d/).map(&:to_i)
|
|
22
|
+
return false unless digits.size >= 13 && digits.size <= 19
|
|
23
|
+
|
|
24
|
+
luhn_valid?(digits)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def replace(match, strategy: :placeholder, mask_char: "*", hmac_salt: nil)
|
|
28
|
+
return super unless strategy == :mask
|
|
29
|
+
|
|
30
|
+
digits = match.scan(/\d/)
|
|
31
|
+
return super if digits.size < 4
|
|
32
|
+
|
|
33
|
+
# Show last 4 digits
|
|
34
|
+
last_four = digits.last(4).join
|
|
35
|
+
"#{mask_char * 12}#{last_four}"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
# Luhn algorithm implementation for checking credit card checksum
|
|
41
|
+
def luhn_valid?(digits)
|
|
42
|
+
checksum = 0
|
|
43
|
+
reverse_digits = digits.reverse
|
|
44
|
+
|
|
45
|
+
reverse_digits.each_with_index do |digit, index|
|
|
46
|
+
if index.odd?
|
|
47
|
+
doubled = digit * 2
|
|
48
|
+
checksum += doubled > 9 ? doubled - 9 : doubled
|
|
49
|
+
else
|
|
50
|
+
checksum += digit
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
(checksum % 10).zero?
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module PiiScrubber
|
|
6
|
+
module Detectors
|
|
7
|
+
class DatabaseUrl < Base
|
|
8
|
+
# Matches connection strings with embedded user/password credentials
|
|
9
|
+
DATABASE_URL_REGEX = /\b(?:postgres(?:ql)?|mysql2?|mongodb(?:\+srv)?|rediss?|amqps?|mssql|sqlserver):\/\/(?:[^:\s\/]+:)?(?:[^@\s\/]+)@[^\s\/:]+(?::\d+)?(?:\/[^\s"']*)?/i
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
super(name: :database_url)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def patterns
|
|
16
|
+
[DATABASE_URL_REGEX]
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def valid?(match)
|
|
20
|
+
# Ensure it contains @ separating credentials from host
|
|
21
|
+
match.include?("@") && match.include?("://")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def replace(match, strategy: :placeholder, mask_char: "*", hmac_salt: nil)
|
|
25
|
+
case strategy
|
|
26
|
+
when :mask
|
|
27
|
+
# Mask password while preserving scheme, username, host and database
|
|
28
|
+
match.sub(%r{://(?:([^:@\s/]+):)?([^@\s/]+)@}) do
|
|
29
|
+
user_part = $1 ? "#{$1}:" : ""
|
|
30
|
+
"://#{user_part}#{mask_char * 8}@"
|
|
31
|
+
end
|
|
32
|
+
when :placeholder
|
|
33
|
+
"[REDACTED:DATABASE_URL]"
|
|
34
|
+
else
|
|
35
|
+
super
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module PiiScrubber
|
|
6
|
+
module Detectors
|
|
7
|
+
class Email < Base
|
|
8
|
+
EMAIL_REGEX = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
super(name: :email)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def patterns
|
|
15
|
+
[EMAIL_REGEX]
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def replace(match, strategy: :placeholder, mask_char: "*", hmac_salt: nil)
|
|
19
|
+
return super unless strategy == :mask
|
|
20
|
+
|
|
21
|
+
user, domain = match.split("@", 2)
|
|
22
|
+
return super unless user && domain
|
|
23
|
+
|
|
24
|
+
masked_user = mask_value(user, mask_char: mask_char)
|
|
25
|
+
domain_parts = domain.split(".")
|
|
26
|
+
masked_domain_name = mask_value(domain_parts[0], mask_char: mask_char)
|
|
27
|
+
tld = domain_parts[1..].join(".")
|
|
28
|
+
|
|
29
|
+
"#{masked_user}@#{masked_domain_name}.#{tld}"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|