e11y 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.rubocop.yml +69 -0
- data/CHANGELOG.md +26 -0
- data/CODE_OF_CONDUCT.md +64 -0
- data/LICENSE.txt +21 -0
- data/README.md +179 -0
- data/Rakefile +37 -0
- data/benchmarks/run_all.rb +33 -0
- data/config/README.md +83 -0
- data/config/loki-local-config.yaml +35 -0
- data/config/prometheus.yml +15 -0
- data/docker-compose.yml +78 -0
- data/docs/00-ICP-AND-TIMELINE.md +483 -0
- data/docs/01-SCALE-REQUIREMENTS.md +858 -0
- data/docs/ADR-001-architecture.md +2617 -0
- data/docs/ADR-002-metrics-yabeda.md +1395 -0
- data/docs/ADR-003-slo-observability.md +3337 -0
- data/docs/ADR-004-adapter-architecture.md +2385 -0
- data/docs/ADR-005-tracing-context.md +1372 -0
- data/docs/ADR-006-security-compliance.md +4143 -0
- data/docs/ADR-007-opentelemetry-integration.md +1385 -0
- data/docs/ADR-008-rails-integration.md +1911 -0
- data/docs/ADR-009-cost-optimization.md +2993 -0
- data/docs/ADR-010-developer-experience.md +2166 -0
- data/docs/ADR-011-testing-strategy.md +1836 -0
- data/docs/ADR-012-event-evolution.md +958 -0
- data/docs/ADR-013-reliability-error-handling.md +2750 -0
- data/docs/ADR-014-event-driven-slo.md +1533 -0
- data/docs/ADR-015-middleware-order.md +1061 -0
- data/docs/ADR-016-self-monitoring-slo.md +1234 -0
- data/docs/API-REFERENCE-L28.md +914 -0
- data/docs/COMPREHENSIVE-CONFIGURATION.md +2366 -0
- data/docs/IMPLEMENTATION_NOTES.md +2804 -0
- data/docs/IMPLEMENTATION_PLAN.md +1971 -0
- data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +586 -0
- data/docs/PLAN.md +148 -0
- data/docs/QUICK-START.md +934 -0
- data/docs/README.md +296 -0
- data/docs/design/00-memory-optimization.md +593 -0
- data/docs/guides/MIGRATION-L27-L28.md +692 -0
- data/docs/guides/PERFORMANCE-BENCHMARKS.md +434 -0
- data/docs/guides/README.md +44 -0
- data/docs/prd/01-overview-vision.md +440 -0
- data/docs/use_cases/README.md +119 -0
- data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +813 -0
- data/docs/use_cases/UC-002-business-event-tracking.md +1953 -0
- data/docs/use_cases/UC-003-pattern-based-metrics.md +1627 -0
- data/docs/use_cases/UC-004-zero-config-slo-tracking.md +728 -0
- data/docs/use_cases/UC-005-sentry-integration.md +759 -0
- data/docs/use_cases/UC-006-trace-context-management.md +905 -0
- data/docs/use_cases/UC-007-pii-filtering.md +2648 -0
- data/docs/use_cases/UC-008-opentelemetry-integration.md +1153 -0
- data/docs/use_cases/UC-009-multi-service-tracing.md +1043 -0
- data/docs/use_cases/UC-010-background-job-tracking.md +1018 -0
- data/docs/use_cases/UC-011-rate-limiting.md +1906 -0
- data/docs/use_cases/UC-012-audit-trail.md +2301 -0
- data/docs/use_cases/UC-013-high-cardinality-protection.md +2127 -0
- data/docs/use_cases/UC-014-adaptive-sampling.md +1940 -0
- data/docs/use_cases/UC-015-cost-optimization.md +735 -0
- data/docs/use_cases/UC-016-rails-logger-migration.md +785 -0
- data/docs/use_cases/UC-017-local-development.md +867 -0
- data/docs/use_cases/UC-018-testing-events.md +1081 -0
- data/docs/use_cases/UC-019-tiered-storage-migration.md +562 -0
- data/docs/use_cases/UC-020-event-versioning.md +708 -0
- data/docs/use_cases/UC-021-error-handling-retry-dlq.md +956 -0
- data/docs/use_cases/UC-022-event-registry.md +648 -0
- data/docs/use_cases/backlog.md +226 -0
- data/e11y.gemspec +76 -0
- data/lib/e11y/adapters/adaptive_batcher.rb +207 -0
- data/lib/e11y/adapters/audit_encrypted.rb +239 -0
- data/lib/e11y/adapters/base.rb +580 -0
- data/lib/e11y/adapters/file.rb +224 -0
- data/lib/e11y/adapters/in_memory.rb +216 -0
- data/lib/e11y/adapters/loki.rb +333 -0
- data/lib/e11y/adapters/otel_logs.rb +203 -0
- data/lib/e11y/adapters/registry.rb +141 -0
- data/lib/e11y/adapters/sentry.rb +230 -0
- data/lib/e11y/adapters/stdout.rb +108 -0
- data/lib/e11y/adapters/yabeda.rb +370 -0
- data/lib/e11y/buffers/adaptive_buffer.rb +339 -0
- data/lib/e11y/buffers/base_buffer.rb +40 -0
- data/lib/e11y/buffers/request_scoped_buffer.rb +246 -0
- data/lib/e11y/buffers/ring_buffer.rb +267 -0
- data/lib/e11y/buffers.rb +14 -0
- data/lib/e11y/console.rb +122 -0
- data/lib/e11y/current.rb +48 -0
- data/lib/e11y/event/base.rb +894 -0
- data/lib/e11y/event/value_sampling_config.rb +84 -0
- data/lib/e11y/events/base_audit_event.rb +43 -0
- data/lib/e11y/events/base_payment_event.rb +33 -0
- data/lib/e11y/events/rails/cache/delete.rb +21 -0
- data/lib/e11y/events/rails/cache/read.rb +23 -0
- data/lib/e11y/events/rails/cache/write.rb +22 -0
- data/lib/e11y/events/rails/database/query.rb +45 -0
- data/lib/e11y/events/rails/http/redirect.rb +21 -0
- data/lib/e11y/events/rails/http/request.rb +26 -0
- data/lib/e11y/events/rails/http/send_file.rb +21 -0
- data/lib/e11y/events/rails/http/start_processing.rb +26 -0
- data/lib/e11y/events/rails/job/completed.rb +22 -0
- data/lib/e11y/events/rails/job/enqueued.rb +22 -0
- data/lib/e11y/events/rails/job/failed.rb +22 -0
- data/lib/e11y/events/rails/job/scheduled.rb +23 -0
- data/lib/e11y/events/rails/job/started.rb +22 -0
- data/lib/e11y/events/rails/log.rb +56 -0
- data/lib/e11y/events/rails/view/render.rb +23 -0
- data/lib/e11y/events.rb +18 -0
- data/lib/e11y/instruments/active_job.rb +201 -0
- data/lib/e11y/instruments/rails_instrumentation.rb +141 -0
- data/lib/e11y/instruments/sidekiq.rb +175 -0
- data/lib/e11y/logger/bridge.rb +205 -0
- data/lib/e11y/metrics/cardinality_protection.rb +172 -0
- data/lib/e11y/metrics/cardinality_tracker.rb +134 -0
- data/lib/e11y/metrics/registry.rb +234 -0
- data/lib/e11y/metrics/relabeling.rb +226 -0
- data/lib/e11y/metrics.rb +102 -0
- data/lib/e11y/middleware/audit_signing.rb +174 -0
- data/lib/e11y/middleware/base.rb +140 -0
- data/lib/e11y/middleware/event_slo.rb +167 -0
- data/lib/e11y/middleware/pii_filter.rb +266 -0
- data/lib/e11y/middleware/pii_filtering.rb +280 -0
- data/lib/e11y/middleware/rate_limiting.rb +214 -0
- data/lib/e11y/middleware/request.rb +163 -0
- data/lib/e11y/middleware/routing.rb +157 -0
- data/lib/e11y/middleware/sampling.rb +254 -0
- data/lib/e11y/middleware/slo.rb +168 -0
- data/lib/e11y/middleware/trace_context.rb +131 -0
- data/lib/e11y/middleware/validation.rb +118 -0
- data/lib/e11y/middleware/versioning.rb +132 -0
- data/lib/e11y/middleware.rb +12 -0
- data/lib/e11y/pii/patterns.rb +90 -0
- data/lib/e11y/pii.rb +13 -0
- data/lib/e11y/pipeline/builder.rb +155 -0
- data/lib/e11y/pipeline/zone_validator.rb +110 -0
- data/lib/e11y/pipeline.rb +12 -0
- data/lib/e11y/presets/audit_event.rb +65 -0
- data/lib/e11y/presets/debug_event.rb +34 -0
- data/lib/e11y/presets/high_value_event.rb +51 -0
- data/lib/e11y/presets.rb +19 -0
- data/lib/e11y/railtie.rb +138 -0
- data/lib/e11y/reliability/circuit_breaker.rb +216 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +277 -0
- data/lib/e11y/reliability/dlq/filter.rb +117 -0
- data/lib/e11y/reliability/retry_handler.rb +207 -0
- data/lib/e11y/reliability/retry_rate_limiter.rb +117 -0
- data/lib/e11y/sampling/error_spike_detector.rb +225 -0
- data/lib/e11y/sampling/load_monitor.rb +161 -0
- data/lib/e11y/sampling/stratified_tracker.rb +92 -0
- data/lib/e11y/sampling/value_extractor.rb +82 -0
- data/lib/e11y/self_monitoring/buffer_monitor.rb +79 -0
- data/lib/e11y/self_monitoring/performance_monitor.rb +97 -0
- data/lib/e11y/self_monitoring/reliability_monitor.rb +146 -0
- data/lib/e11y/slo/event_driven.rb +150 -0
- data/lib/e11y/slo/tracker.rb +119 -0
- data/lib/e11y/version.rb +9 -0
- data/lib/e11y.rb +283 -0
- metadata +452 -0
|
@@ -0,0 +1,2648 @@
|
|
|
1
|
+
# UC-007: PII Filtering (Rails-Compatible)
|
|
2
|
+
|
|
3
|
+
**Status:** MVP Feature (Critical for Production)
|
|
4
|
+
**Complexity:** Intermediate
|
|
5
|
+
**Setup Time:** 20-30 minutes
|
|
6
|
+
**Target Users:** All developers, Security teams, Compliance teams
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## 📋 Overview
|
|
11
|
+
|
|
12
|
+
### Problem Statement
|
|
13
|
+
|
|
14
|
+
**Current Approach (Configuration Duplication):**
|
|
15
|
+
```ruby
|
|
16
|
+
# config/application.rb
|
|
17
|
+
# Rails already has PII filtering
|
|
18
|
+
config.filter_parameters += [:password, :email, :ssn, :credit_card]
|
|
19
|
+
|
|
20
|
+
# config/initializers/e11y.rb
|
|
21
|
+
# Do we need to duplicate for E11y?
|
|
22
|
+
E11y.configure do |config|
|
|
23
|
+
config.pii_filter do
|
|
24
|
+
mask_fields :password, :email, :ssn, :credit_card # ← Duplication! 😞
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Problems:
|
|
29
|
+
# - Configuration duplication
|
|
30
|
+
# - Easy to forget updating both places
|
|
31
|
+
# - Inconsistency risk
|
|
32
|
+
# - More maintenance burden
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### E11y Solution
|
|
36
|
+
|
|
37
|
+
**Rails-compatible PII filtering (zero config):**
|
|
38
|
+
```ruby
|
|
39
|
+
# config/application.rb
|
|
40
|
+
# Configure ONCE in Rails (standard way)
|
|
41
|
+
config.filter_parameters += [:password, :email, :ssn, :credit_card]
|
|
42
|
+
|
|
43
|
+
# config/initializers/e11y.rb
|
|
44
|
+
E11y.configure do |config|
|
|
45
|
+
# NO PII CONFIGURATION NEEDED!
|
|
46
|
+
# E11y automatically uses Rails.filter_parameters ✨
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Track event with PII
|
|
50
|
+
Events::UserRegistered.track(
|
|
51
|
+
email: 'user@example.com', # → Automatically filtered to '[FILTERED]'
|
|
52
|
+
password: 'secret123', # → Automatically filtered to '[FILTERED]'
|
|
53
|
+
name: 'John Doe' # → NOT filtered (not in filter_parameters)
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# Result in logs/adapters:
|
|
57
|
+
# {
|
|
58
|
+
# event_name: 'user.registered',
|
|
59
|
+
# payload: {
|
|
60
|
+
# email: '[FILTERED]', # ← Automatically masked
|
|
61
|
+
# password: '[FILTERED]', # ← Automatically masked
|
|
62
|
+
# name: 'John Doe' # ← Not filtered
|
|
63
|
+
# }
|
|
64
|
+
# }
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## 🎯 Features
|
|
70
|
+
|
|
71
|
+
### 1. Automatic Rails Integration (Zero Config)
|
|
72
|
+
|
|
73
|
+
**Default behavior:**
|
|
74
|
+
```ruby
|
|
75
|
+
# config/application.rb (Rails standard)
|
|
76
|
+
config.filter_parameters += [:password, :email, :ssn]
|
|
77
|
+
|
|
78
|
+
# E11y automatically respects this!
|
|
79
|
+
Events::UserCreated.track(
|
|
80
|
+
user_id: '123',
|
|
81
|
+
email: 'user@example.com',
|
|
82
|
+
password: 'secret'
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# Logged as:
|
|
86
|
+
# {
|
|
87
|
+
# user_id: '123', # ← Not filtered
|
|
88
|
+
# email: '[FILTERED]', # ← Filtered by Rails config
|
|
89
|
+
# password: '[FILTERED]' # ← Filtered by Rails config
|
|
90
|
+
# }
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
### 2. Extended Configuration (Optional)
|
|
96
|
+
|
|
97
|
+
**Add more filters beyond Rails:**
|
|
98
|
+
```ruby
|
|
99
|
+
# config/initializers/e11y.rb
|
|
100
|
+
E11y.configure do |config|
|
|
101
|
+
config.pii_filter do
|
|
102
|
+
# 1. USE RAILS FILTERS (default: true)
|
|
103
|
+
use_rails_filter_parameters true
|
|
104
|
+
|
|
105
|
+
# 2. ADD MORE FIELDS (Rails-compatible syntax)
|
|
106
|
+
filter_parameters :api_key, :token, :auth_token, :secret_key
|
|
107
|
+
|
|
108
|
+
# 3. REGEX FILTERS (like Rails)
|
|
109
|
+
filter_parameters /token/i # Matches: auth_token, api_token, etc.
|
|
110
|
+
filter_parameters /secret/i # Matches: client_secret, api_secret, etc.
|
|
111
|
+
|
|
112
|
+
# 4. WHITELIST (don't filter these, even if in Rails.filter_parameters)
|
|
113
|
+
allow_parameters :user_id, :order_id, :transaction_id
|
|
114
|
+
|
|
115
|
+
# 5. CUSTOM REPLACEMENT (default: '[FILTERED]')
|
|
116
|
+
replacement '[REDACTED]'
|
|
117
|
+
|
|
118
|
+
# 6. KEEP PARTIAL DATA (for debugging)
|
|
119
|
+
keep_partial_data true # 'em***@ex***' instead of '[FILTERED]'
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
### 3. Pattern-Based Filtering (Beyond Rails)
|
|
127
|
+
|
|
128
|
+
**Advanced regex patterns for content scanning:**
|
|
129
|
+
```ruby
|
|
130
|
+
E11y.configure do |config|
|
|
131
|
+
config.pii_filter do
|
|
132
|
+
# EMAIL ADDRESSES (scan content, not just keys)
|
|
133
|
+
filter_pattern /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i,
|
|
134
|
+
replacement: '[EMAIL]'
|
|
135
|
+
|
|
136
|
+
# CREDIT CARDS
|
|
137
|
+
filter_pattern /\b\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}\b/,
|
|
138
|
+
replacement: '[CARD]'
|
|
139
|
+
|
|
140
|
+
# SOCIAL SECURITY NUMBERS
|
|
141
|
+
filter_pattern /\b\d{3}-\d{2}-\d{4}\b/,
|
|
142
|
+
replacement: '[SSN]'
|
|
143
|
+
|
|
144
|
+
# PHONE NUMBERS (US/International)
|
|
145
|
+
filter_pattern /\b(\+\d{1,2}\s?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}\b/,
|
|
146
|
+
replacement: '[PHONE]'
|
|
147
|
+
|
|
148
|
+
# IP ADDRESSES
|
|
149
|
+
filter_pattern /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/,
|
|
150
|
+
replacement: '[IP]'
|
|
151
|
+
|
|
152
|
+
# API KEYS (common formats)
|
|
153
|
+
filter_pattern /[A-Za-z0-9_]{32,}/, # Long alphanumeric strings
|
|
154
|
+
replacement: '[API_KEY]'
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Usage:
|
|
159
|
+
Events::EmailSent.track(
|
|
160
|
+
subject: 'Hello user@example.com!', # → 'Hello [EMAIL]!'
|
|
161
|
+
body: 'Your card 4111-1111-1111-1111 was charged' # → 'Your card [CARD] was charged'
|
|
162
|
+
)
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
### 4. Custom Filter Functions
|
|
168
|
+
|
|
169
|
+
**Full control for complex scenarios:**
|
|
170
|
+
```ruby
|
|
171
|
+
E11y.configure do |config|
|
|
172
|
+
config.pii_filter do
|
|
173
|
+
# Custom filter #1: Mask URLs with secrets
|
|
174
|
+
filter do |key, value|
|
|
175
|
+
if value.is_a?(String) && value.include?('?')
|
|
176
|
+
# Mask query parameters in URLs
|
|
177
|
+
value.gsub(/([?&])(api_key|token|secret)=[^&]+/, '\1\2=[FILTERED]')
|
|
178
|
+
else
|
|
179
|
+
value
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Custom filter #2: Mask long strings (likely secrets)
|
|
184
|
+
filter do |key, value|
|
|
185
|
+
if value.is_a?(String) && value.length > 64 && value.match?(/^[A-Za-z0-9_-]+$/)
|
|
186
|
+
'[LONG_TOKEN]'
|
|
187
|
+
else
|
|
188
|
+
value
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Custom filter #3: Conditional filtering
|
|
193
|
+
filter do |key, value|
|
|
194
|
+
# Only filter emails in production
|
|
195
|
+
if Rails.env.production? && value.to_s.match?(/@/)
|
|
196
|
+
value.gsub(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i, '[EMAIL]')
|
|
197
|
+
else
|
|
198
|
+
value # Don't filter in dev/test
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
---
|
|
206
|
+
|
|
207
|
+
### 5. Deep Scanning (Nested Data)
|
|
208
|
+
|
|
209
|
+
**Scan nested hashes and arrays:**
|
|
210
|
+
```ruby
|
|
211
|
+
E11y.configure do |config|
|
|
212
|
+
config.pii_filter do
|
|
213
|
+
deep_scan true # Default: enabled
|
|
214
|
+
|
|
215
|
+
# Maximum depth (prevent infinite recursion)
|
|
216
|
+
max_depth 10
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Deep scanning in action:
|
|
221
|
+
Events::OrderPlaced.track(
|
|
222
|
+
order_id: '123',
|
|
223
|
+
user: {
|
|
224
|
+
name: 'John Doe',
|
|
225
|
+
contact: {
|
|
226
|
+
email: 'john@example.com', # ← Nested deep, still filtered!
|
|
227
|
+
phone: '+1-555-123-4567'
|
|
228
|
+
},
|
|
229
|
+
billing: {
|
|
230
|
+
card: {
|
|
231
|
+
number: '4111-1111-1111-1111', # ← 3 levels deep, still filtered!
|
|
232
|
+
cvv: '123'
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
},
|
|
236
|
+
items: [
|
|
237
|
+
{ name: 'Product 1', notes: 'Ship to user@example.com' } # ← In array, still filtered!
|
|
238
|
+
]
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
# Result:
|
|
242
|
+
# {
|
|
243
|
+
# order_id: '123',
|
|
244
|
+
# user: {
|
|
245
|
+
# name: 'John Doe',
|
|
246
|
+
# contact: {
|
|
247
|
+
# email: '[FILTERED]', # ← Filtered
|
|
248
|
+
# phone: '[PHONE]' # ← Filtered by pattern
|
|
249
|
+
# },
|
|
250
|
+
# billing: {
|
|
251
|
+
# card: {
|
|
252
|
+
# number: '[CARD]', # ← Filtered by pattern
|
|
253
|
+
# cvv: '[FILTERED]' # ← Filtered by key
|
|
254
|
+
# }
|
|
255
|
+
# }
|
|
256
|
+
# },
|
|
257
|
+
# items: [
|
|
258
|
+
# { name: 'Product 1', notes: 'Ship to [EMAIL]' } # ← Content filtered
|
|
259
|
+
# ]
|
|
260
|
+
# }
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
---
|
|
264
|
+
|
|
265
|
+
### 6. Sampling for Debugging
|
|
266
|
+
|
|
267
|
+
**Log some filtered values for verification:**
|
|
268
|
+
```ruby
|
|
269
|
+
E11y.configure do |config|
|
|
270
|
+
config.pii_filter do
|
|
271
|
+
# Sample 1% of filtered values (for debugging)
|
|
272
|
+
sample_filtered_values 0.01
|
|
273
|
+
|
|
274
|
+
# Log destination
|
|
275
|
+
sample_logger Rails.logger # Or custom logger
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# When filtering happens, 1% of time you'll see in logs:
|
|
280
|
+
# [E11y DEBUG] PII filtered: email = "user@examp..." → [FILTERED]
|
|
281
|
+
# [E11y DEBUG] PII filtered: password = "secre..." → [FILTERED]
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
---
|
|
285
|
+
|
|
286
|
+
## 💻 Implementation Examples
|
|
287
|
+
|
|
288
|
+
### Example 1: User Registration
|
|
289
|
+
|
|
290
|
+
```ruby
|
|
291
|
+
# app/controllers/registrations_controller.rb
|
|
292
|
+
class RegistrationsController < ApplicationController
|
|
293
|
+
def create
|
|
294
|
+
user = User.new(registration_params)
|
|
295
|
+
|
|
296
|
+
if user.save
|
|
297
|
+
# Track registration (PII automatically filtered)
|
|
298
|
+
Events::UserRegistered.track(
|
|
299
|
+
user_id: user.id,
|
|
300
|
+
email: user.email, # ← Filtered
|
|
301
|
+
password: params[:password], # ← Filtered
|
|
302
|
+
referral_code: params[:referral], # ← Not filtered
|
|
303
|
+
ip_address: request.remote_ip # ← Filtered by pattern
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
render json: { status: 'ok' }
|
|
307
|
+
else
|
|
308
|
+
# Track failure (errors may contain PII)
|
|
309
|
+
Events::UserRegistrationFailed.track(
|
|
310
|
+
email: params[:email], # ← Filtered
|
|
311
|
+
errors: user.errors.full_messages,
|
|
312
|
+
severity: :error
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
render json: { errors: user.errors }, status: :unprocessable_entity
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
# Logged events (all PII filtered):
|
|
321
|
+
# {
|
|
322
|
+
# event_name: 'user.registered',
|
|
323
|
+
# payload: {
|
|
324
|
+
# user_id: '123',
|
|
325
|
+
# email: '[FILTERED]',
|
|
326
|
+
# password: '[FILTERED]',
|
|
327
|
+
# referral_code: 'FRIEND10',
|
|
328
|
+
# ip_address: '[IP]'
|
|
329
|
+
# }
|
|
330
|
+
# }
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
---
|
|
334
|
+
|
|
335
|
+
### Example 2: Payment Processing
|
|
336
|
+
|
|
337
|
+
```ruby
|
|
338
|
+
# app/services/process_payment_service.rb
|
|
339
|
+
class ProcessPaymentService
|
|
340
|
+
def call(order, card_params)
|
|
341
|
+
# Track payment attempt (card details filtered)
|
|
342
|
+
Events::PaymentAttempted.track(
|
|
343
|
+
order_id: order.id,
|
|
344
|
+
amount: order.total,
|
|
345
|
+
card_number: card_params[:number], # ← Filtered by pattern
|
|
346
|
+
card_cvv: card_params[:cvv], # ← Filtered by key
|
|
347
|
+
card_holder: card_params[:name], # ← Not filtered (name != PII)
|
|
348
|
+
billing_address: card_params[:address] # ← Deep scanned
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
begin
|
|
352
|
+
result = PaymentGateway.charge(
|
|
353
|
+
amount: order.total,
|
|
354
|
+
card: card_params
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
# Track success
|
|
358
|
+
Events::PaymentSucceeded.track(
|
|
359
|
+
order_id: order.id,
|
|
360
|
+
transaction_id: result.id,
|
|
361
|
+
card_last4: card_params[:number][-4..-1], # Last 4 digits OK
|
|
362
|
+
severity: :success
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
rescue PaymentGateway::Error => e
|
|
366
|
+
# Track failure (error message may contain PII)
|
|
367
|
+
Events::PaymentFailed.track(
|
|
368
|
+
order_id: order.id,
|
|
369
|
+
error_message: e.message, # ← Content filtered
|
|
370
|
+
error_code: e.code,
|
|
371
|
+
severity: :error
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
raise
|
|
375
|
+
end
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
---
|
|
381
|
+
|
|
382
|
+
### Example 3: Support Ticket Creation
|
|
383
|
+
|
|
384
|
+
```ruby
|
|
385
|
+
# app/controllers/support_tickets_controller.rb
|
|
386
|
+
class SupportTicketsController < ApplicationController
|
|
387
|
+
def create
|
|
388
|
+
ticket = SupportTicket.create!(ticket_params)
|
|
389
|
+
|
|
390
|
+
# Track ticket creation (description may contain PII)
|
|
391
|
+
Events::SupportTicketCreated.track(
|
|
392
|
+
ticket_id: ticket.id,
|
|
393
|
+
subject: ticket.subject,
|
|
394
|
+
description: ticket.description, # ← Content scanned for emails, phones, etc.
|
|
395
|
+
category: ticket.category,
|
|
396
|
+
attachments: ticket.attachments.map do |file|
|
|
397
|
+
{
|
|
398
|
+
filename: file.filename,
|
|
399
|
+
size: file.size,
|
|
400
|
+
url: file.url # ← URLs with query strings filtered
|
|
401
|
+
}
|
|
402
|
+
end
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
render json: ticket
|
|
406
|
+
end
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
# If description contains PII:
|
|
410
|
+
# "Please help! My email is john@example.com and phone is 555-1234"
|
|
411
|
+
#
|
|
412
|
+
# Logged as:
|
|
413
|
+
# "Please help! My email is [EMAIL] and phone is [PHONE]"
|
|
414
|
+
```
|
|
415
|
+
|
|
416
|
+
---
|
|
417
|
+
|
|
418
|
+
## 🔧 Configuration API
|
|
419
|
+
|
|
420
|
+
### Full Configuration Example
|
|
421
|
+
|
|
422
|
+
```ruby
|
|
423
|
+
# config/initializers/e11y.rb
|
|
424
|
+
E11y.configure do |config|
|
|
425
|
+
config.pii_filter do
|
|
426
|
+
# === BASIC CONFIGURATION ===
|
|
427
|
+
|
|
428
|
+
# Use Rails filter_parameters (default: true)
|
|
429
|
+
use_rails_filter_parameters true
|
|
430
|
+
|
|
431
|
+
# Add more filters (Rails-compatible syntax)
|
|
432
|
+
filter_parameters :api_key, :token, :auth_token, :secret_key
|
|
433
|
+
filter_parameters /token/i, /secret/i, /key/i
|
|
434
|
+
|
|
435
|
+
# Whitelist (don't filter these)
|
|
436
|
+
allow_parameters :user_id, :order_id, :transaction_id, :session_id
|
|
437
|
+
|
|
438
|
+
# === PATTERN-BASED FILTERING ===
|
|
439
|
+
|
|
440
|
+
# Email addresses
|
|
441
|
+
filter_pattern /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i,
|
|
442
|
+
replacement: '[EMAIL]'
|
|
443
|
+
|
|
444
|
+
# Credit cards
|
|
445
|
+
filter_pattern /\b\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}\b/,
|
|
446
|
+
replacement: '[CARD]'
|
|
447
|
+
|
|
448
|
+
# SSN
|
|
449
|
+
filter_pattern /\b\d{3}-\d{2}-\d{4}\b/,
|
|
450
|
+
replacement: '[SSN]'
|
|
451
|
+
|
|
452
|
+
# Phone numbers
|
|
453
|
+
filter_pattern /\b(\+\d{1,2}\s?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}\b/,
|
|
454
|
+
replacement: '[PHONE]'
|
|
455
|
+
|
|
456
|
+
# IP addresses
|
|
457
|
+
filter_pattern /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/,
|
|
458
|
+
replacement: '[IP]'
|
|
459
|
+
|
|
460
|
+
# === CUSTOM FILTERS ===
|
|
461
|
+
|
|
462
|
+
# Mask query parameters in URLs
|
|
463
|
+
filter do |key, value|
|
|
464
|
+
if value.is_a?(String) && value.include?('?')
|
|
465
|
+
value.gsub(/([?&])(api_key|token|secret)=[^&]+/, '\1\2=[FILTERED]')
|
|
466
|
+
else
|
|
467
|
+
value
|
|
468
|
+
end
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
# === BEHAVIOR ===
|
|
472
|
+
|
|
473
|
+
# Deep scan nested data (default: true)
|
|
474
|
+
deep_scan true
|
|
475
|
+
max_depth 10
|
|
476
|
+
|
|
477
|
+
# Replacement strategy
|
|
478
|
+
replacement '[FILTERED]'
|
|
479
|
+
keep_partial_data true # Show 'em***@ex***' instead of '[FILTERED]'
|
|
480
|
+
|
|
481
|
+
# Sampling (for debugging)
|
|
482
|
+
sample_filtered_values 0.01 # 1%
|
|
483
|
+
sample_logger Rails.logger
|
|
484
|
+
|
|
485
|
+
# Performance
|
|
486
|
+
enabled true # Can disable in dev/test
|
|
487
|
+
cache_compiled_patterns true # Compile regex once
|
|
488
|
+
end
|
|
489
|
+
end
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
---
|
|
493
|
+
|
|
494
|
+
## 🔐 Explicit PII Declaration
|
|
495
|
+
|
|
496
|
+
> **Implementation:** See [ADR-006 Section 3.0.3: Explicit PII Declaration](../ADR-006-security-compliance.md#303-explicit-pii-declaration) for detailed architecture.
|
|
497
|
+
|
|
498
|
+
**Critical Design Principle:** Event classes MUST explicitly declare whether they contain PII. This enables E11y to apply the appropriate filtering tier (see Performance Tiers below) and allows linter validation.
|
|
499
|
+
|
|
500
|
+
### Why Explicit Declaration?
|
|
501
|
+
|
|
502
|
+
**Problem:** Implicit filtering leads to:
|
|
503
|
+
- ❌ Performance waste (filtering events that contain no PII)
|
|
504
|
+
- ❌ Security gaps (missing PII that should be filtered)
|
|
505
|
+
- ❌ No compile-time validation (typos, missing fields)
|
|
506
|
+
|
|
507
|
+
**Solution:** Explicit opt-in declaration at event class level.
|
|
508
|
+
|
|
509
|
+
---
|
|
510
|
+
|
|
511
|
+
### Declaration Syntax: `contains_pii`
|
|
512
|
+
|
|
513
|
+
**Option 1: No PII (Tier 1 - Skip Filtering)**
|
|
514
|
+
|
|
515
|
+
```ruby
|
|
516
|
+
class Events::HealthCheck < E11y::Event::Base
|
|
517
|
+
schema do
|
|
518
|
+
required(:status).filled(:string)
|
|
519
|
+
required(:uptime_ms).filled(:integer)
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
# ✅ Explicit: This event contains NO PII
|
|
523
|
+
contains_pii false
|
|
524
|
+
|
|
525
|
+
# Result:
|
|
526
|
+
# - Tier 1 filtering (0ms overhead)
|
|
527
|
+
# - All fields logged as-is
|
|
528
|
+
# - No pattern scanning
|
|
529
|
+
end
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
**Option 2: Default (Tier 2 - Rails Filters Only)**
|
|
533
|
+
|
|
534
|
+
```ruby
|
|
535
|
+
class Events::OrderCreated < E11y::Event::Base
|
|
536
|
+
schema do
|
|
537
|
+
required(:order_id).filled(:string)
|
|
538
|
+
required(:amount).filled(:float)
|
|
539
|
+
optional(:api_key).filled(:string) # Rails will filter this
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
# No declaration → Tier 2 (Rails filters applied)
|
|
543
|
+
# Keys like :password, :token, :api_key automatically filtered
|
|
544
|
+
end
|
|
545
|
+
```
|
|
546
|
+
|
|
547
|
+
**Option 3: Explicit PII (Tier 3 - Deep Filtering)**
|
|
548
|
+
|
|
549
|
+
```ruby
|
|
550
|
+
class Events::UserRegistered < E11y::Event::Base
|
|
551
|
+
schema do
|
|
552
|
+
required(:email).filled(:string)
|
|
553
|
+
required(:password).filled(:string)
|
|
554
|
+
required(:address).filled(:hash)
|
|
555
|
+
required(:user_id).filled(:string)
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
# ✅ Explicit: This event contains PII
|
|
559
|
+
contains_pii true
|
|
560
|
+
|
|
561
|
+
# MANDATORY: Declare strategy for EVERY schema field
|
|
562
|
+
pii_filtering do
|
|
563
|
+
field :email do
|
|
564
|
+
strategy :hash # Pseudonymize (searchable)
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
field :password do
|
|
568
|
+
strategy :mask # Complete masking
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
field :address do
|
|
572
|
+
strategy :mask # Mask nested data
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
field :user_id do
|
|
576
|
+
strategy :allow # ID is OK to log
|
|
577
|
+
end
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
```
|
|
581
|
+
|
|
582
|
+
---
|
|
583
|
+
|
|
584
|
+
### Per-Field Filtering Strategies
|
|
585
|
+
|
|
586
|
+
When `contains_pii true` is declared, you MUST specify a strategy for each field in the schema:
|
|
587
|
+
|
|
588
|
+
| Strategy | Behavior | Use Case | Example Output |
|
|
589
|
+
|----------|----------|----------|----------------|
|
|
590
|
+
| `:mask` | Replace with `[FILTERED]` | Sensitive data (passwords, SSNs) | `[FILTERED]` |
|
|
591
|
+
| `:hash` | SHA256 hash (one-way) | Searchable identifiers (emails) | `hashed_a1b2c3d4` |
|
|
592
|
+
| `:allow` | No filtering | Non-PII (IDs, amounts) | Original value |
|
|
593
|
+
| `:partial` | Show partial (first/last chars) | Debugging (emails) | `em***@ex***` |
|
|
594
|
+
|
|
595
|
+
**Example: Payment Event with Multiple Strategies**
|
|
596
|
+
|
|
597
|
+
```ruby
|
|
598
|
+
class Events::PaymentProcessed < E11y::Event::Base
|
|
599
|
+
schema do
|
|
600
|
+
required(:order_id).filled(:string)
|
|
601
|
+
required(:amount).filled(:float)
|
|
602
|
+
required(:card_number).filled(:string)
|
|
603
|
+
required(:card_holder).filled(:string)
|
|
604
|
+
required(:user_email).filled(:string)
|
|
605
|
+
required(:ip_address).filled(:string)
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
contains_pii true
|
|
609
|
+
|
|
610
|
+
pii_filtering do
|
|
611
|
+
# Non-PII: allow
|
|
612
|
+
field :order_id do
|
|
613
|
+
strategy :allow # ID is safe to log
|
|
614
|
+
end
|
|
615
|
+
|
|
616
|
+
field :amount do
|
|
617
|
+
strategy :allow # Amount is not PII
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# Sensitive: mask completely
|
|
621
|
+
field :card_number do
|
|
622
|
+
strategy :mask # Never log credit cards
|
|
623
|
+
end
|
|
624
|
+
|
|
625
|
+
field :card_holder do
|
|
626
|
+
strategy :mask # Cardholder name is PII
|
|
627
|
+
end
|
|
628
|
+
|
|
629
|
+
# Searchable: hash
|
|
630
|
+
field :user_email do
|
|
631
|
+
strategy :hash # Pseudonymize for correlation
|
|
632
|
+
end
|
|
633
|
+
|
|
634
|
+
# Debugging: partial
|
|
635
|
+
field :ip_address do
|
|
636
|
+
strategy :partial # Show '192.168.1.x'
|
|
637
|
+
end
|
|
638
|
+
end
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
# Track event:
|
|
642
|
+
Events::PaymentProcessed.track(
|
|
643
|
+
order_id: 'o123',
|
|
644
|
+
amount: 99.99,
|
|
645
|
+
card_number: '4111-1111-1111-1111',
|
|
646
|
+
card_holder: 'John Doe',
|
|
647
|
+
user_email: 'john@example.com',
|
|
648
|
+
ip_address: '192.168.1.100'
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
# Logged as:
|
|
652
|
+
# {
|
|
653
|
+
# order_id: 'o123', # ← Allowed (ID)
|
|
654
|
+
# amount: 99.99, # ← Allowed (not PII)
|
|
655
|
+
# card_number: '[FILTERED]', # ← Masked
|
|
656
|
+
# card_holder: '[FILTERED]', # ← Masked
|
|
657
|
+
# user_email: 'hashed_7a8b9c', # ← Hashed
|
|
658
|
+
# ip_address: '192.168.1.x' # ← Partial
|
|
659
|
+
# }
|
|
660
|
+
```
|
|
661
|
+
|
|
662
|
+
---
|
|
663
|
+
|
|
664
|
+
### Per-Adapter Overrides
|
|
665
|
+
|
|
666
|
+
Different adapters may have different PII requirements (e.g., audit trail needs full data for compliance):
|
|
667
|
+
|
|
668
|
+
```ruby
|
|
669
|
+
class Events::SensitiveUserAction < E11y::Event::Base
|
|
670
|
+
schema do
|
|
671
|
+
required(:user_email).filled(:string)
|
|
672
|
+
required(:action).filled(:string)
|
|
673
|
+
end
|
|
674
|
+
|
|
675
|
+
contains_pii true
|
|
676
|
+
|
|
677
|
+
pii_filtering do
|
|
678
|
+
field :user_email do
|
|
679
|
+
# Default: hash for most adapters
|
|
680
|
+
strategy :hash
|
|
681
|
+
|
|
682
|
+
# Override per adapter
|
|
683
|
+
exclude_adapters [:file_audit] # Audit needs original (GDPR Art. 6(1)(c))
|
|
684
|
+
end
|
|
685
|
+
|
|
686
|
+
field :action do
|
|
687
|
+
strategy :allow # Action type is not PII
|
|
688
|
+
end
|
|
689
|
+
end
|
|
690
|
+
end
|
|
691
|
+
|
|
692
|
+
# Result:
|
|
693
|
+
# - audit_file adapter: { user_email: 'john@example.com' } (original)
|
|
694
|
+
# - elasticsearch: { user_email: 'hashed_a1b2c3' } (hashed)
|
|
695
|
+
# - loki: { user_email: 'hashed_a1b2c3' } (hashed)
|
|
696
|
+
# - sentry: { user_email: 'hashed_a1b2c3' } (hashed)
|
|
697
|
+
```
|
|
698
|
+
|
|
699
|
+
---
|
|
700
|
+
|
|
701
|
+
### Linter Validation
|
|
702
|
+
|
|
703
|
+
When `contains_pii true` is declared, E11y linter validates:
|
|
704
|
+
|
|
705
|
+
1. ✅ **Every schema field has a filtering strategy** (no missing fields)
|
|
706
|
+
2. ✅ **No extra fields** (typos in field names)
|
|
707
|
+
3. ✅ **Valid strategies** (`:mask`, `:hash`, `:allow`, `:partial` only)
|
|
708
|
+
|
|
709
|
+
**Example: Linter catches missing field**
|
|
710
|
+
|
|
711
|
+
```ruby
|
|
712
|
+
class Events::UserLogin < E11y::Event::Base
|
|
713
|
+
schema do
|
|
714
|
+
required(:email).filled(:string)
|
|
715
|
+
required(:password).filled(:string)
|
|
716
|
+
required(:ip_address).filled(:string) # ← MISSING in pii_filtering!
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
contains_pii true
|
|
720
|
+
|
|
721
|
+
pii_filtering do
|
|
722
|
+
field :email do
|
|
723
|
+
strategy :hash
|
|
724
|
+
end
|
|
725
|
+
|
|
726
|
+
field :password do
|
|
727
|
+
strategy :mask
|
|
728
|
+
end
|
|
729
|
+
|
|
730
|
+
# ❌ LINTER ERROR: Field :ip_address declared in schema but missing in pii_filtering!
|
|
731
|
+
end
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
# Fix:
|
|
735
|
+
pii_filtering do
|
|
736
|
+
field :email do
|
|
737
|
+
strategy :hash
|
|
738
|
+
end
|
|
739
|
+
|
|
740
|
+
field :password do
|
|
741
|
+
strategy :mask
|
|
742
|
+
end
|
|
743
|
+
|
|
744
|
+
field :ip_address do # ✅ Added
|
|
745
|
+
strategy :partial
|
|
746
|
+
end
|
|
747
|
+
end
|
|
748
|
+
```
|
|
749
|
+
|
|
750
|
+
---
|
|
751
|
+
|
|
752
|
+
### Default Behavior (No Declaration)
|
|
753
|
+
|
|
754
|
+
If `contains_pii` is not specified, E11y defaults to **Tier 2** (Rails filters only):
|
|
755
|
+
|
|
756
|
+
```ruby
|
|
757
|
+
class Events::OrderPaid < E11y::Event::Base
|
|
758
|
+
schema do
|
|
759
|
+
required(:order_id).filled(:string)
|
|
760
|
+
required(:amount).filled(:float)
|
|
761
|
+
end
|
|
762
|
+
|
|
763
|
+
# No contains_pii declaration
|
|
764
|
+
# → Tier 2: Rails filters applied automatically
|
|
765
|
+
# → Keys like :password, :token, :api_key filtered
|
|
766
|
+
# → No linter validation
|
|
767
|
+
end
|
|
768
|
+
```
|
|
769
|
+
|
|
770
|
+
**Recommended for:** Standard business events where Rails filters provide sufficient coverage (90% of use cases).
|
|
771
|
+
|
|
772
|
+
---
|
|
773
|
+
|
|
774
|
+
### Migration Guide
|
|
775
|
+
|
|
776
|
+
**If you have existing events without explicit declaration:**
|
|
777
|
+
|
|
778
|
+
**Step 1: Audit events**
|
|
779
|
+
```bash
|
|
780
|
+
# List events without PII declaration
|
|
781
|
+
bundle exec rake e11y:audit:pii_declarations
|
|
782
|
+
|
|
783
|
+
# Output:
|
|
784
|
+
# ⚠️ Events without PII declaration (using Tier 2 default):
|
|
785
|
+
# - Events::OrderCreated
|
|
786
|
+
# - Events::PaymentProcessed
|
|
787
|
+
# - Events::UserLogin
|
|
788
|
+
#
|
|
789
|
+
# ✅ Events with PII declaration:
|
|
790
|
+
# - Events::HealthCheck (contains_pii false)
|
|
791
|
+
# - Events::UserRegistered (contains_pii true)
|
|
792
|
+
```
|
|
793
|
+
|
|
794
|
+
**Step 2: Add declarations**
|
|
795
|
+
```ruby
|
|
796
|
+
# For events with NO user data:
|
|
797
|
+
class Events::HealthCheck < E11y::Event::Base
|
|
798
|
+
contains_pii false # ✅ Explicit
|
|
799
|
+
end
|
|
800
|
+
|
|
801
|
+
# For events with PII:
|
|
802
|
+
class Events::UserLogin < E11y::Event::Base
|
|
803
|
+
contains_pii true # ✅ Explicit
|
|
804
|
+
|
|
805
|
+
pii_filtering do
|
|
806
|
+
# ... declare strategies for ALL fields
|
|
807
|
+
end
|
|
808
|
+
end
|
|
809
|
+
|
|
810
|
+
# For standard events (keep default):
|
|
811
|
+
class Events::OrderCreated < E11y::Event::Base
|
|
812
|
+
# No declaration (Tier 2 default is fine)
|
|
813
|
+
end
|
|
814
|
+
```
|
|
815
|
+
|
|
816
|
+
**Step 3: Enable linter in CI**
|
|
817
|
+
```ruby
|
|
818
|
+
# config/environments/test.rb
|
|
819
|
+
config.after_initialize do
|
|
820
|
+
E11y::Linters::PiiDeclarationLinter.validate_all!
|
|
821
|
+
end
|
|
822
|
+
```
|
|
823
|
+
|
|
824
|
+
---
|
|
825
|
+
|
|
826
|
+
### Event Inheritance for PII (NEW - v1.1)
|
|
827
|
+
|
|
828
|
+
> **🎯 CONTRADICTION_01 Resolution:** Use inheritance to share common PII rules across related events.
|
|
829
|
+
|
|
830
|
+
**Base class with common PII rules:**
|
|
831
|
+
|
|
832
|
+
```ruby
|
|
833
|
+
# app/events/base_user_event.rb
|
|
834
|
+
module Events
|
|
835
|
+
class BaseUserEvent < E11y::Event::Base
|
|
836
|
+
# Common for ALL user events
|
|
837
|
+
contains_pii true
|
|
838
|
+
|
|
839
|
+
pii_filtering do
|
|
840
|
+
# Common PII handling
|
|
841
|
+
hashes :email, :phone # Pseudonymize for searchability
|
|
842
|
+
allows :user_id # ID is not PII
|
|
843
|
+
end
|
|
844
|
+
end
|
|
845
|
+
end
|
|
846
|
+
|
|
847
|
+
# Inherit and extend
|
|
848
|
+
class Events::UserRegistered < Events::BaseUserEvent
|
|
849
|
+
schema do
|
|
850
|
+
required(:user_id).filled(:string)
|
|
851
|
+
required(:email).filled(:string)
|
|
852
|
+
required(:password).filled(:string)
|
|
853
|
+
required(:phone).filled(:string)
|
|
854
|
+
end
|
|
855
|
+
|
|
856
|
+
pii_filtering do
|
|
857
|
+
# Inherits: hashes :email, :phone + allows :user_id
|
|
858
|
+
# Add more:
|
|
859
|
+
masks :password # ← Additional field
|
|
860
|
+
end
|
|
861
|
+
end
|
|
862
|
+
|
|
863
|
+
class Events::UserProfileUpdated < Events::BaseUserEvent
|
|
864
|
+
schema do
|
|
865
|
+
required(:user_id).filled(:string)
|
|
866
|
+
required(:email).filled(:string)
|
|
867
|
+
required(:phone).filled(:string)
|
|
868
|
+
required(:address).filled(:hash)
|
|
869
|
+
end
|
|
870
|
+
|
|
871
|
+
pii_filtering do
|
|
872
|
+
# Inherits: hashes :email, :phone + allows :user_id
|
|
873
|
+
# Add more:
|
|
874
|
+
masks :address # ← Additional field
|
|
875
|
+
end
|
|
876
|
+
end
|
|
877
|
+
```
|
|
878
|
+
|
|
879
|
+
**Base class for payment events with PII:**
|
|
880
|
+
|
|
881
|
+
```ruby
|
|
882
|
+
# app/events/base_payment_event.rb
|
|
883
|
+
module Events
|
|
884
|
+
class BasePaymentEvent < E11y::Event::Base
|
|
885
|
+
contains_pii true
|
|
886
|
+
|
|
887
|
+
pii_filtering do
|
|
888
|
+
# Common payment PII handling
|
|
889
|
+
hashes :email, :user_id # Pseudonymize
|
|
890
|
+
allows :order_id, :amount, :currency # Non-PII
|
|
891
|
+
masks :card_number, :cvv # Sensitive
|
|
892
|
+
end
|
|
893
|
+
end
|
|
894
|
+
end
|
|
895
|
+
|
|
896
|
+
# Inherit from base
|
|
897
|
+
class Events::PaymentSucceeded < Events::BasePaymentEvent
|
|
898
|
+
schema do
|
|
899
|
+
required(:transaction_id).filled(:string)
|
|
900
|
+
required(:order_id).filled(:string)
|
|
901
|
+
required(:user_id).filled(:string)
|
|
902
|
+
required(:email).filled(:string)
|
|
903
|
+
required(:amount).filled(:decimal)
|
|
904
|
+
required(:currency).filled(:string)
|
|
905
|
+
required(:card_number).filled(:string)
|
|
906
|
+
end
|
|
907
|
+
# ← Inherits ALL PII rules from BasePaymentEvent!
|
|
908
|
+
end
|
|
909
|
+
|
|
910
|
+
class Events::PaymentFailed < Events::BasePaymentEvent
|
|
911
|
+
schema do
|
|
912
|
+
required(:transaction_id).filled(:string)
|
|
913
|
+
required(:order_id).filled(:string)
|
|
914
|
+
required(:user_id).filled(:string)
|
|
915
|
+
required(:email).filled(:string)
|
|
916
|
+
required(:amount).filled(:decimal)
|
|
917
|
+
required(:error_code).filled(:string)
|
|
918
|
+
end
|
|
919
|
+
# ← Inherits ALL PII rules from BasePaymentEvent!
|
|
920
|
+
end
|
|
921
|
+
```
|
|
922
|
+
|
|
923
|
+
**Benefits:**
|
|
924
|
+
- ✅ DRY (common PII rules shared)
|
|
925
|
+
- ✅ Consistency (all user events handle PII same way)
|
|
926
|
+
- ✅ Easy to update (change base → all events updated)
|
|
927
|
+
- ✅ Linter validates base + child (complete coverage)
|
|
928
|
+
|
|
929
|
+
**Preset modules for PII:**
|
|
930
|
+
|
|
931
|
+
```ruby
|
|
932
|
+
# lib/e11y/presets/pii_aware_event.rb
|
|
933
|
+
module E11y
|
|
934
|
+
module Presets
|
|
935
|
+
module PiiAwareEvent
|
|
936
|
+
extend ActiveSupport::Concern
|
|
937
|
+
included do
|
|
938
|
+
contains_pii true
|
|
939
|
+
|
|
940
|
+
pii_filtering do
|
|
941
|
+
# Common PII patterns
|
|
942
|
+
hashes :email, :phone, :ip_address
|
|
943
|
+
masks :password, :token, :api_key, :secret
|
|
944
|
+
allows :user_id, :order_id, :transaction_id
|
|
945
|
+
end
|
|
946
|
+
end
|
|
947
|
+
end
|
|
948
|
+
end
|
|
949
|
+
end
|
|
950
|
+
|
|
951
|
+
# Usage:
|
|
952
|
+
class Events::UserAction < E11y::Event::Base
|
|
953
|
+
include E11y::Presets::PiiAwareEvent # ← Common PII rules!
|
|
954
|
+
|
|
955
|
+
schema do
|
|
956
|
+
required(:user_id).filled(:string)
|
|
957
|
+
required(:email).filled(:string)
|
|
958
|
+
required(:action).filled(:string)
|
|
959
|
+
end
|
|
960
|
+
|
|
961
|
+
pii_filtering do
|
|
962
|
+
# Inherits: hashes :email + allows :user_id
|
|
963
|
+
# Add more if needed:
|
|
964
|
+
allows :action # ← Additional field
|
|
965
|
+
end
|
|
966
|
+
end
|
|
967
|
+
```
|
|
968
|
+
|
|
969
|
+
---
|
|
970
|
+
|
|
971
|
+
## ⚡ DSL Shortcuts (Rails-Style)
|
|
972
|
+
|
|
973
|
+
> **Implementation:** See [ADR-006 Section 3.4.4: Configuration API (Rails-Style DSL)](../ADR-006-security-compliance.md#344-configuration-api-rails-style-dsl) for detailed architecture.
|
|
974
|
+
|
|
975
|
+
E11y provides **Rails-style DSL shortcuts** to simplify PII declarations. Instead of verbose `field` blocks, use one-liner shortcuts like `masks`, `hashes`, `skips` – similar to Rails validations.
|
|
976
|
+
|
|
977
|
+
### Why DSL Shortcuts?
|
|
978
|
+
|
|
979
|
+
**Problem:** Verbose declarations for simple cases:
|
|
980
|
+
|
|
981
|
+
```ruby
|
|
982
|
+
# ❌ Verbose: 15 lines for 3 fields
|
|
983
|
+
pii_filtering do
|
|
984
|
+
field :password do
|
|
985
|
+
strategy :mask
|
|
986
|
+
end
|
|
987
|
+
|
|
988
|
+
field :token do
|
|
989
|
+
strategy :mask
|
|
990
|
+
end
|
|
991
|
+
|
|
992
|
+
field :secret_key do
|
|
993
|
+
strategy :mask
|
|
994
|
+
end
|
|
995
|
+
end
|
|
996
|
+
```
|
|
997
|
+
|
|
998
|
+
**Solution:** Rails-style shortcuts (like `validates :name, presence: true`):
|
|
999
|
+
|
|
1000
|
+
```ruby
|
|
1001
|
+
# ✅ Concise: 3 lines for 3 fields
|
|
1002
|
+
pii_filtering do
|
|
1003
|
+
masks :password, :token, :secret_key
|
|
1004
|
+
end
|
|
1005
|
+
```
|
|
1006
|
+
|
|
1007
|
+
---
|
|
1008
|
+
|
|
1009
|
+
### Basic Shortcuts
|
|
1010
|
+
|
|
1011
|
+
**`masks(*fields)`** - Complete masking (replace with `[FILTERED]`)
|
|
1012
|
+
|
|
1013
|
+
```ruby
|
|
1014
|
+
class Events::UserLogin < E11y::Event::Base
|
|
1015
|
+
schema do
|
|
1016
|
+
required(:password).filled(:string)
|
|
1017
|
+
required(:token).filled(:string)
|
|
1018
|
+
required(:api_key).filled(:string)
|
|
1019
|
+
end
|
|
1020
|
+
|
|
1021
|
+
contains_pii true
|
|
1022
|
+
|
|
1023
|
+
pii_filtering do
|
|
1024
|
+
# Mask multiple fields at once
|
|
1025
|
+
masks :password, :token, :api_key
|
|
1026
|
+
end
|
|
1027
|
+
end
|
|
1028
|
+
|
|
1029
|
+
# Equivalent to:
|
|
1030
|
+
# field :password do; strategy :mask; end
|
|
1031
|
+
# field :token do; strategy :mask; end
|
|
1032
|
+
# field :api_key do; strategy :mask; end
|
|
1033
|
+
```
|
|
1034
|
+
|
|
1035
|
+
**`hashes(*fields)`** - Pseudonymization (SHA256 hash)
|
|
1036
|
+
|
|
1037
|
+
```ruby
|
|
1038
|
+
class Events::UserRegistered < E11y::Event::Base
|
|
1039
|
+
schema do
|
|
1040
|
+
required(:email).filled(:string)
|
|
1041
|
+
required(:phone).filled(:string)
|
|
1042
|
+
required(:ip_address).filled(:string)
|
|
1043
|
+
end
|
|
1044
|
+
|
|
1045
|
+
contains_pii true
|
|
1046
|
+
|
|
1047
|
+
pii_filtering do
|
|
1048
|
+
# Hash for searchability
|
|
1049
|
+
hashes :email, :phone, :ip_address
|
|
1050
|
+
end
|
|
1051
|
+
end
|
|
1052
|
+
|
|
1053
|
+
# Result:
|
|
1054
|
+
# {
|
|
1055
|
+
# email: 'hashed_a1b2c3d4...', # SHA256 of 'user@example.com'
|
|
1056
|
+
# phone: 'hashed_xyz789...', # SHA256 of '+1-555-1234'
|
|
1057
|
+
# ip_address: 'hashed_abc...' # SHA256 of '192.168.1.100'
|
|
1058
|
+
# }
|
|
1059
|
+
```
|
|
1060
|
+
|
|
1061
|
+
**`allows(*fields)`** - No filtering (explicitly safe)
|
|
1062
|
+
|
|
1063
|
+
```ruby
|
|
1064
|
+
class Events::OrderPaid < E11y::Event::Base
|
|
1065
|
+
schema do
|
|
1066
|
+
required(:order_id).filled(:string)
|
|
1067
|
+
required(:amount).filled(:float)
|
|
1068
|
+
required(:currency).filled(:string)
|
|
1069
|
+
end
|
|
1070
|
+
|
|
1071
|
+
contains_pii true # Event has PII elsewhere
|
|
1072
|
+
|
|
1073
|
+
pii_filtering do
|
|
1074
|
+
# Explicitly mark as non-PII
|
|
1075
|
+
allows :order_id, :amount, :currency
|
|
1076
|
+
end
|
|
1077
|
+
end
|
|
1078
|
+
```
|
|
1079
|
+
|
|
1080
|
+
**`partials(*fields)`** - Partial masking (show first/last chars)
|
|
1081
|
+
|
|
1082
|
+
```ruby
|
|
1083
|
+
class Events::SupportTicket < E11y::Event::Base
|
|
1084
|
+
schema do
|
|
1085
|
+
required(:email).filled(:string)
|
|
1086
|
+
required(:phone).filled(:string)
|
|
1087
|
+
end
|
|
1088
|
+
|
|
1089
|
+
contains_pii true
|
|
1090
|
+
|
|
1091
|
+
pii_filtering do
|
|
1092
|
+
# Show partial for debugging
|
|
1093
|
+
partials :email, :phone
|
|
1094
|
+
end
|
|
1095
|
+
end
|
|
1096
|
+
|
|
1097
|
+
# Result:
|
|
1098
|
+
# {
|
|
1099
|
+
# email: 'em***@ex***', # user@example.com → em***@ex***
|
|
1100
|
+
# phone: '+1-***-***-4567' # +1-555-123-4567 → +1-***-***-4567
|
|
1101
|
+
# }
|
|
1102
|
+
```
|
|
1103
|
+
|
|
1104
|
+
---
|
|
1105
|
+
|
|
1106
|
+
### Combined Example: Payment Processing
|
|
1107
|
+
|
|
1108
|
+
```ruby
|
|
1109
|
+
class Events::PaymentProcessed < E11y::Event::Base
|
|
1110
|
+
schema do
|
|
1111
|
+
# PII fields
|
|
1112
|
+
required(:card_number).filled(:string)
|
|
1113
|
+
required(:card_holder).filled(:string)
|
|
1114
|
+
required(:user_email).filled(:string)
|
|
1115
|
+
required(:billing_address).filled(:hash)
|
|
1116
|
+
|
|
1117
|
+
# Non-PII fields
|
|
1118
|
+
required(:order_id).filled(:string)
|
|
1119
|
+
required(:amount).filled(:float)
|
|
1120
|
+
required(:currency).filled(:string)
|
|
1121
|
+
end
|
|
1122
|
+
|
|
1123
|
+
contains_pii true
|
|
1124
|
+
|
|
1125
|
+
pii_filtering do
|
|
1126
|
+
# Sensitive: complete masking
|
|
1127
|
+
masks :card_number, :card_holder, :billing_address
|
|
1128
|
+
|
|
1129
|
+
# Searchable: hashing
|
|
1130
|
+
hashes :user_email
|
|
1131
|
+
|
|
1132
|
+
# Non-PII: explicitly allowed
|
|
1133
|
+
allows :order_id, :amount, :currency
|
|
1134
|
+
end
|
|
1135
|
+
end
|
|
1136
|
+
|
|
1137
|
+
# Compare to verbose version (15+ lines):
|
|
1138
|
+
# pii_filtering do
|
|
1139
|
+
# field :card_number do; strategy :mask; end
|
|
1140
|
+
# field :card_holder do; strategy :mask; end
|
|
1141
|
+
# field :billing_address do; strategy :mask; end
|
|
1142
|
+
# field :user_email do; strategy :hash; end
|
|
1143
|
+
# field :order_id do; strategy :allow; end
|
|
1144
|
+
# field :amount do; strategy :allow; end
|
|
1145
|
+
# field :currency do; strategy :allow; end
|
|
1146
|
+
# end
|
|
1147
|
+
```
|
|
1148
|
+
|
|
1149
|
+
---
|
|
1150
|
+
|
|
1151
|
+
### Advanced Shortcuts
|
|
1152
|
+
|
|
1153
|
+
**Per-Adapter Exclusions**
|
|
1154
|
+
|
|
1155
|
+
```ruby
|
|
1156
|
+
class Events::SensitiveAction < E11y::Event::Base
|
|
1157
|
+
schema do
|
|
1158
|
+
required(:user_email).filled(:string)
|
|
1159
|
+
required(:action).filled(:string)
|
|
1160
|
+
end
|
|
1161
|
+
|
|
1162
|
+
contains_pii true
|
|
1163
|
+
|
|
1164
|
+
pii_filtering do
|
|
1165
|
+
# Hash email, but keep original in audit
|
|
1166
|
+
hashes :user_email, exclude_adapters: [:file_audit]
|
|
1167
|
+
|
|
1168
|
+
# Action is not PII
|
|
1169
|
+
allows :action
|
|
1170
|
+
end
|
|
1171
|
+
end
|
|
1172
|
+
|
|
1173
|
+
# Result:
|
|
1174
|
+
# audit_file: { user_email: 'john@example.com' } (original)
|
|
1175
|
+
# elasticsearch: { user_email: 'hashed_a1b2c3' } (hashed)
|
|
1176
|
+
# loki: { user_email: 'hashed_a1b2c3' } (hashed)
|
|
1177
|
+
```
|
|
1178
|
+
|
|
1179
|
+
**Conditional Filtering (Rails-style)**
|
|
1180
|
+
|
|
1181
|
+
```ruby
|
|
1182
|
+
class Events::UserAction < E11y::Event::Base
|
|
1183
|
+
schema do
|
|
1184
|
+
required(:email).filled(:string)
|
|
1185
|
+
required(:admin_flag).filled(:bool)
|
|
1186
|
+
end
|
|
1187
|
+
|
|
1188
|
+
contains_pii true
|
|
1189
|
+
|
|
1190
|
+
pii_filtering do
|
|
1191
|
+
# Only mask in production
|
|
1192
|
+
masks_if -> { Rails.env.production? }, :email
|
|
1193
|
+
|
|
1194
|
+
# Admin flag is not PII
|
|
1195
|
+
allows :admin_flag
|
|
1196
|
+
end
|
|
1197
|
+
end
|
|
1198
|
+
```
|
|
1199
|
+
|
|
1200
|
+
**Grouping with `with_strategy`**
|
|
1201
|
+
|
|
1202
|
+
```ruby
|
|
1203
|
+
class Events::UserProfile < E11y::Event::Base
|
|
1204
|
+
schema do
|
|
1205
|
+
required(:password).filled(:string)
|
|
1206
|
+
required(:token).filled(:string)
|
|
1207
|
+
required(:secret_key).filled(:string)
|
|
1208
|
+
required(:email).filled(:string)
|
|
1209
|
+
required(:phone).filled(:string)
|
|
1210
|
+
end
|
|
1211
|
+
|
|
1212
|
+
contains_pii true
|
|
1213
|
+
|
|
1214
|
+
pii_filtering do
|
|
1215
|
+
# Group fields by strategy
|
|
1216
|
+
with_strategy :mask do
|
|
1217
|
+
field :password
|
|
1218
|
+
field :token
|
|
1219
|
+
field :secret_key
|
|
1220
|
+
end
|
|
1221
|
+
|
|
1222
|
+
with_strategy :hash do
|
|
1223
|
+
field :email
|
|
1224
|
+
field :phone
|
|
1225
|
+
end
|
|
1226
|
+
end
|
|
1227
|
+
end
|
|
1228
|
+
|
|
1229
|
+
# Equivalent to:
|
|
1230
|
+
# masks :password, :token, :secret_key
|
|
1231
|
+
# hashes :email, :phone
|
|
1232
|
+
```
|
|
1233
|
+
|
|
1234
|
+
**Bulk Operations**
|
|
1235
|
+
|
|
1236
|
+
```ruby
|
|
1237
|
+
class Events::ComplexEvent < E11y::Event::Base
|
|
1238
|
+
schema do
|
|
1239
|
+
required(:password).filled(:string)
|
|
1240
|
+
required(:token).filled(:string)
|
|
1241
|
+
required(:email).filled(:string)
|
|
1242
|
+
required(:phone).filled(:string)
|
|
1243
|
+
required(:order_id).filled(:string)
|
|
1244
|
+
required(:amount).filled(:float)
|
|
1245
|
+
end
|
|
1246
|
+
|
|
1247
|
+
contains_pii true
|
|
1248
|
+
|
|
1249
|
+
pii_filtering do
|
|
1250
|
+
# Mask everything EXCEPT safe fields
|
|
1251
|
+
masks_all_except :order_id, :amount
|
|
1252
|
+
end
|
|
1253
|
+
end
|
|
1254
|
+
|
|
1255
|
+
# Result: password, token, email, phone → masked
|
|
1256
|
+
# order_id, amount → allowed
|
|
1257
|
+
```
|
|
1258
|
+
|
|
1259
|
+
---
|
|
1260
|
+
|
|
1261
|
+
### Cheat Sheet: Shortcuts vs. Strategies
|
|
1262
|
+
|
|
1263
|
+
| Shortcut | Strategy | Output Example | Use Case |
|
|
1264
|
+
|----------|----------|----------------|----------|
|
|
1265
|
+
| `masks` | `:mask` | `[FILTERED]` | Passwords, secrets, credit cards |
|
|
1266
|
+
| `hashes` | `:hash` | `hashed_a1b2c3` | Emails, phones (searchable) |
|
|
1267
|
+
| `allows` | `:allow` | Original value | IDs, amounts (non-PII) |
|
|
1268
|
+
| `partials` | `:partial` | `em***@ex***` | Debugging (show partial) |
|
|
1269
|
+
|
|
1270
|
+
---
|
|
1271
|
+
|
|
1272
|
+
### When to Use Shortcuts vs. Verbose DSL
|
|
1273
|
+
|
|
1274
|
+
**Use Shortcuts:**
|
|
1275
|
+
```ruby
|
|
1276
|
+
# ✅ GOOD: Simple cases with same strategy
|
|
1277
|
+
pii_filtering do
|
|
1278
|
+
masks :password, :token, :api_key
|
|
1279
|
+
hashes :email, :phone
|
|
1280
|
+
allows :order_id, :amount
|
|
1281
|
+
end
|
|
1282
|
+
```
|
|
1283
|
+
|
|
1284
|
+
**Use Verbose DSL:**
|
|
1285
|
+
```ruby
|
|
1286
|
+
# ✅ GOOD: Complex per-field configuration
|
|
1287
|
+
pii_filtering do
|
|
1288
|
+
field :email do
|
|
1289
|
+
strategy :hash
|
|
1290
|
+
hash_algorithm :sha256
|
|
1291
|
+
hash_salt ENV['PII_SALT']
|
|
1292
|
+
exclude_adapters [:file_audit]
|
|
1293
|
+
end
|
|
1294
|
+
|
|
1295
|
+
field :ip_address do
|
|
1296
|
+
strategy :partial
|
|
1297
|
+
custom_for_adapter :loki do
|
|
1298
|
+
->(value) { value.split('.')[0..2].join('.') + '.x' }
|
|
1299
|
+
end
|
|
1300
|
+
end
|
|
1301
|
+
end
|
|
1302
|
+
```
|
|
1303
|
+
|
|
1304
|
+
---
|
|
1305
|
+
|
|
1306
|
+
### Migration: Verbose → Shortcuts
|
|
1307
|
+
|
|
1308
|
+
**Before (Verbose):**
|
|
1309
|
+
```ruby
|
|
1310
|
+
class Events::UserLogin < E11y::Event::Base
|
|
1311
|
+
contains_pii true
|
|
1312
|
+
|
|
1313
|
+
pii_filtering do
|
|
1314
|
+
field :password do
|
|
1315
|
+
strategy :mask
|
|
1316
|
+
end
|
|
1317
|
+
|
|
1318
|
+
field :email do
|
|
1319
|
+
strategy :hash
|
|
1320
|
+
end
|
|
1321
|
+
|
|
1322
|
+
field :user_id do
|
|
1323
|
+
strategy :allow
|
|
1324
|
+
end
|
|
1325
|
+
end
|
|
1326
|
+
end
|
|
1327
|
+
```
|
|
1328
|
+
|
|
1329
|
+
**After (Shortcuts):**
|
|
1330
|
+
```ruby
|
|
1331
|
+
class Events::UserLogin < E11y::Event::Base
|
|
1332
|
+
contains_pii true
|
|
1333
|
+
|
|
1334
|
+
pii_filtering do
|
|
1335
|
+
masks :password
|
|
1336
|
+
hashes :email
|
|
1337
|
+
allows :user_id
|
|
1338
|
+
end
|
|
1339
|
+
end
|
|
1340
|
+
|
|
1341
|
+
# Or even shorter:
|
|
1342
|
+
# pii_filtering do
|
|
1343
|
+
# masks :password
|
|
1344
|
+
# hashes :email
|
|
1345
|
+
# allows :user_id
|
|
1346
|
+
# end
|
|
1347
|
+
```
|
|
1348
|
+
|
|
1349
|
+
---
|
|
1350
|
+
|
|
1351
|
+
### Best Practices
|
|
1352
|
+
|
|
1353
|
+
**1. Use shortcuts for simple cases**
|
|
1354
|
+
```ruby
|
|
1355
|
+
# ✅ GOOD: Clear and concise
|
|
1356
|
+
masks :password, :token
|
|
1357
|
+
hashes :email, :phone
|
|
1358
|
+
```
|
|
1359
|
+
|
|
1360
|
+
**2. Group related fields**
|
|
1361
|
+
```ruby
|
|
1362
|
+
# ✅ GOOD: Grouped by purpose
|
|
1363
|
+
pii_filtering do
|
|
1364
|
+
# Credentials: mask completely
|
|
1365
|
+
masks :password, :token, :api_key
|
|
1366
|
+
|
|
1367
|
+
# Identifiers: hash for searchability
|
|
1368
|
+
hashes :email, :phone, :user_id
|
|
1369
|
+
|
|
1370
|
+
# Business data: allow
|
|
1371
|
+
allows :order_id, :amount, :currency
|
|
1372
|
+
end
|
|
1373
|
+
```
|
|
1374
|
+
|
|
1375
|
+
**3. Use verbose DSL for complex config**
|
|
1376
|
+
```ruby
|
|
1377
|
+
# ✅ GOOD: Complex per-adapter rules need verbose syntax
|
|
1378
|
+
field :email do
|
|
1379
|
+
strategy :hash
|
|
1380
|
+
exclude_adapters [:file_audit]
|
|
1381
|
+
custom_for_adapter :loki do
|
|
1382
|
+
->(value) { mask_domain(value) }
|
|
1383
|
+
end
|
|
1384
|
+
end
|
|
1385
|
+
```
|
|
1386
|
+
|
|
1387
|
+
**4. Don't mix shortcuts and verbose for same strategy**
|
|
1388
|
+
```ruby
|
|
1389
|
+
# ❌ BAD: Mixing shortcuts and verbose
|
|
1390
|
+
masks :password
|
|
1391
|
+
field :token do; strategy :mask; end # ← Should use shortcut
|
|
1392
|
+
|
|
1393
|
+
# ✅ GOOD: Consistent style
|
|
1394
|
+
masks :password, :token
|
|
1395
|
+
```
|
|
1396
|
+
|
|
1397
|
+
---
|
|
1398
|
+
|
|
1399
|
+
## 🔍 Linter Enforcement
|
|
1400
|
+
|
|
1401
|
+
> **Implementation:** See [ADR-006 Section 3.0.5: PII Declaration Linter](../ADR-006-security-compliance.md#305-pii-declaration-linter) for detailed architecture.
|
|
1402
|
+
|
|
1403
|
+
E11y includes a **PII Declaration Linter** that validates PII handling at boot time and in CI. This catches missing declarations, typos, and incomplete coverage BEFORE code reaches production.
|
|
1404
|
+
|
|
1405
|
+
### Why Linter Enforcement?
|
|
1406
|
+
|
|
1407
|
+
**Problem:** Manual PII declaration is error-prone:
|
|
1408
|
+
- ❌ Forget to declare a field → PII leaks to logs
|
|
1409
|
+
- ❌ Typo in field name → Declaration doesn't apply
|
|
1410
|
+
- ❌ Add new field, forget PII strategy → Security gap
|
|
1411
|
+
|
|
1412
|
+
**Solution:** Linter validates at boot time (development/test) and in CI.
|
|
1413
|
+
|
|
1414
|
+
---
|
|
1415
|
+
|
|
1416
|
+
### What the Linter Checks
|
|
1417
|
+
|
|
1418
|
+
When `contains_pii true` is declared, the linter enforces:
|
|
1419
|
+
|
|
1420
|
+
1. ✅ **Every schema field has a filtering strategy** (completeness)
|
|
1421
|
+
2. ✅ **No extra fields in pii_filtering** (no typos)
|
|
1422
|
+
3. ✅ **Valid strategies only** (`:mask`, `:hash`, `:allow`, `:partial`)
|
|
1423
|
+
4. ✅ **Adapter exclusions are valid** (adapter exists)
|
|
1424
|
+
|
|
1425
|
+
**Example: Linter Catches Missing Field**
|
|
1426
|
+
|
|
1427
|
+
```ruby
|
|
1428
|
+
class Events::UserLogin < E11y::Event::Base
|
|
1429
|
+
schema do
|
|
1430
|
+
required(:email).filled(:string)
|
|
1431
|
+
required(:password).filled(:string)
|
|
1432
|
+
required(:ip_address).filled(:string) # ← Missing in pii_filtering!
|
|
1433
|
+
end
|
|
1434
|
+
|
|
1435
|
+
contains_pii true
|
|
1436
|
+
|
|
1437
|
+
pii_filtering do
|
|
1438
|
+
field :email do
|
|
1439
|
+
strategy :hash
|
|
1440
|
+
end
|
|
1441
|
+
|
|
1442
|
+
field :password do
|
|
1443
|
+
strategy :mask
|
|
1444
|
+
end
|
|
1445
|
+
|
|
1446
|
+
# ❌ Missing: :ip_address
|
|
1447
|
+
end
|
|
1448
|
+
end
|
|
1449
|
+
|
|
1450
|
+
# Boot output:
|
|
1451
|
+
# ❌ E11y::Linters::PiiDeclarationError:
|
|
1452
|
+
# Missing PII declaration for Events::UserLogin
|
|
1453
|
+
#
|
|
1454
|
+
# Schema fields: [:email, :password, :ip_address]
|
|
1455
|
+
# Declared fields: [:email, :password]
|
|
1456
|
+
# Missing: [:ip_address]
|
|
1457
|
+
#
|
|
1458
|
+
# Fix: Add pii_filtering for :ip_address
|
|
1459
|
+
```
|
|
1460
|
+
|
|
1461
|
+
**Example: Linter Catches Typo**
|
|
1462
|
+
|
|
1463
|
+
```ruby
|
|
1464
|
+
class Events::PaymentProcessed < E11y::Event::Base
|
|
1465
|
+
schema do
|
|
1466
|
+
required(:card_number).filled(:string)
|
|
1467
|
+
required(:amount).filled(:float)
|
|
1468
|
+
end
|
|
1469
|
+
|
|
1470
|
+
contains_pii true
|
|
1471
|
+
|
|
1472
|
+
pii_filtering do
|
|
1473
|
+
field :card_numbre do # ← Typo: numbre instead of number
|
|
1474
|
+
strategy :mask
|
|
1475
|
+
end
|
|
1476
|
+
|
|
1477
|
+
field :amount do
|
|
1478
|
+
strategy :allow
|
|
1479
|
+
end
|
|
1480
|
+
end
|
|
1481
|
+
end
|
|
1482
|
+
|
|
1483
|
+
# Boot output:
|
|
1484
|
+
# ❌ E11y::Linters::PiiDeclarationError:
|
|
1485
|
+
# Invalid PII declarations for Events::PaymentProcessed
|
|
1486
|
+
#
|
|
1487
|
+
# Schema fields: [:card_number, :amount]
|
|
1488
|
+
# Declared fields: [:card_numbre, :amount]
|
|
1489
|
+
# Extra: [:card_numbre] # ← Not in schema (typo?)
|
|
1490
|
+
# Missing: [:card_number] # ← Not declared
|
|
1491
|
+
#
|
|
1492
|
+
# Fix: Check field names match schema exactly
|
|
1493
|
+
```
|
|
1494
|
+
|
|
1495
|
+
---
|
|
1496
|
+
|
|
1497
|
+
### Running the Linter
|
|
1498
|
+
|
|
1499
|
+
**Option 1: Boot-Time Validation (Development/Test)**
|
|
1500
|
+
|
|
1501
|
+
```ruby
|
|
1502
|
+
# config/initializers/e11y.rb
|
|
1503
|
+
E11y.configure do |config|
|
|
1504
|
+
# ... other config ...
|
|
1505
|
+
|
|
1506
|
+
# Validate PII declarations at boot
|
|
1507
|
+
if Rails.env.development? || Rails.env.test?
|
|
1508
|
+
config.after_initialize do
|
|
1509
|
+
E11y::Linters::PiiDeclarationLinter.validate_all!
|
|
1510
|
+
end
|
|
1511
|
+
end
|
|
1512
|
+
end
|
|
1513
|
+
|
|
1514
|
+
# Result: App won't boot if PII declarations invalid
|
|
1515
|
+
# $ rails server
|
|
1516
|
+
# => Booting Puma
|
|
1517
|
+
# => Rails 7.1.2 application starting in development
|
|
1518
|
+
# => Run `bin/rails server --help` for more startup options
|
|
1519
|
+
# ❌ E11y::Linters::PiiDeclarationError: Missing PII declaration for Events::UserLogin
|
|
1520
|
+
# ... (detailed error message) ...
|
|
1521
|
+
```
|
|
1522
|
+
|
|
1523
|
+
**Option 2: Rake Task (CI/Manual)**
|
|
1524
|
+
|
|
1525
|
+
```bash
|
|
1526
|
+
# Run PII linter manually
|
|
1527
|
+
bundle exec rake e11y:lint:pii
|
|
1528
|
+
|
|
1529
|
+
# Output:
|
|
1530
|
+
# Checking PII declarations...
|
|
1531
|
+
# ================================================================================
|
|
1532
|
+
# ✅ Events::UserRegistered - All 4 fields declared
|
|
1533
|
+
# ✅ Events::PaymentProcessed - All 6 fields declared
|
|
1534
|
+
# ⚪ Events::HealthCheck - No PII (skipped)
|
|
1535
|
+
# ⚪ Events::OrderCreated - No PII declaration (Tier 2 default)
|
|
1536
|
+
# ❌ Events::UserLogin - Missing declarations
|
|
1537
|
+
# ================================================================================
|
|
1538
|
+
#
|
|
1539
|
+
# ❌ ERRORS:
|
|
1540
|
+
#
|
|
1541
|
+
# Missing PII declaration for Events::UserLogin
|
|
1542
|
+
# Schema fields: [:email, :password, :ip_address]
|
|
1543
|
+
# Declared fields: [:email, :password]
|
|
1544
|
+
# Missing: [:ip_address]
|
|
1545
|
+
#
|
|
1546
|
+
# Fix: Add pii_filtering for :ip_address
|
|
1547
|
+
#
|
|
1548
|
+
# Exit code: 1 (fails CI build)
|
|
1549
|
+
```
|
|
1550
|
+
|
|
1551
|
+
**Option 3: RSpec Matcher (Unit Tests)**
|
|
1552
|
+
|
|
1553
|
+
```ruby
|
|
1554
|
+
# spec/support/e11y_pii_matchers.rb
|
|
1555
|
+
RSpec::Matchers.define :have_complete_pii_declaration do
|
|
1556
|
+
match do |event_class|
|
|
1557
|
+
return true unless event_class.contains_pii?
|
|
1558
|
+
|
|
1559
|
+
E11y::Linters::PiiDeclarationLinter.validate!(event_class)
|
|
1560
|
+
true
|
|
1561
|
+
rescue E11y::Linters::PiiDeclarationError => e
|
|
1562
|
+
@error_message = e.message
|
|
1563
|
+
false
|
|
1564
|
+
end
|
|
1565
|
+
|
|
1566
|
+
failure_message do |event_class|
|
|
1567
|
+
"Expected #{event_class.name} to have complete PII declaration, but:\n#{@error_message}"
|
|
1568
|
+
end
|
|
1569
|
+
end
|
|
1570
|
+
|
|
1571
|
+
# spec/events/user_login_spec.rb
|
|
1572
|
+
RSpec.describe Events::UserLogin do
|
|
1573
|
+
it { is_expected.to have_complete_pii_declaration }
|
|
1574
|
+
end
|
|
1575
|
+
|
|
1576
|
+
# Test output if declaration incomplete:
|
|
1577
|
+
# ❌ Expected Events::UserLogin to have complete PII declaration, but:
|
|
1578
|
+
# Missing PII declaration for Events::UserLogin
|
|
1579
|
+
# Schema fields: [:email, :password, :ip_address]
|
|
1580
|
+
# Declared fields: [:email, :password]
|
|
1581
|
+
# Missing: [:ip_address]
|
|
1582
|
+
```
|
|
1583
|
+
|
|
1584
|
+
---
|
|
1585
|
+
|
|
1586
|
+
### Linter Configuration
|
|
1587
|
+
|
|
1588
|
+
**Enable/Disable Linter**
|
|
1589
|
+
|
|
1590
|
+
```ruby
|
|
1591
|
+
E11y.configure do |config|
|
|
1592
|
+
config.pii_linter do
|
|
1593
|
+
# Enable in development/test (default: true)
|
|
1594
|
+
enabled Rails.env.development? || Rails.env.test?
|
|
1595
|
+
|
|
1596
|
+
# Fail on errors (default: true)
|
|
1597
|
+
fail_on_error true
|
|
1598
|
+
|
|
1599
|
+
# Log warnings for default (Tier 2) events (default: false)
|
|
1600
|
+
warn_on_default_tier false
|
|
1601
|
+
end
|
|
1602
|
+
end
|
|
1603
|
+
```
|
|
1604
|
+
|
|
1605
|
+
**Custom Linter Rules**
|
|
1606
|
+
|
|
1607
|
+
```ruby
|
|
1608
|
+
E11y.configure do |config|
|
|
1609
|
+
config.pii_linter do
|
|
1610
|
+
# Enforce explicit declaration for ALL events (even Tier 2)
|
|
1611
|
+
require_explicit_declaration true # Default: false
|
|
1612
|
+
|
|
1613
|
+
# Allowed strategies (customize if needed)
|
|
1614
|
+
allowed_strategies [:mask, :hash, :allow, :partial]
|
|
1615
|
+
|
|
1616
|
+
# Forbidden strategies (never allow)
|
|
1617
|
+
forbidden_strategies [:skip] # Force explicit :allow instead
|
|
1618
|
+
end
|
|
1619
|
+
end
|
|
1620
|
+
```
|
|
1621
|
+
|
|
1622
|
+
---
|
|
1623
|
+
|
|
1624
|
+
### CI Integration
|
|
1625
|
+
|
|
1626
|
+
**GitHub Actions Example**
|
|
1627
|
+
|
|
1628
|
+
```yaml
|
|
1629
|
+
# .github/workflows/ci.yml
|
|
1630
|
+
name: CI
|
|
1631
|
+
|
|
1632
|
+
on: [push, pull_request]
|
|
1633
|
+
|
|
1634
|
+
jobs:
|
|
1635
|
+
test:
|
|
1636
|
+
runs-on: ubuntu-latest
|
|
1637
|
+
|
|
1638
|
+
steps:
|
|
1639
|
+
- uses: actions/checkout@v3
|
|
1640
|
+
- uses: ruby/setup-ruby@v1
|
|
1641
|
+
with:
|
|
1642
|
+
ruby-version: 3.2
|
|
1643
|
+
bundler-cache: true
|
|
1644
|
+
|
|
1645
|
+
# Run PII linter BEFORE tests
|
|
1646
|
+
- name: Validate PII Declarations
|
|
1647
|
+
run: bundle exec rake e11y:lint:pii
|
|
1648
|
+
|
|
1649
|
+
# Run tests only if linter passes
|
|
1650
|
+
- name: Run tests
|
|
1651
|
+
run: bundle exec rspec
|
|
1652
|
+
```
|
|
1653
|
+
|
|
1654
|
+
**GitLab CI Example**
|
|
1655
|
+
|
|
1656
|
+
```yaml
|
|
1657
|
+
# .gitlab-ci.yml
|
|
1658
|
+
test:
|
|
1659
|
+
stage: test
|
|
1660
|
+
script:
|
|
1661
|
+
# Fail fast if PII declarations invalid
|
|
1662
|
+
- bundle exec rake e11y:lint:pii
|
|
1663
|
+
- bundle exec rspec
|
|
1664
|
+
```
|
|
1665
|
+
|
|
1666
|
+
---
|
|
1667
|
+
|
|
1668
|
+
### Audit Report
|
|
1669
|
+
|
|
1670
|
+
Generate a report of all PII declarations:
|
|
1671
|
+
|
|
1672
|
+
```bash
|
|
1673
|
+
# Generate PII audit report
|
|
1674
|
+
bundle exec rake e11y:audit:pii_declarations
|
|
1675
|
+
|
|
1676
|
+
# Output:
|
|
1677
|
+
# E11y PII Declaration Audit Report
|
|
1678
|
+
# ================================================================================
|
|
1679
|
+
# Generated: 2026-01-14 10:30:00 UTC
|
|
1680
|
+
# Total Events: 42
|
|
1681
|
+
#
|
|
1682
|
+
# 📊 SUMMARY:
|
|
1683
|
+
# - contains_pii true: 8 events (19%)
|
|
1684
|
+
# - contains_pii false: 5 events (12%)
|
|
1685
|
+
# - No declaration: 29 events (69%, using Tier 2 default)
|
|
1686
|
+
#
|
|
1687
|
+
# 📋 TIER 3 EVENTS (contains_pii true):
|
|
1688
|
+
#
|
|
1689
|
+
# ✅ Events::UserRegistered
|
|
1690
|
+
# Fields: [:email, :password, :address, :user_id]
|
|
1691
|
+
# Strategies: hash(1), mask(2), allow(1)
|
|
1692
|
+
# Adapters excluded: [:file_audit] for [:email, :address]
|
|
1693
|
+
#
|
|
1694
|
+
# ✅ Events::PaymentProcessed
|
|
1695
|
+
# Fields: [:card_number, :amount, :user_email]
|
|
1696
|
+
# Strategies: mask(1), allow(1), hash(1)
|
|
1697
|
+
# Adapters excluded: none
|
|
1698
|
+
#
|
|
1699
|
+
# ... (more events) ...
|
|
1700
|
+
#
|
|
1701
|
+
# 📋 TIER 1 EVENTS (contains_pii false):
|
|
1702
|
+
#
|
|
1703
|
+
# ⚪ Events::HealthCheck
|
|
1704
|
+
# Fields: [:status, :uptime_ms]
|
|
1705
|
+
# PII filtering: SKIPPED (performance optimized)
|
|
1706
|
+
#
|
|
1707
|
+
# ... (more events) ...
|
|
1708
|
+
#
|
|
1709
|
+
# ⚠️ TIER 2 EVENTS (no declaration, default filtering):
|
|
1710
|
+
#
|
|
1711
|
+
# 🔵 Events::OrderCreated
|
|
1712
|
+
# Fields: [:order_id, :amount, :api_key]
|
|
1713
|
+
# PII filtering: Rails filters only (Tier 2 default)
|
|
1714
|
+
# Recommendation: Keep default (sufficient for standard events)
|
|
1715
|
+
#
|
|
1716
|
+
# ... (more events) ...
|
|
1717
|
+
#
|
|
1718
|
+
# ================================================================================
|
|
1719
|
+
#
|
|
1720
|
+
# 💡 RECOMMENDATIONS:
|
|
1721
|
+
# - 29 events use Tier 2 default (Rails filters)
|
|
1722
|
+
# - Consider adding contains_pii false to high-frequency events (health checks)
|
|
1723
|
+
# - All Tier 3 events have complete declarations ✅
|
|
1724
|
+
```
|
|
1725
|
+
|
|
1726
|
+
---
|
|
1727
|
+
|
|
1728
|
+
### Best Practices
|
|
1729
|
+
|
|
1730
|
+
**1. Enable linter in development/test**
|
|
1731
|
+
```ruby
|
|
1732
|
+
# ✅ GOOD: Catch errors early
|
|
1733
|
+
config.after_initialize do
|
|
1734
|
+
E11y::Linters::PiiDeclarationLinter.validate_all!
|
|
1735
|
+
end
|
|
1736
|
+
```
|
|
1737
|
+
|
|
1738
|
+
**2. Run linter in CI before tests**
|
|
1739
|
+
```bash
|
|
1740
|
+
# ✅ GOOD: Fail fast
|
|
1741
|
+
bundle exec rake e11y:lint:pii && bundle exec rspec
|
|
1742
|
+
```
|
|
1743
|
+
|
|
1744
|
+
**3. Use RSpec matchers for new events**
|
|
1745
|
+
```ruby
|
|
1746
|
+
# ✅ GOOD: Test-driven PII declarations
|
|
1747
|
+
RSpec.describe Events::NewEvent do
|
|
1748
|
+
it { is_expected.to have_complete_pii_declaration }
|
|
1749
|
+
end
|
|
1750
|
+
```
|
|
1751
|
+
|
|
1752
|
+
**4. Review audit report periodically**
|
|
1753
|
+
```bash
|
|
1754
|
+
# ✅ GOOD: Ensure no PII leaks over time
|
|
1755
|
+
bundle exec rake e11y:audit:pii_declarations > pii_audit_$(date +%Y%m%d).txt
|
|
1756
|
+
```
|
|
1757
|
+
|
|
1758
|
+
**5. Don't disable linter in production**
|
|
1759
|
+
```ruby
|
|
1760
|
+
# ❌ BAD: Linter should not run in production (performance)
|
|
1761
|
+
# Boot-time validation is for dev/test only
|
|
1762
|
+
|
|
1763
|
+
# ✅ GOOD: Enable only in non-production
|
|
1764
|
+
if Rails.env.development? || Rails.env.test?
|
|
1765
|
+
E11y::Linters::PiiDeclarationLinter.validate_all!
|
|
1766
|
+
end
|
|
1767
|
+
```
|
|
1768
|
+
|
|
1769
|
+
---
|
|
1770
|
+
|
|
1771
|
+
## ⚡ Performance Tiers
|
|
1772
|
+
|
|
1773
|
+
> **Implementation:** See [ADR-006 Section 3.0: PII Filtering Strategy](../ADR-006-security-compliance.md#30-pii-filtering-strategy) for detailed architecture.
|
|
1774
|
+
|
|
1775
|
+
E11y uses a **3-tier filtering strategy** to balance security and performance. Filtering ALL events by default would create massive overhead (1M events × 0.2ms = 200 seconds CPU/day). Instead, events are categorized into 3 tiers based on PII content.
|
|
1776
|
+
|
|
1777
|
+
### Overview: 3-Tier Strategy
|
|
1778
|
+
|
|
1779
|
+
| Tier | Strategy | Overhead | Use Case | Events/sec |
|
|
1780
|
+
|------|----------|----------|----------|------------|
|
|
1781
|
+
| **Tier 1** | Skip filtering | 0ms | Health checks, metrics, internal events | 500 |
|
|
1782
|
+
| **Tier 2** | Rails filters only | ~0.05ms | Standard events (known PII keys) | 400 |
|
|
1783
|
+
| **Tier 3** | Deep filtering | ~0.2ms | User data, payments, complex nested | 100 |
|
|
1784
|
+
|
|
1785
|
+
**Performance Budget:**
|
|
1786
|
+
```
|
|
1787
|
+
500 events/sec × 0ms = 0ms CPU/sec (Tier 1)
|
|
1788
|
+
400 events/sec × 0.05ms = 20ms CPU/sec (Tier 2)
|
|
1789
|
+
100 events/sec × 0.2ms = 20ms CPU/sec (Tier 3)
|
|
1790
|
+
----
|
|
1791
|
+
Total: 40ms CPU/sec = 4% CPU on single core ✅
|
|
1792
|
+
```
|
|
1793
|
+
|
|
1794
|
+
---
|
|
1795
|
+
|
|
1796
|
+
### Tier 1: No PII (Skip Filtering)
|
|
1797
|
+
|
|
1798
|
+
**Use when:** Event contains NO personal data (health checks, metrics, system events).
|
|
1799
|
+
|
|
1800
|
+
**How to declare:**
|
|
1801
|
+
```ruby
|
|
1802
|
+
class Events::HealthCheck < E11y::Event::Base
|
|
1803
|
+
schema do
|
|
1804
|
+
required(:status).filled(:string)
|
|
1805
|
+
required(:uptime_ms).filled(:integer)
|
|
1806
|
+
end
|
|
1807
|
+
|
|
1808
|
+
# ✅ Explicit: This event contains NO PII
|
|
1809
|
+
contains_pii false # Skip all PII filtering
|
|
1810
|
+
end
|
|
1811
|
+
|
|
1812
|
+
# Result: 0ms overhead per event
|
|
1813
|
+
```
|
|
1814
|
+
|
|
1815
|
+
**Performance:**
|
|
1816
|
+
```ruby
|
|
1817
|
+
# Benchmark: 1000 events
|
|
1818
|
+
Benchmark.ips do |x|
|
|
1819
|
+
x.report('Tier 1 - No PII') do
|
|
1820
|
+
Events::HealthCheck.track(status: 'ok', uptime_ms: 12345)
|
|
1821
|
+
end
|
|
1822
|
+
end
|
|
1823
|
+
|
|
1824
|
+
# Results:
|
|
1825
|
+
# Tier 1 - No PII: 10,000 i/s (100μs per event)
|
|
1826
|
+
# Overhead: 0ms (no filtering)
|
|
1827
|
+
```
|
|
1828
|
+
|
|
1829
|
+
**When to use:**
|
|
1830
|
+
- ✅ Health checks
|
|
1831
|
+
- ✅ Performance metrics
|
|
1832
|
+
- ✅ System heartbeats
|
|
1833
|
+
- ✅ Resource usage events
|
|
1834
|
+
- ❌ Anything with user data
|
|
1835
|
+
|
|
1836
|
+
---
|
|
1837
|
+
|
|
1838
|
+
### Tier 2: Rails Filters Only (Default)
|
|
1839
|
+
|
|
1840
|
+
**Use when:** Event has simple PII (passwords, tokens, API keys) already in `Rails.filter_parameters`.
|
|
1841
|
+
|
|
1842
|
+
**How to declare:**
|
|
1843
|
+
```ruby
|
|
1844
|
+
class Events::OrderCreated < E11y::Event::Base
|
|
1845
|
+
schema do
|
|
1846
|
+
required(:order_id).filled(:string)
|
|
1847
|
+
required(:amount).filled(:float)
|
|
1848
|
+
optional(:api_key).filled(:string)
|
|
1849
|
+
end
|
|
1850
|
+
|
|
1851
|
+
# No declaration → Rails filters applied automatically (Tier 2)
|
|
1852
|
+
# Filters keys like: password, token, secret, api_key
|
|
1853
|
+
end
|
|
1854
|
+
|
|
1855
|
+
# Result: ~0.05ms overhead per event
|
|
1856
|
+
```
|
|
1857
|
+
|
|
1858
|
+
**How it works:**
|
|
1859
|
+
```ruby
|
|
1860
|
+
# Rails config (single source of truth)
|
|
1861
|
+
Rails.application.config.filter_parameters += [:password, :email, :token]
|
|
1862
|
+
|
|
1863
|
+
# E11y automatically applies these filters
|
|
1864
|
+
Events::OrderCreated.track(
|
|
1865
|
+
order_id: 'o123',
|
|
1866
|
+
amount: 99.99,
|
|
1867
|
+
api_key: 'sk_live_xxx' # ← Filtered by Rails config
|
|
1868
|
+
)
|
|
1869
|
+
|
|
1870
|
+
# Logged as:
|
|
1871
|
+
# {
|
|
1872
|
+
# order_id: 'o123',
|
|
1873
|
+
# amount: 99.99,
|
|
1874
|
+
# api_key: '[FILTERED]' # ← Rails filter applied
|
|
1875
|
+
# }
|
|
1876
|
+
```
|
|
1877
|
+
|
|
1878
|
+
**Performance:**
|
|
1879
|
+
```ruby
|
|
1880
|
+
# Benchmark: 1000 events
|
|
1881
|
+
Benchmark.ips do |x|
|
|
1882
|
+
x.report('Tier 2 - Rails filters') do
|
|
1883
|
+
Events::OrderCreated.track(order_id: 'o123', api_key: 'secret')
|
|
1884
|
+
end
|
|
1885
|
+
end
|
|
1886
|
+
|
|
1887
|
+
# Results:
|
|
1888
|
+
# Tier 2 - Rails filters: 8,000 i/s (125μs per event)
|
|
1889
|
+
# Overhead: ~0.05ms (simple key matching)
|
|
1890
|
+
```
|
|
1891
|
+
|
|
1892
|
+
**When to use:**
|
|
1893
|
+
- ✅ Standard business events (orders, payments)
|
|
1894
|
+
- ✅ Simple PII (known keys: password, token, email)
|
|
1895
|
+
- ✅ Most application events (90% of use cases)
|
|
1896
|
+
- ❌ Complex nested data with PII in content
|
|
1897
|
+
|
|
1898
|
+
---
|
|
1899
|
+
|
|
1900
|
+
### Tier 3: Deep Filtering (Explicit PII)
|
|
1901
|
+
|
|
1902
|
+
**Use when:** Event contains complex PII (nested data, emails in content, credit cards).
|
|
1903
|
+
|
|
1904
|
+
**How to declare:**
|
|
1905
|
+
```ruby
|
|
1906
|
+
class Events::UserRegistered < E11y::Event::Base
|
|
1907
|
+
schema do
|
|
1908
|
+
required(:email).filled(:string)
|
|
1909
|
+
required(:password).filled(:string)
|
|
1910
|
+
required(:address).filled(:hash)
|
|
1911
|
+
required(:user_id).filled(:string)
|
|
1912
|
+
end
|
|
1913
|
+
|
|
1914
|
+
# ✅ Explicit: This event contains PII
|
|
1915
|
+
contains_pii true # Tier 3: Deep filtering + content scanning
|
|
1916
|
+
|
|
1917
|
+
pii_filtering do
|
|
1918
|
+
field :email do
|
|
1919
|
+
strategy :hash # Pseudonymize for searchability
|
|
1920
|
+
end
|
|
1921
|
+
|
|
1922
|
+
field :password do
|
|
1923
|
+
strategy :mask # Complete masking
|
|
1924
|
+
end
|
|
1925
|
+
|
|
1926
|
+
field :address do
|
|
1927
|
+
strategy :mask # Mask nested hash
|
|
1928
|
+
end
|
|
1929
|
+
|
|
1930
|
+
field :user_id do
|
|
1931
|
+
strategy :allow # ID is OK to log
|
|
1932
|
+
end
|
|
1933
|
+
end
|
|
1934
|
+
end
|
|
1935
|
+
|
|
1936
|
+
# Result: ~0.2ms overhead per event
|
|
1937
|
+
```
|
|
1938
|
+
|
|
1939
|
+
**What Deep Filtering does:**
|
|
1940
|
+
1. **Key-based filtering:** Filters fields by name (like Tier 2)
|
|
1941
|
+
2. **Pattern scanning:** Scans string content for emails, credit cards, SSNs
|
|
1942
|
+
3. **Nested traversal:** Recursively filters hashes and arrays
|
|
1943
|
+
4. **Custom filters:** Applies per-field strategies (mask/hash/allow)
|
|
1944
|
+
|
|
1945
|
+
**Performance:**
|
|
1946
|
+
```ruby
|
|
1947
|
+
# Benchmark: 1000 events with nested data
|
|
1948
|
+
Benchmark.ips do |x|
|
|
1949
|
+
x.report('Tier 3 - Deep filtering') do
|
|
1950
|
+
Events::UserRegistered.track(
|
|
1951
|
+
email: 'user@example.com',
|
|
1952
|
+
password: 'secret123',
|
|
1953
|
+
address: { street: '123 Main', city: 'NYC' }
|
|
1954
|
+
)
|
|
1955
|
+
end
|
|
1956
|
+
end
|
|
1957
|
+
|
|
1958
|
+
# Results:
|
|
1959
|
+
# Tier 3 - Deep filtering: 5,000 i/s (200μs per event)
|
|
1960
|
+
# Overhead: ~0.2ms (deep traversal + pattern matching)
|
|
1961
|
+
```
|
|
1962
|
+
|
|
1963
|
+
**When to use:**
|
|
1964
|
+
- ✅ User registration/profile updates
|
|
1965
|
+
- ✅ Payment processing (credit cards)
|
|
1966
|
+
- ✅ Support tickets (PII in content)
|
|
1967
|
+
- ✅ Complex nested data structures
|
|
1968
|
+
- ⚠️ Use sparingly (higher overhead)
|
|
1969
|
+
|
|
1970
|
+
---
|
|
1971
|
+
|
|
1972
|
+
### Choosing the Right Tier
|
|
1973
|
+
|
|
1974
|
+
**Decision Tree:**
|
|
1975
|
+
|
|
1976
|
+
```
|
|
1977
|
+
Does event contain ANY user data?
|
|
1978
|
+
├─ NO → Tier 1 (contains_pii false)
|
|
1979
|
+
│ └─ Examples: health checks, metrics, system events
|
|
1980
|
+
│
|
|
1981
|
+
└─ YES → Does data have nested structures or PII in content?
|
|
1982
|
+
├─ NO → Tier 2 (default, no declaration)
|
|
1983
|
+
│ └─ Examples: orders, standard business events
|
|
1984
|
+
│
|
|
1985
|
+
└─ YES → Tier 3 (contains_pii true)
|
|
1986
|
+
└─ Examples: user profiles, payments, support tickets
|
|
1987
|
+
```
|
|
1988
|
+
|
|
1989
|
+
**Performance Comparison:**
|
|
1990
|
+
|
|
1991
|
+
```ruby
|
|
1992
|
+
# Tracking 1000 events of each tier:
|
|
1993
|
+
|
|
1994
|
+
# Tier 1: 100ms (no filtering)
|
|
1995
|
+
1000.times { Events::HealthCheck.track(status: 'ok') }
|
|
1996
|
+
|
|
1997
|
+
# Tier 2: 150ms (+50ms overhead from Rails filters)
|
|
1998
|
+
1000.times { Events::OrderCreated.track(order_id: 'o1', api_key: 'secret') }
|
|
1999
|
+
|
|
2000
|
+
# Tier 3: 300ms (+200ms overhead from deep filtering)
|
|
2001
|
+
1000.times { Events::UserRegistered.track(email: 'u@x.com', address: {...}) }
|
|
2002
|
+
```
|
|
2003
|
+
|
|
2004
|
+
**Best Practices:**
|
|
2005
|
+
|
|
2006
|
+
1. ✅ **Default to Tier 2:** Most events don't need deep filtering
|
|
2007
|
+
2. ✅ **Use Tier 1 for high-frequency events:** Health checks, metrics (avoid overhead)
|
|
2008
|
+
3. ✅ **Reserve Tier 3 for true PII events:** User data, payments, support tickets
|
|
2009
|
+
4. ⚠️ **Monitor performance impact:** Use self-monitoring metrics (see below)
|
|
2010
|
+
|
|
2011
|
+
---
|
|
2012
|
+
|
|
2013
|
+
## 📊 Monitoring
|
|
2014
|
+
|
|
2015
|
+
### Self-Monitoring Metrics
|
|
2016
|
+
|
|
2017
|
+
```ruby
|
|
2018
|
+
# Track PII filtering effectiveness
|
|
2019
|
+
E11y.configure do |config|
|
|
2020
|
+
config.self_monitoring do
|
|
2021
|
+
# Count filtered fields
|
|
2022
|
+
counter :pii_fields_filtered_total,
|
|
2023
|
+
tags: [:field_name, :filter_type]
|
|
2024
|
+
|
|
2025
|
+
# Count pattern matches
|
|
2026
|
+
counter :pii_patterns_matched_total,
|
|
2027
|
+
tags: [:pattern_name]
|
|
2028
|
+
|
|
2029
|
+
# Track performance impact
|
|
2030
|
+
histogram :pii_filter_duration_ms,
|
|
2031
|
+
tags: [:event_name],
|
|
2032
|
+
buckets: [0.1, 0.5, 1.0, 5.0, 10.0]
|
|
2033
|
+
end
|
|
2034
|
+
end
|
|
2035
|
+
|
|
2036
|
+
# Prometheus queries:
|
|
2037
|
+
# - How many emails filtered per day?
|
|
2038
|
+
# sum(increase(e11y_pii_fields_filtered_total{field_name="email"}[1d]))
|
|
2039
|
+
#
|
|
2040
|
+
# - Which events have most PII?
|
|
2041
|
+
# topk(10, sum by (event_name) (e11y_pii_fields_filtered_total))
|
|
2042
|
+
#
|
|
2043
|
+
# - Performance impact?
|
|
2044
|
+
# histogram_quantile(0.99, e11y_pii_filter_duration_ms_bucket)
|
|
2045
|
+
```
|
|
2046
|
+
|
|
2047
|
+
---
|
|
2048
|
+
|
|
2049
|
+
## 🧪 Testing
|
|
2050
|
+
|
|
2051
|
+
### RSpec Examples
|
|
2052
|
+
|
|
2053
|
+
```ruby
|
|
2054
|
+
# spec/e11y/pii_filtering_spec.rb
|
|
2055
|
+
RSpec.describe 'E11y PII Filtering' do
|
|
2056
|
+
before do
|
|
2057
|
+
# Configure Rails filters
|
|
2058
|
+
Rails.application.config.filter_parameters += [:email, :password]
|
|
2059
|
+
|
|
2060
|
+
E11y.configure do |config|
|
|
2061
|
+
config.pii_filter do
|
|
2062
|
+
use_rails_filter_parameters true
|
|
2063
|
+
filter_pattern /\d{4}-\d{4}-\d{4}-\d{4}/, replacement: '[CARD]'
|
|
2064
|
+
end
|
|
2065
|
+
end
|
|
2066
|
+
end
|
|
2067
|
+
|
|
2068
|
+
it 'filters Rails filter_parameters' do
|
|
2069
|
+
Events::UserCreated.track(
|
|
2070
|
+
email: 'user@example.com',
|
|
2071
|
+
password: 'secret123'
|
|
2072
|
+
)
|
|
2073
|
+
|
|
2074
|
+
event = E11y::Buffer.pop
|
|
2075
|
+
expect(event[:payload][:email]).to eq('[FILTERED]')
|
|
2076
|
+
expect(event[:payload][:password]).to eq('[FILTERED]')
|
|
2077
|
+
end
|
|
2078
|
+
|
|
2079
|
+
it 'filters by pattern (credit cards)' do
|
|
2080
|
+
Events::PaymentProcessed.track(
|
|
2081
|
+
card_number: '4111-1111-1111-1111',
|
|
2082
|
+
amount: 99.99
|
|
2083
|
+
)
|
|
2084
|
+
|
|
2085
|
+
event = E11y::Buffer.pop
|
|
2086
|
+
expect(event[:payload][:card_number]).to eq('[CARD]')
|
|
2087
|
+
expect(event[:payload][:amount]).to eq(99.99) # Not filtered
|
|
2088
|
+
end
|
|
2089
|
+
|
|
2090
|
+
it 'deep scans nested data' do
|
|
2091
|
+
Events::OrderPlaced.track(
|
|
2092
|
+
order_id: '123',
|
|
2093
|
+
user: {
|
|
2094
|
+
contact: {
|
|
2095
|
+
email: 'nested@example.com'
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
)
|
|
2099
|
+
|
|
2100
|
+
event = E11y::Buffer.pop
|
|
2101
|
+
expect(event[:payload][:user][:contact][:email]).to eq('[FILTERED]')
|
|
2102
|
+
end
|
|
2103
|
+
|
|
2104
|
+
it 'respects whitelist' do
|
|
2105
|
+
E11y.configure do |config|
|
|
2106
|
+
config.pii_filter do
|
|
2107
|
+
allow_parameters :user_id
|
|
2108
|
+
end
|
|
2109
|
+
end
|
|
2110
|
+
|
|
2111
|
+
# Even if 'user_id' is in Rails.filter_parameters
|
|
2112
|
+
Rails.application.config.filter_parameters += [:user_id]
|
|
2113
|
+
|
|
2114
|
+
Events::UserAction.track(user_id: '123')
|
|
2115
|
+
|
|
2116
|
+
event = E11y::Buffer.pop
|
|
2117
|
+
expect(event[:payload][:user_id]).to eq('123') # NOT filtered
|
|
2118
|
+
end
|
|
2119
|
+
end
|
|
2120
|
+
```
|
|
2121
|
+
|
|
2122
|
+
---
|
|
2123
|
+
|
|
2124
|
+
## 💡 Best Practices
|
|
2125
|
+
|
|
2126
|
+
### ✅ DO
|
|
2127
|
+
|
|
2128
|
+
**1. Use Rails filter_parameters as single source of truth**
|
|
2129
|
+
```ruby
|
|
2130
|
+
# ✅ GOOD: Configure once in Rails
|
|
2131
|
+
config.filter_parameters += [:password, :email, :ssn]
|
|
2132
|
+
# E11y automatically respects this
|
|
2133
|
+
```
|
|
2134
|
+
|
|
2135
|
+
**2. Add pattern-based filtering for content scanning**
|
|
2136
|
+
```ruby
|
|
2137
|
+
# ✅ GOOD: Catch PII in content, not just keys
|
|
2138
|
+
filter_pattern /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i,
|
|
2139
|
+
replacement: '[EMAIL]'
|
|
2140
|
+
```
|
|
2141
|
+
|
|
2142
|
+
**3. Whitelist IDs (not PII)**
|
|
2143
|
+
```ruby
|
|
2144
|
+
# ✅ GOOD: IDs are OK to log
|
|
2145
|
+
allow_parameters :user_id, :order_id, :transaction_id
|
|
2146
|
+
```
|
|
2147
|
+
|
|
2148
|
+
**4. Test PII filtering**
|
|
2149
|
+
```ruby
|
|
2150
|
+
# ✅ GOOD: Verify filtering works
|
|
2151
|
+
it 'filters PII' do
|
|
2152
|
+
Events::SomeEvent.track(email: 'test@example.com')
|
|
2153
|
+
expect(event[:payload][:email]).to eq('[FILTERED]')
|
|
2154
|
+
end
|
|
2155
|
+
```
|
|
2156
|
+
|
|
2157
|
+
---
|
|
2158
|
+
|
|
2159
|
+
### ❌ DON'T
|
|
2160
|
+
|
|
2161
|
+
**1. Don't duplicate configuration**
|
|
2162
|
+
```ruby
|
|
2163
|
+
# ❌ BAD: Duplication
|
|
2164
|
+
config.filter_parameters += [:email] # Rails
|
|
2165
|
+
config.pii_filter do
|
|
2166
|
+
filter_parameters :email # E11y ← Unnecessary!
|
|
2167
|
+
end
|
|
2168
|
+
|
|
2169
|
+
# ✅ GOOD: Configure once
|
|
2170
|
+
config.filter_parameters += [:email] # Rails only
|
|
2171
|
+
```
|
|
2172
|
+
|
|
2173
|
+
**2. Don't over-whitelist**
|
|
2174
|
+
```ruby
|
|
2175
|
+
# ❌ BAD: Whitelisting actual PII
|
|
2176
|
+
allow_parameters :email, :phone, :address # ← These ARE PII!
|
|
2177
|
+
|
|
2178
|
+
# ✅ GOOD: Only whitelist non-PII identifiers
|
|
2179
|
+
allow_parameters :user_id, :order_id # ← IDs, not PII
|
|
2180
|
+
```
|
|
2181
|
+
|
|
2182
|
+
**3. Don't disable deep scanning without good reason**
|
|
2183
|
+
```ruby
|
|
2184
|
+
# ❌ BAD: PII in nested data will leak
|
|
2185
|
+
config.pii_filter do
|
|
2186
|
+
deep_scan false # ← PII in nested hashes won't be filtered!
|
|
2187
|
+
end
|
|
2188
|
+
|
|
2189
|
+
# ✅ GOOD: Keep deep scanning enabled (default)
|
|
2190
|
+
config.pii_filter do
|
|
2191
|
+
deep_scan true # Default, catches nested PII
|
|
2192
|
+
end
|
|
2193
|
+
```
|
|
2194
|
+
|
|
2195
|
+
**4. Don't use same PII rules for all adapters**
|
|
2196
|
+
```ruby
|
|
2197
|
+
# ❌ BAD: Same strict rules everywhere
|
|
2198
|
+
config.pii_filter do
|
|
2199
|
+
mask_fields :email, :ip_address
|
|
2200
|
+
# Applied to ALL adapters (audit, OTel, Sentry, etc.)
|
|
2201
|
+
end
|
|
2202
|
+
|
|
2203
|
+
# ✅ GOOD: Per-adapter rules based on purpose
|
|
2204
|
+
config.pii_filter do
|
|
2205
|
+
# Default (most adapters)
|
|
2206
|
+
mask_fields :email, :ip_address
|
|
2207
|
+
|
|
2208
|
+
# Per-adapter overrides
|
|
2209
|
+
adapter_overrides do
|
|
2210
|
+
# Audit: keep PII (compliance requirement)
|
|
2211
|
+
adapter :audit_file do
|
|
2212
|
+
skip_filtering true
|
|
2213
|
+
end
|
|
2214
|
+
|
|
2215
|
+
# OTel: pseudonymize (queryable but privacy-safe)
|
|
2216
|
+
adapter :otlp do
|
|
2217
|
+
pseudonymize_fields :email, :ip_address
|
|
2218
|
+
end
|
|
2219
|
+
|
|
2220
|
+
# Sentry: strict masking (external service)
|
|
2221
|
+
adapter :sentry do
|
|
2222
|
+
mask_fields :email, :ip_address, :user_id
|
|
2223
|
+
end
|
|
2224
|
+
end
|
|
2225
|
+
end
|
|
2226
|
+
```
|
|
2227
|
+
|
|
2228
|
+
---
|
|
2229
|
+
|
|
2230
|
+
## 🎯 Per-Adapter PII Filtering
|
|
2231
|
+
|
|
2232
|
+
**Problem:** Different adapters have different compliance requirements.
|
|
2233
|
+
|
|
2234
|
+
### Use Case: Audit Trail vs. Observability
|
|
2235
|
+
|
|
2236
|
+
```ruby
|
|
2237
|
+
# Event goes to multiple adapters
|
|
2238
|
+
class UserPermissionChanged < E11y::AuditEvent
|
|
2239
|
+
adapters [:audit_file, :elasticsearch, :loki, :sentry]
|
|
2240
|
+
|
|
2241
|
+
schema do
|
|
2242
|
+
required(:user_email).filled(:string)
|
|
2243
|
+
required(:ip_address).filled(:string)
|
|
2244
|
+
required(:old_role).filled(:string)
|
|
2245
|
+
required(:new_role).filled(:string)
|
|
2246
|
+
end
|
|
2247
|
+
end
|
|
2248
|
+
|
|
2249
|
+
# Different PII treatment per adapter:
|
|
2250
|
+
# - audit_file: KEEP all PII (compliance)
|
|
2251
|
+
# - elasticsearch: PSEUDONYMIZE (queryable but safe)
|
|
2252
|
+
# - loki: MASK (observability only)
|
|
2253
|
+
# - sentry: MASK (external service)
|
|
2254
|
+
```
|
|
2255
|
+
|
|
2256
|
+
### Configuration: Global Per-Adapter Rules
|
|
2257
|
+
|
|
2258
|
+
```ruby
|
|
2259
|
+
E11y.configure do |config|
|
|
2260
|
+
config.pii_filter do
|
|
2261
|
+
# Default (most adapters): strict masking
|
|
2262
|
+
mask_fields :email, :ip_address, :phone, :ssn
|
|
2263
|
+
|
|
2264
|
+
# Per-adapter overrides
|
|
2265
|
+
adapter_overrides do
|
|
2266
|
+
# === Audit Log: No Filtering ===
|
|
2267
|
+
adapter :audit_file do
|
|
2268
|
+
skip_filtering true
|
|
2269
|
+
|
|
2270
|
+
# Reason: Legal requirement to keep original data
|
|
2271
|
+
# Justification: GDPR Art. 6(1)(c) - "legal obligation"
|
|
2272
|
+
# Mitigation: Encryption + access control
|
|
2273
|
+
end
|
|
2274
|
+
|
|
2275
|
+
# === Elasticsearch: Pseudonymization ===
|
|
2276
|
+
adapter :elasticsearch do
|
|
2277
|
+
# Don't mask, but hash (one-way)
|
|
2278
|
+
pseudonymize_fields :email, :ip_address
|
|
2279
|
+
hash_algorithm :sha256
|
|
2280
|
+
hash_salt ENV['PII_HASH_SALT']
|
|
2281
|
+
|
|
2282
|
+
# Result: same user always same hash (queryable!)
|
|
2283
|
+
# email: 'john@example.com' → 'hashed_a1b2c3d4'
|
|
2284
|
+
# But can't reverse the hash
|
|
2285
|
+
end
|
|
2286
|
+
|
|
2287
|
+
# === OpenTelemetry: Pseudonymization ===
|
|
2288
|
+
adapter :otlp do
|
|
2289
|
+
pseudonymize_fields :email, :ip_address
|
|
2290
|
+
hash_algorithm :sha256
|
|
2291
|
+
|
|
2292
|
+
# Reason: OTel Semantic Conventions need some PII
|
|
2293
|
+
# But we can't send raw PII to external collector
|
|
2294
|
+
end
|
|
2295
|
+
|
|
2296
|
+
# === Sentry: Strict Masking ===
|
|
2297
|
+
adapter :sentry do
|
|
2298
|
+
# External service: mask EVERYTHING
|
|
2299
|
+
mask_fields :email, :ip_address, :phone, :ssn, :user_id
|
|
2300
|
+
|
|
2301
|
+
# Reason: Sentry is 3rd party, minimize data sharing
|
|
2302
|
+
end
|
|
2303
|
+
|
|
2304
|
+
# === Loki: Default Masking ===
|
|
2305
|
+
adapter :loki do
|
|
2306
|
+
# Use default rules (mask_fields from above)
|
|
2307
|
+
end
|
|
2308
|
+
end
|
|
2309
|
+
end
|
|
2310
|
+
end
|
|
2311
|
+
```
|
|
2312
|
+
|
|
2313
|
+
### Configuration: Per-Event Per-Adapter Rules
|
|
2314
|
+
|
|
2315
|
+
```ruby
|
|
2316
|
+
# More granular: override at event level
|
|
2317
|
+
class SensitiveUserAction < E11y::Event::Base
|
|
2318
|
+
adapters [:audit_file, :elasticsearch, :sentry]
|
|
2319
|
+
|
|
2320
|
+
schema do
|
|
2321
|
+
required(:user_email).filled(:string)
|
|
2322
|
+
required(:action).filled(:string)
|
|
2323
|
+
end
|
|
2324
|
+
|
|
2325
|
+
# Override PII rules just for THIS event
|
|
2326
|
+
pii_rules do
|
|
2327
|
+
# Audit: keep everything
|
|
2328
|
+
adapter :audit_file do
|
|
2329
|
+
skip_filtering true
|
|
2330
|
+
end
|
|
2331
|
+
|
|
2332
|
+
# Elasticsearch: hash email
|
|
2333
|
+
adapter :elasticsearch do
|
|
2334
|
+
pseudonymize_fields :user_email
|
|
2335
|
+
end
|
|
2336
|
+
|
|
2337
|
+
# Sentry: mask email
|
|
2338
|
+
adapter :sentry do
|
|
2339
|
+
mask_fields :user_email
|
|
2340
|
+
end
|
|
2341
|
+
end
|
|
2342
|
+
end
|
|
2343
|
+
```
|
|
2344
|
+
|
|
2345
|
+
### Implementation: How It Works
|
|
2346
|
+
|
|
2347
|
+
```ruby
|
|
2348
|
+
# Internal pipeline
|
|
2349
|
+
def write_event_to_adapter(event, adapter)
|
|
2350
|
+
# 1. Get PII rules for this adapter
|
|
2351
|
+
pii_rules = get_pii_rules_for_adapter(adapter)
|
|
2352
|
+
|
|
2353
|
+
# 2. Clone event (don't modify original)
|
|
2354
|
+
filtered_event = event.deep_dup
|
|
2355
|
+
|
|
2356
|
+
# 3. Apply adapter-specific filtering
|
|
2357
|
+
case pii_rules.strategy
|
|
2358
|
+
when :skip
|
|
2359
|
+
# No filtering
|
|
2360
|
+
when :mask
|
|
2361
|
+
filtered_event = pii_filter.mask(filtered_event, pii_rules.fields)
|
|
2362
|
+
when :pseudonymize
|
|
2363
|
+
filtered_event = pii_filter.pseudonymize(filtered_event, pii_rules.fields)
|
|
2364
|
+
end
|
|
2365
|
+
|
|
2366
|
+
# 4. Write to adapter
|
|
2367
|
+
adapter.write(filtered_event)
|
|
2368
|
+
end
|
|
2369
|
+
```
|
|
2370
|
+
|
|
2371
|
+
### Result: Same Event, Different PII Treatment
|
|
2372
|
+
|
|
2373
|
+
```ruby
|
|
2374
|
+
# Original event:
|
|
2375
|
+
event = {
|
|
2376
|
+
user_email: 'john@example.com',
|
|
2377
|
+
ip_address: '192.168.1.100',
|
|
2378
|
+
action: 'role_changed'
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
# Written to adapters:
|
|
2382
|
+
# audit_file: { user_email: 'john@example.com', ip_address: '192.168.1.100', ... }
|
|
2383
|
+
# elasticsearch: { user_email: 'hashed_a1b2c3', ip_address: 'hashed_xyz789', ... }
|
|
2384
|
+
# loki: { user_email: '[FILTERED]', ip_address: '[FILTERED]', ... }
|
|
2385
|
+
# sentry: { user_email: '[FILTERED]', ip_address: '[FILTERED]', ... }
|
|
2386
|
+
```
|
|
2387
|
+
|
|
2388
|
+
### Benefits
|
|
2389
|
+
|
|
2390
|
+
1. ✅ **Compliance:** Audit log has original data (legal requirement)
|
|
2391
|
+
2. ✅ **Privacy:** External services get masked data (GDPR)
|
|
2392
|
+
3. ✅ **Queryability:** Pseudonymized data in ES (can group by user)
|
|
2393
|
+
4. ✅ **Security:** Layered approach (different rules for different risks)
|
|
2394
|
+
|
|
2395
|
+
---
|
|
2396
|
+
|
|
2397
|
+
## 📚 Related Use Cases
|
|
2398
|
+
|
|
2399
|
+
- **[UC-002: Business Event Tracking](./UC-002-business-event-tracking.md)** - Event definitions
|
|
2400
|
+
- **[UC-012: Audit Trail](./UC-012-audit-trail.md)** - Compliance logging (skip PII filtering)
|
|
2401
|
+
- **[UC-005: Sentry Integration](./UC-005-sentry-integration.md)** - PII in error reports (strict masking)
|
|
2402
|
+
- **[UC-008: OpenTelemetry Integration](./UC-008-opentelemetry-integration.md)** - OTel semantic conventions (pseudonymization)
|
|
2403
|
+
|
|
2404
|
+
---
|
|
2405
|
+
|
|
2406
|
+
## 🔒 Validations (NEW - v1.1)
|
|
2407
|
+
|
|
2408
|
+
> **🎯 Pattern:** Validate PII configuration at class load time.
|
|
2409
|
+
|
|
2410
|
+
### PII Strategy Validation
|
|
2411
|
+
|
|
2412
|
+
**Problem:** Invalid PII strategies → runtime errors.
|
|
2413
|
+
|
|
2414
|
+
**Solution:** Validate strategy against whitelist:
|
|
2415
|
+
|
|
2416
|
+
```ruby
|
|
2417
|
+
# Gem implementation (automatic):
|
|
2418
|
+
VALID_PII_STRATEGIES = [:mask, :hash, :remove, :allow]
|
|
2419
|
+
|
|
2420
|
+
def self.pii_filtering(&block)
|
|
2421
|
+
# Validate strategies during DSL execution
|
|
2422
|
+
# Raises ArgumentError if invalid strategy used
|
|
2423
|
+
end
|
|
2424
|
+
|
|
2425
|
+
# Result:
|
|
2426
|
+
class Events::UserRegistered < E11y::Event::Base
|
|
2427
|
+
contains_pii true
|
|
2428
|
+
pii_filtering do
|
|
2429
|
+
encrypts :email # ← ERROR: "Invalid PII strategy: :encrypts. Valid: mask, hash, remove, allow"
|
|
2430
|
+
end
|
|
2431
|
+
end
|
|
2432
|
+
```
|
|
2433
|
+
|
|
2434
|
+
### PII Field Existence Validation
|
|
2435
|
+
|
|
2436
|
+
**Problem:** Typos in PII field names → fields not filtered.
|
|
2437
|
+
|
|
2438
|
+
**Solution:** Validate against schema fields:
|
|
2439
|
+
|
|
2440
|
+
```ruby
|
|
2441
|
+
# Gem implementation (automatic):
|
|
2442
|
+
def self.pii_filtering(&block)
|
|
2443
|
+
# After schema is defined, validate PII fields exist
|
|
2444
|
+
pii_fields = extract_pii_fields_from_block(block)
|
|
2445
|
+
schema_fields = self.schema.keys
|
|
2446
|
+
|
|
2447
|
+
invalid_fields = pii_fields - schema_fields
|
|
2448
|
+
if invalid_fields.any?
|
|
2449
|
+
raise ArgumentError, "PII fields not in schema: #{invalid_fields.join(', ')}"
|
|
2450
|
+
end
|
|
2451
|
+
end
|
|
2452
|
+
|
|
2453
|
+
# Result:
|
|
2454
|
+
class Events::UserRegistered < E11y::Event::Base
|
|
2455
|
+
schema do
|
|
2456
|
+
required(:email).filled(:string)
|
|
2457
|
+
required(:name).filled(:string)
|
|
2458
|
+
end
|
|
2459
|
+
|
|
2460
|
+
contains_pii true
|
|
2461
|
+
pii_filtering do
|
|
2462
|
+
masks :email, :username # ← ERROR: "PII fields not in schema: username"
|
|
2463
|
+
end
|
|
2464
|
+
end
|
|
2465
|
+
```
|
|
2466
|
+
|
|
2467
|
+
---
|
|
2468
|
+
|
|
2469
|
+
## 🌍 Environment-Specific PII Configuration (NEW - v1.1)
|
|
2470
|
+
|
|
2471
|
+
> **🎯 Pattern:** Different PII strategies per environment.
|
|
2472
|
+
|
|
2473
|
+
### Example 1: Strict Masking in Production
|
|
2474
|
+
|
|
2475
|
+
```ruby
|
|
2476
|
+
class Events::UserRegistered < E11y::Event::Base
|
|
2477
|
+
schema do
|
|
2478
|
+
required(:email).filled(:string)
|
|
2479
|
+
required(:name).filled(:string)
|
|
2480
|
+
required(:ip_address).filled(:string)
|
|
2481
|
+
end
|
|
2482
|
+
|
|
2483
|
+
contains_pii true
|
|
2484
|
+
pii_filtering do
|
|
2485
|
+
if Rails.env.production?
|
|
2486
|
+
# Production: strict masking
|
|
2487
|
+
masks :email, :name, :ip_address
|
|
2488
|
+
else
|
|
2489
|
+
# Dev/Test: allow for debugging
|
|
2490
|
+
allows :email, :name, :ip_address
|
|
2491
|
+
end
|
|
2492
|
+
end
|
|
2493
|
+
end
|
|
2494
|
+
```
|
|
2495
|
+
|
|
2496
|
+
### Example 2: Jurisdiction-Specific Hashing
|
|
2497
|
+
|
|
2498
|
+
```ruby
|
|
2499
|
+
class Events::PaymentProcessed < E11y::Event::Base
|
|
2500
|
+
schema do
|
|
2501
|
+
required(:user_id).filled(:string)
|
|
2502
|
+
required(:credit_card_last4).filled(:string)
|
|
2503
|
+
end
|
|
2504
|
+
|
|
2505
|
+
contains_pii true
|
|
2506
|
+
pii_filtering do
|
|
2507
|
+
case ENV['JURISDICTION']
|
|
2508
|
+
when 'EU'
|
|
2509
|
+
# GDPR: pseudonymization (reversible)
|
|
2510
|
+
hashes :user_id, algorithm: :sha256, salt: ENV['PII_SALT']
|
|
2511
|
+
masks :credit_card_last4
|
|
2512
|
+
when 'US'
|
|
2513
|
+
# US: allow user_id (not PII), mask card
|
|
2514
|
+
allows :user_id
|
|
2515
|
+
masks :credit_card_last4
|
|
2516
|
+
else
|
|
2517
|
+
# Default: strict masking
|
|
2518
|
+
masks :user_id, :credit_card_last4
|
|
2519
|
+
end
|
|
2520
|
+
end
|
|
2521
|
+
end
|
|
2522
|
+
```
|
|
2523
|
+
|
|
2524
|
+
---
|
|
2525
|
+
|
|
2526
|
+
## 📊 Precedence Rules for PII (NEW - v1.1)
|
|
2527
|
+
|
|
2528
|
+
> **🎯 Pattern:** PII configuration precedence (most specific wins).
|
|
2529
|
+
|
|
2530
|
+
### Precedence Order (Highest to Lowest)
|
|
2531
|
+
|
|
2532
|
+
```
|
|
2533
|
+
1. Event-level pii_filtering block (highest)
|
|
2534
|
+
↓
|
|
2535
|
+
2. Preset module PII config
|
|
2536
|
+
↓
|
|
2537
|
+
3. Base class PII config
|
|
2538
|
+
↓
|
|
2539
|
+
4. Rails.application.config.filter_parameters
|
|
2540
|
+
↓
|
|
2541
|
+
5. Global E11y.config.pii_filter (lowest)
|
|
2542
|
+
```
|
|
2543
|
+
|
|
2544
|
+
### Example: Mixing Inheritance + Presets for PII
|
|
2545
|
+
|
|
2546
|
+
```ruby
|
|
2547
|
+
# Global config (lowest priority)
|
|
2548
|
+
E11y.configure do |config|
|
|
2549
|
+
config.pii_filter do
|
|
2550
|
+
use_rails_filter_parameters true # Use Rails config
|
|
2551
|
+
masks :password, :ssn # Additional global masks
|
|
2552
|
+
end
|
|
2553
|
+
end
|
|
2554
|
+
|
|
2555
|
+
# Rails config (used by global)
|
|
2556
|
+
Rails.application.config.filter_parameters += [:email, :phone]
|
|
2557
|
+
|
|
2558
|
+
# Base class (medium priority)
|
|
2559
|
+
class Events::BaseUserEvent < E11y::Event::Base
|
|
2560
|
+
contains_pii true
|
|
2561
|
+
pii_filtering do
|
|
2562
|
+
hashes :user_id, :email # Override global (hash instead of mask)
|
|
2563
|
+
allows :name # Allow name (not PII in this context)
|
|
2564
|
+
end
|
|
2565
|
+
end
|
|
2566
|
+
|
|
2567
|
+
# Preset module (higher priority)
|
|
2568
|
+
module E11y::Presets::PiiAwareEvent
|
|
2569
|
+
extend ActiveSupport::Concern
|
|
2570
|
+
included do
|
|
2571
|
+
contains_pii true
|
|
2572
|
+
pii_filtering do
|
|
2573
|
+
masks :ip_address, :session_id # Additional masks
|
|
2574
|
+
end
|
|
2575
|
+
end
|
|
2576
|
+
end
|
|
2577
|
+
|
|
2578
|
+
# Event (highest priority)
|
|
2579
|
+
class Events::UserLogin < Events::BaseUserEvent
|
|
2580
|
+
include E11y::Presets::PiiAwareEvent
|
|
2581
|
+
|
|
2582
|
+
pii_filtering do
|
|
2583
|
+
allows :email # Override base (allow email for login events)
|
|
2584
|
+
end
|
|
2585
|
+
|
|
2586
|
+
# Final PII config:
|
|
2587
|
+
# - user_id: hashed (from base)
|
|
2588
|
+
# - email: allowed (event-level override)
|
|
2589
|
+
# - name: allowed (from base)
|
|
2590
|
+
# - ip_address: masked (from preset)
|
|
2591
|
+
# - session_id: masked (from preset)
|
|
2592
|
+
# - password: masked (from global)
|
|
2593
|
+
# - ssn: masked (from global)
|
|
2594
|
+
# - phone: masked (from Rails config)
|
|
2595
|
+
end
|
|
2596
|
+
```
|
|
2597
|
+
|
|
2598
|
+
### PII Precedence Rules Table
|
|
2599
|
+
|
|
2600
|
+
| Field | Global | Rails Config | Base Class | Preset | Event-Level | Winner |
|
|
2601
|
+
|-------|--------|--------------|------------|--------|-------------|--------|
|
|
2602
|
+
| `email` | `mask` | `mask` | `hash` | - | `allow` | **`allow`** (event) |
|
|
2603
|
+
| `user_id` | - | - | `hash` | - | - | **`hash`** (base) |
|
|
2604
|
+
| `ip_address` | - | - | - | `mask` | - | **`mask`** (preset) |
|
|
2605
|
+
| `password` | `mask` | - | - | - | - | **`mask`** (global) |
|
|
2606
|
+
| `phone` | - | `mask` | - | - | - | **`mask`** (Rails) |
|
|
2607
|
+
|
|
2608
|
+
---
|
|
2609
|
+
|
|
2610
|
+
## 🔒 GDPR Compliance
|
|
2611
|
+
|
|
2612
|
+
### Key GDPR Requirements Met
|
|
2613
|
+
|
|
2614
|
+
1. ✅ **Data Minimization** - Only log what's needed (filter PII)
|
|
2615
|
+
2. ✅ **Purpose Limitation** - Logs for observability only
|
|
2616
|
+
3. ✅ **Storage Limitation** - Set retention policies in adapters
|
|
2617
|
+
4. ✅ **Integrity & Confidentiality** - PII filtered at source
|
|
2618
|
+
5. ✅ **Accountability** - Audit which PII was filtered (sampling)
|
|
2619
|
+
|
|
2620
|
+
### Configuration for GDPR
|
|
2621
|
+
|
|
2622
|
+
```ruby
|
|
2623
|
+
E11y.configure do |config|
|
|
2624
|
+
config.pii_filter do
|
|
2625
|
+
# GDPR-compliant defaults
|
|
2626
|
+
use_rails_filter_parameters true
|
|
2627
|
+
|
|
2628
|
+
# Filter all personal data
|
|
2629
|
+
filter_parameters :email, :name, :address, :phone, :ssn,
|
|
2630
|
+
:birth_date, :ip_address
|
|
2631
|
+
|
|
2632
|
+
# Content scanning
|
|
2633
|
+
filter_pattern /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i,
|
|
2634
|
+
replacement: '[EMAIL]'
|
|
2635
|
+
filter_pattern /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/,
|
|
2636
|
+
replacement: '[IP]'
|
|
2637
|
+
|
|
2638
|
+
# Sampling for compliance verification (not PII itself)
|
|
2639
|
+
sample_filtered_values 0.001 # 0.1% for audit
|
|
2640
|
+
end
|
|
2641
|
+
end
|
|
2642
|
+
```
|
|
2643
|
+
|
|
2644
|
+
---
|
|
2645
|
+
|
|
2646
|
+
**Document Version:** 1.1 (Unified DSL)
|
|
2647
|
+
**Last Updated:** January 16, 2026
|
|
2648
|
+
**Status:** ✅ Complete - Consistent with DSL-SPECIFICATION.md v1.1.0
|