e11y 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.rubocop.yml +69 -0
- data/CHANGELOG.md +26 -0
- data/CODE_OF_CONDUCT.md +64 -0
- data/LICENSE.txt +21 -0
- data/README.md +179 -0
- data/Rakefile +37 -0
- data/benchmarks/run_all.rb +33 -0
- data/config/README.md +83 -0
- data/config/loki-local-config.yaml +35 -0
- data/config/prometheus.yml +15 -0
- data/docker-compose.yml +78 -0
- data/docs/00-ICP-AND-TIMELINE.md +483 -0
- data/docs/01-SCALE-REQUIREMENTS.md +858 -0
- data/docs/ADR-001-architecture.md +2617 -0
- data/docs/ADR-002-metrics-yabeda.md +1395 -0
- data/docs/ADR-003-slo-observability.md +3337 -0
- data/docs/ADR-004-adapter-architecture.md +2385 -0
- data/docs/ADR-005-tracing-context.md +1372 -0
- data/docs/ADR-006-security-compliance.md +4143 -0
- data/docs/ADR-007-opentelemetry-integration.md +1385 -0
- data/docs/ADR-008-rails-integration.md +1911 -0
- data/docs/ADR-009-cost-optimization.md +2993 -0
- data/docs/ADR-010-developer-experience.md +2166 -0
- data/docs/ADR-011-testing-strategy.md +1836 -0
- data/docs/ADR-012-event-evolution.md +958 -0
- data/docs/ADR-013-reliability-error-handling.md +2750 -0
- data/docs/ADR-014-event-driven-slo.md +1533 -0
- data/docs/ADR-015-middleware-order.md +1061 -0
- data/docs/ADR-016-self-monitoring-slo.md +1234 -0
- data/docs/API-REFERENCE-L28.md +914 -0
- data/docs/COMPREHENSIVE-CONFIGURATION.md +2366 -0
- data/docs/IMPLEMENTATION_NOTES.md +2804 -0
- data/docs/IMPLEMENTATION_PLAN.md +1971 -0
- data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +586 -0
- data/docs/PLAN.md +148 -0
- data/docs/QUICK-START.md +934 -0
- data/docs/README.md +296 -0
- data/docs/design/00-memory-optimization.md +593 -0
- data/docs/guides/MIGRATION-L27-L28.md +692 -0
- data/docs/guides/PERFORMANCE-BENCHMARKS.md +434 -0
- data/docs/guides/README.md +44 -0
- data/docs/prd/01-overview-vision.md +440 -0
- data/docs/use_cases/README.md +119 -0
- data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +813 -0
- data/docs/use_cases/UC-002-business-event-tracking.md +1953 -0
- data/docs/use_cases/UC-003-pattern-based-metrics.md +1627 -0
- data/docs/use_cases/UC-004-zero-config-slo-tracking.md +728 -0
- data/docs/use_cases/UC-005-sentry-integration.md +759 -0
- data/docs/use_cases/UC-006-trace-context-management.md +905 -0
- data/docs/use_cases/UC-007-pii-filtering.md +2648 -0
- data/docs/use_cases/UC-008-opentelemetry-integration.md +1153 -0
- data/docs/use_cases/UC-009-multi-service-tracing.md +1043 -0
- data/docs/use_cases/UC-010-background-job-tracking.md +1018 -0
- data/docs/use_cases/UC-011-rate-limiting.md +1906 -0
- data/docs/use_cases/UC-012-audit-trail.md +2301 -0
- data/docs/use_cases/UC-013-high-cardinality-protection.md +2127 -0
- data/docs/use_cases/UC-014-adaptive-sampling.md +1940 -0
- data/docs/use_cases/UC-015-cost-optimization.md +735 -0
- data/docs/use_cases/UC-016-rails-logger-migration.md +785 -0
- data/docs/use_cases/UC-017-local-development.md +867 -0
- data/docs/use_cases/UC-018-testing-events.md +1081 -0
- data/docs/use_cases/UC-019-tiered-storage-migration.md +562 -0
- data/docs/use_cases/UC-020-event-versioning.md +708 -0
- data/docs/use_cases/UC-021-error-handling-retry-dlq.md +956 -0
- data/docs/use_cases/UC-022-event-registry.md +648 -0
- data/docs/use_cases/backlog.md +226 -0
- data/e11y.gemspec +76 -0
- data/lib/e11y/adapters/adaptive_batcher.rb +207 -0
- data/lib/e11y/adapters/audit_encrypted.rb +239 -0
- data/lib/e11y/adapters/base.rb +580 -0
- data/lib/e11y/adapters/file.rb +224 -0
- data/lib/e11y/adapters/in_memory.rb +216 -0
- data/lib/e11y/adapters/loki.rb +333 -0
- data/lib/e11y/adapters/otel_logs.rb +203 -0
- data/lib/e11y/adapters/registry.rb +141 -0
- data/lib/e11y/adapters/sentry.rb +230 -0
- data/lib/e11y/adapters/stdout.rb +108 -0
- data/lib/e11y/adapters/yabeda.rb +370 -0
- data/lib/e11y/buffers/adaptive_buffer.rb +339 -0
- data/lib/e11y/buffers/base_buffer.rb +40 -0
- data/lib/e11y/buffers/request_scoped_buffer.rb +246 -0
- data/lib/e11y/buffers/ring_buffer.rb +267 -0
- data/lib/e11y/buffers.rb +14 -0
- data/lib/e11y/console.rb +122 -0
- data/lib/e11y/current.rb +48 -0
- data/lib/e11y/event/base.rb +894 -0
- data/lib/e11y/event/value_sampling_config.rb +84 -0
- data/lib/e11y/events/base_audit_event.rb +43 -0
- data/lib/e11y/events/base_payment_event.rb +33 -0
- data/lib/e11y/events/rails/cache/delete.rb +21 -0
- data/lib/e11y/events/rails/cache/read.rb +23 -0
- data/lib/e11y/events/rails/cache/write.rb +22 -0
- data/lib/e11y/events/rails/database/query.rb +45 -0
- data/lib/e11y/events/rails/http/redirect.rb +21 -0
- data/lib/e11y/events/rails/http/request.rb +26 -0
- data/lib/e11y/events/rails/http/send_file.rb +21 -0
- data/lib/e11y/events/rails/http/start_processing.rb +26 -0
- data/lib/e11y/events/rails/job/completed.rb +22 -0
- data/lib/e11y/events/rails/job/enqueued.rb +22 -0
- data/lib/e11y/events/rails/job/failed.rb +22 -0
- data/lib/e11y/events/rails/job/scheduled.rb +23 -0
- data/lib/e11y/events/rails/job/started.rb +22 -0
- data/lib/e11y/events/rails/log.rb +56 -0
- data/lib/e11y/events/rails/view/render.rb +23 -0
- data/lib/e11y/events.rb +18 -0
- data/lib/e11y/instruments/active_job.rb +201 -0
- data/lib/e11y/instruments/rails_instrumentation.rb +141 -0
- data/lib/e11y/instruments/sidekiq.rb +175 -0
- data/lib/e11y/logger/bridge.rb +205 -0
- data/lib/e11y/metrics/cardinality_protection.rb +172 -0
- data/lib/e11y/metrics/cardinality_tracker.rb +134 -0
- data/lib/e11y/metrics/registry.rb +234 -0
- data/lib/e11y/metrics/relabeling.rb +226 -0
- data/lib/e11y/metrics.rb +102 -0
- data/lib/e11y/middleware/audit_signing.rb +174 -0
- data/lib/e11y/middleware/base.rb +140 -0
- data/lib/e11y/middleware/event_slo.rb +167 -0
- data/lib/e11y/middleware/pii_filter.rb +266 -0
- data/lib/e11y/middleware/pii_filtering.rb +280 -0
- data/lib/e11y/middleware/rate_limiting.rb +214 -0
- data/lib/e11y/middleware/request.rb +163 -0
- data/lib/e11y/middleware/routing.rb +157 -0
- data/lib/e11y/middleware/sampling.rb +254 -0
- data/lib/e11y/middleware/slo.rb +168 -0
- data/lib/e11y/middleware/trace_context.rb +131 -0
- data/lib/e11y/middleware/validation.rb +118 -0
- data/lib/e11y/middleware/versioning.rb +132 -0
- data/lib/e11y/middleware.rb +12 -0
- data/lib/e11y/pii/patterns.rb +90 -0
- data/lib/e11y/pii.rb +13 -0
- data/lib/e11y/pipeline/builder.rb +155 -0
- data/lib/e11y/pipeline/zone_validator.rb +110 -0
- data/lib/e11y/pipeline.rb +12 -0
- data/lib/e11y/presets/audit_event.rb +65 -0
- data/lib/e11y/presets/debug_event.rb +34 -0
- data/lib/e11y/presets/high_value_event.rb +51 -0
- data/lib/e11y/presets.rb +19 -0
- data/lib/e11y/railtie.rb +138 -0
- data/lib/e11y/reliability/circuit_breaker.rb +216 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +277 -0
- data/lib/e11y/reliability/dlq/filter.rb +117 -0
- data/lib/e11y/reliability/retry_handler.rb +207 -0
- data/lib/e11y/reliability/retry_rate_limiter.rb +117 -0
- data/lib/e11y/sampling/error_spike_detector.rb +225 -0
- data/lib/e11y/sampling/load_monitor.rb +161 -0
- data/lib/e11y/sampling/stratified_tracker.rb +92 -0
- data/lib/e11y/sampling/value_extractor.rb +82 -0
- data/lib/e11y/self_monitoring/buffer_monitor.rb +79 -0
- data/lib/e11y/self_monitoring/performance_monitor.rb +97 -0
- data/lib/e11y/self_monitoring/reliability_monitor.rb +146 -0
- data/lib/e11y/slo/event_driven.rb +150 -0
- data/lib/e11y/slo/tracker.rb +119 -0
- data/lib/e11y/version.rb +9 -0
- data/lib/e11y.rb +283 -0
- metadata +452 -0
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module E11y
|
|
4
|
+
module Middleware
|
|
5
|
+
# PII Filter Middleware - 3-Tier Strategy
|
|
6
|
+
#
|
|
7
|
+
# Filters Personally Identifiable Information (PII) from event payloads
|
|
8
|
+
# before they reach adapters. Implements ADR-006 3-tier security model.
|
|
9
|
+
#
|
|
10
|
+
# **Three-Tier Strategy:**
|
|
11
|
+
# - Tier 1: No PII (`contains_pii false`) - Skip filtering (0ms overhead)
|
|
12
|
+
# - Tier 2: Default - Rails filters only (~0.05ms overhead)
|
|
13
|
+
# - Tier 3: Explicit PII (`contains_pii true`) - Deep filtering (~0.2ms overhead)
|
|
14
|
+
#
|
|
15
|
+
# @example Basic Usage (Tier 2 - Default)
|
|
16
|
+
# class Events::OrderCreated < E11y::Event::Base
|
|
17
|
+
# schema do
|
|
18
|
+
# required(:order_id).filled(:string)
|
|
19
|
+
# optional(:api_key).filled(:string) # Rails will filter this
|
|
20
|
+
# end
|
|
21
|
+
# end
|
|
22
|
+
#
|
|
23
|
+
# @example Tier 1: No PII (High Performance)
|
|
24
|
+
# class Events::HealthCheck < E11y::Event::Base
|
|
25
|
+
# contains_pii false # Skip all filtering
|
|
26
|
+
# end
|
|
27
|
+
#
|
|
28
|
+
# @example Tier 3: Explicit PII (Deep Filtering)
|
|
29
|
+
# class Events::UserRegistered < E11y::Event::Base
|
|
30
|
+
# contains_pii true
|
|
31
|
+
#
|
|
32
|
+
# pii_filtering do
|
|
33
|
+
# masks :password
|
|
34
|
+
# hashes :email
|
|
35
|
+
# allows :user_id
|
|
36
|
+
# end
|
|
37
|
+
# end
|
|
38
|
+
#
|
|
39
|
+
# @see ADR-006 PII Security & Compliance
|
|
40
|
+
# @see UC-007 PII Filtering
|
|
41
|
+
# @see E11y::PII::Patterns
|
|
42
|
+
class PIIFilter < Base
|
|
43
|
+
middleware_zone :security
|
|
44
|
+
|
|
45
|
+
# Initialize PII filtering middleware
|
|
46
|
+
#
|
|
47
|
+
# @param app [Proc] Next middleware in chain
|
|
48
|
+
# @param config [Hash] Configuration options
|
|
49
|
+
def initialize(app, config = {})
|
|
50
|
+
super(app)
|
|
51
|
+
@config = config
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Process event and filter PII based on tier
|
|
55
|
+
#
|
|
56
|
+
# @param event_data [Hash] Event data with payload
|
|
57
|
+
# @return [Hash] Processed event data
|
|
58
|
+
def call(event_data)
|
|
59
|
+
# Determine filtering tier
|
|
60
|
+
tier = determine_tier(event_data)
|
|
61
|
+
|
|
62
|
+
case tier
|
|
63
|
+
when :tier1
|
|
64
|
+
# Tier 1: No PII - Skip filtering (0ms overhead)
|
|
65
|
+
@app.call(event_data)
|
|
66
|
+
when :tier2
|
|
67
|
+
# Tier 2: Rails filters only (~0.05ms overhead)
|
|
68
|
+
filtered_data = apply_rails_filters(event_data)
|
|
69
|
+
@app.call(filtered_data)
|
|
70
|
+
when :tier3
|
|
71
|
+
# Tier 3: Deep filtering (~0.2ms overhead)
|
|
72
|
+
filtered_data = apply_deep_filtering(event_data)
|
|
73
|
+
@app.call(filtered_data)
|
|
74
|
+
else
|
|
75
|
+
@app.call(event_data)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# Determine PII filtering tier for event
|
|
82
|
+
#
|
|
83
|
+
# @param event_data [Hash] Event data
|
|
84
|
+
# @return [Symbol] :tier1, :tier2, or :tier3
|
|
85
|
+
def determine_tier(event_data)
|
|
86
|
+
event_class = event_data[:event_class]
|
|
87
|
+
return :tier2 unless event_class.respond_to?(:pii_tier)
|
|
88
|
+
|
|
89
|
+
# Return tier directly from event class
|
|
90
|
+
event_class.pii_tier
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Apply Rails filter_parameters (Tier 2)
|
|
94
|
+
#
|
|
95
|
+
# @param event_data [Hash] Event data
|
|
96
|
+
# @return [Hash] Filtered event data
|
|
97
|
+
def apply_rails_filters(event_data)
|
|
98
|
+
# Clone to avoid modifying original
|
|
99
|
+
filtered_data = deep_dup(event_data)
|
|
100
|
+
|
|
101
|
+
# Apply Rails parameter filter
|
|
102
|
+
filter = parameter_filter
|
|
103
|
+
filtered_data[:payload] = filter.filter(filtered_data[:payload])
|
|
104
|
+
|
|
105
|
+
filtered_data
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Apply deep PII filtering (Tier 3)
|
|
109
|
+
#
|
|
110
|
+
# @param event_data [Hash] Event data
|
|
111
|
+
# @return [Hash] Filtered event data
|
|
112
|
+
def apply_deep_filtering(event_data)
|
|
113
|
+
event_class = event_data[:event_class]
|
|
114
|
+
return event_data unless event_class
|
|
115
|
+
|
|
116
|
+
# Clone to avoid modifying original
|
|
117
|
+
filtered_data = deep_dup(event_data)
|
|
118
|
+
|
|
119
|
+
# Get PII filtering config from event class
|
|
120
|
+
pii_config = event_class.pii_filtering_config if event_class.respond_to?(:pii_filtering_config)
|
|
121
|
+
return filtered_data unless pii_config
|
|
122
|
+
|
|
123
|
+
# Apply field-level strategies
|
|
124
|
+
filtered_data[:payload] = apply_field_strategies(
|
|
125
|
+
filtered_data[:payload],
|
|
126
|
+
pii_config
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Apply pattern-based filtering
|
|
130
|
+
filtered_data[:payload] = apply_pattern_filtering(
|
|
131
|
+
filtered_data[:payload]
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
filtered_data
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Apply field-level filtering strategies
|
|
138
|
+
#
|
|
139
|
+
# @param payload [Hash] Payload to filter
|
|
140
|
+
# @param config [Hash] PII configuration
|
|
141
|
+
# @return [Hash] Filtered payload
|
|
142
|
+
def apply_field_strategies(payload, config)
|
|
143
|
+
return payload unless config
|
|
144
|
+
|
|
145
|
+
filtered = {}
|
|
146
|
+
|
|
147
|
+
payload.each do |key, value|
|
|
148
|
+
strategy = config.dig(:fields, key, :strategy) || :allow
|
|
149
|
+
|
|
150
|
+
filtered[key] = case strategy
|
|
151
|
+
when :mask
|
|
152
|
+
"[FILTERED]"
|
|
153
|
+
when :hash
|
|
154
|
+
hash_value(value)
|
|
155
|
+
when :partial
|
|
156
|
+
partial_mask(value)
|
|
157
|
+
when :redact
|
|
158
|
+
nil
|
|
159
|
+
when :allow
|
|
160
|
+
value
|
|
161
|
+
else
|
|
162
|
+
value
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
filtered
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Apply pattern-based filtering to string values
|
|
170
|
+
#
|
|
171
|
+
# @param data [Object] Data to filter (recursively)
|
|
172
|
+
# @return [Object] Filtered data
|
|
173
|
+
def apply_pattern_filtering(data)
|
|
174
|
+
case data
|
|
175
|
+
when Hash
|
|
176
|
+
data.transform_values { |v| apply_pattern_filtering(v) }
|
|
177
|
+
when Array
|
|
178
|
+
data.map { |v| apply_pattern_filtering(v) }
|
|
179
|
+
when String
|
|
180
|
+
filter_string_patterns(data)
|
|
181
|
+
else
|
|
182
|
+
data
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Filter PII patterns in string
|
|
187
|
+
#
|
|
188
|
+
# @param str [String] String to filter
|
|
189
|
+
# @return [String] Filtered string
|
|
190
|
+
def filter_string_patterns(str)
|
|
191
|
+
result = str.dup
|
|
192
|
+
|
|
193
|
+
# Apply all PII patterns
|
|
194
|
+
E11y::PII::Patterns::ALL.each do |pattern|
|
|
195
|
+
result = result.gsub(pattern, "[FILTERED]")
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
result
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Hash value using SHA256
|
|
202
|
+
#
|
|
203
|
+
# @param value [Object] Value to hash
|
|
204
|
+
# @return [String] Hashed value
|
|
205
|
+
def hash_value(value)
|
|
206
|
+
return "[FILTERED]" if value.nil?
|
|
207
|
+
|
|
208
|
+
require "digest"
|
|
209
|
+
"hashed_#{Digest::SHA256.hexdigest(value.to_s)[0..15]}"
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Partial mask (show first/last chars)
|
|
213
|
+
#
|
|
214
|
+
# @param value [String] Value to mask
|
|
215
|
+
# @return [String] Partially masked value
|
|
216
|
+
def partial_mask(value)
|
|
217
|
+
return "[FILTERED]" unless value.is_a?(String)
|
|
218
|
+
return "[FILTERED]" if value.length < 4
|
|
219
|
+
|
|
220
|
+
if value.include?("@")
|
|
221
|
+
# Email: show first 2 chars before @, last 3 chars after @
|
|
222
|
+
local, domain = value.split("@", 2)
|
|
223
|
+
"#{local[0..1]}***#{domain[-3..]}"
|
|
224
|
+
else
|
|
225
|
+
# Generic: show first/last 2 chars
|
|
226
|
+
"#{value[0..1]}***#{value[-2..]}"
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Deep duplicate data structure
|
|
231
|
+
#
|
|
232
|
+
# @param data [Object] Data to duplicate
|
|
233
|
+
# @return [Object] Duplicated data
|
|
234
|
+
def deep_dup(data)
|
|
235
|
+
case data
|
|
236
|
+
when Hash
|
|
237
|
+
data.transform_values { |v| deep_dup(v) }
|
|
238
|
+
when Array
|
|
239
|
+
data.map { |v| deep_dup(v) }
|
|
240
|
+
when String, Symbol, Integer, Float, TrueClass, FalseClass, NilClass
|
|
241
|
+
data
|
|
242
|
+
else
|
|
243
|
+
begin
|
|
244
|
+
data.dup
|
|
245
|
+
rescue StandardError
|
|
246
|
+
data
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# Get Rails parameter filter
|
|
252
|
+
#
|
|
253
|
+
# @return [ActiveSupport::ParameterFilter] Parameter filter
|
|
254
|
+
def parameter_filter
|
|
255
|
+
@parameter_filter ||= if defined?(Rails)
|
|
256
|
+
ActiveSupport::ParameterFilter.new(
|
|
257
|
+
Rails.application.config.filter_parameters
|
|
258
|
+
)
|
|
259
|
+
else
|
|
260
|
+
# Fallback for non-Rails environments
|
|
261
|
+
ActiveSupport::ParameterFilter.new([])
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
end
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module E11y
|
|
4
|
+
module Middleware
|
|
5
|
+
# PII Filtering Middleware - 3-Tier Strategy
|
|
6
|
+
#
|
|
7
|
+
# Filters Personally Identifiable Information (PII) from event payloads
|
|
8
|
+
# before they reach adapters. Implements ADR-006 3-tier security model.
|
|
9
|
+
#
|
|
10
|
+
# **Three-Tier Strategy:**
|
|
11
|
+
# - Tier 1: No PII (`contains_pii false`) - Skip filtering (0ms overhead)
|
|
12
|
+
# - Tier 2: Default - Rails filters only (~0.05ms overhead)
|
|
13
|
+
# - Tier 3: Explicit PII (`contains_pii true`) - Deep filtering (~0.2ms overhead)
|
|
14
|
+
#
|
|
15
|
+
# @example Basic Usage (Tier 2 - Default)
|
|
16
|
+
# class Events::OrderCreated < E11y::Event::Base
|
|
17
|
+
# schema do
|
|
18
|
+
# required(:order_id).filled(:string)
|
|
19
|
+
# optional(:api_key).filled(:string) # Rails will filter this
|
|
20
|
+
# end
|
|
21
|
+
# end
|
|
22
|
+
#
|
|
23
|
+
# @example Tier 1: No PII (High Performance)
|
|
24
|
+
# class Events::HealthCheck < E11y::Event::Base
|
|
25
|
+
# contains_pii false # Skip all filtering
|
|
26
|
+
# end
|
|
27
|
+
#
|
|
28
|
+
# @example Tier 3: Explicit PII (Deep Filtering)
|
|
29
|
+
# class Events::UserRegistered < E11y::Event::Base
|
|
30
|
+
# contains_pii true
|
|
31
|
+
#
|
|
32
|
+
# pii_filtering do
|
|
33
|
+
# masks :password
|
|
34
|
+
# hashes :email
|
|
35
|
+
# allows :user_id
|
|
36
|
+
# end
|
|
37
|
+
# end
|
|
38
|
+
#
|
|
39
|
+
# @see ADR-006 PII Security & Compliance
|
|
40
|
+
# @see UC-007 PII Filtering
|
|
41
|
+
# @see E11y::PII::Patterns
|
|
42
|
+
class PIIFiltering < Base
|
|
43
|
+
middleware_zone :security
|
|
44
|
+
|
|
45
|
+
# Initialize PII filtering middleware
|
|
46
|
+
#
|
|
47
|
+
# @param app [Proc] Next middleware in chain
|
|
48
|
+
# @param config [Hash] Configuration options
|
|
49
|
+
def initialize(app, config = {})
|
|
50
|
+
super(app)
|
|
51
|
+
@config = config
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Process event and filter PII based on tier
|
|
55
|
+
#
|
|
56
|
+
# @param event_data [Hash] Event data with payload
|
|
57
|
+
# @return [Hash] Processed event data
|
|
58
|
+
def call(event_data)
|
|
59
|
+
# Determine filtering tier
|
|
60
|
+
tier = determine_tier(event_data)
|
|
61
|
+
|
|
62
|
+
case tier
|
|
63
|
+
when :tier1
|
|
64
|
+
# Tier 1: No PII - Skip filtering (0ms overhead)
|
|
65
|
+
@app.call(event_data)
|
|
66
|
+
when :tier2
|
|
67
|
+
# Tier 2: Rails filters only (~0.05ms overhead)
|
|
68
|
+
filtered_data = apply_rails_filters(event_data)
|
|
69
|
+
@app.call(filtered_data)
|
|
70
|
+
when :tier3
|
|
71
|
+
# Tier 3: Deep filtering (~0.2ms overhead)
|
|
72
|
+
filtered_data = apply_deep_filtering(event_data)
|
|
73
|
+
@app.call(filtered_data)
|
|
74
|
+
else
|
|
75
|
+
@app.call(event_data)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# Determine PII filtering tier for event
|
|
82
|
+
#
|
|
83
|
+
# @param event_data [Hash] Event data
|
|
84
|
+
# @return [Symbol] :tier1, :tier2, or :tier3
|
|
85
|
+
def determine_tier(event_data)
|
|
86
|
+
event_class = event_data[:event_class]
|
|
87
|
+
return :tier2 unless event_class
|
|
88
|
+
|
|
89
|
+
# Check explicit declaration
|
|
90
|
+
if event_class.respond_to?(:pii_tier)
|
|
91
|
+
case event_class.pii_tier
|
|
92
|
+
when :none
|
|
93
|
+
:tier1
|
|
94
|
+
when :explicit
|
|
95
|
+
:tier3
|
|
96
|
+
else
|
|
97
|
+
:tier2
|
|
98
|
+
end
|
|
99
|
+
else
|
|
100
|
+
# Default: Tier 2 (Rails filters)
|
|
101
|
+
:tier2
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Apply Rails filter_parameters (Tier 2)
|
|
106
|
+
#
|
|
107
|
+
# @param event_data [Hash] Event data
|
|
108
|
+
# @return [Hash] Filtered event data
|
|
109
|
+
def apply_rails_filters(event_data)
|
|
110
|
+
return event_data unless defined?(Rails)
|
|
111
|
+
|
|
112
|
+
# Clone to avoid modifying original
|
|
113
|
+
filtered_data = deep_dup(event_data)
|
|
114
|
+
|
|
115
|
+
# Apply Rails parameter filter
|
|
116
|
+
filter = parameter_filter
|
|
117
|
+
filtered_data[:payload] = filter.filter(filtered_data[:payload])
|
|
118
|
+
|
|
119
|
+
filtered_data
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Apply deep PII filtering (Tier 3)
|
|
123
|
+
#
|
|
124
|
+
# @param event_data [Hash] Event data
|
|
125
|
+
# @return [Hash] Filtered event data
|
|
126
|
+
def apply_deep_filtering(event_data)
|
|
127
|
+
event_class = event_data[:event_class]
|
|
128
|
+
return event_data unless event_class
|
|
129
|
+
|
|
130
|
+
# Clone to avoid modifying original
|
|
131
|
+
filtered_data = deep_dup(event_data)
|
|
132
|
+
|
|
133
|
+
# Get PII filtering config from event class
|
|
134
|
+
pii_config = event_class.pii_filtering_config if event_class.respond_to?(:pii_filtering_config)
|
|
135
|
+
return filtered_data unless pii_config
|
|
136
|
+
|
|
137
|
+
# Apply field-level strategies
|
|
138
|
+
filtered_data[:payload] = apply_field_strategies(
|
|
139
|
+
filtered_data[:payload],
|
|
140
|
+
pii_config
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
# Apply pattern-based filtering
|
|
144
|
+
filtered_data[:payload] = apply_pattern_filtering(
|
|
145
|
+
filtered_data[:payload]
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
filtered_data
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Apply field-level filtering strategies
|
|
152
|
+
#
|
|
153
|
+
# @param payload [Hash] Payload to filter
|
|
154
|
+
# @param config [Hash] PII configuration
|
|
155
|
+
# @return [Hash] Filtered payload
|
|
156
|
+
def apply_field_strategies(payload, config)
|
|
157
|
+
return payload unless config
|
|
158
|
+
|
|
159
|
+
filtered = {}
|
|
160
|
+
|
|
161
|
+
payload.each do |key, value|
|
|
162
|
+
strategy = config.dig(:fields, key, :strategy) || :allow
|
|
163
|
+
|
|
164
|
+
filtered[key] = case strategy
|
|
165
|
+
when :mask
|
|
166
|
+
"[FILTERED]"
|
|
167
|
+
when :hash
|
|
168
|
+
hash_value(value)
|
|
169
|
+
when :partial
|
|
170
|
+
partial_mask(value)
|
|
171
|
+
when :redact
|
|
172
|
+
nil
|
|
173
|
+
when :allow
|
|
174
|
+
value
|
|
175
|
+
else
|
|
176
|
+
value
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
filtered
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Apply pattern-based filtering to string values
|
|
184
|
+
#
|
|
185
|
+
# @param data [Object] Data to filter (recursively)
|
|
186
|
+
# @return [Object] Filtered data
|
|
187
|
+
def apply_pattern_filtering(data)
|
|
188
|
+
case data
|
|
189
|
+
when Hash
|
|
190
|
+
data.transform_values { |v| apply_pattern_filtering(v) }
|
|
191
|
+
when Array
|
|
192
|
+
data.map { |v| apply_pattern_filtering(v) }
|
|
193
|
+
when String
|
|
194
|
+
filter_string_patterns(data)
|
|
195
|
+
else
|
|
196
|
+
data
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Filter PII patterns in string
|
|
201
|
+
#
|
|
202
|
+
# @param str [String] String to filter
|
|
203
|
+
# @return [String] Filtered string
|
|
204
|
+
def filter_string_patterns(str)
|
|
205
|
+
result = str.dup
|
|
206
|
+
|
|
207
|
+
# Apply all PII patterns
|
|
208
|
+
E11y::PII::Patterns::ALL.each do |pattern|
|
|
209
|
+
result = result.gsub(pattern, "[FILTERED]")
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
result
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Hash value using SHA256
|
|
216
|
+
#
|
|
217
|
+
# @param value [Object] Value to hash
|
|
218
|
+
# @return [String] Hashed value
|
|
219
|
+
def hash_value(value)
|
|
220
|
+
return "[FILTERED]" if value.nil?
|
|
221
|
+
|
|
222
|
+
require "digest"
|
|
223
|
+
"hashed_#{Digest::SHA256.hexdigest(value.to_s)[0..15]}"
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Partial mask (show first/last chars)
|
|
227
|
+
#
|
|
228
|
+
# @param value [String] Value to mask
|
|
229
|
+
# @return [String] Partially masked value
|
|
230
|
+
def partial_mask(value)
|
|
231
|
+
return "[FILTERED]" unless value.is_a?(String)
|
|
232
|
+
return "[FILTERED]" if value.length < 4
|
|
233
|
+
|
|
234
|
+
if value.include?("@")
|
|
235
|
+
# Email: show first 2 chars before @, last 2 after @
|
|
236
|
+
local, domain = value.split("@", 2)
|
|
237
|
+
"#{local[0..1]}***@#{domain[-3..-1]}"
|
|
238
|
+
else
|
|
239
|
+
# Generic: show first/last 2 chars
|
|
240
|
+
"#{value[0..1]}***#{value[-2..-1]}"
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Deep duplicate data structure
|
|
245
|
+
#
|
|
246
|
+
# @param data [Object] Data to duplicate
|
|
247
|
+
# @return [Object] Duplicated data
|
|
248
|
+
def deep_dup(data)
|
|
249
|
+
case data
|
|
250
|
+
when Hash
|
|
251
|
+
data.transform_values { |v| deep_dup(v) }
|
|
252
|
+
when Array
|
|
253
|
+
data.map { |v| deep_dup(v) }
|
|
254
|
+
when String, Symbol, Integer, Float, TrueClass, FalseClass, NilClass
|
|
255
|
+
data
|
|
256
|
+
else
|
|
257
|
+
begin
|
|
258
|
+
data.dup
|
|
259
|
+
rescue StandardError
|
|
260
|
+
data
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# Get Rails parameter filter
|
|
266
|
+
#
|
|
267
|
+
# @return [ActiveSupport::ParameterFilter] Parameter filter
|
|
268
|
+
def parameter_filter
|
|
269
|
+
@parameter_filter ||= if defined?(Rails)
|
|
270
|
+
ActiveSupport::ParameterFilter.new(
|
|
271
|
+
Rails.application.config.filter_parameters
|
|
272
|
+
)
|
|
273
|
+
else
|
|
274
|
+
# Fallback for non-Rails environments
|
|
275
|
+
ActiveSupport::ParameterFilter.new([])
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
end
|