e11y 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/.rubocop.yml +20 -0
- data/CHANGELOG.md +151 -13
- data/README.md +1138 -104
- data/RELEASE.md +254 -0
- data/Rakefile +377 -0
- data/benchmarks/OPTIMIZATION.md +246 -0
- data/benchmarks/README.md +103 -0
- data/benchmarks/allocation_profiling.rb +253 -0
- data/benchmarks/e11y_benchmarks.rb +447 -0
- data/benchmarks/ruby_baseline_allocations.rb +175 -0
- data/benchmarks/run_all.rb +9 -21
- data/docs/00-ICP-AND-TIMELINE.md +2 -2
- data/docs/ADR-001-architecture.md +1 -1
- data/docs/ADR-004-adapter-architecture.md +247 -0
- data/docs/ADR-009-cost-optimization.md +231 -115
- data/docs/ADR-017-multi-rails-compatibility.md +103 -0
- data/docs/ADR-INDEX.md +99 -0
- data/docs/CONTRIBUTING.md +312 -0
- data/docs/IMPLEMENTATION_PLAN.md +1 -1
- data/docs/QUICK-START.md +0 -6
- data/docs/use_cases/UC-019-retention-based-routing.md +584 -0
- data/e11y.gemspec +28 -17
- data/lib/e11y/adapters/adaptive_batcher.rb +3 -0
- data/lib/e11y/adapters/audit_encrypted.rb +10 -4
- data/lib/e11y/adapters/base.rb +15 -0
- data/lib/e11y/adapters/file.rb +4 -1
- data/lib/e11y/adapters/in_memory.rb +6 -0
- data/lib/e11y/adapters/loki.rb +9 -0
- data/lib/e11y/adapters/otel_logs.rb +11 -9
- data/lib/e11y/adapters/sentry.rb +9 -0
- data/lib/e11y/adapters/yabeda.rb +54 -10
- data/lib/e11y/buffers.rb +8 -8
- data/lib/e11y/console.rb +52 -60
- data/lib/e11y/event/base.rb +75 -10
- data/lib/e11y/event/value_sampling_config.rb +10 -4
- data/lib/e11y/events/rails/http/request.rb +1 -1
- data/lib/e11y/instruments/active_job.rb +6 -3
- data/lib/e11y/instruments/rails_instrumentation.rb +51 -28
- data/lib/e11y/instruments/sidekiq.rb +7 -7
- data/lib/e11y/logger/bridge.rb +24 -54
- data/lib/e11y/metrics/cardinality_protection.rb +257 -12
- data/lib/e11y/metrics/cardinality_tracker.rb +17 -0
- data/lib/e11y/metrics/registry.rb +6 -2
- data/lib/e11y/metrics/relabeling.rb +0 -56
- data/lib/e11y/metrics.rb +6 -1
- data/lib/e11y/middleware/audit_signing.rb +12 -9
- data/lib/e11y/middleware/pii_filter.rb +18 -10
- data/lib/e11y/middleware/request.rb +10 -4
- data/lib/e11y/middleware/routing.rb +117 -90
- data/lib/e11y/middleware/sampling.rb +47 -28
- data/lib/e11y/middleware/trace_context.rb +40 -11
- data/lib/e11y/middleware/validation.rb +20 -2
- data/lib/e11y/middleware/versioning.rb +1 -1
- data/lib/e11y/pii.rb +7 -7
- data/lib/e11y/railtie.rb +24 -20
- data/lib/e11y/reliability/circuit_breaker.rb +3 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +16 -5
- data/lib/e11y/reliability/dlq/filter.rb +3 -0
- data/lib/e11y/reliability/retry_handler.rb +4 -0
- data/lib/e11y/sampling/error_spike_detector.rb +16 -5
- data/lib/e11y/sampling/load_monitor.rb +13 -4
- data/lib/e11y/self_monitoring/reliability_monitor.rb +3 -0
- data/lib/e11y/version.rb +1 -1
- data/lib/e11y.rb +86 -9
- metadata +83 -38
- data/docs/use_cases/UC-019-tiered-storage-migration.md +0 -562
- data/lib/e11y/middleware/pii_filtering.rb +0 -280
- data/lib/e11y/middleware/slo.rb +0 -168
|
@@ -7,10 +7,11 @@ module E11y
|
|
|
7
7
|
module Metrics
|
|
8
8
|
# Cardinality protection for metrics labels.
|
|
9
9
|
#
|
|
10
|
-
# Implements
|
|
10
|
+
# Implements 4-layer defense system to prevent cardinality explosions:
|
|
11
11
|
# 1. Universal Denylist - Block high-cardinality fields (user_id, order_id, etc.)
|
|
12
12
|
# 2. Per-Metric Limits - Track unique values per metric, drop if exceeded
|
|
13
13
|
# 3. Dynamic Monitoring - Alert when approaching limits
|
|
14
|
+
# 4. Dynamic Actions - Auto-relabeling, alerting, or dropping on overflow
|
|
14
15
|
#
|
|
15
16
|
# Now supports optional relabeling to reduce cardinality while preserving signal.
|
|
16
17
|
#
|
|
@@ -27,8 +28,16 @@ module E11y
|
|
|
27
28
|
# safe_labels = protection.filter(labels, 'http.requests')
|
|
28
29
|
# # => { http_status: '2xx', path: '/api/users' }
|
|
29
30
|
#
|
|
31
|
+
# @example With overflow strategy
|
|
32
|
+
# protection = E11y::Metrics::CardinalityProtection.new(
|
|
33
|
+
# overflow_strategy: :alert,
|
|
34
|
+
# alert_threshold: 0.8
|
|
35
|
+
# )
|
|
36
|
+
#
|
|
30
37
|
# @see ADR-002 §4 (Cardinality Protection)
|
|
31
38
|
# @see UC-013 (High Cardinality Protection)
|
|
39
|
+
# rubocop:disable Metrics/ClassLength
|
|
40
|
+
# Cardinality protection is a cohesive 4-layer defense system against metric explosions
|
|
32
41
|
class CardinalityProtection
|
|
33
42
|
# Universal denylist - high-cardinality fields that should NEVER be labels
|
|
34
43
|
UNIVERSAL_DENYLIST = %i[
|
|
@@ -55,7 +64,16 @@ module E11y
|
|
|
55
64
|
# Default per-metric cardinality limit
|
|
56
65
|
DEFAULT_CARDINALITY_LIMIT = 1000
|
|
57
66
|
|
|
58
|
-
|
|
67
|
+
# Overflow strategies (Layer 4: Dynamic Actions)
|
|
68
|
+
OVERFLOW_STRATEGIES = %i[drop alert relabel].freeze
|
|
69
|
+
|
|
70
|
+
# Default overflow strategy
|
|
71
|
+
DEFAULT_OVERFLOW_STRATEGY = :drop
|
|
72
|
+
|
|
73
|
+
# Default alert threshold (80% of limit)
|
|
74
|
+
DEFAULT_ALERT_THRESHOLD = 0.8
|
|
75
|
+
|
|
76
|
+
attr_reader :tracker, :relabeler, :overflow_strategy, :alert_threshold
|
|
59
77
|
|
|
60
78
|
# Initialize cardinality protection
|
|
61
79
|
# @param config [Hash] Configuration options
|
|
@@ -63,16 +81,35 @@ module E11y
|
|
|
63
81
|
# @option config [Array<Symbol>] :additional_denylist Additional fields to deny
|
|
64
82
|
# @option config [Boolean] :enabled (true) Enable/disable protection
|
|
65
83
|
# @option config [Boolean] :relabeling_enabled (true) Enable/disable relabeling
|
|
84
|
+
# @option config [Symbol] :overflow_strategy (:drop) Strategy when limit exceeded (:drop, :alert, :relabel)
|
|
85
|
+
# @option config [Float] :alert_threshold (0.8) Alert when cardinality reaches this ratio
|
|
86
|
+
# @option config [Proc] :alert_callback Optional callback when alert triggered
|
|
87
|
+
# @option config [Boolean] :auto_relabel (false) Auto-relabel to [OTHER] on overflow
|
|
88
|
+
# rubocop:disable Metrics/AbcSize
|
|
89
|
+
# Cardinality protection initialization requires extracting multiple config options and setting up components
|
|
66
90
|
def initialize(config = {})
|
|
67
91
|
@cardinality_limit = config.fetch(:cardinality_limit, DEFAULT_CARDINALITY_LIMIT)
|
|
68
92
|
@enabled = config.fetch(:enabled, true)
|
|
69
93
|
@relabeling_enabled = config.fetch(:relabeling_enabled, true)
|
|
70
94
|
@denylist = Set.new(UNIVERSAL_DENYLIST + (config[:additional_denylist] || []))
|
|
71
95
|
|
|
96
|
+
# Layer 4: Dynamic Actions configuration
|
|
97
|
+
@overflow_strategy = config.fetch(:overflow_strategy, DEFAULT_OVERFLOW_STRATEGY)
|
|
98
|
+
@alert_threshold = config.fetch(:alert_threshold, DEFAULT_ALERT_THRESHOLD)
|
|
99
|
+
@alert_callback = config[:alert_callback]
|
|
100
|
+
@auto_relabel = config.fetch(:auto_relabel, false)
|
|
101
|
+
|
|
102
|
+
validate_config!
|
|
103
|
+
|
|
72
104
|
# Use extracted components
|
|
73
105
|
@tracker = CardinalityTracker.new(limit: @cardinality_limit)
|
|
74
106
|
@relabeler = Relabeling.new
|
|
107
|
+
|
|
108
|
+
# Track overflow metrics
|
|
109
|
+
@overflow_counts = Hash.new(0)
|
|
110
|
+
@overflow_mutex = Mutex.new
|
|
75
111
|
end
|
|
112
|
+
# rubocop:enable Metrics/AbcSize
|
|
76
113
|
|
|
77
114
|
# Define relabeling rule for a label
|
|
78
115
|
#
|
|
@@ -91,11 +128,11 @@ module E11y
|
|
|
91
128
|
|
|
92
129
|
# Filter labels to prevent cardinality explosions
|
|
93
130
|
#
|
|
94
|
-
# Applies
|
|
131
|
+
# Applies 4-layer defense + optional relabeling:
|
|
95
132
|
# 1. Relabel high-cardinality values (if enabled)
|
|
96
133
|
# 2. Drop denylisted fields
|
|
97
134
|
# 3. Track and limit per-metric cardinality
|
|
98
|
-
# 4.
|
|
135
|
+
# 4. Dynamic Actions on overflow (drop/alert/relabel)
|
|
99
136
|
#
|
|
100
137
|
# @param labels [Hash] Raw labels from event
|
|
101
138
|
# @param metric_name [String] Metric name for tracking
|
|
@@ -106,21 +143,24 @@ module E11y
|
|
|
106
143
|
safe_labels = {}
|
|
107
144
|
|
|
108
145
|
labels.each do |key, value|
|
|
109
|
-
#
|
|
146
|
+
# Layer 1: Relabel if rule exists (reduces cardinality)
|
|
110
147
|
relabeled_value = @relabeling_enabled ? @relabeler.apply(key, value) : value
|
|
111
148
|
|
|
112
|
-
#
|
|
149
|
+
# Layer 2: Denylist - drop high-cardinality fields
|
|
113
150
|
next if should_deny?(key)
|
|
114
151
|
|
|
115
|
-
#
|
|
152
|
+
# Layer 3: Per-Metric Cardinality Limit
|
|
116
153
|
if @tracker.track(metric_name, key, relabeled_value)
|
|
117
154
|
safe_labels[key] = relabeled_value
|
|
118
155
|
else
|
|
119
|
-
#
|
|
120
|
-
|
|
156
|
+
# Layer 4: Dynamic Actions on overflow
|
|
157
|
+
handle_overflow(metric_name, key, relabeled_value, safe_labels)
|
|
121
158
|
end
|
|
122
159
|
end
|
|
123
160
|
|
|
161
|
+
# Check if approaching alert threshold (after tracking new values)
|
|
162
|
+
check_alert_threshold(metric_name)
|
|
163
|
+
|
|
124
164
|
safe_labels
|
|
125
165
|
end
|
|
126
166
|
|
|
@@ -150,10 +190,29 @@ module E11y
|
|
|
150
190
|
def reset!
|
|
151
191
|
@tracker.reset_all!
|
|
152
192
|
@relabeler.reset!
|
|
193
|
+
@overflow_mutex.synchronize do
|
|
194
|
+
@overflow_counts.clear
|
|
195
|
+
end
|
|
153
196
|
end
|
|
154
197
|
|
|
155
198
|
private
|
|
156
199
|
|
|
200
|
+
# Validate configuration
|
|
201
|
+
# @raise [ArgumentError] If configuration is invalid
|
|
202
|
+
def validate_config!
|
|
203
|
+
unless OVERFLOW_STRATEGIES.include?(@overflow_strategy)
|
|
204
|
+
raise ArgumentError,
|
|
205
|
+
"Invalid overflow_strategy: #{@overflow_strategy}. " \
|
|
206
|
+
"Must be one of: #{OVERFLOW_STRATEGIES.join(', ')}"
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
return if @alert_threshold.is_a?(Numeric) && @alert_threshold.positive? && @alert_threshold <= 1.0
|
|
210
|
+
|
|
211
|
+
raise ArgumentError,
|
|
212
|
+
"Invalid alert_threshold: #{@alert_threshold}. " \
|
|
213
|
+
"Must be a number between 0 and 1.0"
|
|
214
|
+
end
|
|
215
|
+
|
|
157
216
|
# Check if label should be denied (Layer 1: Denylist)
|
|
158
217
|
# @param key [Symbol] Label key
|
|
159
218
|
# @return [Boolean] True if should be denied
|
|
@@ -161,12 +220,198 @@ module E11y
|
|
|
161
220
|
@denylist.include?(key)
|
|
162
221
|
end
|
|
163
222
|
|
|
164
|
-
#
|
|
223
|
+
# Check if approaching alert threshold (Layer 3: Monitoring)
|
|
224
|
+
# @param metric_name [String] Metric name
|
|
225
|
+
# rubocop:disable Metrics/MethodLength
|
|
226
|
+
# Alert threshold checking requires calculating ratio, checking conditions, and sending detailed alerts
|
|
227
|
+
def check_alert_threshold(metric_name)
|
|
228
|
+
return unless @alert_threshold
|
|
229
|
+
|
|
230
|
+
current_cardinality = @tracker.cardinalities(metric_name).values.sum
|
|
231
|
+
ratio = current_cardinality.to_f / @cardinality_limit
|
|
232
|
+
|
|
233
|
+
return unless ratio >= @alert_threshold
|
|
234
|
+
|
|
235
|
+
# Only alert once per threshold crossing
|
|
236
|
+
alert_key = "#{metric_name}:#{@alert_threshold}"
|
|
237
|
+
return if @overflow_counts[alert_key].positive?
|
|
238
|
+
|
|
239
|
+
@overflow_mutex.synchronize do
|
|
240
|
+
@overflow_counts[alert_key] += 1
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
send_alert(
|
|
244
|
+
metric_name: metric_name,
|
|
245
|
+
message: "Cardinality approaching limit",
|
|
246
|
+
current: current_cardinality,
|
|
247
|
+
limit: @cardinality_limit,
|
|
248
|
+
ratio: ratio,
|
|
249
|
+
severity: :warn
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
# Track metric
|
|
253
|
+
track_cardinality_metric(metric_name, :threshold_exceeded, current_cardinality)
|
|
254
|
+
end
|
|
255
|
+
# rubocop:enable Metrics/MethodLength
|
|
256
|
+
|
|
257
|
+
# Handle overflow when cardinality limit exceeded (Layer 4: Dynamic Actions)
|
|
258
|
+
# @param metric_name [String] Metric name
|
|
259
|
+
# @param key [Symbol] Label key
|
|
260
|
+
# @param value [Object] Label value
|
|
261
|
+
# @param safe_labels [Hash] Current safe labels hash (may be modified)
|
|
262
|
+
def handle_overflow(metric_name, key, value, safe_labels)
|
|
263
|
+
# Increment overflow counter
|
|
264
|
+
overflow_key = "#{metric_name}:#{key}"
|
|
265
|
+
@overflow_mutex.synchronize do
|
|
266
|
+
@overflow_counts[overflow_key] += 1
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
case @overflow_strategy
|
|
270
|
+
when :drop
|
|
271
|
+
handle_drop(metric_name, key, value)
|
|
272
|
+
when :alert
|
|
273
|
+
handle_alert(metric_name, key, value)
|
|
274
|
+
when :relabel
|
|
275
|
+
handle_relabel(metric_name, key, value, safe_labels)
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# Track overflow metric
|
|
279
|
+
track_cardinality_metric(metric_name, @overflow_strategy, @overflow_counts[overflow_key])
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# Handle drop strategy - silently drop label
|
|
165
283
|
# @param metric_name [String] Metric name
|
|
166
284
|
# @param key [Symbol] Label key
|
|
167
|
-
|
|
168
|
-
|
|
285
|
+
# @param value [Object] Label value
|
|
286
|
+
def handle_drop(metric_name, key, value)
|
|
287
|
+
# Silent drop (most efficient)
|
|
288
|
+
# Optionally log at debug level
|
|
289
|
+
return unless defined?(::Rails) && ::Rails.logger&.debug?
|
|
290
|
+
|
|
291
|
+
::Rails.logger.debug(
|
|
292
|
+
"[E11y] Cardinality limit exceeded: #{metric_name}:#{key}=#{value} (dropped)"
|
|
293
|
+
)
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Handle alert strategy - alert ops team and drop
|
|
297
|
+
# @param metric_name [String] Metric name
|
|
298
|
+
# @param key [Symbol] Label key
|
|
299
|
+
# @param value [Object] Label value
|
|
300
|
+
def handle_alert(metric_name, key, value)
|
|
301
|
+
current_cardinality = @tracker.cardinalities(metric_name)[key] || 0
|
|
302
|
+
|
|
303
|
+
send_alert(
|
|
304
|
+
metric_name: metric_name,
|
|
305
|
+
label_key: key,
|
|
306
|
+
label_value: value,
|
|
307
|
+
message: "Cardinality limit exceeded",
|
|
308
|
+
current: current_cardinality,
|
|
309
|
+
limit: @cardinality_limit,
|
|
310
|
+
overflow_count: @overflow_counts["#{metric_name}:#{key}"],
|
|
311
|
+
severity: :error
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
# Also log warning
|
|
315
|
+
warn "E11y Metrics: Cardinality limit exceeded for #{metric_name}:#{key} " \
|
|
316
|
+
"(limit: #{@cardinality_limit}, current: #{current_cardinality})"
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# Handle relabel strategy - relabel to [OTHER]
|
|
320
|
+
# @param metric_name [String] Metric name
|
|
321
|
+
# @param key [Symbol] Label key
|
|
322
|
+
# @param value [Object] Label value
|
|
323
|
+
# @param safe_labels [Hash] Current safe labels hash (modified in place)
|
|
324
|
+
def handle_relabel(metric_name, key, value, safe_labels)
|
|
325
|
+
# Relabel to [OTHER] to preserve some signal
|
|
326
|
+
other_value = "[OTHER]"
|
|
327
|
+
|
|
328
|
+
# Force-track [OTHER] as a special aggregate value
|
|
329
|
+
# This bypasses limit checks since [OTHER] represents multiple overflow values
|
|
330
|
+
@tracker.force_track(metric_name, key, other_value)
|
|
331
|
+
|
|
332
|
+
# Add [OTHER] to safe_labels
|
|
333
|
+
safe_labels[key] = other_value
|
|
334
|
+
|
|
335
|
+
return unless defined?(::Rails) && ::Rails.logger&.debug?
|
|
336
|
+
|
|
337
|
+
::Rails.logger.debug(
|
|
338
|
+
"[E11y] Cardinality limit exceeded: #{metric_name}:#{key}=#{value} " \
|
|
339
|
+
"(relabeled to [OTHER])"
|
|
340
|
+
)
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# Send alert to configured destinations
|
|
344
|
+
# @param data [Hash] Alert data
|
|
345
|
+
def send_alert(data)
|
|
346
|
+
# Call custom callback if provided
|
|
347
|
+
@alert_callback&.call(data)
|
|
348
|
+
|
|
349
|
+
# Send to Sentry if available
|
|
350
|
+
send_sentry_alert(data) if sentry_available?
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# Send alert to Sentry
|
|
354
|
+
# @param data [Hash] Alert data
|
|
355
|
+
def send_sentry_alert(data)
|
|
356
|
+
require "sentry-ruby" if defined?(Sentry)
|
|
357
|
+
|
|
358
|
+
::Sentry.with_scope do |scope|
|
|
359
|
+
scope.set_tags(
|
|
360
|
+
metric_name: data[:metric_name].to_s,
|
|
361
|
+
label_key: data[:label_key].to_s,
|
|
362
|
+
overflow_strategy: @overflow_strategy.to_s
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
scope.set_extras(data)
|
|
366
|
+
|
|
367
|
+
level = data[:severity] == :error ? :error : :warning
|
|
368
|
+
|
|
369
|
+
::Sentry.capture_message(
|
|
370
|
+
"[E11y] #{data[:message]}: #{data[:metric_name]}",
|
|
371
|
+
level: level
|
|
372
|
+
)
|
|
373
|
+
end
|
|
374
|
+
rescue LoadError, NameError
|
|
375
|
+
# Sentry not available, skip
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
# Check if Sentry is available
|
|
379
|
+
# @return [Boolean]
|
|
380
|
+
def sentry_available?
|
|
381
|
+
defined?(::Sentry) && ::Sentry.initialized?
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
# Track cardinality metric via Yabeda
|
|
385
|
+
# @param metric_name [String] Metric name
|
|
386
|
+
# @param action [Symbol] Action type (:threshold_exceeded, :drop, :alert, :relabel)
|
|
387
|
+
# @param value [Integer] Metric value
|
|
388
|
+
# rubocop:disable Metrics/MethodLength
|
|
389
|
+
# Cardinality tracking requires incrementing overflow counters and updating gauge metrics
|
|
390
|
+
def track_cardinality_metric(metric_name, action, value)
|
|
391
|
+
return unless defined?(E11y::Metrics)
|
|
392
|
+
|
|
393
|
+
# Track overflow actions
|
|
394
|
+
E11y::Metrics.increment(
|
|
395
|
+
:e11y_cardinality_overflow_total,
|
|
396
|
+
{
|
|
397
|
+
metric: metric_name,
|
|
398
|
+
action: action.to_s,
|
|
399
|
+
strategy: @overflow_strategy.to_s
|
|
400
|
+
}
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
# Track current cardinality
|
|
404
|
+
E11y::Metrics.gauge(
|
|
405
|
+
:e11y_cardinality_current,
|
|
406
|
+
value,
|
|
407
|
+
{ metric: metric_name }
|
|
408
|
+
)
|
|
409
|
+
rescue StandardError => e
|
|
410
|
+
# Don't fail on metrics tracking errors
|
|
411
|
+
warn "E11y: Failed to track cardinality metric: #{e.message}"
|
|
169
412
|
end
|
|
413
|
+
# rubocop:enable Metrics/MethodLength
|
|
170
414
|
end
|
|
415
|
+
# rubocop:enable Metrics/ClassLength
|
|
171
416
|
end
|
|
172
417
|
end
|
|
@@ -54,6 +54,23 @@ module E11y
|
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
+
# Force-track a label value, bypassing limit checks
|
|
58
|
+
#
|
|
59
|
+
# Used for special aggregate values like "[OTHER]" that need to be tracked
|
|
60
|
+
# even when limit is exceeded.
|
|
61
|
+
# Thread-safe operation.
|
|
62
|
+
#
|
|
63
|
+
# @param metric_name [String] Metric name
|
|
64
|
+
# @param label_key [Symbol, String] Label key
|
|
65
|
+
# @param label_value [Object] Label value to track
|
|
66
|
+
# @return [void]
|
|
67
|
+
def force_track(metric_name, label_key, label_value)
|
|
68
|
+
@mutex.synchronize do
|
|
69
|
+
value_set = @tracker[metric_name][label_key]
|
|
70
|
+
value_set.add(label_value) unless value_set.include?(label_value)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
57
74
|
# Check if metric+label has exceeded cardinality limit
|
|
58
75
|
#
|
|
59
76
|
# @param metric_name [String] Metric name
|
|
@@ -23,6 +23,8 @@ module E11y
|
|
|
23
23
|
# @example Find matching metrics
|
|
24
24
|
# metrics = registry.find_matching('order.paid')
|
|
25
25
|
# # => [{ type: :counter, name: :orders_total, ... }]
|
|
26
|
+
# rubocop:disable Metrics/ClassLength
|
|
27
|
+
# Registry is a cohesive singleton managing metric definitions and pattern matching
|
|
26
28
|
class Registry
|
|
27
29
|
include Singleton
|
|
28
30
|
|
|
@@ -161,7 +163,8 @@ module E11y
|
|
|
161
163
|
# @param new_config [Hash] New metric configuration
|
|
162
164
|
# @raise [TypeConflictError] if types don't match
|
|
163
165
|
# @raise [LabelConflictError] if labels don't match
|
|
164
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
166
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
167
|
+
# Conflict validation requires checking type and labels with detailed error messages
|
|
165
168
|
def validate_no_conflicts!(existing, new_config)
|
|
166
169
|
# Check 1: Type must match
|
|
167
170
|
if existing[:type] != new_config[:type]
|
|
@@ -212,7 +215,7 @@ module E11y
|
|
|
212
215
|
Using existing buckets.
|
|
213
216
|
WARNING
|
|
214
217
|
end
|
|
215
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
218
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
216
219
|
|
|
217
220
|
# Compile glob pattern to regex
|
|
218
221
|
# @param pattern [String] Glob pattern (e.g., "order.*", "user.*.created")
|
|
@@ -230,5 +233,6 @@ module E11y
|
|
|
230
233
|
Regexp.new("\\A#{regex_pattern}\\z")
|
|
231
234
|
end
|
|
232
235
|
end
|
|
236
|
+
# rubocop:enable Metrics/ClassLength
|
|
233
237
|
end
|
|
234
238
|
end
|
|
@@ -165,62 +165,6 @@ module E11y
|
|
|
165
165
|
def size
|
|
166
166
|
@mutex.synchronize { @rules.size }
|
|
167
167
|
end
|
|
168
|
-
|
|
169
|
-
# Predefined common relabeling rules
|
|
170
|
-
module CommonRules
|
|
171
|
-
# HTTP status code to status class (1xx, 2xx, 3xx, 4xx, 5xx)
|
|
172
|
-
#
|
|
173
|
-
# @param value [Integer, String] HTTP status code
|
|
174
|
-
# @return [String] Status class
|
|
175
|
-
def self.http_status_class(value)
|
|
176
|
-
code = value.to_i
|
|
177
|
-
return "unknown" if code < 100 || code >= 600
|
|
178
|
-
|
|
179
|
-
"#{code / 100}xx"
|
|
180
|
-
end
|
|
181
|
-
|
|
182
|
-
# Path normalization - replace numeric IDs and UUIDs with placeholders
|
|
183
|
-
#
|
|
184
|
-
# @param value [String] URL path
|
|
185
|
-
# @return [String] Normalized path
|
|
186
|
-
def self.normalize_path(value)
|
|
187
|
-
value.to_s
|
|
188
|
-
.gsub(%r{/[a-f0-9-]{36}}, "/:uuid") # UUIDs (must be before :id to avoid partial match)
|
|
189
|
-
.gsub(%r{/[a-f0-9]{32}}, "/:hash") # MD5 hashes (must be before :id)
|
|
190
|
-
.gsub(%r{/\d+}, "/:id") # /users/123 -> /users/:id
|
|
191
|
-
end
|
|
192
|
-
|
|
193
|
-
# Region to region group (us-east-1 -> us, eu-west-2 -> eu)
|
|
194
|
-
#
|
|
195
|
-
# @param value [String] AWS-style region
|
|
196
|
-
# @return [String] Region group
|
|
197
|
-
def self.region_group(value)
|
|
198
|
-
case value.to_s
|
|
199
|
-
when /^us-/ then "us"
|
|
200
|
-
when /^eu-/ then "eu"
|
|
201
|
-
when /^ap-/ then "ap"
|
|
202
|
-
when /^sa-/ then "sa"
|
|
203
|
-
when /^ca-/ then "ca"
|
|
204
|
-
when /^af-/ then "af"
|
|
205
|
-
when /^me-/ then "me"
|
|
206
|
-
else "other"
|
|
207
|
-
end
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
# Duration classification (ms to fast/medium/slow)
|
|
211
|
-
#
|
|
212
|
-
# @param value [Numeric] Duration in milliseconds
|
|
213
|
-
# @return [String] Classification
|
|
214
|
-
def self.duration_class(value)
|
|
215
|
-
ms = value.to_f
|
|
216
|
-
case ms
|
|
217
|
-
when 0..100 then "fast"
|
|
218
|
-
when 101..1000 then "medium"
|
|
219
|
-
when 1001..5000 then "slow"
|
|
220
|
-
else "very_slow"
|
|
221
|
-
end
|
|
222
|
-
end
|
|
223
|
-
end
|
|
224
168
|
end
|
|
225
169
|
end
|
|
226
170
|
end
|
data/lib/e11y/metrics.rb
CHANGED
|
@@ -91,7 +91,12 @@ module E11y
|
|
|
91
91
|
def detect_backend
|
|
92
92
|
# Check if Yabeda adapter is configured
|
|
93
93
|
# Use class name string to avoid LoadError if Yabeda gem not installed
|
|
94
|
-
|
|
94
|
+
# rubocop:disable Style/ClassEqualityComparison
|
|
95
|
+
# Reason: instance_of?(::E11y::Adapters::Yabeda) would trigger LoadError when gem not installed
|
|
96
|
+
yabeda_adapter = E11y.config.adapters.values.find do |adapter|
|
|
97
|
+
adapter.class.name == "E11y::Adapters::Yabeda"
|
|
98
|
+
end
|
|
99
|
+
# rubocop:enable Style/ClassEqualityComparison
|
|
95
100
|
return yabeda_adapter if yabeda_adapter
|
|
96
101
|
|
|
97
102
|
# No backend configured → noop
|
|
@@ -39,14 +39,17 @@ module E11y
|
|
|
39
39
|
class AuditSigning < Base
|
|
40
40
|
middleware_zone :security
|
|
41
41
|
|
|
42
|
-
# HMAC signing key (from ENV or generated)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
42
|
+
# Get HMAC signing key (from ENV or generated)
|
|
43
|
+
# @return [String] Signing key
|
|
44
|
+
def self.signing_key
|
|
45
|
+
@signing_key ||= ENV.fetch("E11Y_AUDIT_SIGNING_KEY") do
|
|
46
|
+
# Development fallback (NOT for production!)
|
|
47
|
+
if defined?(::Rails) && ::Rails.env.production?
|
|
48
|
+
raise E11y::Error, "E11Y_AUDIT_SIGNING_KEY must be set in production"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
"development_key_#{SecureRandom.hex(32)}"
|
|
47
52
|
end
|
|
48
|
-
|
|
49
|
-
"development_key_#{SecureRandom.hex(32)}"
|
|
50
53
|
end
|
|
51
54
|
|
|
52
55
|
# Initialize audit signing middleware
|
|
@@ -79,7 +82,7 @@ module E11y
|
|
|
79
82
|
|
|
80
83
|
return false unless expected_signature && canonical
|
|
81
84
|
|
|
82
|
-
actual_signature = OpenSSL::HMAC.hexdigest("SHA256",
|
|
85
|
+
actual_signature = OpenSSL::HMAC.hexdigest("SHA256", signing_key, canonical)
|
|
83
86
|
actual_signature == expected_signature
|
|
84
87
|
end
|
|
85
88
|
# rubocop:enable Naming/PredicateMethod
|
|
@@ -152,7 +155,7 @@ module E11y
|
|
|
152
155
|
# @param data [String] Data to sign
|
|
153
156
|
# @return [String] Hex-encoded signature
|
|
154
157
|
def generate_signature(data)
|
|
155
|
-
OpenSSL::HMAC.hexdigest("SHA256",
|
|
158
|
+
OpenSSL::HMAC.hexdigest("SHA256", self.class.signing_key, data)
|
|
156
159
|
end
|
|
157
160
|
|
|
158
161
|
# Sort hash recursively for deterministic JSON
|
|
@@ -39,6 +39,8 @@ module E11y
|
|
|
39
39
|
# @see ADR-006 PII Security & Compliance
|
|
40
40
|
# @see UC-007 PII Filtering
|
|
41
41
|
# @see E11y::PII::Patterns
|
|
42
|
+
# rubocop:disable Metrics/ClassLength
|
|
43
|
+
# PII filter is a cohesive security component with 3-tier filtering strategy
|
|
42
44
|
class PIIFilter < Base
|
|
43
45
|
middleware_zone :security
|
|
44
46
|
|
|
@@ -55,6 +57,8 @@ module E11y
|
|
|
55
57
|
#
|
|
56
58
|
# @param event_data [Hash] Event data with payload
|
|
57
59
|
# @return [Hash] Processed event data
|
|
60
|
+
# rubocop:disable Lint/DuplicateBranch
|
|
61
|
+
# Unknown tiers intentionally fallback to no filtering (same as tier1)
|
|
58
62
|
def call(event_data)
|
|
59
63
|
# Determine filtering tier
|
|
60
64
|
tier = determine_tier(event_data)
|
|
@@ -75,6 +79,7 @@ module E11y
|
|
|
75
79
|
@app.call(event_data)
|
|
76
80
|
end
|
|
77
81
|
end
|
|
82
|
+
# rubocop:enable Lint/DuplicateBranch
|
|
78
83
|
|
|
79
84
|
private
|
|
80
85
|
|
|
@@ -99,8 +104,7 @@ module E11y
|
|
|
99
104
|
filtered_data = deep_dup(event_data)
|
|
100
105
|
|
|
101
106
|
# Apply Rails parameter filter
|
|
102
|
-
|
|
103
|
-
filtered_data[:payload] = filter.filter(filtered_data[:payload])
|
|
107
|
+
filtered_data[:payload] = parameter_filter.filter(filtered_data[:payload])
|
|
104
108
|
|
|
105
109
|
filtered_data
|
|
106
110
|
end
|
|
@@ -139,6 +143,8 @@ module E11y
|
|
|
139
143
|
# @param payload [Hash] Payload to filter
|
|
140
144
|
# @param config [Hash] PII configuration
|
|
141
145
|
# @return [Hash] Filtered payload
|
|
146
|
+
# rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
|
|
147
|
+
# Field strategies require case/when for each PII filtering strategy type
|
|
142
148
|
def apply_field_strategies(payload, config)
|
|
143
149
|
return payload unless config
|
|
144
150
|
|
|
@@ -147,6 +153,8 @@ module E11y
|
|
|
147
153
|
payload.each do |key, value|
|
|
148
154
|
strategy = config.dig(:fields, key, :strategy) || :allow
|
|
149
155
|
|
|
156
|
+
# rubocop:disable Lint/DuplicateBranch
|
|
157
|
+
# Unknown strategies intentionally fallback to allow (same as :allow)
|
|
150
158
|
filtered[key] = case strategy
|
|
151
159
|
when :mask
|
|
152
160
|
"[FILTERED]"
|
|
@@ -161,10 +169,12 @@ module E11y
|
|
|
161
169
|
else
|
|
162
170
|
value
|
|
163
171
|
end
|
|
172
|
+
# rubocop:enable Lint/DuplicateBranch
|
|
164
173
|
end
|
|
165
174
|
|
|
166
175
|
filtered
|
|
167
176
|
end
|
|
177
|
+
# rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength
|
|
168
178
|
|
|
169
179
|
# Apply pattern-based filtering to string values
|
|
170
180
|
#
|
|
@@ -250,17 +260,15 @@ module E11y
|
|
|
250
260
|
|
|
251
261
|
# Get Rails parameter filter
|
|
252
262
|
#
|
|
263
|
+
# Uses Rails.application.config.filter_parameters for PII filtering.
|
|
264
|
+
#
|
|
253
265
|
# @return [ActiveSupport::ParameterFilter] Parameter filter
|
|
254
266
|
def parameter_filter
|
|
255
|
-
@parameter_filter ||=
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
)
|
|
259
|
-
else
|
|
260
|
-
# Fallback for non-Rails environments
|
|
261
|
-
ActiveSupport::ParameterFilter.new([])
|
|
262
|
-
end
|
|
267
|
+
@parameter_filter ||= ActiveSupport::ParameterFilter.new(
|
|
268
|
+
Rails.application.config.filter_parameters
|
|
269
|
+
)
|
|
263
270
|
end
|
|
264
271
|
end
|
|
272
|
+
# rubocop:enable Metrics/ClassLength
|
|
265
273
|
end
|
|
266
274
|
end
|
|
@@ -32,6 +32,8 @@ module E11y
|
|
|
32
32
|
# Process request
|
|
33
33
|
# @param env [Hash] Rack environment
|
|
34
34
|
# @return [Array] Rack response [status, headers, body]
|
|
35
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
36
|
+
# Rack middleware request processing requires sequential setup of tracing, context, buffer, and SLO tracking
|
|
35
37
|
def call(env)
|
|
36
38
|
request = Rack::Request.new(env)
|
|
37
39
|
|
|
@@ -50,7 +52,7 @@ module E11y
|
|
|
50
52
|
E11y::Current.request_path = request.path
|
|
51
53
|
|
|
52
54
|
# Start request-scoped buffer (for debug events)
|
|
53
|
-
E11y::Buffers::RequestScopedBuffer.
|
|
55
|
+
E11y::Buffers::RequestScopedBuffer.initialize! if E11y.config.request_buffer&.enabled
|
|
54
56
|
|
|
55
57
|
# Track request start time for SLO
|
|
56
58
|
start_time = Time.now
|
|
@@ -68,20 +70,21 @@ module E11y
|
|
|
68
70
|
[status, headers, body]
|
|
69
71
|
rescue StandardError
|
|
70
72
|
# Flush request buffer on error (includes debug events)
|
|
71
|
-
E11y::Buffers::RequestScopedBuffer.flush_on_error
|
|
73
|
+
E11y::Buffers::RequestScopedBuffer.flush_on_error if E11y.config.request_buffer&.enabled
|
|
72
74
|
|
|
73
75
|
raise # Re-raise original exception
|
|
74
76
|
ensure
|
|
75
|
-
#
|
|
77
|
+
# Discard request buffer on success (not on error, already flushed above)
|
|
76
78
|
# We need to check if we're here from normal completion or exception
|
|
77
79
|
# If there was an exception, buffer was already flushed in rescue block
|
|
78
80
|
if !$ERROR_INFO && E11y.config.request_buffer&.enabled # No exception occurred
|
|
79
|
-
E11y::Buffers::RequestScopedBuffer.
|
|
81
|
+
E11y::Buffers::RequestScopedBuffer.discard
|
|
80
82
|
end
|
|
81
83
|
|
|
82
84
|
# Reset context
|
|
83
85
|
E11y::Current.reset
|
|
84
86
|
end
|
|
87
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
85
88
|
|
|
86
89
|
private
|
|
87
90
|
|
|
@@ -138,6 +141,8 @@ module E11y
|
|
|
138
141
|
# @param start_time [Time] Request start time
|
|
139
142
|
# @return [void]
|
|
140
143
|
# @api private
|
|
144
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
|
|
145
|
+
# SLO tracking requires extracting controller/action, calculating duration, and error handling
|
|
141
146
|
def track_http_request_slo(env, status, start_time)
|
|
142
147
|
return unless E11y.config.slo_tracking&.enabled
|
|
143
148
|
|
|
@@ -158,6 +163,7 @@ module E11y
|
|
|
158
163
|
# Don't fail if SLO tracking fails
|
|
159
164
|
warn "[E11y] SLO tracking error: #{e.message}"
|
|
160
165
|
end
|
|
166
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
|
|
161
167
|
end
|
|
162
168
|
end
|
|
163
169
|
end
|