e11y 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.rubocop.yml +69 -0
- data/CHANGELOG.md +26 -0
- data/CODE_OF_CONDUCT.md +64 -0
- data/LICENSE.txt +21 -0
- data/README.md +179 -0
- data/Rakefile +37 -0
- data/benchmarks/run_all.rb +33 -0
- data/config/README.md +83 -0
- data/config/loki-local-config.yaml +35 -0
- data/config/prometheus.yml +15 -0
- data/docker-compose.yml +78 -0
- data/docs/00-ICP-AND-TIMELINE.md +483 -0
- data/docs/01-SCALE-REQUIREMENTS.md +858 -0
- data/docs/ADR-001-architecture.md +2617 -0
- data/docs/ADR-002-metrics-yabeda.md +1395 -0
- data/docs/ADR-003-slo-observability.md +3337 -0
- data/docs/ADR-004-adapter-architecture.md +2385 -0
- data/docs/ADR-005-tracing-context.md +1372 -0
- data/docs/ADR-006-security-compliance.md +4143 -0
- data/docs/ADR-007-opentelemetry-integration.md +1385 -0
- data/docs/ADR-008-rails-integration.md +1911 -0
- data/docs/ADR-009-cost-optimization.md +2993 -0
- data/docs/ADR-010-developer-experience.md +2166 -0
- data/docs/ADR-011-testing-strategy.md +1836 -0
- data/docs/ADR-012-event-evolution.md +958 -0
- data/docs/ADR-013-reliability-error-handling.md +2750 -0
- data/docs/ADR-014-event-driven-slo.md +1533 -0
- data/docs/ADR-015-middleware-order.md +1061 -0
- data/docs/ADR-016-self-monitoring-slo.md +1234 -0
- data/docs/API-REFERENCE-L28.md +914 -0
- data/docs/COMPREHENSIVE-CONFIGURATION.md +2366 -0
- data/docs/IMPLEMENTATION_NOTES.md +2804 -0
- data/docs/IMPLEMENTATION_PLAN.md +1971 -0
- data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +586 -0
- data/docs/PLAN.md +148 -0
- data/docs/QUICK-START.md +934 -0
- data/docs/README.md +296 -0
- data/docs/design/00-memory-optimization.md +593 -0
- data/docs/guides/MIGRATION-L27-L28.md +692 -0
- data/docs/guides/PERFORMANCE-BENCHMARKS.md +434 -0
- data/docs/guides/README.md +44 -0
- data/docs/prd/01-overview-vision.md +440 -0
- data/docs/use_cases/README.md +119 -0
- data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +813 -0
- data/docs/use_cases/UC-002-business-event-tracking.md +1953 -0
- data/docs/use_cases/UC-003-pattern-based-metrics.md +1627 -0
- data/docs/use_cases/UC-004-zero-config-slo-tracking.md +728 -0
- data/docs/use_cases/UC-005-sentry-integration.md +759 -0
- data/docs/use_cases/UC-006-trace-context-management.md +905 -0
- data/docs/use_cases/UC-007-pii-filtering.md +2648 -0
- data/docs/use_cases/UC-008-opentelemetry-integration.md +1153 -0
- data/docs/use_cases/UC-009-multi-service-tracing.md +1043 -0
- data/docs/use_cases/UC-010-background-job-tracking.md +1018 -0
- data/docs/use_cases/UC-011-rate-limiting.md +1906 -0
- data/docs/use_cases/UC-012-audit-trail.md +2301 -0
- data/docs/use_cases/UC-013-high-cardinality-protection.md +2127 -0
- data/docs/use_cases/UC-014-adaptive-sampling.md +1940 -0
- data/docs/use_cases/UC-015-cost-optimization.md +735 -0
- data/docs/use_cases/UC-016-rails-logger-migration.md +785 -0
- data/docs/use_cases/UC-017-local-development.md +867 -0
- data/docs/use_cases/UC-018-testing-events.md +1081 -0
- data/docs/use_cases/UC-019-tiered-storage-migration.md +562 -0
- data/docs/use_cases/UC-020-event-versioning.md +708 -0
- data/docs/use_cases/UC-021-error-handling-retry-dlq.md +956 -0
- data/docs/use_cases/UC-022-event-registry.md +648 -0
- data/docs/use_cases/backlog.md +226 -0
- data/e11y.gemspec +76 -0
- data/lib/e11y/adapters/adaptive_batcher.rb +207 -0
- data/lib/e11y/adapters/audit_encrypted.rb +239 -0
- data/lib/e11y/adapters/base.rb +580 -0
- data/lib/e11y/adapters/file.rb +224 -0
- data/lib/e11y/adapters/in_memory.rb +216 -0
- data/lib/e11y/adapters/loki.rb +333 -0
- data/lib/e11y/adapters/otel_logs.rb +203 -0
- data/lib/e11y/adapters/registry.rb +141 -0
- data/lib/e11y/adapters/sentry.rb +230 -0
- data/lib/e11y/adapters/stdout.rb +108 -0
- data/lib/e11y/adapters/yabeda.rb +370 -0
- data/lib/e11y/buffers/adaptive_buffer.rb +339 -0
- data/lib/e11y/buffers/base_buffer.rb +40 -0
- data/lib/e11y/buffers/request_scoped_buffer.rb +246 -0
- data/lib/e11y/buffers/ring_buffer.rb +267 -0
- data/lib/e11y/buffers.rb +14 -0
- data/lib/e11y/console.rb +122 -0
- data/lib/e11y/current.rb +48 -0
- data/lib/e11y/event/base.rb +894 -0
- data/lib/e11y/event/value_sampling_config.rb +84 -0
- data/lib/e11y/events/base_audit_event.rb +43 -0
- data/lib/e11y/events/base_payment_event.rb +33 -0
- data/lib/e11y/events/rails/cache/delete.rb +21 -0
- data/lib/e11y/events/rails/cache/read.rb +23 -0
- data/lib/e11y/events/rails/cache/write.rb +22 -0
- data/lib/e11y/events/rails/database/query.rb +45 -0
- data/lib/e11y/events/rails/http/redirect.rb +21 -0
- data/lib/e11y/events/rails/http/request.rb +26 -0
- data/lib/e11y/events/rails/http/send_file.rb +21 -0
- data/lib/e11y/events/rails/http/start_processing.rb +26 -0
- data/lib/e11y/events/rails/job/completed.rb +22 -0
- data/lib/e11y/events/rails/job/enqueued.rb +22 -0
- data/lib/e11y/events/rails/job/failed.rb +22 -0
- data/lib/e11y/events/rails/job/scheduled.rb +23 -0
- data/lib/e11y/events/rails/job/started.rb +22 -0
- data/lib/e11y/events/rails/log.rb +56 -0
- data/lib/e11y/events/rails/view/render.rb +23 -0
- data/lib/e11y/events.rb +18 -0
- data/lib/e11y/instruments/active_job.rb +201 -0
- data/lib/e11y/instruments/rails_instrumentation.rb +141 -0
- data/lib/e11y/instruments/sidekiq.rb +175 -0
- data/lib/e11y/logger/bridge.rb +205 -0
- data/lib/e11y/metrics/cardinality_protection.rb +172 -0
- data/lib/e11y/metrics/cardinality_tracker.rb +134 -0
- data/lib/e11y/metrics/registry.rb +234 -0
- data/lib/e11y/metrics/relabeling.rb +226 -0
- data/lib/e11y/metrics.rb +102 -0
- data/lib/e11y/middleware/audit_signing.rb +174 -0
- data/lib/e11y/middleware/base.rb +140 -0
- data/lib/e11y/middleware/event_slo.rb +167 -0
- data/lib/e11y/middleware/pii_filter.rb +266 -0
- data/lib/e11y/middleware/pii_filtering.rb +280 -0
- data/lib/e11y/middleware/rate_limiting.rb +214 -0
- data/lib/e11y/middleware/request.rb +163 -0
- data/lib/e11y/middleware/routing.rb +157 -0
- data/lib/e11y/middleware/sampling.rb +254 -0
- data/lib/e11y/middleware/slo.rb +168 -0
- data/lib/e11y/middleware/trace_context.rb +131 -0
- data/lib/e11y/middleware/validation.rb +118 -0
- data/lib/e11y/middleware/versioning.rb +132 -0
- data/lib/e11y/middleware.rb +12 -0
- data/lib/e11y/pii/patterns.rb +90 -0
- data/lib/e11y/pii.rb +13 -0
- data/lib/e11y/pipeline/builder.rb +155 -0
- data/lib/e11y/pipeline/zone_validator.rb +110 -0
- data/lib/e11y/pipeline.rb +12 -0
- data/lib/e11y/presets/audit_event.rb +65 -0
- data/lib/e11y/presets/debug_event.rb +34 -0
- data/lib/e11y/presets/high_value_event.rb +51 -0
- data/lib/e11y/presets.rb +19 -0
- data/lib/e11y/railtie.rb +138 -0
- data/lib/e11y/reliability/circuit_breaker.rb +216 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +277 -0
- data/lib/e11y/reliability/dlq/filter.rb +117 -0
- data/lib/e11y/reliability/retry_handler.rb +207 -0
- data/lib/e11y/reliability/retry_rate_limiter.rb +117 -0
- data/lib/e11y/sampling/error_spike_detector.rb +225 -0
- data/lib/e11y/sampling/load_monitor.rb +161 -0
- data/lib/e11y/sampling/stratified_tracker.rb +92 -0
- data/lib/e11y/sampling/value_extractor.rb +82 -0
- data/lib/e11y/self_monitoring/buffer_monitor.rb +79 -0
- data/lib/e11y/self_monitoring/performance_monitor.rb +97 -0
- data/lib/e11y/self_monitoring/reliability_monitor.rb +146 -0
- data/lib/e11y/slo/event_driven.rb +150 -0
- data/lib/e11y/slo/tracker.rb +119 -0
- data/lib/e11y/version.rb +9 -0
- data/lib/e11y.rb +283 -0
- metadata +452 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "delegate"
|
|
4
|
+
|
|
5
|
+
module E11y
|
|
6
|
+
module Logger
|
|
7
|
+
# Rails.logger Bridge (SimpleDelegator wrapper)
|
|
8
|
+
#
|
|
9
|
+
# Transparent wrapper around Rails.logger that:
|
|
10
|
+
# 1. Delegates all calls to the original logger (preserves Rails behavior)
|
|
11
|
+
# 2. Optionally tracks log calls as E11y events (when enabled)
|
|
12
|
+
#
|
|
13
|
+
# **Why SimpleDelegator instead of full replacement:**
|
|
14
|
+
# - ✅ Simpler: No need to reimplement entire Logger API
|
|
15
|
+
# - ✅ Safer: Preserves all Rails.logger behavior
|
|
16
|
+
# - ✅ Flexible: Can be enabled/disabled without breaking anything
|
|
17
|
+
# - ✅ Rails Way: Extends functionality without replacing core components
|
|
18
|
+
#
|
|
19
|
+
# @example Basic usage
|
|
20
|
+
# # Automatically enabled by E11y::Railtie if config.logger_bridge.enabled = true
|
|
21
|
+
# Rails.logger = E11y::Logger::Bridge.new(Rails.logger)
|
|
22
|
+
#
|
|
23
|
+
# @example Manual setup
|
|
24
|
+
# E11y.configure do |config|
|
|
25
|
+
# config.logger_bridge.enabled = true
|
|
26
|
+
# config.logger_bridge.track_to_e11y = true # Send logs to E11y events (optional)
|
|
27
|
+
# end
|
|
28
|
+
#
|
|
29
|
+
# @see ADR-008 §7 (Rails.logger Migration)
|
|
30
|
+
# @see UC-016 (Rails Logger Migration)
|
|
31
|
+
class Bridge < SimpleDelegator
|
|
32
|
+
# Setup Rails.logger bridge
|
|
33
|
+
#
|
|
34
|
+
# Wraps Rails.logger with E11y::Logger::Bridge.
|
|
35
|
+
#
|
|
36
|
+
# @return [void]
|
|
37
|
+
def self.setup!
|
|
38
|
+
return unless E11y.config.logger_bridge&.enabled
|
|
39
|
+
return unless defined?(Rails)
|
|
40
|
+
|
|
41
|
+
# Wrap Rails.logger (preserves original behavior)
|
|
42
|
+
Rails.logger = Bridge.new(Rails.logger)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Initialize bridge wrapper
|
|
46
|
+
# @param original_logger [Logger] Original Rails logger
|
|
47
|
+
def initialize(original_logger)
|
|
48
|
+
super
|
|
49
|
+
@severity_mapping = {
|
|
50
|
+
::Logger::DEBUG => :debug,
|
|
51
|
+
::Logger::INFO => :info,
|
|
52
|
+
::Logger::WARN => :warn,
|
|
53
|
+
::Logger::ERROR => :error,
|
|
54
|
+
::Logger::FATAL => :fatal,
|
|
55
|
+
::Logger::UNKNOWN => :warn
|
|
56
|
+
}
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Intercept logger methods to optionally track to E11y
|
|
60
|
+
# All calls are delegated to the original logger via SimpleDelegator
|
|
61
|
+
|
|
62
|
+
# Log debug message
|
|
63
|
+
# @param message [String, nil] Log message
|
|
64
|
+
# @yield Block that returns log message
|
|
65
|
+
# @return [true] Always returns true (Logger API)
|
|
66
|
+
def debug(message = nil, &)
|
|
67
|
+
track_to_e11y(:debug, message, &) if should_track_severity?(:debug)
|
|
68
|
+
super # Delegate to original logger
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Log info message
|
|
72
|
+
# @param message [String, nil] Log message
|
|
73
|
+
# @yield Block that returns log message
|
|
74
|
+
# @return [true] Always returns true (Logger API)
|
|
75
|
+
def info(message = nil, &)
|
|
76
|
+
track_to_e11y(:info, message, &) if should_track_severity?(:info)
|
|
77
|
+
super # Delegate to original logger
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Log warn message
|
|
81
|
+
# @param message [String, nil] Log message
|
|
82
|
+
# @yield Block that returns log message
|
|
83
|
+
# @return [true] Always returns true (Logger API)
|
|
84
|
+
def warn(message = nil, &)
|
|
85
|
+
track_to_e11y(:warn, message, &) if should_track_severity?(:warn)
|
|
86
|
+
super # Delegate to original logger
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Log error message
|
|
90
|
+
# @param message [String, nil] Log message
|
|
91
|
+
# @yield Block that returns log message
|
|
92
|
+
# @return [true] Always returns true (Logger API)
|
|
93
|
+
def error(message = nil, &)
|
|
94
|
+
track_to_e11y(:error, message, &) if should_track_severity?(:error)
|
|
95
|
+
super # Delegate to original logger
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Log fatal message
|
|
99
|
+
# @param message [String, nil] Log message
|
|
100
|
+
# @yield Block that returns log message
|
|
101
|
+
# @return [true] Always returns true (Logger API)
|
|
102
|
+
def fatal(message = nil, &)
|
|
103
|
+
track_to_e11y(:fatal, message, &) if should_track_severity?(:fatal)
|
|
104
|
+
super # Delegate to original logger
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Generic log method
|
|
108
|
+
# @param severity [Integer] Logger severity constant
|
|
109
|
+
# @param message [String, nil] Log message
|
|
110
|
+
# @param progname [String, nil] Program name
|
|
111
|
+
# @yield Block that returns log message
|
|
112
|
+
# @return [true] Always returns true (Logger API)
|
|
113
|
+
def add(severity, message = nil, progname = nil, &)
|
|
114
|
+
e11y_severity = @severity_mapping[severity] || :info
|
|
115
|
+
track_to_e11y(e11y_severity, message || progname, &) if should_track_severity?(e11y_severity)
|
|
116
|
+
super # Delegate to original logger
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
alias log add
|
|
120
|
+
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
# Check if E11y tracking is enabled for specific severity
|
|
124
|
+
# Supports both boolean and per-severity Hash configuration
|
|
125
|
+
#
|
|
126
|
+
# @param severity [Symbol] E11y severity (:debug, :info, :warn, :error, :fatal)
|
|
127
|
+
# @return [Boolean]
|
|
128
|
+
#
|
|
129
|
+
# @example Boolean config (all or nothing)
|
|
130
|
+
# config.logger_bridge.track_to_e11y = true # Track all
|
|
131
|
+
# config.logger_bridge.track_to_e11y = false # Track none
|
|
132
|
+
#
|
|
133
|
+
# @example Per-severity config (granular control)
|
|
134
|
+
# config.logger_bridge.track_to_e11y = {
|
|
135
|
+
# debug: false,
|
|
136
|
+
# info: true,
|
|
137
|
+
# warn: true,
|
|
138
|
+
# error: true,
|
|
139
|
+
# fatal: true
|
|
140
|
+
# }
|
|
141
|
+
def should_track_severity?(severity)
|
|
142
|
+
config = E11y.config.logger_bridge&.track_to_e11y
|
|
143
|
+
return false unless config
|
|
144
|
+
|
|
145
|
+
case config
|
|
146
|
+
when TrueClass
|
|
147
|
+
true # Track all severities
|
|
148
|
+
when FalseClass
|
|
149
|
+
false # Track none
|
|
150
|
+
when Hash
|
|
151
|
+
config[severity] || false # Check per-severity config
|
|
152
|
+
else
|
|
153
|
+
false # Unknown config type
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Track log message as E11y event
|
|
158
|
+
# @param severity [Symbol] E11y severity
|
|
159
|
+
# @param message [String, nil] Log message
|
|
160
|
+
# @yield Block that returns log message
|
|
161
|
+
# @return [void]
|
|
162
|
+
def track_to_e11y(severity, message = nil, &block)
|
|
163
|
+
# Extract message
|
|
164
|
+
msg = message || (block_given? ? block.call : nil)
|
|
165
|
+
return if msg.nil? || (msg.respond_to?(:empty?) && msg.empty?)
|
|
166
|
+
|
|
167
|
+
# Track to E11y using severity-specific class
|
|
168
|
+
require "e11y/events/rails/log"
|
|
169
|
+
event_class = event_class_for_severity(severity)
|
|
170
|
+
event_class.track(
|
|
171
|
+
message: msg.to_s,
|
|
172
|
+
caller_location: extract_caller_location
|
|
173
|
+
)
|
|
174
|
+
rescue StandardError => e
|
|
175
|
+
# Silently ignore E11y tracking errors (don't break logging!)
|
|
176
|
+
# In development/test, you might want to log this
|
|
177
|
+
warn "E11y logger tracking failed: #{e.message}" if defined?(Rails) && Rails.env.development?
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Get event class for severity
|
|
181
|
+
# @param severity [Symbol] E11y severity
|
|
182
|
+
# @return [Class] Event class
|
|
183
|
+
def event_class_for_severity(severity)
|
|
184
|
+
case severity
|
|
185
|
+
when :debug then E11y::Events::Rails::Log::Debug
|
|
186
|
+
when :info then E11y::Events::Rails::Log::Info
|
|
187
|
+
when :warn then E11y::Events::Rails::Log::Warn
|
|
188
|
+
when :error then E11y::Events::Rails::Log::Error
|
|
189
|
+
when :fatal then E11y::Events::Rails::Log::Fatal
|
|
190
|
+
else E11y::Events::Rails::Log::Info # Fallback
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Extract caller location (first caller outside E11y)
|
|
195
|
+
# @return [String, nil] Caller location string
|
|
196
|
+
def extract_caller_location
|
|
197
|
+
caller_locations.find do |loc|
|
|
198
|
+
!loc.path.include?("e11y")
|
|
199
|
+
end&.then do |loc|
|
|
200
|
+
"#{loc.path}:#{loc.lineno}:in `#{loc.label}'"
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "cardinality_tracker"
|
|
4
|
+
require_relative "relabeling"
|
|
5
|
+
|
|
6
|
+
module E11y
|
|
7
|
+
module Metrics
|
|
8
|
+
# Cardinality protection for metrics labels.
|
|
9
|
+
#
|
|
10
|
+
# Implements 3-layer defense system to prevent cardinality explosions:
|
|
11
|
+
# 1. Universal Denylist - Block high-cardinality fields (user_id, order_id, etc.)
|
|
12
|
+
# 2. Per-Metric Limits - Track unique values per metric, drop if exceeded
|
|
13
|
+
# 3. Dynamic Monitoring - Alert when approaching limits
|
|
14
|
+
#
|
|
15
|
+
# Now supports optional relabeling to reduce cardinality while preserving signal.
|
|
16
|
+
#
|
|
17
|
+
# @example Basic usage
|
|
18
|
+
# protection = E11y::Metrics::CardinalityProtection.new
|
|
19
|
+
# labels = { user_id: '123', status: 'paid', currency: 'USD' }
|
|
20
|
+
# safe_labels = protection.filter(labels, 'orders.total')
|
|
21
|
+
# # => { status: 'paid', currency: 'USD' } (user_id dropped)
|
|
22
|
+
#
|
|
23
|
+
# @example With relabeling
|
|
24
|
+
# protection = E11y::Metrics::CardinalityProtection.new
|
|
25
|
+
# protection.relabel(:http_status) { |v| "#{v.to_i / 100}xx" }
|
|
26
|
+
# labels = { http_status: 200, path: '/api/users' }
|
|
27
|
+
# safe_labels = protection.filter(labels, 'http.requests')
|
|
28
|
+
# # => { http_status: '2xx', path: '/api/users' }
|
|
29
|
+
#
|
|
30
|
+
# @see ADR-002 §4 (Cardinality Protection)
|
|
31
|
+
# @see UC-013 (High Cardinality Protection)
|
|
32
|
+
class CardinalityProtection
|
|
33
|
+
# Universal denylist - high-cardinality fields that should NEVER be labels
|
|
34
|
+
UNIVERSAL_DENYLIST = %i[
|
|
35
|
+
id
|
|
36
|
+
user_id
|
|
37
|
+
order_id
|
|
38
|
+
session_id
|
|
39
|
+
request_id
|
|
40
|
+
trace_id
|
|
41
|
+
span_id
|
|
42
|
+
email
|
|
43
|
+
phone
|
|
44
|
+
ip_address
|
|
45
|
+
token
|
|
46
|
+
api_key
|
|
47
|
+
password
|
|
48
|
+
uuid
|
|
49
|
+
guid
|
|
50
|
+
timestamp
|
|
51
|
+
created_at
|
|
52
|
+
updated_at
|
|
53
|
+
].freeze
|
|
54
|
+
|
|
55
|
+
# Default per-metric cardinality limit
|
|
56
|
+
DEFAULT_CARDINALITY_LIMIT = 1000
|
|
57
|
+
|
|
58
|
+
attr_reader :tracker, :relabeler
|
|
59
|
+
|
|
60
|
+
# Initialize cardinality protection
|
|
61
|
+
# @param config [Hash] Configuration options
|
|
62
|
+
# @option config [Integer] :cardinality_limit (1000) Max unique label combinations per metric
|
|
63
|
+
# @option config [Array<Symbol>] :additional_denylist Additional fields to deny
|
|
64
|
+
# @option config [Boolean] :enabled (true) Enable/disable protection
|
|
65
|
+
# @option config [Boolean] :relabeling_enabled (true) Enable/disable relabeling
|
|
66
|
+
def initialize(config = {})
|
|
67
|
+
@cardinality_limit = config.fetch(:cardinality_limit, DEFAULT_CARDINALITY_LIMIT)
|
|
68
|
+
@enabled = config.fetch(:enabled, true)
|
|
69
|
+
@relabeling_enabled = config.fetch(:relabeling_enabled, true)
|
|
70
|
+
@denylist = Set.new(UNIVERSAL_DENYLIST + (config[:additional_denylist] || []))
|
|
71
|
+
|
|
72
|
+
# Use extracted components
|
|
73
|
+
@tracker = CardinalityTracker.new(limit: @cardinality_limit)
|
|
74
|
+
@relabeler = Relabeling.new
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Define relabeling rule for a label
|
|
78
|
+
#
|
|
79
|
+
# @param label_key [Symbol, String] Label key to relabel
|
|
80
|
+
# @yield [value] Block that transforms label value
|
|
81
|
+
# @return [void]
|
|
82
|
+
#
|
|
83
|
+
# @example HTTP status to class
|
|
84
|
+
# protection.relabel(:http_status) { |v| "#{v.to_i / 100}xx" }
|
|
85
|
+
#
|
|
86
|
+
# @example Path normalization
|
|
87
|
+
# protection.relabel(:path) { |v| v.gsub(/\/\d+/, '/:id') }
|
|
88
|
+
def relabel(label_key, &)
|
|
89
|
+
@relabeler.define(label_key, &)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Filter labels to prevent cardinality explosions
|
|
93
|
+
#
|
|
94
|
+
# Applies 3-layer defense + optional relabeling:
|
|
95
|
+
# 1. Relabel high-cardinality values (if enabled)
|
|
96
|
+
# 2. Drop denylisted fields
|
|
97
|
+
# 3. Track and limit per-metric cardinality
|
|
98
|
+
# 4. Alert on limit exceeded
|
|
99
|
+
#
|
|
100
|
+
# @param labels [Hash] Raw labels from event
|
|
101
|
+
# @param metric_name [String] Metric name for tracking
|
|
102
|
+
# @return [Hash] Filtered safe labels
|
|
103
|
+
def filter(labels, metric_name)
|
|
104
|
+
return labels unless @enabled
|
|
105
|
+
|
|
106
|
+
safe_labels = {}
|
|
107
|
+
|
|
108
|
+
labels.each do |key, value|
|
|
109
|
+
# Step 1: Relabel if rule exists (reduces cardinality)
|
|
110
|
+
relabeled_value = @relabeling_enabled ? @relabeler.apply(key, value) : value
|
|
111
|
+
|
|
112
|
+
# Step 2: Denylist - drop high-cardinality fields
|
|
113
|
+
next if should_deny?(key)
|
|
114
|
+
|
|
115
|
+
# Step 3: Per-Metric Cardinality Limit
|
|
116
|
+
if @tracker.track(metric_name, key, relabeled_value)
|
|
117
|
+
safe_labels[key] = relabeled_value
|
|
118
|
+
else
|
|
119
|
+
# Step 4: Alert when limit exceeded
|
|
120
|
+
warn_cardinality_exceeded(metric_name, key)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
safe_labels
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Check if cardinality limit is exceeded for a metric
|
|
128
|
+
# @param metric_name [String] Metric name
|
|
129
|
+
# @return [Boolean] True if ANY label exceeded limit
|
|
130
|
+
def cardinality_exceeded?(metric_name)
|
|
131
|
+
# Check if any label has exceeded limit
|
|
132
|
+
@tracker.cardinalities(metric_name).values.any? { |count| count >= @cardinality_limit }
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Get current cardinality for a metric (all labels)
|
|
136
|
+
# @param metric_name [String] Metric name
|
|
137
|
+
# @return [Hash{Symbol => Integer}] Label key => cardinality
|
|
138
|
+
def cardinality(metric_name)
|
|
139
|
+
@tracker.cardinalities(metric_name)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Get all metrics with their cardinalities
|
|
143
|
+
# @return [Hash{String => Hash{Symbol => Integer}}] Metric => Label => cardinality
|
|
144
|
+
def cardinalities
|
|
145
|
+
@tracker.all_cardinalities
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Reset cardinality tracking (for testing)
|
|
149
|
+
# @return [void]
|
|
150
|
+
def reset!
|
|
151
|
+
@tracker.reset_all!
|
|
152
|
+
@relabeler.reset!
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
private
|
|
156
|
+
|
|
157
|
+
# Check if label should be denied (Layer 1: Denylist)
|
|
158
|
+
# @param key [Symbol] Label key
|
|
159
|
+
# @return [Boolean] True if should be denied
|
|
160
|
+
def should_deny?(key)
|
|
161
|
+
@denylist.include?(key)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Warn about cardinality limit exceeded (Layer 3: Monitoring)
|
|
165
|
+
# @param metric_name [String] Metric name
|
|
166
|
+
# @param key [Symbol] Label key
|
|
167
|
+
def warn_cardinality_exceeded(metric_name, key)
|
|
168
|
+
warn "E11y Metrics: Cardinality limit exceeded for #{metric_name}:#{key} (limit: #{@cardinality_limit})"
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module E11y
|
|
4
|
+
module Metrics
|
|
5
|
+
# Thread-safe cardinality tracker for metrics labels.
|
|
6
|
+
#
|
|
7
|
+
# Tracks unique label value combinations per metric to detect
|
|
8
|
+
# cardinality explosions. Separated from CardinalityProtection
|
|
9
|
+
# for single responsibility and easier testing.
|
|
10
|
+
#
|
|
11
|
+
# @example Track label values
|
|
12
|
+
# tracker = CardinalityTracker.new(limit: 100)
|
|
13
|
+
# tracker.track('orders.total', :status, 'paid')
|
|
14
|
+
# tracker.track('orders.total', :status, 'failed')
|
|
15
|
+
# tracker.cardinality('orders.total', :status) # => 2
|
|
16
|
+
#
|
|
17
|
+
# @see CardinalityProtection
|
|
18
|
+
class CardinalityTracker
|
|
19
|
+
# @return [Integer] Default cardinality limit per metric
|
|
20
|
+
DEFAULT_LIMIT = 1000
|
|
21
|
+
|
|
22
|
+
# Initialize tracker
|
|
23
|
+
#
|
|
24
|
+
# @param limit [Integer] Maximum unique values per metric+label
|
|
25
|
+
def initialize(limit: DEFAULT_LIMIT)
|
|
26
|
+
@limit = limit
|
|
27
|
+
@tracker = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = Set.new } }
|
|
28
|
+
@mutex = Mutex.new
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Track a label value for a metric
|
|
32
|
+
#
|
|
33
|
+
# Records unique label values per metric+label combination.
|
|
34
|
+
# Thread-safe operation.
|
|
35
|
+
#
|
|
36
|
+
# @param metric_name [String] Metric name
|
|
37
|
+
# @param label_key [Symbol, String] Label key
|
|
38
|
+
# @param label_value [Object] Label value to track
|
|
39
|
+
# @return [Boolean] true if within limit, false if limit exceeded
|
|
40
|
+
def track(metric_name, label_key, label_value)
|
|
41
|
+
@mutex.synchronize do
|
|
42
|
+
value_set = @tracker[metric_name][label_key]
|
|
43
|
+
|
|
44
|
+
# Allow if already tracked (existing value)
|
|
45
|
+
return true if value_set.include?(label_value)
|
|
46
|
+
|
|
47
|
+
# Check if adding new value would exceed limit
|
|
48
|
+
if value_set.size >= @limit
|
|
49
|
+
false
|
|
50
|
+
else
|
|
51
|
+
value_set.add(label_value)
|
|
52
|
+
true
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Check if metric+label has exceeded cardinality limit
|
|
58
|
+
#
|
|
59
|
+
# @param metric_name [String] Metric name
|
|
60
|
+
# @param label_key [Symbol, String] Label key
|
|
61
|
+
# @return [Boolean] true if at or above limit
|
|
62
|
+
def exceeded?(metric_name, label_key)
|
|
63
|
+
@mutex.synchronize do
|
|
64
|
+
@tracker.dig(metric_name, label_key)&.size.to_i >= @limit
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Get current cardinality for metric+label
|
|
69
|
+
#
|
|
70
|
+
# @param metric_name [String] Metric name
|
|
71
|
+
# @param label_key [Symbol, String] Label key
|
|
72
|
+
# @return [Integer] Number of unique values tracked
|
|
73
|
+
def cardinality(metric_name, label_key)
|
|
74
|
+
@mutex.synchronize do
|
|
75
|
+
@tracker.dig(metric_name, label_key)&.size || 0
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Get cardinalities for all labels of a metric
|
|
80
|
+
#
|
|
81
|
+
# @param metric_name [String] Metric name
|
|
82
|
+
# @return [Hash{Symbol => Integer}] Label key => cardinality
|
|
83
|
+
def cardinalities(metric_name)
|
|
84
|
+
@mutex.synchronize do
|
|
85
|
+
metric_data = @tracker[metric_name]
|
|
86
|
+
metric_data.transform_values(&:size)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Get all tracked cardinalities across all metrics
|
|
91
|
+
#
|
|
92
|
+
# @return [Hash{String => Hash{Symbol => Integer}}] Nested hash of metric => label => cardinality
|
|
93
|
+
def all_cardinalities
|
|
94
|
+
@mutex.synchronize do
|
|
95
|
+
result = {}
|
|
96
|
+
@tracker.each do |metric_name, labels|
|
|
97
|
+
label_cardinalities = labels.transform_values(&:size)
|
|
98
|
+
# Only include metrics with non-zero cardinalities
|
|
99
|
+
result[metric_name] = label_cardinalities if label_cardinalities.values.any?(&:positive?)
|
|
100
|
+
end
|
|
101
|
+
result
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Reset tracking for specific metric
|
|
106
|
+
#
|
|
107
|
+
# @param metric_name [String] Metric name to reset
|
|
108
|
+
# @return [void]
|
|
109
|
+
def reset_metric!(metric_name)
|
|
110
|
+
@mutex.synchronize do
|
|
111
|
+
@tracker.delete(metric_name)
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Reset all tracking data
|
|
116
|
+
#
|
|
117
|
+
# @return [void]
|
|
118
|
+
def reset_all!
|
|
119
|
+
@mutex.synchronize do
|
|
120
|
+
@tracker.clear
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Get total number of tracked metrics
|
|
125
|
+
#
|
|
126
|
+
# @return [Integer] Number of unique metrics being tracked
|
|
127
|
+
def metrics_count
|
|
128
|
+
@mutex.synchronize do
|
|
129
|
+
@tracker.size
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|