e11y 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.rubocop.yml +69 -0
- data/CHANGELOG.md +26 -0
- data/CODE_OF_CONDUCT.md +64 -0
- data/LICENSE.txt +21 -0
- data/README.md +179 -0
- data/Rakefile +37 -0
- data/benchmarks/run_all.rb +33 -0
- data/config/README.md +83 -0
- data/config/loki-local-config.yaml +35 -0
- data/config/prometheus.yml +15 -0
- data/docker-compose.yml +78 -0
- data/docs/00-ICP-AND-TIMELINE.md +483 -0
- data/docs/01-SCALE-REQUIREMENTS.md +858 -0
- data/docs/ADR-001-architecture.md +2617 -0
- data/docs/ADR-002-metrics-yabeda.md +1395 -0
- data/docs/ADR-003-slo-observability.md +3337 -0
- data/docs/ADR-004-adapter-architecture.md +2385 -0
- data/docs/ADR-005-tracing-context.md +1372 -0
- data/docs/ADR-006-security-compliance.md +4143 -0
- data/docs/ADR-007-opentelemetry-integration.md +1385 -0
- data/docs/ADR-008-rails-integration.md +1911 -0
- data/docs/ADR-009-cost-optimization.md +2993 -0
- data/docs/ADR-010-developer-experience.md +2166 -0
- data/docs/ADR-011-testing-strategy.md +1836 -0
- data/docs/ADR-012-event-evolution.md +958 -0
- data/docs/ADR-013-reliability-error-handling.md +2750 -0
- data/docs/ADR-014-event-driven-slo.md +1533 -0
- data/docs/ADR-015-middleware-order.md +1061 -0
- data/docs/ADR-016-self-monitoring-slo.md +1234 -0
- data/docs/API-REFERENCE-L28.md +914 -0
- data/docs/COMPREHENSIVE-CONFIGURATION.md +2366 -0
- data/docs/IMPLEMENTATION_NOTES.md +2804 -0
- data/docs/IMPLEMENTATION_PLAN.md +1971 -0
- data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +586 -0
- data/docs/PLAN.md +148 -0
- data/docs/QUICK-START.md +934 -0
- data/docs/README.md +296 -0
- data/docs/design/00-memory-optimization.md +593 -0
- data/docs/guides/MIGRATION-L27-L28.md +692 -0
- data/docs/guides/PERFORMANCE-BENCHMARKS.md +434 -0
- data/docs/guides/README.md +44 -0
- data/docs/prd/01-overview-vision.md +440 -0
- data/docs/use_cases/README.md +119 -0
- data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +813 -0
- data/docs/use_cases/UC-002-business-event-tracking.md +1953 -0
- data/docs/use_cases/UC-003-pattern-based-metrics.md +1627 -0
- data/docs/use_cases/UC-004-zero-config-slo-tracking.md +728 -0
- data/docs/use_cases/UC-005-sentry-integration.md +759 -0
- data/docs/use_cases/UC-006-trace-context-management.md +905 -0
- data/docs/use_cases/UC-007-pii-filtering.md +2648 -0
- data/docs/use_cases/UC-008-opentelemetry-integration.md +1153 -0
- data/docs/use_cases/UC-009-multi-service-tracing.md +1043 -0
- data/docs/use_cases/UC-010-background-job-tracking.md +1018 -0
- data/docs/use_cases/UC-011-rate-limiting.md +1906 -0
- data/docs/use_cases/UC-012-audit-trail.md +2301 -0
- data/docs/use_cases/UC-013-high-cardinality-protection.md +2127 -0
- data/docs/use_cases/UC-014-adaptive-sampling.md +1940 -0
- data/docs/use_cases/UC-015-cost-optimization.md +735 -0
- data/docs/use_cases/UC-016-rails-logger-migration.md +785 -0
- data/docs/use_cases/UC-017-local-development.md +867 -0
- data/docs/use_cases/UC-018-testing-events.md +1081 -0
- data/docs/use_cases/UC-019-tiered-storage-migration.md +562 -0
- data/docs/use_cases/UC-020-event-versioning.md +708 -0
- data/docs/use_cases/UC-021-error-handling-retry-dlq.md +956 -0
- data/docs/use_cases/UC-022-event-registry.md +648 -0
- data/docs/use_cases/backlog.md +226 -0
- data/e11y.gemspec +76 -0
- data/lib/e11y/adapters/adaptive_batcher.rb +207 -0
- data/lib/e11y/adapters/audit_encrypted.rb +239 -0
- data/lib/e11y/adapters/base.rb +580 -0
- data/lib/e11y/adapters/file.rb +224 -0
- data/lib/e11y/adapters/in_memory.rb +216 -0
- data/lib/e11y/adapters/loki.rb +333 -0
- data/lib/e11y/adapters/otel_logs.rb +203 -0
- data/lib/e11y/adapters/registry.rb +141 -0
- data/lib/e11y/adapters/sentry.rb +230 -0
- data/lib/e11y/adapters/stdout.rb +108 -0
- data/lib/e11y/adapters/yabeda.rb +370 -0
- data/lib/e11y/buffers/adaptive_buffer.rb +339 -0
- data/lib/e11y/buffers/base_buffer.rb +40 -0
- data/lib/e11y/buffers/request_scoped_buffer.rb +246 -0
- data/lib/e11y/buffers/ring_buffer.rb +267 -0
- data/lib/e11y/buffers.rb +14 -0
- data/lib/e11y/console.rb +122 -0
- data/lib/e11y/current.rb +48 -0
- data/lib/e11y/event/base.rb +894 -0
- data/lib/e11y/event/value_sampling_config.rb +84 -0
- data/lib/e11y/events/base_audit_event.rb +43 -0
- data/lib/e11y/events/base_payment_event.rb +33 -0
- data/lib/e11y/events/rails/cache/delete.rb +21 -0
- data/lib/e11y/events/rails/cache/read.rb +23 -0
- data/lib/e11y/events/rails/cache/write.rb +22 -0
- data/lib/e11y/events/rails/database/query.rb +45 -0
- data/lib/e11y/events/rails/http/redirect.rb +21 -0
- data/lib/e11y/events/rails/http/request.rb +26 -0
- data/lib/e11y/events/rails/http/send_file.rb +21 -0
- data/lib/e11y/events/rails/http/start_processing.rb +26 -0
- data/lib/e11y/events/rails/job/completed.rb +22 -0
- data/lib/e11y/events/rails/job/enqueued.rb +22 -0
- data/lib/e11y/events/rails/job/failed.rb +22 -0
- data/lib/e11y/events/rails/job/scheduled.rb +23 -0
- data/lib/e11y/events/rails/job/started.rb +22 -0
- data/lib/e11y/events/rails/log.rb +56 -0
- data/lib/e11y/events/rails/view/render.rb +23 -0
- data/lib/e11y/events.rb +18 -0
- data/lib/e11y/instruments/active_job.rb +201 -0
- data/lib/e11y/instruments/rails_instrumentation.rb +141 -0
- data/lib/e11y/instruments/sidekiq.rb +175 -0
- data/lib/e11y/logger/bridge.rb +205 -0
- data/lib/e11y/metrics/cardinality_protection.rb +172 -0
- data/lib/e11y/metrics/cardinality_tracker.rb +134 -0
- data/lib/e11y/metrics/registry.rb +234 -0
- data/lib/e11y/metrics/relabeling.rb +226 -0
- data/lib/e11y/metrics.rb +102 -0
- data/lib/e11y/middleware/audit_signing.rb +174 -0
- data/lib/e11y/middleware/base.rb +140 -0
- data/lib/e11y/middleware/event_slo.rb +167 -0
- data/lib/e11y/middleware/pii_filter.rb +266 -0
- data/lib/e11y/middleware/pii_filtering.rb +280 -0
- data/lib/e11y/middleware/rate_limiting.rb +214 -0
- data/lib/e11y/middleware/request.rb +163 -0
- data/lib/e11y/middleware/routing.rb +157 -0
- data/lib/e11y/middleware/sampling.rb +254 -0
- data/lib/e11y/middleware/slo.rb +168 -0
- data/lib/e11y/middleware/trace_context.rb +131 -0
- data/lib/e11y/middleware/validation.rb +118 -0
- data/lib/e11y/middleware/versioning.rb +132 -0
- data/lib/e11y/middleware.rb +12 -0
- data/lib/e11y/pii/patterns.rb +90 -0
- data/lib/e11y/pii.rb +13 -0
- data/lib/e11y/pipeline/builder.rb +155 -0
- data/lib/e11y/pipeline/zone_validator.rb +110 -0
- data/lib/e11y/pipeline.rb +12 -0
- data/lib/e11y/presets/audit_event.rb +65 -0
- data/lib/e11y/presets/debug_event.rb +34 -0
- data/lib/e11y/presets/high_value_event.rb +51 -0
- data/lib/e11y/presets.rb +19 -0
- data/lib/e11y/railtie.rb +138 -0
- data/lib/e11y/reliability/circuit_breaker.rb +216 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +277 -0
- data/lib/e11y/reliability/dlq/filter.rb +117 -0
- data/lib/e11y/reliability/retry_handler.rb +207 -0
- data/lib/e11y/reliability/retry_rate_limiter.rb +117 -0
- data/lib/e11y/sampling/error_spike_detector.rb +225 -0
- data/lib/e11y/sampling/load_monitor.rb +161 -0
- data/lib/e11y/sampling/stratified_tracker.rb +92 -0
- data/lib/e11y/sampling/value_extractor.rb +82 -0
- data/lib/e11y/self_monitoring/buffer_monitor.rb +79 -0
- data/lib/e11y/self_monitoring/performance_monitor.rb +97 -0
- data/lib/e11y/self_monitoring/reliability_monitor.rb +146 -0
- data/lib/e11y/slo/event_driven.rb +150 -0
- data/lib/e11y/slo/tracker.rb +119 -0
- data/lib/e11y/version.rb +9 -0
- data/lib/e11y.rb +283 -0
- metadata +452 -0
|
@@ -0,0 +1,1627 @@
|
|
|
1
|
+
# UC-003: Pattern-Based Metrics
|
|
2
|
+
|
|
3
|
+
**Status:** Implemented (2026-01-20)
|
|
4
|
+
**Complexity:** Intermediate
|
|
5
|
+
**Setup Time:** 15-30 minutes
|
|
6
|
+
**Target Users:** DevOps, SRE, Backend Developers
|
|
7
|
+
|
|
8
|
+
**Implementation:** Rails Way with Event::Base DSL - see [IMPLEMENTATION_NOTES.md](../IMPLEMENTATION_NOTES.md#2026-01-20-metrics-architecture-refactoring---rails-way-)
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## ๐ Overview
|
|
13
|
+
|
|
14
|
+
### Problem Statement
|
|
15
|
+
|
|
16
|
+
**Current Approach (Manual Metrics):**
|
|
17
|
+
```ruby
|
|
18
|
+
# โ Manual duplication
|
|
19
|
+
Rails.logger.info "Order #{order.id} paid"
|
|
20
|
+
OrderMetrics.increment('orders.paid.total', tags: { currency: 'USD' })
|
|
21
|
+
OrderMetrics.observe('orders.paid.amount', order.amount, tags: { currency: 'USD' })
|
|
22
|
+
|
|
23
|
+
# Problems:
|
|
24
|
+
# - Duplication (log + metrics)
|
|
25
|
+
# - Inconsistent naming (order vs orders)
|
|
26
|
+
# - Typos in tags (curency vs currency)
|
|
27
|
+
# - Missing tags (forgot payment_method)
|
|
28
|
+
# - Maintenance burden (update both places)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### E11y Solution (Rails Way)
|
|
32
|
+
|
|
33
|
+
**Option 1: Event-Level Metrics (Recommended)**
|
|
34
|
+
```ruby
|
|
35
|
+
# Define metrics directly in event class
|
|
36
|
+
class Events::OrderPaid < E11y::Event::Base
|
|
37
|
+
schema do
|
|
38
|
+
required(:order_id).filled(:string)
|
|
39
|
+
required(:amount).filled(:float)
|
|
40
|
+
required(:currency).filled(:string)
|
|
41
|
+
required(:payment_method).filled(:string)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Metrics DSL - defined once, validated at boot
|
|
45
|
+
metrics do
|
|
46
|
+
counter :orders_paid_total, tags: [:currency, :payment_method]
|
|
47
|
+
histogram :orders_paid_amount, value: :amount, tags: [:currency], buckets: [10, 50, 100, 500, 1000, 5000]
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Track event - metrics automatically updated
|
|
52
|
+
Events::OrderPaid.track(
|
|
53
|
+
order_id: '123',
|
|
54
|
+
amount: 99.99,
|
|
55
|
+
currency: 'USD',
|
|
56
|
+
payment_method: 'stripe'
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# Result:
|
|
60
|
+
# โ
orders_paid_total{currency="USD",payment_method="stripe"} = 1
|
|
61
|
+
# โ
orders_paid_amount_bucket{currency="USD",le="100"} = 1
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
**Option 2: Global Pattern-Based Metrics**
|
|
65
|
+
```ruby
|
|
66
|
+
# config/initializers/e11y.rb
|
|
67
|
+
E11y::Metrics::Registry.instance.register(
|
|
68
|
+
type: :counter,
|
|
69
|
+
pattern: 'order.paid', # Exact match
|
|
70
|
+
name: :orders_paid_total,
|
|
71
|
+
tags: [:currency, :payment_method],
|
|
72
|
+
source: 'config/initializers/e11y.rb'
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
E11y::Metrics::Registry.instance.register(
|
|
76
|
+
type: :histogram,
|
|
77
|
+
pattern: 'order.paid',
|
|
78
|
+
name: :orders_paid_amount,
|
|
79
|
+
value: :amount,
|
|
80
|
+
tags: [:currency],
|
|
81
|
+
buckets: [10, 50, 100, 500, 1000, 5000],
|
|
82
|
+
source: 'config/initializers/e11y.rb'
|
|
83
|
+
)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
**Option 3: Base Class for Shared Metrics**
|
|
87
|
+
```ruby
|
|
88
|
+
# Shared metric for all order events
|
|
89
|
+
class BaseOrderEvent < E11y::Event::Base
|
|
90
|
+
metrics do
|
|
91
|
+
counter :orders_total, tags: [:currency, :status]
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
class Events::OrderPaid < BaseOrderEvent
|
|
96
|
+
# Inherits orders_total + adds own metric
|
|
97
|
+
metrics do
|
|
98
|
+
histogram :order_amount, value: :amount, tags: [:currency]
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## ๐ Rails Integration & Boot-Time Validation
|
|
106
|
+
|
|
107
|
+
### Automatic Validation on Rails Boot
|
|
108
|
+
|
|
109
|
+
**E11y automatically validates metrics when Rails starts:**
|
|
110
|
+
|
|
111
|
+
```ruby
|
|
112
|
+
# lib/e11y/railtie.rb (built-in, no configuration needed)
|
|
113
|
+
class Railtie < Rails::Railtie
|
|
114
|
+
initializer "e11y.validate_metrics", after: :load_config_initializers do
|
|
115
|
+
Rails.application.config.after_initialize do
|
|
116
|
+
E11y::Metrics::Registry.instance.validate_all!
|
|
117
|
+
Rails.logger.info "E11y: Metrics validated successfully (#{registry.size} metrics)"
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
**What gets validated:**
|
|
124
|
+
1. **Metric type conflicts**: Same metric name with different types (counter vs histogram)
|
|
125
|
+
2. **Label conflicts**: Same metric name with different labels
|
|
126
|
+
3. **Bucket conflicts**: Same histogram with different buckets (warning only)
|
|
127
|
+
|
|
128
|
+
**Example: Catching conflicts at boot time**
|
|
129
|
+
|
|
130
|
+
```ruby
|
|
131
|
+
# app/events/order_created.rb
|
|
132
|
+
class Events::OrderCreated < E11y::Event::Base
|
|
133
|
+
metrics do
|
|
134
|
+
counter :orders_total, tags: [:currency, :status]
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# app/events/order_paid.rb
|
|
139
|
+
class Events::OrderPaid < E11y::Event::Base
|
|
140
|
+
metrics do
|
|
141
|
+
counter :orders_total, tags: [:currency] # โ Different labels!
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Rails boot output:
|
|
146
|
+
# E11y::Metrics::Registry::LabelConflictError:
|
|
147
|
+
# Metric "orders_total" label conflict!
|
|
148
|
+
#
|
|
149
|
+
# Existing: [:currency, :status] (from Events::OrderCreated.metrics)
|
|
150
|
+
# New: [:currency] (from Events::OrderPaid.metrics)
|
|
151
|
+
#
|
|
152
|
+
# Fix: Use the same labels everywhere or rename the metric.
|
|
153
|
+
#
|
|
154
|
+
# Example:
|
|
155
|
+
# # Event 1
|
|
156
|
+
# counter :orders_total, tags: [:currency, :status]
|
|
157
|
+
#
|
|
158
|
+
# # Event 2 (must match!)
|
|
159
|
+
# counter :orders_total, tags: [:currency, :status]
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
**Benefits:**
|
|
163
|
+
- โ
**Fail-fast**: Errors caught at boot time, not in production
|
|
164
|
+
- โ
**Clear error messages**: Shows conflicting definitions with source locations
|
|
165
|
+
- โ
**Zero runtime overhead**: Validation happens once at boot
|
|
166
|
+
- โ
**Automatic**: No manual validation calls needed in Rails
|
|
167
|
+
|
|
168
|
+
**Non-Rails Projects:**
|
|
169
|
+
|
|
170
|
+
```ruby
|
|
171
|
+
# config/boot.rb or similar
|
|
172
|
+
require 'e11y'
|
|
173
|
+
|
|
174
|
+
# After loading all event classes
|
|
175
|
+
E11y::Metrics::Registry.instance.validate_all!
|
|
176
|
+
puts "E11y: Metrics validated successfully"
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
---
|
|
180
|
+
|
|
181
|
+
## ๐ฏ Use Case Scenarios
|
|
182
|
+
|
|
183
|
+
### Scenario 1: Multi-Domain Metrics
|
|
184
|
+
|
|
185
|
+
**Business domains:** Orders, Users, Payments
|
|
186
|
+
|
|
187
|
+
```ruby
|
|
188
|
+
# config/initializers/e11y.rb
|
|
189
|
+
E11y.configure do |config|
|
|
190
|
+
config.metrics do
|
|
191
|
+
# === GLOBAL METRICS ===
|
|
192
|
+
# All events counter
|
|
193
|
+
counter_for pattern: '*',
|
|
194
|
+
name: 'business_events.total',
|
|
195
|
+
tags: [:event_name, :severity]
|
|
196
|
+
|
|
197
|
+
# === ORDERS DOMAIN ===
|
|
198
|
+
counter_for pattern: 'order.*',
|
|
199
|
+
name: 'orders.events.total',
|
|
200
|
+
tags: [:event_name]
|
|
201
|
+
|
|
202
|
+
histogram_for pattern: 'order.paid',
|
|
203
|
+
name: 'orders.amount',
|
|
204
|
+
value: ->(e) { e.payload[:amount] },
|
|
205
|
+
tags: [:currency],
|
|
206
|
+
buckets: [10, 50, 100, 500, 1000, 5000, 10000]
|
|
207
|
+
|
|
208
|
+
# Success rate (special metric)
|
|
209
|
+
success_rate_for pattern: 'order.*',
|
|
210
|
+
name: 'orders.success_rate'
|
|
211
|
+
|
|
212
|
+
# === USERS DOMAIN ===
|
|
213
|
+
counter_for pattern: 'user.*',
|
|
214
|
+
name: 'users.events.total',
|
|
215
|
+
tags: [:event_name]
|
|
216
|
+
|
|
217
|
+
# Funnel metric
|
|
218
|
+
funnel_for pattern: 'user.*',
|
|
219
|
+
name: 'users.registration_funnel',
|
|
220
|
+
steps: ['registration.started', 'email.verified', 'profile.completed', 'first.login']
|
|
221
|
+
|
|
222
|
+
# === PAYMENTS DOMAIN ===
|
|
223
|
+
counter_for pattern: 'payment.*',
|
|
224
|
+
name: 'payments.events.total',
|
|
225
|
+
tags: [:event_name, :payment_method]
|
|
226
|
+
|
|
227
|
+
histogram_for pattern: 'payment.succeeded',
|
|
228
|
+
name: 'payments.duration_ms',
|
|
229
|
+
value: ->(e) { e.duration_ms },
|
|
230
|
+
tags: [:payment_method],
|
|
231
|
+
buckets: [100, 250, 500, 1000, 2000, 5000]
|
|
232
|
+
|
|
233
|
+
success_rate_for pattern: 'payment.*',
|
|
234
|
+
name: 'payments.success_rate',
|
|
235
|
+
tags: [:payment_method]
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
**Result in Prometheus:**
|
|
241
|
+
```promql
|
|
242
|
+
# Global
|
|
243
|
+
business_events_total{event_name="order.paid",severity="success"} 1234
|
|
244
|
+
|
|
245
|
+
# Orders
|
|
246
|
+
orders_events_total{event_name="order.paid"} 1234
|
|
247
|
+
orders_amount_sum{currency="USD"} 123456.78
|
|
248
|
+
orders_success_rate 0.998 # 99.8%
|
|
249
|
+
|
|
250
|
+
# Users
|
|
251
|
+
users_events_total{event_name="registration.started"} 5000
|
|
252
|
+
users_registration_funnel{step="email.verified"} 4500 # 90% conversion
|
|
253
|
+
users_registration_funnel{step="profile.completed"} 4000 # 80% conversion
|
|
254
|
+
|
|
255
|
+
# Payments
|
|
256
|
+
payments_events_total{event_name="payment.succeeded",payment_method="stripe"} 1200
|
|
257
|
+
payments_duration_ms_bucket{payment_method="stripe",le="500"} 1100 # p95 < 500ms
|
|
258
|
+
payments_success_rate{payment_method="stripe"} 0.997 # 99.7%
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
---
|
|
262
|
+
|
|
263
|
+
### Scenario 2: Cardinality-Safe Labels
|
|
264
|
+
|
|
265
|
+
**Problem:** High-cardinality labels (user_id, order_id) cause metric explosions
|
|
266
|
+
|
|
267
|
+
**Solution:** Pattern-based extraction with aggregation
|
|
268
|
+
|
|
269
|
+
```ruby
|
|
270
|
+
E11y.configure do |config|
|
|
271
|
+
config.metrics do
|
|
272
|
+
# โ BAD: user_id as label (1M users = 1M series)
|
|
273
|
+
# counter_for pattern: 'user.action',
|
|
274
|
+
# tags: [:user_id] # โ DON'T DO THIS!
|
|
275
|
+
|
|
276
|
+
# โ
GOOD: Aggregate to user_segment (3 values = 3 series)
|
|
277
|
+
counter_for pattern: 'user.action',
|
|
278
|
+
name: 'users.actions.total',
|
|
279
|
+
tags: [:action_type, :user_segment],
|
|
280
|
+
tag_extractors: {
|
|
281
|
+
user_segment: ->(event) {
|
|
282
|
+
# Aggregate user_id โ user_segment
|
|
283
|
+
user = User.find(event.payload[:user_id])
|
|
284
|
+
user.segment # 'free', 'paid', 'enterprise'
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
# โ
GOOD: Bucket amounts (not exact values)
|
|
289
|
+
histogram_for pattern: 'order.paid',
|
|
290
|
+
name: 'orders.amount_bucket',
|
|
291
|
+
value: ->(e) {
|
|
292
|
+
# Bucket: small, medium, large
|
|
293
|
+
amount = e.payload[:amount]
|
|
294
|
+
case amount
|
|
295
|
+
when 0..50 then 'small'
|
|
296
|
+
when 51..200 then 'medium'
|
|
297
|
+
else 'large'
|
|
298
|
+
end
|
|
299
|
+
},
|
|
300
|
+
tags: [:currency]
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
**Result:**
|
|
306
|
+
```promql
|
|
307
|
+
# Before (BAD): 1M series
|
|
308
|
+
users_actions_total{user_id="user_1",action_type="click"} 1
|
|
309
|
+
users_actions_total{user_id="user_2",action_type="click"} 1
|
|
310
|
+
# ... 1 million more
|
|
311
|
+
|
|
312
|
+
# After (GOOD): 6 series (3 segments ร 2 action types)
|
|
313
|
+
users_actions_total{user_segment="free",action_type="click"} 500000
|
|
314
|
+
users_actions_total{user_segment="paid",action_type="click"} 400000
|
|
315
|
+
users_actions_total{user_segment="enterprise",action_type="click"} 100000
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
---
|
|
319
|
+
|
|
320
|
+
### Scenario 3: Custom Metric Types
|
|
321
|
+
|
|
322
|
+
**Built-in metric types:**
|
|
323
|
+
- `counter_for` - monotonically increasing
|
|
324
|
+
- `histogram_for` - distribution (with buckets)
|
|
325
|
+
- `gauge_for` - point-in-time value
|
|
326
|
+
- `success_rate_for` - ratio of :success / (:success + :error)
|
|
327
|
+
- `funnel_for` - multi-step conversion tracking
|
|
328
|
+
|
|
329
|
+
```ruby
|
|
330
|
+
E11y.configure do |config|
|
|
331
|
+
config.metrics do
|
|
332
|
+
# === COUNTER ===
|
|
333
|
+
counter_for pattern: 'email.sent',
|
|
334
|
+
name: 'emails.sent.total',
|
|
335
|
+
tags: [:template, :status]
|
|
336
|
+
|
|
337
|
+
# === HISTOGRAM ===
|
|
338
|
+
histogram_for pattern: 'api.request',
|
|
339
|
+
name: 'api.request.duration_seconds',
|
|
340
|
+
value: ->(e) { e.duration_ms / 1000.0 },
|
|
341
|
+
tags: [:controller, :action],
|
|
342
|
+
buckets: [0.01, 0.05, 0.1, 0.5, 1.0, 5.0]
|
|
343
|
+
|
|
344
|
+
# === GAUGE (current state) ===
|
|
345
|
+
gauge_for pattern: 'queue.size',
|
|
346
|
+
name: 'sidekiq.queue.size',
|
|
347
|
+
value: ->(e) { e.payload[:size] },
|
|
348
|
+
tags: [:queue_name]
|
|
349
|
+
|
|
350
|
+
# === SUCCESS RATE (auto-calculated) ===
|
|
351
|
+
success_rate_for pattern: 'payment.*',
|
|
352
|
+
name: 'payments.success_rate',
|
|
353
|
+
tags: [:payment_method]
|
|
354
|
+
# Automatically:
|
|
355
|
+
# - Counts events with severity :success
|
|
356
|
+
# - Counts events with severity :error
|
|
357
|
+
# - Calculates: success / (success + error)
|
|
358
|
+
|
|
359
|
+
# === FUNNEL (multi-step conversion) ===
|
|
360
|
+
funnel_for pattern: 'checkout.*',
|
|
361
|
+
name: 'checkout.funnel',
|
|
362
|
+
steps: [
|
|
363
|
+
'checkout.cart_viewed',
|
|
364
|
+
'checkout.shipping_info_entered',
|
|
365
|
+
'checkout.payment_info_entered',
|
|
366
|
+
'checkout.order_placed'
|
|
367
|
+
]
|
|
368
|
+
# Automatically tracks conversion at each step
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
---
|
|
374
|
+
|
|
375
|
+
## ๐ง Configuration API
|
|
376
|
+
|
|
377
|
+
### Pattern Syntax
|
|
378
|
+
|
|
379
|
+
> **Implementation:** See [ADR-002 Section 3.1: Pattern Matching](../ADR-002-metrics-yabeda.md#31-pattern-matching) for detailed architecture.
|
|
380
|
+
|
|
381
|
+
E11y uses **glob-style pattern matching** to determine which events should generate which metrics. Patterns are compiled to regex at initialization for efficient matching at runtime.
|
|
382
|
+
|
|
383
|
+
#### Basic Patterns
|
|
384
|
+
|
|
385
|
+
```ruby
|
|
386
|
+
# Exact match (no wildcards)
|
|
387
|
+
counter_for pattern: 'order.paid'
|
|
388
|
+
# Matches: 'order.paid' only
|
|
389
|
+
# Does NOT match: 'order.paid.usd', 'orders.paid'
|
|
390
|
+
|
|
391
|
+
# Wildcard suffix (*)
|
|
392
|
+
counter_for pattern: 'order.*'
|
|
393
|
+
# Matches: 'order.paid', 'order.refunded', 'order.cancelled'
|
|
394
|
+
# Does NOT match: 'order' (needs at least one char after dot), 'orders.paid'
|
|
395
|
+
|
|
396
|
+
# Wildcard prefix
|
|
397
|
+
counter_for pattern: '*.paid'
|
|
398
|
+
# Matches: 'order.paid', 'invoice.paid', 'subscription.paid'
|
|
399
|
+
|
|
400
|
+
# Multi-level wildcard
|
|
401
|
+
counter_for pattern: 'api.*.request'
|
|
402
|
+
# Matches: 'api.users.request', 'api.orders.request'
|
|
403
|
+
# Does NOT match: 'api.v1.users.request' (multi-segment, use 'api.**.request' for that)
|
|
404
|
+
|
|
405
|
+
# Global wildcard (all events)
|
|
406
|
+
counter_for pattern: '*'
|
|
407
|
+
# Matches: ANY event name
|
|
408
|
+
# Use for: Global event counters, observability dashboards
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
#### Advanced Patterns
|
|
412
|
+
|
|
413
|
+
**1. Brace Expansion (Multiple Values)**
|
|
414
|
+
|
|
415
|
+
```ruby
|
|
416
|
+
# Match multiple specific values
|
|
417
|
+
counter_for pattern: 'payment.{processed,failed,pending}'
|
|
418
|
+
# Matches: 'payment.processed', 'payment.failed', 'payment.pending'
|
|
419
|
+
# Does NOT match: 'payment.refunded'
|
|
420
|
+
|
|
421
|
+
# Use case: Track specific event types
|
|
422
|
+
counter_for pattern: 'order.{paid,refunded}',
|
|
423
|
+
name: 'orders_financial_events_total',
|
|
424
|
+
tags: [:currency]
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
**2. Combining Wildcards and Braces**
|
|
428
|
+
|
|
429
|
+
```ruby
|
|
430
|
+
# Wildcard + brace expansion
|
|
431
|
+
counter_for pattern: 'api.{v1,v2}.*.request'
|
|
432
|
+
# Matches: 'api.v1.users.request', 'api.v2.orders.request'
|
|
433
|
+
# Does NOT match: 'api.v3.users.request'
|
|
434
|
+
|
|
435
|
+
# Use case: Track requests across multiple API versions
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
**3. Multiple Patterns (OR Logic)**
|
|
439
|
+
|
|
440
|
+
```ruby
|
|
441
|
+
# Array of patterns (any match)
|
|
442
|
+
counter_for patterns: [
|
|
443
|
+
'order.paid',
|
|
444
|
+
'subscription.renewed',
|
|
445
|
+
'invoice.paid'
|
|
446
|
+
],
|
|
447
|
+
name: 'revenue_events_total',
|
|
448
|
+
tags: [:currency]
|
|
449
|
+
|
|
450
|
+
# Matches: Any of the listed events
|
|
451
|
+
# Use case: Aggregate different revenue sources into one metric
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
---
|
|
455
|
+
|
|
456
|
+
#### Pattern Compilation (How It Works)
|
|
457
|
+
|
|
458
|
+
Under the hood, E11y compiles glob patterns to Ruby regex at initialization:
|
|
459
|
+
|
|
460
|
+
**Compilation Algorithm:**
|
|
461
|
+
|
|
462
|
+
```ruby
|
|
463
|
+
# lib/e11y/metrics/pattern_matcher.rb
|
|
464
|
+
def compile_pattern(pattern_string)
|
|
465
|
+
# Step 1: Escape dots (literal character)
|
|
466
|
+
# 'order.paid' โ 'order\.paid'
|
|
467
|
+
regex = pattern_string.gsub('.', '\.')
|
|
468
|
+
|
|
469
|
+
# Step 2: Convert wildcards to regex
|
|
470
|
+
# 'order.*' โ 'order\..+' (one or more chars)
|
|
471
|
+
regex = regex.gsub('*', '.+')
|
|
472
|
+
|
|
473
|
+
# Step 3: Convert brace expansion to regex groups
|
|
474
|
+
# 'payment.{processed,failed}' โ 'payment\.(processed|failed)'
|
|
475
|
+
regex = regex.gsub('{', '(')
|
|
476
|
+
.gsub('}', ')')
|
|
477
|
+
.gsub(',', '|')
|
|
478
|
+
|
|
479
|
+
# Step 4: Anchor pattern (exact match required)
|
|
480
|
+
# 'order\.paid' โ '^order\.paid$'
|
|
481
|
+
/^#{regex}$/
|
|
482
|
+
end
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
**Examples:**
|
|
486
|
+
|
|
487
|
+
| Glob Pattern | Compiled Regex | Matches | Does NOT Match |
|
|
488
|
+
|--------------|----------------|---------|----------------|
|
|
489
|
+
| `order.paid` | `/^order\.paid$/` | `order.paid` | `order.paid.usd`, `orders.paid` |
|
|
490
|
+
| `order.*` | `/^order\..+$/` | `order.paid`, `order.refunded` | `order`, `orders.paid` |
|
|
491
|
+
| `*.paid` | `/^.+\.paid$/` | `order.paid`, `invoice.paid` | `paid`, `order.paid.usd` |
|
|
492
|
+
| `payment.{processed,failed}` | `/^payment\.(processed\|failed)$/` | `payment.processed`, `payment.failed` | `payment.pending` |
|
|
493
|
+
| `api.*.request` | `/^api\..+\.request$/` | `api.users.request` | `api.v1.users.request` |
|
|
494
|
+
| `*` | `/^.+$/` | ANY non-empty string | (empty string) |
|
|
495
|
+
|
|
496
|
+
**Performance:**
|
|
497
|
+
- Compilation happens **once at boot** (not per event)
|
|
498
|
+
- Runtime matching is fast regex match: ~0.1ฮผs per pattern
|
|
499
|
+
- Recommended: <20 patterns per metric config (to keep matching fast)
|
|
500
|
+
|
|
501
|
+
---
|
|
502
|
+
|
|
503
|
+
#### Pattern Matching Behavior
|
|
504
|
+
|
|
505
|
+
**1. First Match vs. All Matches**
|
|
506
|
+
|
|
507
|
+
E11y processes **all matching patterns** for each event:
|
|
508
|
+
|
|
509
|
+
```ruby
|
|
510
|
+
# config/initializers/e11y.rb
|
|
511
|
+
E11y.configure do |config|
|
|
512
|
+
config.metrics do
|
|
513
|
+
# Pattern 1: Global counter
|
|
514
|
+
counter_for pattern: '*',
|
|
515
|
+
name: 'business_events_total',
|
|
516
|
+
tags: [:event_name]
|
|
517
|
+
|
|
518
|
+
# Pattern 2: Orders counter
|
|
519
|
+
counter_for pattern: 'order.*',
|
|
520
|
+
name: 'orders_events_total',
|
|
521
|
+
tags: [:event_name]
|
|
522
|
+
|
|
523
|
+
# Pattern 3: Specific order event
|
|
524
|
+
counter_for pattern: 'order.paid',
|
|
525
|
+
name: 'orders_paid_total',
|
|
526
|
+
tags: [:currency]
|
|
527
|
+
end
|
|
528
|
+
end
|
|
529
|
+
|
|
530
|
+
# When Events::OrderPaid.track(...) is called:
|
|
531
|
+
# โ
Increments 'business_events_total' (matched '*')
|
|
532
|
+
# โ
Increments 'orders_events_total' (matched 'order.*')
|
|
533
|
+
# โ
Increments 'orders_paid_total' (matched 'order.paid')
|
|
534
|
+
# Result: 3 metrics updated from 1 event
|
|
535
|
+
```
|
|
536
|
+
|
|
537
|
+
**2. Case Sensitivity**
|
|
538
|
+
|
|
539
|
+
Patterns are **case-sensitive**:
|
|
540
|
+
|
|
541
|
+
```ruby
|
|
542
|
+
counter_for pattern: 'Order.Paid' # โ Won't match 'order.paid'
|
|
543
|
+
counter_for pattern: 'order.paid' # โ
Matches 'order.paid'
|
|
544
|
+
```
|
|
545
|
+
|
|
546
|
+
**Recommendation:** Use lowercase event names consistently (e.g., `order.paid`, not `Order.Paid`).
|
|
547
|
+
|
|
548
|
+
---
|
|
549
|
+
|
|
550
|
+
#### Pattern Testing
|
|
551
|
+
|
|
552
|
+
**In Rails Console:**
|
|
553
|
+
|
|
554
|
+
```ruby
|
|
555
|
+
# Test pattern matching without tracking events
|
|
556
|
+
matcher = E11y::Metrics::PatternMatcher.new(E11y.config)
|
|
557
|
+
|
|
558
|
+
# Check if event matches any patterns
|
|
559
|
+
event_name = 'order.paid'
|
|
560
|
+
matched_patterns = matcher.match(event_name)
|
|
561
|
+
|
|
562
|
+
puts "Event '#{event_name}' matches:"
|
|
563
|
+
matched_patterns.each do |pattern_config|
|
|
564
|
+
puts " - #{pattern_config[:name]} (pattern: #{pattern_config[:pattern]})"
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
# Output:
|
|
568
|
+
# Event 'order.paid' matches:
|
|
569
|
+
# - business_events_total (pattern: *)
|
|
570
|
+
# - orders_events_total (pattern: order.*)
|
|
571
|
+
# - orders_paid_total (pattern: order.paid)
|
|
572
|
+
```
|
|
573
|
+
|
|
574
|
+
**In Tests:**
|
|
575
|
+
|
|
576
|
+
```ruby
|
|
577
|
+
# spec/lib/e11y/metrics/pattern_matcher_spec.rb
|
|
578
|
+
RSpec.describe E11y::Metrics::PatternMatcher do
|
|
579
|
+
describe '#match' do
|
|
580
|
+
it 'matches exact pattern' do
|
|
581
|
+
matcher = described_class.new([{ pattern: 'order.paid' }])
|
|
582
|
+
expect(matcher.match('order.paid')).not_to be_empty
|
|
583
|
+
expect(matcher.match('order.refunded')).to be_empty
|
|
584
|
+
end
|
|
585
|
+
|
|
586
|
+
it 'matches wildcard pattern' do
|
|
587
|
+
matcher = described_class.new([{ pattern: 'order.*' }])
|
|
588
|
+
expect(matcher.match('order.paid')).not_to be_empty
|
|
589
|
+
expect(matcher.match('order.refunded')).not_to be_empty
|
|
590
|
+
expect(matcher.match('user.signup')).to be_empty
|
|
591
|
+
end
|
|
592
|
+
|
|
593
|
+
it 'matches brace expansion' do
|
|
594
|
+
matcher = described_class.new([{ pattern: 'payment.{processed,failed}' }])
|
|
595
|
+
expect(matcher.match('payment.processed')).not_to be_empty
|
|
596
|
+
expect(matcher.match('payment.failed')).not_to be_empty
|
|
597
|
+
expect(matcher.match('payment.pending')).to be_empty
|
|
598
|
+
end
|
|
599
|
+
end
|
|
600
|
+
end
|
|
601
|
+
```
|
|
602
|
+
|
|
603
|
+
---
|
|
604
|
+
|
|
605
|
+
#### Common Pitfalls
|
|
606
|
+
|
|
607
|
+
**โ BAD: Overlapping patterns without distinct tags**
|
|
608
|
+
|
|
609
|
+
```ruby
|
|
610
|
+
counter_for pattern: 'order.*', name: 'orders_total'
|
|
611
|
+
counter_for pattern: 'order.paid', name: 'orders_paid_total'
|
|
612
|
+
|
|
613
|
+
# Problem: 'order.paid' event increments BOTH counters
|
|
614
|
+
# Result: orders_total = orders_paid_total (redundant)
|
|
615
|
+
```
|
|
616
|
+
|
|
617
|
+
**โ
GOOD: Use distinct tags or aggregate metrics**
|
|
618
|
+
|
|
619
|
+
```ruby
|
|
620
|
+
# Option 1: Use tags to differentiate
|
|
621
|
+
counter_for pattern: 'order.*',
|
|
622
|
+
name: 'orders_total',
|
|
623
|
+
tags: [:event_type] # event_type: 'paid', 'refunded', etc.
|
|
624
|
+
|
|
625
|
+
# Option 2: Separate metrics for specific events
|
|
626
|
+
counter_for pattern: 'order.paid', name: 'orders_paid_total'
|
|
627
|
+
counter_for pattern: 'order.refunded', name: 'orders_refunded_total'
|
|
628
|
+
```
|
|
629
|
+
|
|
630
|
+
**โ BAD: Wildcard without dot (too broad)**
|
|
631
|
+
|
|
632
|
+
```ruby
|
|
633
|
+
counter_for pattern: 'order*'
|
|
634
|
+
# Matches: 'orders', 'order_paid', 'order.paid'
|
|
635
|
+
# Problem: Matches event names you might not expect
|
|
636
|
+
```
|
|
637
|
+
|
|
638
|
+
**โ
GOOD: Explicit dot with wildcard**
|
|
639
|
+
|
|
640
|
+
```ruby
|
|
641
|
+
counter_for pattern: 'order.*'
|
|
642
|
+
# Matches: 'order.paid', 'order.refunded'
|
|
643
|
+
# Does NOT match: 'orders', 'order_paid'
|
|
644
|
+
```
|
|
645
|
+
|
|
646
|
+
---
|
|
647
|
+
|
|
648
|
+
### Tag Extraction
|
|
649
|
+
|
|
650
|
+
```ruby
|
|
651
|
+
counter_for pattern: 'order.*',
|
|
652
|
+
name: 'orders.events.total',
|
|
653
|
+
tags: [:currency, :payment_method],
|
|
654
|
+
tag_extractors: {
|
|
655
|
+
# Simple: extract from payload
|
|
656
|
+
currency: ->(e) { e.payload[:currency] },
|
|
657
|
+
|
|
658
|
+
# Complex: aggregate/transform
|
|
659
|
+
payment_method: ->(e) {
|
|
660
|
+
method = e.payload[:payment_method]
|
|
661
|
+
method == 'apple_pay' ? 'mobile' : method
|
|
662
|
+
},
|
|
663
|
+
|
|
664
|
+
# From context
|
|
665
|
+
region: ->(e) { e.context[:region] }
|
|
666
|
+
}
|
|
667
|
+
```
|
|
668
|
+
|
|
669
|
+
#### Label Extraction Algorithm
|
|
670
|
+
|
|
671
|
+
> **Implementation:** See [ADR-002 Section 3.3: Label Extraction](../ADR-002-metrics-yabeda.md#33-label-extraction) for detailed architecture.
|
|
672
|
+
|
|
673
|
+
E11y uses a **safe label extraction algorithm** that protects against high-cardinality explosions while allowing flexible label customization. Understanding this algorithm helps debug unexpected label values and optimize metric performance.
|
|
674
|
+
|
|
675
|
+
**Extraction Flow (4 Steps):**
|
|
676
|
+
|
|
677
|
+
```
|
|
678
|
+
Event โ Label Extractor
|
|
679
|
+
โ
|
|
680
|
+
1. FOR each tag in `tags` list
|
|
681
|
+
โ
|
|
682
|
+
2. Extract raw value from event.payload[tag] or tag_extractors[tag].call(event)
|
|
683
|
+
โ
|
|
684
|
+
3. Apply Cardinality Protection (4-layer defense)
|
|
685
|
+
โโ Layer 1: Denylist โ DROP if forbidden (e.g., user_id, session_id)
|
|
686
|
+
โโ Layer 2: Allowlist โ KEEP if safe (e.g., status, method)
|
|
687
|
+
โโ Layer 3: Limit Check โ KEEP if cardinality < limit
|
|
688
|
+
โโ Layer 4: Dynamic Action โ hash/drop/alert if over limit
|
|
689
|
+
โ
|
|
690
|
+
4. Add protected label to metric (skip if nil/dropped)
|
|
691
|
+
```
|
|
692
|
+
|
|
693
|
+
**Example: Safe vs. Unsafe Extraction**
|
|
694
|
+
|
|
695
|
+
```ruby
|
|
696
|
+
# Event tracked:
|
|
697
|
+
Events::OrderPaid.track(
|
|
698
|
+
order_id: '123456', # โ High cardinality (unique per order)
|
|
699
|
+
user_id: 'user-789', # โ High cardinality (unique per user)
|
|
700
|
+
status: 'paid', # โ Low cardinality (3 values: paid/pending/failed)
|
|
701
|
+
payment_method: 'card', # โ Low cardinality (5 values: card/paypal/crypto/...)
|
|
702
|
+
amount: 99.99
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
# Metric configuration:
|
|
706
|
+
counter_for pattern: 'order.paid',
|
|
707
|
+
name: 'orders.total',
|
|
708
|
+
tags: [:order_id, :status, :payment_method]
|
|
709
|
+
|
|
710
|
+
# Label extraction with protection:
|
|
711
|
+
# โ order_id โ DROPPED (Layer 1: in FORBIDDEN_LABELS denylist)
|
|
712
|
+
# โ
status โ KEPT (Layer 2: in SAFE_LABELS allowlist)
|
|
713
|
+
# โ
payment_method โ KEPT (Layer 2: in SAFE_LABELS allowlist)
|
|
714
|
+
|
|
715
|
+
# Resulting Prometheus metric:
|
|
716
|
+
# orders_total{status="paid", payment_method="card"} 1
|
|
717
|
+
# (order_id NOT included in labels!)
|
|
718
|
+
```
|
|
719
|
+
|
|
720
|
+
**Why Cardinality Protection Matters:**
|
|
721
|
+
|
|
722
|
+
```ruby
|
|
723
|
+
# โ WITHOUT protection (dangerous!):
|
|
724
|
+
counter_for pattern: 'order.*', tags: [:order_id]
|
|
725
|
+
# Result: 1,000,000 orders = 1,000,000 metric series
|
|
726
|
+
# Memory: 1M ร 3KB = 3GB RAM ๐ฅ
|
|
727
|
+
# Prometheus query time: >30s ๐ข
|
|
728
|
+
|
|
729
|
+
# โ
WITH protection (safe):
|
|
730
|
+
counter_for pattern: 'order.*', tags: [:status]
|
|
731
|
+
# Result: 1,000,000 orders = 3 metric series (paid/pending/failed)
|
|
732
|
+
# Memory: 3 ร 3KB = 9KB RAM โ
|
|
733
|
+
# Prometheus query time: <10ms โก
|
|
734
|
+
```
|
|
735
|
+
|
|
736
|
+
**Custom Tag Extractors with Protection:**
|
|
737
|
+
|
|
738
|
+
Tag extractors can apply custom logic, but cardinality protection is ALWAYS applied after extraction:
|
|
739
|
+
|
|
740
|
+
```ruby
|
|
741
|
+
counter_for pattern: 'api.request',
|
|
742
|
+
tags: [:endpoint_family, :status_class],
|
|
743
|
+
tag_extractors: {
|
|
744
|
+
# Aggregate high-cardinality paths into families
|
|
745
|
+
endpoint_family: ->(e) {
|
|
746
|
+
path = e.payload[:path]
|
|
747
|
+
case path
|
|
748
|
+
when %r{^/api/users/\d+} then '/api/users/:id'
|
|
749
|
+
when %r{^/api/orders/\d+} then '/api/orders/:id'
|
|
750
|
+
else path
|
|
751
|
+
end
|
|
752
|
+
},
|
|
753
|
+
|
|
754
|
+
# Aggregate HTTP status codes into classes
|
|
755
|
+
status_class: ->(e) {
|
|
756
|
+
status = e.payload[:status]
|
|
757
|
+
case status
|
|
758
|
+
when 200..299 then '2xx'
|
|
759
|
+
when 400..499 then '4xx'
|
|
760
|
+
when 500..599 then '5xx'
|
|
761
|
+
else 'other'
|
|
762
|
+
end
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
# Extraction flow:
|
|
767
|
+
# 1. Extract raw value via tag_extractors lambda
|
|
768
|
+
# 2. Apply cardinality protection (check against limits)
|
|
769
|
+
# 3. Add to metric labels
|
|
770
|
+
```
|
|
771
|
+
|
|
772
|
+
**Debugging Label Extraction:**
|
|
773
|
+
|
|
774
|
+
```ruby
|
|
775
|
+
# Enable label extraction logging in development:
|
|
776
|
+
E11y.configure do |config|
|
|
777
|
+
config.metrics do
|
|
778
|
+
log_label_extraction true # Log what labels are extracted/dropped
|
|
779
|
+
end
|
|
780
|
+
end
|
|
781
|
+
|
|
782
|
+
# Output:
|
|
783
|
+
# [E11y::Metrics] Event: order.paid
|
|
784
|
+
# โ Extracting labels: [:order_id, :status, :payment_method]
|
|
785
|
+
# โ order_id: "123456" โ DROPPED (Layer 1: Denylist)
|
|
786
|
+
# โ status: "paid" โ KEPT (Layer 2: Allowlist)
|
|
787
|
+
# โ payment_method: "card" โ KEPT (Layer 2: Allowlist)
|
|
788
|
+
# โ Final labels: {status: "paid", payment_method: "card"}
|
|
789
|
+
```
|
|
790
|
+
|
|
791
|
+
**Testing Label Extraction:**
|
|
792
|
+
|
|
793
|
+
```ruby
|
|
794
|
+
# RSpec: Verify labels are extracted correctly
|
|
795
|
+
RSpec.describe 'Order metrics labels', type: :metrics do
|
|
796
|
+
it 'extracts safe labels only' do
|
|
797
|
+
expect {
|
|
798
|
+
Events::OrderPaid.track(
|
|
799
|
+
order_id: '123456',
|
|
800
|
+
status: 'paid',
|
|
801
|
+
payment_method: 'card'
|
|
802
|
+
)
|
|
803
|
+
}.to change {
|
|
804
|
+
Yabeda.e11y.orders_total.values[{status: 'paid', payment_method: 'card'}]
|
|
805
|
+
}.by(1)
|
|
806
|
+
|
|
807
|
+
# Verify high-cardinality label NOT present:
|
|
808
|
+
expect(
|
|
809
|
+
Yabeda.e11y.orders_total.values.keys.any? { |labels| labels.key?(:order_id) }
|
|
810
|
+
).to be false
|
|
811
|
+
end
|
|
812
|
+
end
|
|
813
|
+
```
|
|
814
|
+
|
|
815
|
+
---
|
|
816
|
+
|
|
817
|
+
### Conditional Metrics
|
|
818
|
+
|
|
819
|
+
```ruby
|
|
820
|
+
# Only create metric if condition met
|
|
821
|
+
counter_for pattern: 'order.*',
|
|
822
|
+
name: 'high_value_orders.total',
|
|
823
|
+
if: ->(event) { event.payload[:amount] > 1000 },
|
|
824
|
+
tags: [:currency]
|
|
825
|
+
|
|
826
|
+
# Different metrics based on condition
|
|
827
|
+
counter_for pattern: 'user.signup',
|
|
828
|
+
name: ->(event) {
|
|
829
|
+
event.payload[:plan] == 'free' ? 'users.free.total' : 'users.paid.total'
|
|
830
|
+
}
|
|
831
|
+
```
|
|
832
|
+
|
|
833
|
+
### Value Extraction
|
|
834
|
+
|
|
835
|
+
```ruby
|
|
836
|
+
# Simple: extract field
|
|
837
|
+
histogram_for pattern: 'order.paid',
|
|
838
|
+
value: ->(e) { e.payload[:amount] }
|
|
839
|
+
|
|
840
|
+
# Transform: convert units
|
|
841
|
+
histogram_for pattern: 'api.request',
|
|
842
|
+
value: ->(e) { e.duration_ms / 1000.0 } # ms โ seconds
|
|
843
|
+
|
|
844
|
+
# Aggregate: bucket values
|
|
845
|
+
histogram_for pattern: 'order.paid',
|
|
846
|
+
value: ->(e) {
|
|
847
|
+
amount = e.payload[:amount]
|
|
848
|
+
case amount
|
|
849
|
+
when 0..50 then 25 # Small: avg 25
|
|
850
|
+
when 51..200 then 125 # Medium: avg 125
|
|
851
|
+
else 500 # Large: avg 500
|
|
852
|
+
end
|
|
853
|
+
}
|
|
854
|
+
```
|
|
855
|
+
|
|
856
|
+
---
|
|
857
|
+
|
|
858
|
+
## ๐ง Implementation Details
|
|
859
|
+
|
|
860
|
+
> **Implementation:** See [ADR-002 Section 2.2: Component Architecture](../ADR-002-metrics-yabeda.md#22-component-architecture) and [Section 2.3: Data Flow](../ADR-002-metrics-yabeda.md#23-data-flow) for detailed architecture.
|
|
861
|
+
|
|
862
|
+
### Metrics Middleware Architecture
|
|
863
|
+
|
|
864
|
+
E11y pattern-based metrics are implemented as **middleware** in the event processing pipeline. Understanding how metrics middleware works helps with debugging, custom metrics, and performance optimization.
|
|
865
|
+
|
|
866
|
+
**Pipeline Integration:**
|
|
867
|
+
```
|
|
868
|
+
Event.track()
|
|
869
|
+
โ Schema Validation
|
|
870
|
+
โ Context Enrichment
|
|
871
|
+
โ Rate Limiting
|
|
872
|
+
โ Metrics Middleware โ YOU ARE HERE
|
|
873
|
+
โโ Pattern Matcher
|
|
874
|
+
โโ Label Extractor
|
|
875
|
+
โโ Cardinality Protection (4 layers)
|
|
876
|
+
โโ Yabeda Integration
|
|
877
|
+
โ PII Filtering
|
|
878
|
+
โ Adapter Routing
|
|
879
|
+
โ Write to Adapters (Loki, OTel, Sentry)
|
|
880
|
+
```
|
|
881
|
+
|
|
882
|
+
**Key Point:** Metrics middleware processes events **before** PII filtering, so it has access to original labels. However, cardinality protection filters out high-cardinality fields (like `user_id`) before they reach Prometheus.
|
|
883
|
+
|
|
884
|
+
---
|
|
885
|
+
|
|
886
|
+
### Middleware Flow (Step-by-Step)
|
|
887
|
+
|
|
888
|
+
**1. Pattern Matching**
|
|
889
|
+
|
|
890
|
+
```ruby
|
|
891
|
+
# Event: Events::OrderPaid.track(order_id: '123', amount: 99.99, currency: 'USD')
|
|
892
|
+
|
|
893
|
+
# Middleware receives event data
|
|
894
|
+
event_data = {
|
|
895
|
+
event_name: 'order.paid',
|
|
896
|
+
payload: { order_id: '123', amount: 99.99, currency: 'USD' }
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
# Pattern matcher finds matching metric configs
|
|
900
|
+
matched_metrics = pattern_matcher.match(event_data[:event_name])
|
|
901
|
+
# => [
|
|
902
|
+
# { type: :counter, pattern: 'order.paid', name: 'orders_paid_total', tags: [:currency] },
|
|
903
|
+
# { type: :histogram, pattern: 'order.paid', name: 'orders_paid_amount', tags: [:currency], value: ... }
|
|
904
|
+
# ]
|
|
905
|
+
```
|
|
906
|
+
|
|
907
|
+
**Pattern Matching Algorithm:**
|
|
908
|
+
|
|
909
|
+
```ruby
|
|
910
|
+
# lib/e11y/metrics/pattern_matcher.rb
|
|
911
|
+
module E11y
|
|
912
|
+
module Metrics
|
|
913
|
+
class PatternMatcher
|
|
914
|
+
def initialize(config)
|
|
915
|
+
@patterns = config.metrics.patterns
|
|
916
|
+
end
|
|
917
|
+
|
|
918
|
+
def match(event_name)
|
|
919
|
+
@patterns.select do |pattern_config|
|
|
920
|
+
pattern = pattern_config[:pattern]
|
|
921
|
+
|
|
922
|
+
case pattern
|
|
923
|
+
when String
|
|
924
|
+
# Exact match or wildcard
|
|
925
|
+
match_pattern?(event_name, pattern)
|
|
926
|
+
when Regexp
|
|
927
|
+
# Regex match
|
|
928
|
+
pattern.match?(event_name)
|
|
929
|
+
when Array
|
|
930
|
+
# Multiple patterns
|
|
931
|
+
pattern.any? { |p| match_pattern?(event_name, p) }
|
|
932
|
+
end
|
|
933
|
+
end
|
|
934
|
+
end
|
|
935
|
+
|
|
936
|
+
private
|
|
937
|
+
|
|
938
|
+
def match_pattern?(event_name, pattern)
|
|
939
|
+
# Convert glob pattern to regex
|
|
940
|
+
# 'order.*' โ /^order\..+$/
|
|
941
|
+
# 'order.paid' โ /^order\.paid$/
|
|
942
|
+
# '*' โ /^.+$/
|
|
943
|
+
|
|
944
|
+
regex_pattern = pattern
|
|
945
|
+
.gsub('.', '\.') # Escape dots
|
|
946
|
+
.gsub('*', '.+') # * โ one or more chars
|
|
947
|
+
.then { |p| /^#{p}$/ }
|
|
948
|
+
|
|
949
|
+
regex_pattern.match?(event_name)
|
|
950
|
+
end
|
|
951
|
+
end
|
|
952
|
+
end
|
|
953
|
+
end
|
|
954
|
+
```
|
|
955
|
+
|
|
956
|
+
**Examples:**
|
|
957
|
+
```ruby
|
|
958
|
+
match_pattern?('order.paid', 'order.paid') # => true (exact match)
|
|
959
|
+
match_pattern?('order.paid', 'order.*') # => true (wildcard)
|
|
960
|
+
match_pattern?('order.paid', '*') # => true (global wildcard)
|
|
961
|
+
match_pattern?('order.paid', 'user.*') # => false (no match)
|
|
962
|
+
match_pattern?('order.refunded', 'order.*') # => true (wildcard match)
|
|
963
|
+
```
|
|
964
|
+
|
|
965
|
+
---
|
|
966
|
+
|
|
967
|
+
**2. Label Extraction**
|
|
968
|
+
|
|
969
|
+
```ruby
|
|
970
|
+
# For matched metric: counter 'orders_paid_total' with tags: [:currency]
|
|
971
|
+
|
|
972
|
+
# Extract labels from event payload
|
|
973
|
+
extractor = LabelExtractor.new(metric_config, event_data)
|
|
974
|
+
labels = extractor.extract
|
|
975
|
+
# => { currency: 'USD' }
|
|
976
|
+
```
|
|
977
|
+
|
|
978
|
+
**Label Extraction Logic:**
|
|
979
|
+
|
|
980
|
+
```ruby
|
|
981
|
+
# lib/e11y/metrics/label_extractor.rb
|
|
982
|
+
module E11y
|
|
983
|
+
module Metrics
|
|
984
|
+
class LabelExtractor
|
|
985
|
+
def initialize(metric_config, event_data)
|
|
986
|
+
@config = metric_config
|
|
987
|
+
@event_data = event_data
|
|
988
|
+
@payload = event_data[:payload]
|
|
989
|
+
end
|
|
990
|
+
|
|
991
|
+
def extract
|
|
992
|
+
tags = @config[:tags] || []
|
|
993
|
+
extractors = @config[:tag_extractors] || {}
|
|
994
|
+
|
|
995
|
+
labels = {}
|
|
996
|
+
|
|
997
|
+
tags.each do |tag_name|
|
|
998
|
+
# Check if custom extractor defined
|
|
999
|
+
if extractors[tag_name]
|
|
1000
|
+
labels[tag_name] = extractors[tag_name].call(@event_data)
|
|
1001
|
+
else
|
|
1002
|
+
# Default: extract from payload
|
|
1003
|
+
labels[tag_name] = @payload[tag_name]
|
|
1004
|
+
end
|
|
1005
|
+
end
|
|
1006
|
+
|
|
1007
|
+
# Remove nil values
|
|
1008
|
+
labels.compact
|
|
1009
|
+
end
|
|
1010
|
+
end
|
|
1011
|
+
end
|
|
1012
|
+
end
|
|
1013
|
+
```
|
|
1014
|
+
|
|
1015
|
+
**Custom Extractors Example:**
|
|
1016
|
+
|
|
1017
|
+
```ruby
|
|
1018
|
+
E11y.configure do |config|
|
|
1019
|
+
config.metrics do
|
|
1020
|
+
counter_for pattern: 'user.action',
|
|
1021
|
+
name: 'users_actions_total',
|
|
1022
|
+
tags: [:action_type, :user_segment],
|
|
1023
|
+
tag_extractors: {
|
|
1024
|
+
# Custom logic for user_segment
|
|
1025
|
+
user_segment: ->(event) {
|
|
1026
|
+
user_id = event.payload[:user_id]
|
|
1027
|
+
user = User.find(user_id)
|
|
1028
|
+
user.segment # 'free', 'paid', 'enterprise'
|
|
1029
|
+
},
|
|
1030
|
+
|
|
1031
|
+
# Simple extraction (could omit - same as default)
|
|
1032
|
+
action_type: ->(event) {
|
|
1033
|
+
event.payload[:action_type]
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
end
|
|
1037
|
+
end
|
|
1038
|
+
```
|
|
1039
|
+
|
|
1040
|
+
---
|
|
1041
|
+
|
|
1042
|
+
**3. Cardinality Protection (4 Layers)**
|
|
1043
|
+
|
|
1044
|
+
After label extraction, cardinality protection filters out high-cardinality labels:
|
|
1045
|
+
|
|
1046
|
+
```ruby
|
|
1047
|
+
# Extracted labels: { currency: 'USD', order_id: '123' }
|
|
1048
|
+
|
|
1049
|
+
# Pass through 4 protection layers
|
|
1050
|
+
protected_labels = cardinality_protection.filter(labels)
|
|
1051
|
+
# => { currency: 'USD' } # order_id dropped!
|
|
1052
|
+
```
|
|
1053
|
+
|
|
1054
|
+
**4-Layer Protection:**
|
|
1055
|
+
|
|
1056
|
+
| Layer | Purpose | Example |
|
|
1057
|
+
|-------|---------|---------|
|
|
1058
|
+
| **Layer 1: Denylist** | Block known high-cardinality fields | `order_id`, `user_id`, `transaction_id` |
|
|
1059
|
+
| **Layer 2: Allowlist** | Allow only safe, known fields | `currency`, `status`, `payment_method` |
|
|
1060
|
+
| **Layer 3: Per-Metric Limits** | Limit unique values per metric | Max 2000 unique combos for `orders_paid_total` |
|
|
1061
|
+
| **Layer 4: Dynamic Monitoring** | Alert on cardinality spikes | Alert if new metric > 1000 series/min |
|
|
1062
|
+
|
|
1063
|
+
**Implementation:**
|
|
1064
|
+
|
|
1065
|
+
```ruby
|
|
1066
|
+
# lib/e11y/metrics/cardinality_protection.rb
|
|
1067
|
+
module E11y
|
|
1068
|
+
module Metrics
|
|
1069
|
+
class CardinalityProtection
|
|
1070
|
+
def initialize(config)
|
|
1071
|
+
@denylist = config.cardinality_denylist || DEFAULT_DENYLIST
|
|
1072
|
+
@allowlist = config.cardinality_allowlist || []
|
|
1073
|
+
@per_metric_limits = config.per_metric_limits || {}
|
|
1074
|
+
@enable_dynamic_monitoring = config.dynamic_monitoring || true
|
|
1075
|
+
end
|
|
1076
|
+
|
|
1077
|
+
def filter(metric_name, labels)
|
|
1078
|
+
filtered = labels.dup
|
|
1079
|
+
|
|
1080
|
+
# Layer 1: Denylist
|
|
1081
|
+
@denylist.each { |field| filtered.delete(field) }
|
|
1082
|
+
|
|
1083
|
+
# Layer 2: Allowlist (if configured)
|
|
1084
|
+
if @allowlist.any?
|
|
1085
|
+
filtered.select! { |k, v| @allowlist.include?(k) }
|
|
1086
|
+
end
|
|
1087
|
+
|
|
1088
|
+
# Layer 3: Per-metric limits
|
|
1089
|
+
if limit = @per_metric_limits[metric_name]
|
|
1090
|
+
check_cardinality_limit(metric_name, filtered, limit)
|
|
1091
|
+
end
|
|
1092
|
+
|
|
1093
|
+
# Layer 4: Dynamic monitoring
|
|
1094
|
+
if @enable_dynamic_monitoring
|
|
1095
|
+
monitor_cardinality_spike(metric_name, filtered)
|
|
1096
|
+
end
|
|
1097
|
+
|
|
1098
|
+
filtered
|
|
1099
|
+
end
|
|
1100
|
+
|
|
1101
|
+
private
|
|
1102
|
+
|
|
1103
|
+
DEFAULT_DENYLIST = [
|
|
1104
|
+
:user_id, :order_id, :transaction_id, :session_id,
|
|
1105
|
+
:request_id, :trace_id, :ip_address, :email,
|
|
1106
|
+
:phone, :uuid, :token, :api_key
|
|
1107
|
+
].freeze
|
|
1108
|
+
|
|
1109
|
+
def check_cardinality_limit(metric_name, labels, limit)
|
|
1110
|
+
# Check current cardinality from Redis
|
|
1111
|
+
key = "e11y:metrics:cardinality:#{metric_name}"
|
|
1112
|
+
label_combo = labels.to_json
|
|
1113
|
+
|
|
1114
|
+
redis.zincrby(key, 1, label_combo)
|
|
1115
|
+
current_cardinality = redis.zcard(key)
|
|
1116
|
+
|
|
1117
|
+
if current_cardinality > limit[:max_cardinality]
|
|
1118
|
+
handle_limit_exceeded(metric_name, labels, current_cardinality, limit)
|
|
1119
|
+
end
|
|
1120
|
+
end
|
|
1121
|
+
|
|
1122
|
+
def handle_limit_exceeded(metric_name, labels, current, limit)
|
|
1123
|
+
case limit[:overflow_strategy]
|
|
1124
|
+
when :drop
|
|
1125
|
+
# Drop this label combination
|
|
1126
|
+
raise CardinalityLimitExceeded, "Metric #{metric_name} exceeded limit"
|
|
1127
|
+
when :alert
|
|
1128
|
+
# Track but alert
|
|
1129
|
+
alert_cardinality_spike(metric_name, current, limit[:max_cardinality])
|
|
1130
|
+
end
|
|
1131
|
+
end
|
|
1132
|
+
|
|
1133
|
+
def monitor_cardinality_spike(metric_name, labels)
|
|
1134
|
+
# Track new unique label combinations
|
|
1135
|
+
Yabeda.e11y_internal.metrics_cardinality_current.set(
|
|
1136
|
+
labels.size,
|
|
1137
|
+
metric_name: metric_name
|
|
1138
|
+
)
|
|
1139
|
+
end
|
|
1140
|
+
end
|
|
1141
|
+
end
|
|
1142
|
+
end
|
|
1143
|
+
```
|
|
1144
|
+
|
|
1145
|
+
---
|
|
1146
|
+
|
|
1147
|
+
**4. Yabeda Integration**
|
|
1148
|
+
|
|
1149
|
+
After cardinality protection, safe labels are used to increment/observe Yabeda metrics:
|
|
1150
|
+
|
|
1151
|
+
```ruby
|
|
1152
|
+
# Safe labels: { currency: 'USD' }
|
|
1153
|
+
|
|
1154
|
+
# Increment counter
|
|
1155
|
+
Yabeda.e11y.orders_paid_total.increment(
|
|
1156
|
+
{ currency: 'USD' }
|
|
1157
|
+
)
|
|
1158
|
+
|
|
1159
|
+
# Observe histogram
|
|
1160
|
+
Yabeda.e11y.orders_paid_amount.observe(
|
|
1161
|
+
99.99, # value
|
|
1162
|
+
{ currency: 'USD' } # labels
|
|
1163
|
+
)
|
|
1164
|
+
```
|
|
1165
|
+
|
|
1166
|
+
**Yabeda Metric Registration:**
|
|
1167
|
+
|
|
1168
|
+
Metrics are registered in Yabeda when E11y initializes:
|
|
1169
|
+
|
|
1170
|
+
```ruby
|
|
1171
|
+
# lib/e11y/metrics/registry.rb
|
|
1172
|
+
module E11y
|
|
1173
|
+
module Metrics
|
|
1174
|
+
class Registry
|
|
1175
|
+
def initialize(config)
|
|
1176
|
+
@config = config
|
|
1177
|
+
@registered_metrics = {}
|
|
1178
|
+
end
|
|
1179
|
+
|
|
1180
|
+
def register_all
|
|
1181
|
+
@config.metrics.patterns.each do |pattern_config|
|
|
1182
|
+
register_metric(pattern_config)
|
|
1183
|
+
end
|
|
1184
|
+
end
|
|
1185
|
+
|
|
1186
|
+
private
|
|
1187
|
+
|
|
1188
|
+
def register_metric(config)
|
|
1189
|
+
metric_name = config[:name]
|
|
1190
|
+
|
|
1191
|
+
return if @registered_metrics[metric_name]
|
|
1192
|
+
|
|
1193
|
+
case config[:type]
|
|
1194
|
+
when :counter
|
|
1195
|
+
Yabeda.configure do
|
|
1196
|
+
counter metric_name.to_sym,
|
|
1197
|
+
comment: config[:comment] || "Auto-generated from pattern #{config[:pattern]}",
|
|
1198
|
+
tags: config[:tags] || []
|
|
1199
|
+
end
|
|
1200
|
+
|
|
1201
|
+
when :histogram
|
|
1202
|
+
Yabeda.configure do
|
|
1203
|
+
histogram metric_name.to_sym,
|
|
1204
|
+
comment: config[:comment] || "Auto-generated from pattern #{config[:pattern]}",
|
|
1205
|
+
unit: config[:unit] || :milliseconds,
|
|
1206
|
+
buckets: config[:buckets] || [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5],
|
|
1207
|
+
tags: config[:tags] || []
|
|
1208
|
+
end
|
|
1209
|
+
|
|
1210
|
+
when :gauge
|
|
1211
|
+
Yabeda.configure do
|
|
1212
|
+
gauge metric_name.to_sym,
|
|
1213
|
+
comment: config[:comment] || "Auto-generated from pattern #{config[:pattern]}",
|
|
1214
|
+
tags: config[:tags] || []
|
|
1215
|
+
end
|
|
1216
|
+
end
|
|
1217
|
+
|
|
1218
|
+
@registered_metrics[metric_name] = config
|
|
1219
|
+
end
|
|
1220
|
+
end
|
|
1221
|
+
end
|
|
1222
|
+
end
|
|
1223
|
+
```
|
|
1224
|
+
|
|
1225
|
+
---
|
|
1226
|
+
|
|
1227
|
+
### Complete Middleware Implementation
|
|
1228
|
+
|
|
1229
|
+
**Full Middleware Class:**
|
|
1230
|
+
|
|
1231
|
+
```ruby
|
|
1232
|
+
# lib/e11y/middleware/metrics.rb
|
|
1233
|
+
module E11y
|
|
1234
|
+
module Middleware
|
|
1235
|
+
class Metrics < Base
|
|
1236
|
+
def initialize(app, config)
|
|
1237
|
+
super(app)
|
|
1238
|
+
@config = config
|
|
1239
|
+
@pattern_matcher = PatternMatcher.new(config)
|
|
1240
|
+
@label_extractor = LabelExtractor
|
|
1241
|
+
@cardinality_protection = CardinalityProtection.new(config)
|
|
1242
|
+
end
|
|
1243
|
+
|
|
1244
|
+
def call(event_data)
|
|
1245
|
+
# 1. Match patterns for this event
|
|
1246
|
+
matched_metrics = @pattern_matcher.match(event_data[:event_name])
|
|
1247
|
+
|
|
1248
|
+
# 2. Process each matched metric
|
|
1249
|
+
matched_metrics.each do |metric_config|
|
|
1250
|
+
process_metric(metric_config, event_data)
|
|
1251
|
+
end
|
|
1252
|
+
|
|
1253
|
+
# 3. Continue pipeline
|
|
1254
|
+
super(event_data)
|
|
1255
|
+
end
|
|
1256
|
+
|
|
1257
|
+
private
|
|
1258
|
+
|
|
1259
|
+
def process_metric(metric_config, event_data)
|
|
1260
|
+
# Extract labels
|
|
1261
|
+
extractor = @label_extractor.new(metric_config, event_data)
|
|
1262
|
+
raw_labels = extractor.extract
|
|
1263
|
+
|
|
1264
|
+
# Apply cardinality protection
|
|
1265
|
+
safe_labels = @cardinality_protection.filter(
|
|
1266
|
+
metric_config[:name],
|
|
1267
|
+
raw_labels
|
|
1268
|
+
)
|
|
1269
|
+
|
|
1270
|
+
# Update Yabeda metric
|
|
1271
|
+
update_yabeda_metric(metric_config, event_data, safe_labels)
|
|
1272
|
+
|
|
1273
|
+
rescue CardinalityLimitExceeded => e
|
|
1274
|
+
# Log warning but don't fail event processing
|
|
1275
|
+
E11y.logger.warn(
|
|
1276
|
+
"[E11y Metrics] Cardinality limit exceeded: #{e.message}"
|
|
1277
|
+
)
|
|
1278
|
+
|
|
1279
|
+
# Track dropped metric
|
|
1280
|
+
Yabeda.e11y_internal.metrics_dropped_total.increment(
|
|
1281
|
+
metric_name: metric_config[:name],
|
|
1282
|
+
reason: 'cardinality_limit'
|
|
1283
|
+
)
|
|
1284
|
+
end
|
|
1285
|
+
|
|
1286
|
+
def update_yabeda_metric(config, event_data, labels)
|
|
1287
|
+
metric_name = config[:name].to_sym
|
|
1288
|
+
|
|
1289
|
+
case config[:type]
|
|
1290
|
+
when :counter
|
|
1291
|
+
Yabeda.e11y.public_send(metric_name).increment(labels)
|
|
1292
|
+
|
|
1293
|
+
when :histogram
|
|
1294
|
+
value = extract_value(config[:value], event_data)
|
|
1295
|
+
Yabeda.e11y.public_send(metric_name).observe(value, labels)
|
|
1296
|
+
|
|
1297
|
+
when :gauge
|
|
1298
|
+
value = extract_value(config[:value], event_data)
|
|
1299
|
+
Yabeda.e11y.public_send(metric_name).set(value, labels)
|
|
1300
|
+
end
|
|
1301
|
+
end
|
|
1302
|
+
|
|
1303
|
+
def extract_value(value_extractor, event_data)
|
|
1304
|
+
case value_extractor
|
|
1305
|
+
when Proc
|
|
1306
|
+
value_extractor.call(event_data)
|
|
1307
|
+
when Symbol
|
|
1308
|
+
event_data[:payload][value_extractor]
|
|
1309
|
+
when String
|
|
1310
|
+
event_data[:payload][value_extractor.to_sym]
|
|
1311
|
+
else
|
|
1312
|
+
1 # Default value for counters
|
|
1313
|
+
end
|
|
1314
|
+
end
|
|
1315
|
+
end
|
|
1316
|
+
end
|
|
1317
|
+
end
|
|
1318
|
+
```
|
|
1319
|
+
|
|
1320
|
+
---
|
|
1321
|
+
|
|
1322
|
+
### Performance Characteristics
|
|
1323
|
+
|
|
1324
|
+
**Latency:**
|
|
1325
|
+
|
|
1326
|
+
```ruby
|
|
1327
|
+
# Benchmark: Metrics middleware overhead
|
|
1328
|
+
Benchmark.ips do |x|
|
|
1329
|
+
x.report('Event without metrics') do
|
|
1330
|
+
Events::OrderPaid.track(order_id: '123', amount: 99.99)
|
|
1331
|
+
end
|
|
1332
|
+
|
|
1333
|
+
x.report('Event with 1 metric') do
|
|
1334
|
+
# Pattern: 'order.paid' โ counter
|
|
1335
|
+
Events::OrderPaid.track(order_id: '123', amount: 99.99)
|
|
1336
|
+
end
|
|
1337
|
+
|
|
1338
|
+
x.report('Event with 3 metrics') do
|
|
1339
|
+
# Patterns: counter + histogram + gauge
|
|
1340
|
+
Events::OrderPaid.track(order_id: '123', amount: 99.99)
|
|
1341
|
+
end
|
|
1342
|
+
|
|
1343
|
+
x.compare!
|
|
1344
|
+
end
|
|
1345
|
+
|
|
1346
|
+
# Results:
|
|
1347
|
+
# Without metrics: 100,000 i/s (10ฮผs per event)
|
|
1348
|
+
# With 1 metric: 95,000 i/s (10.5ฮผs per event) โ +0.5ฮผs overhead
|
|
1349
|
+
# With 3 metrics: 90,000 i/s (11ฮผs per event) โ +1ฮผs overhead
|
|
1350
|
+
#
|
|
1351
|
+
# Overhead per metric: ~0.3-0.5ฮผs
|
|
1352
|
+
```
|
|
1353
|
+
|
|
1354
|
+
**Breakdown:**
|
|
1355
|
+
- Pattern matching: ~0.1ฮผs
|
|
1356
|
+
- Label extraction: ~0.1ฮผs
|
|
1357
|
+
- Cardinality check: ~0.1ฮผs
|
|
1358
|
+
- Yabeda update: ~0.2ฮผs
|
|
1359
|
+
- **Total: ~0.5ฮผs per metric**
|
|
1360
|
+
|
|
1361
|
+
**Memory:**
|
|
1362
|
+
```ruby
|
|
1363
|
+
# Metrics middleware memory usage:
|
|
1364
|
+
# - Pattern matcher: ~10KB (patterns cache)
|
|
1365
|
+
# - Cardinality tracker: ~5MB (Redis keys for 10k metrics)
|
|
1366
|
+
# - Yabeda metrics: ~1KB per metric ร 100 metrics = 100KB
|
|
1367
|
+
#
|
|
1368
|
+
# Total: ~5.1MB (negligible)
|
|
1369
|
+
```
|
|
1370
|
+
|
|
1371
|
+
---
|
|
1372
|
+
|
|
1373
|
+
### Debugging Metrics
|
|
1374
|
+
|
|
1375
|
+
**1. Check if pattern matches:**
|
|
1376
|
+
|
|
1377
|
+
```ruby
|
|
1378
|
+
# In Rails console
|
|
1379
|
+
event_name = 'order.paid'
|
|
1380
|
+
matcher = E11y::Metrics::PatternMatcher.new(E11y.config)
|
|
1381
|
+
matched = matcher.match(event_name)
|
|
1382
|
+
|
|
1383
|
+
puts "Matched metrics for '#{event_name}':"
|
|
1384
|
+
matched.each do |m|
|
|
1385
|
+
puts " - #{m[:type]}: #{m[:name]} (pattern: #{m[:pattern]})"
|
|
1386
|
+
end
|
|
1387
|
+
```
|
|
1388
|
+
|
|
1389
|
+
**2. Check label extraction:**
|
|
1390
|
+
|
|
1391
|
+
```ruby
|
|
1392
|
+
event_data = {
|
|
1393
|
+
event_name: 'order.paid',
|
|
1394
|
+
payload: { order_id: '123', amount: 99.99, currency: 'USD' }
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
metric_config = { tags: [:currency, :order_id] }
|
|
1398
|
+
extractor = E11y::Metrics::LabelExtractor.new(metric_config, event_data)
|
|
1399
|
+
labels = extractor.extract
|
|
1400
|
+
|
|
1401
|
+
puts "Extracted labels: #{labels.inspect}"
|
|
1402
|
+
# => { currency: 'USD', order_id: '123' }
|
|
1403
|
+
```
|
|
1404
|
+
|
|
1405
|
+
**3. Check cardinality protection:**
|
|
1406
|
+
|
|
1407
|
+
```ruby
|
|
1408
|
+
protection = E11y::Metrics::CardinalityProtection.new(E11y.config)
|
|
1409
|
+
raw_labels = { currency: 'USD', order_id: '123' }
|
|
1410
|
+
safe_labels = protection.filter('orders_paid_total', raw_labels)
|
|
1411
|
+
|
|
1412
|
+
puts "Raw labels: #{raw_labels.inspect}"
|
|
1413
|
+
puts "Safe labels: #{safe_labels.inspect}"
|
|
1414
|
+
# Raw: { currency: 'USD', order_id: '123' }
|
|
1415
|
+
# Safe: { currency: 'USD' } # order_id filtered out!
|
|
1416
|
+
```
|
|
1417
|
+
|
|
1418
|
+
**4. Verify metric exists in Prometheus:**
|
|
1419
|
+
|
|
1420
|
+
```bash
|
|
1421
|
+
# Check if metric is registered
|
|
1422
|
+
curl http://localhost:9394/metrics | grep orders_paid_total
|
|
1423
|
+
|
|
1424
|
+
# Expected output:
|
|
1425
|
+
# orders_paid_total{currency="USD"} 42
|
|
1426
|
+
```
|
|
1427
|
+
|
|
1428
|
+
---
|
|
1429
|
+
|
|
1430
|
+
## ๐ Advanced Use Cases
|
|
1431
|
+
|
|
1432
|
+
### Percentile Tracking
|
|
1433
|
+
|
|
1434
|
+
```ruby
|
|
1435
|
+
# Track p50, p95, p99 latency
|
|
1436
|
+
histogram_for pattern: 'api.request',
|
|
1437
|
+
name: 'api.request.duration_seconds',
|
|
1438
|
+
value: ->(e) { e.duration_ms / 1000.0 },
|
|
1439
|
+
buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0],
|
|
1440
|
+
tags: [:controller, :action]
|
|
1441
|
+
|
|
1442
|
+
# Query in Prometheus
|
|
1443
|
+
histogram_quantile(0.50, rate(api_request_duration_seconds_bucket[5m])) # p50
|
|
1444
|
+
histogram_quantile(0.95, rate(api_request_duration_seconds_bucket[5m])) # p95
|
|
1445
|
+
histogram_quantile(0.99, rate(api_request_duration_seconds_bucket[5m])) # p99
|
|
1446
|
+
```
|
|
1447
|
+
|
|
1448
|
+
### Multi-Dimensional Aggregation
|
|
1449
|
+
|
|
1450
|
+
```ruby
|
|
1451
|
+
# Break down by multiple dimensions
|
|
1452
|
+
counter_for pattern: 'order.paid',
|
|
1453
|
+
name: 'orders.paid.total',
|
|
1454
|
+
tags: [:currency, :payment_method, :country, :plan_tier]
|
|
1455
|
+
|
|
1456
|
+
# Query in Prometheus (flexible slicing)
|
|
1457
|
+
sum(orders_paid_total{currency="USD"}) # By currency
|
|
1458
|
+
sum(orders_paid_total{payment_method="stripe"}) # By payment method
|
|
1459
|
+
sum(orders_paid_total{currency="USD",payment_method="stripe"}) # Combined
|
|
1460
|
+
sum by (country) (orders_paid_total) # Group by country
|
|
1461
|
+
```
|
|
1462
|
+
|
|
1463
|
+
### Time-Based Metrics
|
|
1464
|
+
|
|
1465
|
+
```ruby
|
|
1466
|
+
# Track events by time-of-day
|
|
1467
|
+
counter_for pattern: 'user.login',
|
|
1468
|
+
name: 'users.logins.total',
|
|
1469
|
+
tags: [:hour_of_day],
|
|
1470
|
+
tag_extractors: {
|
|
1471
|
+
hour_of_day: ->(e) { e.timestamp.hour }
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
# Track events by day-of-week
|
|
1475
|
+
counter_for pattern: 'order.paid',
|
|
1476
|
+
name: 'orders.paid.total',
|
|
1477
|
+
tags: [:day_of_week],
|
|
1478
|
+
tag_extractors: {
|
|
1479
|
+
day_of_week: ->(e) { e.timestamp.strftime('%A') } # Monday, Tuesday, ...
|
|
1480
|
+
}
|
|
1481
|
+
```
|
|
1482
|
+
|
|
1483
|
+
---
|
|
1484
|
+
|
|
1485
|
+
## ๐ก Best Practices
|
|
1486
|
+
|
|
1487
|
+
### โ
DO
|
|
1488
|
+
|
|
1489
|
+
**1. Keep cardinality low (<100 unique combinations per label)**
|
|
1490
|
+
```ruby
|
|
1491
|
+
# โ
GOOD: status has 4 values
|
|
1492
|
+
tags: [:status] # pending, paid, shipped, delivered
|
|
1493
|
+
|
|
1494
|
+
# โ BAD: user_id has 1M values
|
|
1495
|
+
tags: [:user_id] # DON'T!
|
|
1496
|
+
```
|
|
1497
|
+
|
|
1498
|
+
**2. Use meaningful metric names**
|
|
1499
|
+
```ruby
|
|
1500
|
+
# โ
GOOD: Clear, follows Prometheus conventions
|
|
1501
|
+
name: 'orders.paid.total'
|
|
1502
|
+
name: 'api.request.duration_seconds'
|
|
1503
|
+
name: 'payments.success_rate'
|
|
1504
|
+
|
|
1505
|
+
# โ BAD: Vague, non-standard
|
|
1506
|
+
name: 'counter1'
|
|
1507
|
+
name: 'latency'
|
|
1508
|
+
name: 'stuff'
|
|
1509
|
+
```
|
|
1510
|
+
|
|
1511
|
+
**3. Tag consistently across domains**
|
|
1512
|
+
```ruby
|
|
1513
|
+
# โ
GOOD: Consistent tags
|
|
1514
|
+
tags: [:currency] # All financial events
|
|
1515
|
+
tags: [:plan_tier] # All user events
|
|
1516
|
+
tags: [:region] # All geo events
|
|
1517
|
+
|
|
1518
|
+
# โ BAD: Inconsistent
|
|
1519
|
+
tags: [:curr] # Some events
|
|
1520
|
+
tags: [:currency] # Other events
|
|
1521
|
+
```
|
|
1522
|
+
|
|
1523
|
+
**4. Use appropriate metric types**
|
|
1524
|
+
```ruby
|
|
1525
|
+
# โ
GOOD: Counter for cumulative events
|
|
1526
|
+
counter_for pattern: 'order.paid'
|
|
1527
|
+
|
|
1528
|
+
# โ
GOOD: Histogram for distributions
|
|
1529
|
+
histogram_for pattern: 'api.request', value: ->(e) { e.duration_ms }
|
|
1530
|
+
|
|
1531
|
+
# โ
GOOD: Gauge for point-in-time values
|
|
1532
|
+
gauge_for pattern: 'queue.size', value: ->(e) { e.payload[:size] }
|
|
1533
|
+
```
|
|
1534
|
+
|
|
1535
|
+
---
|
|
1536
|
+
|
|
1537
|
+
### โ DON'T
|
|
1538
|
+
|
|
1539
|
+
**1. Don't use high-cardinality tags**
|
|
1540
|
+
```ruby
|
|
1541
|
+
# โ BAD: Will create millions of series
|
|
1542
|
+
tags: [:user_id, :order_id, :session_id]
|
|
1543
|
+
|
|
1544
|
+
# โ
GOOD: Aggregate
|
|
1545
|
+
tags: [:user_segment] # free, paid, enterprise (3 values)
|
|
1546
|
+
```
|
|
1547
|
+
|
|
1548
|
+
**2. Don't create duplicate metrics**
|
|
1549
|
+
```ruby
|
|
1550
|
+
# โ BAD: Same metric twice
|
|
1551
|
+
counter_for pattern: 'order.paid', name: 'orders.total'
|
|
1552
|
+
counter_for pattern: 'order.*', name: 'orders.total' # Duplicate!
|
|
1553
|
+
|
|
1554
|
+
# โ
GOOD: One metric per pattern
|
|
1555
|
+
counter_for pattern: 'order.*', name: 'orders.events.total'
|
|
1556
|
+
```
|
|
1557
|
+
|
|
1558
|
+
**3. Don't ignore buckets for histograms**
|
|
1559
|
+
```ruby
|
|
1560
|
+
# โ BAD: Default buckets may not fit your data
|
|
1561
|
+
histogram_for pattern: 'api.request', value: ->(e) { e.duration_ms }
|
|
1562
|
+
|
|
1563
|
+
# โ
GOOD: Explicit buckets for your scale
|
|
1564
|
+
histogram_for pattern: 'api.request',
|
|
1565
|
+
value: ->(e) { e.duration_ms },
|
|
1566
|
+
buckets: [10, 50, 100, 500, 1000, 5000] # milliseconds
|
|
1567
|
+
```
|
|
1568
|
+
|
|
1569
|
+
---
|
|
1570
|
+
|
|
1571
|
+
## ๐งช Testing
|
|
1572
|
+
|
|
1573
|
+
```ruby
|
|
1574
|
+
# spec/e11y/metrics_spec.rb
|
|
1575
|
+
RSpec.describe 'E11y Metrics' do
|
|
1576
|
+
before do
|
|
1577
|
+
E11y.configure do |config|
|
|
1578
|
+
config.metrics do
|
|
1579
|
+
counter_for pattern: 'order.paid',
|
|
1580
|
+
name: 'orders.paid.total',
|
|
1581
|
+
tags: [:currency]
|
|
1582
|
+
end
|
|
1583
|
+
end
|
|
1584
|
+
end
|
|
1585
|
+
|
|
1586
|
+
it 'creates counter metric from event' do
|
|
1587
|
+
# Track event
|
|
1588
|
+
Events::OrderPaid.track(
|
|
1589
|
+
order_id: '123',
|
|
1590
|
+
amount: 99.99,
|
|
1591
|
+
currency: 'USD'
|
|
1592
|
+
)
|
|
1593
|
+
|
|
1594
|
+
# Verify metric was created
|
|
1595
|
+
metric = Yabeda.orders.paid.total
|
|
1596
|
+
expect(metric.values).to include(
|
|
1597
|
+
{ currency: 'USD' } => 1
|
|
1598
|
+
)
|
|
1599
|
+
end
|
|
1600
|
+
|
|
1601
|
+
it 'aggregates multiple events' do
|
|
1602
|
+
3.times do
|
|
1603
|
+
Events::OrderPaid.track(
|
|
1604
|
+
order_id: SecureRandom.uuid,
|
|
1605
|
+
amount: rand(100),
|
|
1606
|
+
currency: 'USD'
|
|
1607
|
+
)
|
|
1608
|
+
end
|
|
1609
|
+
|
|
1610
|
+
metric = Yabeda.orders.paid.total
|
|
1611
|
+
expect(metric.values[{ currency: 'USD' }]).to eq(3)
|
|
1612
|
+
end
|
|
1613
|
+
end
|
|
1614
|
+
```
|
|
1615
|
+
|
|
1616
|
+
---
|
|
1617
|
+
|
|
1618
|
+
## ๐ Related Use Cases
|
|
1619
|
+
|
|
1620
|
+
- **[UC-002: Business Event Tracking](./UC-002-business-event-tracking.md)** - Event definitions
|
|
1621
|
+
- **[UC-013: High Cardinality Protection](./UC-013-high-cardinality-protection.md)** - Prevent metric explosions
|
|
1622
|
+
|
|
1623
|
+
---
|
|
1624
|
+
|
|
1625
|
+
**Document Version:** 1.0
|
|
1626
|
+
**Last Updated:** January 12, 2026
|
|
1627
|
+
**Status:** โ
Complete
|