e11y 0.2.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +130 -10
  3. data/CHANGELOG.md +56 -1
  4. data/CLAUDE.md +168 -0
  5. data/CONTRIBUTING.md +640 -0
  6. data/README.md +134 -702
  7. data/RELEASE.md +18 -3
  8. data/Rakefile +108 -29
  9. data/config/README.md +1 -1
  10. data/config/loki-local-config.yaml +12 -0
  11. data/config/otel-collector-config.yaml +44 -0
  12. data/cucumber.yml +1 -0
  13. data/docker-compose.yml +18 -2
  14. data/docs/ADAPTERS.md +76 -0
  15. data/docs/ADAPTIVE_SAMPLING.md +59 -0
  16. data/docs/COMPARISON.md +104 -0
  17. data/docs/CONFIGURATION.md +52 -0
  18. data/docs/DISTRIBUTED_TRACING.md +44 -0
  19. data/docs/LIMITATIONS.md +13 -0
  20. data/docs/METRICS_DSL.md +84 -0
  21. data/docs/PERFORMANCE.md +60 -0
  22. data/docs/PII_FILTERING.md +40 -0
  23. data/docs/PRESETS.md +65 -0
  24. data/docs/QUICK-START.md +546 -587
  25. data/docs/RAILS_INTEGRATION.md +29 -0
  26. data/docs/SCHEMA_VALIDATION.md +63 -0
  27. data/docs/SLO-PROMQL-ALERTS.md +161 -0
  28. data/docs/TESTING.md +69 -0
  29. data/docs/{ADR-001-architecture.md → architecture/ADR-001-architecture.md} +35 -64
  30. data/docs/{ADR-002-metrics-yabeda.md → architecture/ADR-002-metrics-yabeda.md} +62 -236
  31. data/docs/{ADR-003-slo-observability.md → architecture/ADR-003-slo-observability.md} +27 -466
  32. data/docs/{ADR-004-adapter-architecture.md → architecture/ADR-004-adapter-architecture.md} +163 -146
  33. data/docs/{ADR-005-tracing-context.md → architecture/ADR-005-tracing-context.md} +10 -9
  34. data/docs/{ADR-006-security-compliance.md → architecture/ADR-006-security-compliance.md} +184 -191
  35. data/docs/{ADR-007-opentelemetry-integration.md → architecture/ADR-007-opentelemetry-integration.md} +3 -21
  36. data/docs/{ADR-008-rails-integration.md → architecture/ADR-008-rails-integration.md} +209 -339
  37. data/docs/{ADR-009-cost-optimization.md → architecture/ADR-009-cost-optimization.md} +45 -54
  38. data/docs/architecture/ADR-010-developer-experience.md +522 -0
  39. data/docs/{ADR-011-testing-strategy.md → architecture/ADR-011-testing-strategy.md} +41 -83
  40. data/docs/{ADR-013-reliability-error-handling.md → architecture/ADR-013-reliability-error-handling.md} +37 -12
  41. data/docs/{ADR-014-event-driven-slo.md → architecture/ADR-014-event-driven-slo.md} +12 -24
  42. data/docs/{ADR-015-middleware-order.md → architecture/ADR-015-middleware-order.md} +23 -41
  43. data/docs/{ADR-016-self-monitoring-slo.md → architecture/ADR-016-self-monitoring-slo.md} +52 -349
  44. data/docs/{ADR-017-multi-rails-compatibility.md → architecture/ADR-017-multi-rails-compatibility.md} +4 -11
  45. data/docs/architecture/ADR-018-memory-optimization.md +366 -0
  46. data/docs/{ADR-INDEX.md → architecture/ADR-INDEX.md} +11 -6
  47. data/docs/{00-ICP-AND-TIMELINE.md → prd/00-ICP-AND-TIMELINE.md} +6 -6
  48. data/docs/{01-SCALE-REQUIREMENTS.md → prd/01-SCALE-REQUIREMENTS.md} +6 -6
  49. data/docs/prd/01-overview-vision.md +19 -14
  50. data/docs/use_cases/README.md +22 -23
  51. data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +50 -44
  52. data/docs/use_cases/UC-002-business-event-tracking.md +26 -95
  53. data/docs/use_cases/UC-003-event-metrics.md +66 -0
  54. data/docs/use_cases/UC-004-zero-config-slo-tracking.md +42 -101
  55. data/docs/use_cases/UC-005-sentry-integration.md +13 -15
  56. data/docs/use_cases/UC-006-trace-context-management.md +30 -28
  57. data/docs/use_cases/UC-007-pii-filtering.md +35 -87
  58. data/docs/use_cases/UC-008-opentelemetry-integration.md +51 -89
  59. data/docs/use_cases/UC-009-multi-service-tracing.md +4 -4
  60. data/docs/use_cases/UC-010-background-job-tracking.md +5 -5
  61. data/docs/use_cases/UC-011-rate-limiting.md +95 -168
  62. data/docs/use_cases/UC-012-audit-trail.md +21 -46
  63. data/docs/use_cases/UC-013-high-cardinality-protection.md +29 -167
  64. data/docs/use_cases/UC-014-adaptive-sampling.md +2 -2
  65. data/docs/use_cases/UC-015-cost-optimization.md +46 -99
  66. data/docs/use_cases/UC-016-rails-logger-migration.md +39 -213
  67. data/docs/use_cases/UC-017-local-development.md +203 -777
  68. data/docs/use_cases/UC-018-testing-events.md +3 -3
  69. data/docs/use_cases/UC-019-retention-based-routing.md +53 -106
  70. data/docs/use_cases/UC-020-event-versioning.md +8 -9
  71. data/docs/use_cases/UC-021-error-handling-retry-dlq.md +18 -22
  72. data/docs/use_cases/UC-022-event-registry.md +15 -21
  73. data/docs/use_cases/backlog.md +119 -87
  74. data/e11y.gemspec +2 -2
  75. data/gems/e11y-devtools/README.md +136 -0
  76. data/gems/e11y-devtools/config/routes.rb +8 -0
  77. data/gems/e11y-devtools/e11y-devtools.gemspec +25 -0
  78. data/gems/e11y-devtools/exe/e11y +34 -0
  79. data/gems/e11y-devtools/lib/e11y/devtools/mcp/server.rb +96 -0
  80. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tool_base.rb +25 -0
  81. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/clear.rb +31 -0
  82. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/errors.rb +35 -0
  83. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/event_detail.rb +33 -0
  84. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/events_by_trace.rb +33 -0
  85. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/interactions.rb +40 -0
  86. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/recent_events.rb +34 -0
  87. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/search.rb +34 -0
  88. data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/stats.rb +30 -0
  89. data/gems/e11y-devtools/lib/e11y/devtools/overlay/assets/overlay.js +115 -0
  90. data/gems/e11y-devtools/lib/e11y/devtools/overlay/controller.rb +54 -0
  91. data/gems/e11y-devtools/lib/e11y/devtools/overlay/engine.rb +26 -0
  92. data/gems/e11y-devtools/lib/e11y/devtools/overlay/middleware.rb +80 -0
  93. data/gems/e11y-devtools/lib/e11y/devtools/overlay/rails_controller.rb +42 -0
  94. data/gems/e11y-devtools/lib/e11y/devtools/tui/app.rb +262 -0
  95. data/gems/e11y-devtools/lib/e11y/devtools/tui/grouping.rb +66 -0
  96. data/gems/e11y-devtools/lib/e11y/devtools/tui/widgets/event_detail.rb +62 -0
  97. data/gems/e11y-devtools/lib/e11y/devtools/tui/widgets/event_list.rb +70 -0
  98. data/gems/e11y-devtools/lib/e11y/devtools/tui/widgets/interaction_list.rb +47 -0
  99. data/gems/e11y-devtools/lib/e11y/devtools/version.rb +8 -0
  100. data/gems/e11y-devtools/lib/e11y/devtools.rb +13 -0
  101. data/gems/e11y-devtools/spec/e11y/devtools/mcp/tools_spec.rb +107 -0
  102. data/gems/e11y-devtools/spec/e11y/devtools/overlay/controller_spec.rb +58 -0
  103. data/gems/e11y-devtools/spec/e11y/devtools/overlay/middleware_spec.rb +46 -0
  104. data/gems/e11y-devtools/spec/e11y/devtools/tui/app_spec.rb +85 -0
  105. data/gems/e11y-devtools/spec/e11y/devtools/tui/grouping_spec.rb +64 -0
  106. data/gems/e11y-devtools/spec/spec_helper.rb +5 -0
  107. data/gems/e11y-devtools/spec/tui/widgets/event_list_spec.rb +44 -0
  108. data/gems/e11y-devtools/spec/tui/widgets/interaction_list_spec.rb +62 -0
  109. data/lib/e11y/adapters/audit_encrypted.rb +53 -11
  110. data/lib/e11y/adapters/base.rb +33 -34
  111. data/lib/e11y/adapters/dev_log/file_store.rb +143 -0
  112. data/lib/e11y/adapters/dev_log/query.rb +219 -0
  113. data/lib/e11y/adapters/dev_log.rb +118 -0
  114. data/lib/e11y/adapters/file.rb +3 -6
  115. data/lib/e11y/adapters/in_memory.rb +52 -5
  116. data/lib/e11y/adapters/in_memory_test.rb +29 -0
  117. data/lib/e11y/adapters/loki.rb +58 -23
  118. data/lib/e11y/adapters/null.rb +82 -0
  119. data/lib/e11y/adapters/opentelemetry_collector.rb +183 -0
  120. data/lib/e11y/adapters/otel_logs.rb +136 -23
  121. data/lib/e11y/adapters/sentry.rb +4 -7
  122. data/lib/e11y/adapters/stdout.rb +73 -7
  123. data/lib/e11y/adapters/yabeda.rb +153 -29
  124. data/lib/e11y/buffers/adaptive_buffer.rb +3 -17
  125. data/lib/e11y/buffers/{request_scoped_buffer.rb → ephemeral_buffer.rb} +72 -58
  126. data/lib/e11y/buffers/ring_buffer.rb +3 -16
  127. data/lib/e11y/configuration.rb +272 -0
  128. data/lib/e11y/console.rb +10 -17
  129. data/lib/e11y/current.rb +53 -1
  130. data/lib/e11y/debug/pipeline_inspector.rb +96 -0
  131. data/lib/e11y/documentation/generator.rb +48 -0
  132. data/lib/e11y/event/base.rb +176 -82
  133. data/lib/e11y/event/value_sampling_config.rb +1 -5
  134. data/lib/e11y/events/rails/database/query.rb +1 -4
  135. data/lib/e11y/events/rails/job/failed.rb +2 -0
  136. data/lib/e11y/instruments/active_job.rb +46 -12
  137. data/lib/e11y/instruments/rails_instrumentation.rb +49 -24
  138. data/lib/e11y/instruments/sidekiq.rb +137 -31
  139. data/lib/e11y/linters/base.rb +11 -0
  140. data/lib/e11y/linters/pii/pii_declaration_linter.rb +120 -0
  141. data/lib/e11y/linters/slo/config_consistency_linter.rb +76 -0
  142. data/lib/e11y/linters/slo/explicit_declaration_linter.rb +36 -0
  143. data/lib/e11y/linters/slo/slo_status_from_linter.rb +41 -0
  144. data/lib/e11y/logger/bridge.rb +26 -7
  145. data/lib/e11y/metrics/cardinality_protection.rb +10 -15
  146. data/lib/e11y/metrics/cardinality_tracker.rb +16 -6
  147. data/lib/e11y/metrics/registry.rb +3 -5
  148. data/lib/e11y/metrics/test_backend.rb +62 -0
  149. data/lib/e11y/metrics.rb +56 -10
  150. data/lib/e11y/middleware/adapter_resolver.rb +40 -0
  151. data/lib/e11y/middleware/audit_signing.rb +43 -6
  152. data/lib/e11y/middleware/baggage_protection.rb +75 -0
  153. data/lib/e11y/middleware/dev_log_source.rb +24 -0
  154. data/lib/e11y/middleware/event_slo.rb +23 -9
  155. data/lib/e11y/middleware/otel_span.rb +23 -0
  156. data/lib/e11y/middleware/pii_filter.rb +104 -75
  157. data/lib/e11y/middleware/rate_limiting.rb +54 -27
  158. data/lib/e11y/middleware/request.rb +70 -23
  159. data/lib/e11y/middleware/routing.rb +78 -21
  160. data/lib/e11y/middleware/sampling.rb +66 -17
  161. data/lib/e11y/middleware/self_monitoring_emit.rb +39 -0
  162. data/lib/e11y/middleware/trace_context.rb +45 -10
  163. data/lib/e11y/middleware/track_latency.rb +34 -0
  164. data/lib/e11y/middleware/validation.rb +7 -16
  165. data/lib/e11y/middleware/versioning.rb +26 -22
  166. data/lib/e11y/opentelemetry/semantic_conventions.rb +109 -0
  167. data/lib/e11y/opentelemetry/span_creator.rb +142 -0
  168. data/lib/e11y/pii/patterns.rb +12 -1
  169. data/lib/e11y/pipeline/builder.rb +1 -1
  170. data/lib/e11y/presets/audit_event.rb +13 -2
  171. data/lib/e11y/railtie.rb +52 -15
  172. data/lib/e11y/registry.rb +306 -0
  173. data/lib/e11y/reliability/circuit_breaker.rb +19 -21
  174. data/lib/e11y/reliability/dlq/base.rb +71 -0
  175. data/lib/e11y/reliability/dlq/file_adapter.rb +301 -0
  176. data/lib/e11y/reliability/dlq/file_storage.rb +63 -34
  177. data/lib/e11y/reliability/dlq/filter.rb +37 -54
  178. data/lib/e11y/reliability/retry_handler.rb +26 -29
  179. data/lib/e11y/reliability/retry_rate_limiter.rb +3 -11
  180. data/lib/e11y/sampling/error_spike_detector.rb +0 -2
  181. data/lib/e11y/sampling/load_monitor.rb +5 -9
  182. data/lib/e11y/sampling/stratified_tracker.rb +18 -0
  183. data/lib/e11y/self_monitoring/buffer_monitor.rb +2 -0
  184. data/lib/e11y/self_monitoring/performance_monitor.rb +19 -61
  185. data/lib/e11y/self_monitoring/reliability_monitor.rb +4 -74
  186. data/lib/e11y/slo/config_loader.rb +40 -0
  187. data/lib/e11y/slo/config_validator.rb +58 -0
  188. data/lib/e11y/slo/dashboard_generator.rb +122 -0
  189. data/lib/e11y/slo/event_driven.rb +8 -0
  190. data/lib/e11y/slo/tracker.rb +31 -4
  191. data/lib/e11y/testing/have_tracked_event_matcher.rb +190 -0
  192. data/lib/e11y/testing/rspec_matchers.rb +21 -0
  193. data/lib/e11y/testing/snapshot_matcher.rb +86 -0
  194. data/lib/e11y/trace_context/sampler.rb +35 -0
  195. data/lib/e11y/tracing/faraday_middleware.rb +31 -0
  196. data/lib/e11y/tracing/net_http_patch.rb +33 -0
  197. data/lib/e11y/tracing/propagator.rb +116 -0
  198. data/lib/e11y/tracing.rb +47 -0
  199. data/lib/e11y/version.rb +1 -1
  200. data/lib/e11y/versioning/version_extractor.rb +32 -0
  201. data/lib/e11y.rb +141 -265
  202. data/lib/generators/e11y/event/event_generator.rb +22 -0
  203. data/lib/generators/e11y/event/templates/event.rb.tt +16 -0
  204. data/lib/generators/e11y/grafana_dashboard/grafana_dashboard_generator.rb +30 -0
  205. data/lib/generators/e11y/grafana_dashboard/templates/e11y_dashboard.json +81 -0
  206. data/lib/generators/e11y/install/install_generator.rb +34 -0
  207. data/lib/generators/e11y/install/templates/e11y.rb +239 -0
  208. data/lib/generators/e11y/prometheus_alerts/prometheus_alerts_generator.rb +29 -0
  209. data/lib/generators/e11y/prometheus_alerts/templates/e11y_alerts.yml +28 -0
  210. data/lib/tasks/e11y_docs.rake +30 -0
  211. data/lib/tasks/e11y_events.rake +71 -0
  212. data/lib/tasks/e11y_lint.rake +91 -0
  213. data/lib/tasks/e11y_slo.rake +29 -0
  214. metadata +129 -39
  215. data/docs/ADR-010-developer-experience.md +0 -2166
  216. data/docs/API-REFERENCE-L28.md +0 -914
  217. data/docs/COMPREHENSIVE-CONFIGURATION.md +0 -2366
  218. data/docs/CONTRIBUTING.md +0 -312
  219. data/docs/IMPLEMENTATION_NOTES.md +0 -2804
  220. data/docs/IMPLEMENTATION_PLAN.md +0 -1971
  221. data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +0 -586
  222. data/docs/PLAN.md +0 -148
  223. data/docs/README.md +0 -296
  224. data/docs/design/00-memory-optimization.md +0 -593
  225. data/docs/guides/MIGRATION-L27-L28.md +0 -692
  226. data/docs/guides/PERFORMANCE-BENCHMARKS.md +0 -434
  227. data/docs/guides/README.md +0 -44
  228. data/docs/use_cases/UC-003-pattern-based-metrics.md +0 -1627
  229. data/lib/e11y/adapters/registry.rb +0 -141
  230. /data/docs/{ADR-012-event-evolution.md → architecture/ADR-012-event-evolution.md} +0 -0
@@ -41,11 +41,8 @@ module E11y
41
41
  # batch_timeout: 5
42
42
  # )
43
43
  #
44
- # @example With Registry
45
- # E11y::Adapters::Registry.register(
46
- # :loki_logger,
47
- # E11y::Adapters::Loki.new(url: ENV["LOKI_URL"])
48
- # )
44
+ # @example Configuration
45
+ # config.adapters[:loki] = E11y::Adapters::Loki.new(url: ENV["LOKI_URL"])
49
46
  #
50
47
  # @example With Cardinality Protection (C04 Resolution - Enterprise)
51
48
  # # Enable for high-traffic environments to prevent label explosion
@@ -82,8 +79,9 @@ module E11y
82
79
  # @option config [Integer] :batch_timeout (5) Max seconds to wait before flushing batch
83
80
  # @option config [Boolean] :compress (true) Enable gzip compression
84
81
  # @option config [String] :tenant_id (nil) Loki tenant ID (X-Scope-OrgID header)
85
- # @option config [Boolean] :enable_cardinality_protection (false) Enable cardinality protection for labels (C04)
86
- # @option config [Integer] :max_label_cardinality (100) Max unique values per label when protection enabled
82
+ # @option config [Boolean] :enable_cardinality_protection (true) Enable cardinality protection for labels (C04)
83
+ # @option config [Integer] :max_label_cardinality (1000) Max unique values per label when protection enabled.
84
+ # Labels = event_name + severity only (payload stays in log line). 1000 covers ~1000 event types.
87
85
  # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
88
86
  # Adapter initialization requires many instance variable assignments
89
87
  def initialize(config = {})
@@ -91,20 +89,22 @@ module E11y
91
89
  @labels = config.fetch(:labels, {})
92
90
  @batch_size = config.fetch(:batch_size, DEFAULT_BATCH_SIZE)
93
91
  @batch_timeout = config.fetch(:batch_timeout, DEFAULT_BATCH_TIMEOUT)
92
+ @timeout = config.fetch(:timeout, 5)
93
+ @health_check_timeout = [@timeout, 2].min
94
94
  @compress = config.fetch(:compress, true)
95
95
  @tenant_id = config[:tenant_id]
96
- @enable_cardinality_protection = config.fetch(:enable_cardinality_protection, false)
97
- @max_label_cardinality = config.fetch(:max_label_cardinality, 100)
96
+ @enable_cardinality_protection = config.fetch(:enable_cardinality_protection, true)
97
+ @max_label_cardinality = config.fetch(:max_label_cardinality, 1000)
98
98
 
99
99
  @buffer = []
100
100
  @buffer_mutex = Mutex.new
101
101
  @connection = nil
102
102
  @last_flush = Time.now
103
103
 
104
- # C04: Optional cardinality protection (disabled by default for logs)
104
+ # C04: Cardinality protection for labels (enabled by default per ADR-009 §8)
105
105
  if @enable_cardinality_protection
106
106
  @cardinality_protection = E11y::Metrics::CardinalityProtection.new(
107
- max_unique_values: @max_label_cardinality
107
+ cardinality_limit: @max_label_cardinality
108
108
  )
109
109
  end
110
110
 
@@ -155,11 +155,22 @@ module E11y
155
155
  end
156
156
  end
157
157
 
158
- # Check if adapter is healthy
158
+ # Loki health check endpoint
159
+ READY_PATH = "/ready"
160
+
161
+ # Check if adapter is healthy (Loki server reachable)
162
+ #
163
+ # Performs actual HTTP GET to /ready. Returns false on connection failure,
164
+ # timeout, or non-2xx response.
159
165
  #
160
- # @return [Boolean] True if connection is established
166
+ # @return [Boolean] True if Loki responds with 2xx
161
167
  def healthy?
162
- @connection&.respond_to?(:get)
168
+ return false unless @connection
169
+
170
+ response = @connection.get(READY_PATH)
171
+ (200..299).cover?(response.status)
172
+ rescue Faraday::Error, Errno::ECONNREFUSED, Errno::ETIMEDOUT
173
+ false
163
174
  end
164
175
 
165
176
  # Adapter capabilities
@@ -194,7 +205,6 @@ module E11y
194
205
  #
195
206
  # @see ADR-004 Section 7.1 (Retry Policy via gem-level middleware)
196
207
  # @see ADR-004 Section 6.1 (Connection pooling via HTTP client)
197
- # rubocop:disable Metrics/MethodLength
198
208
  # HTTP client configuration requires detailed retry and connection settings
199
209
  def build_connection!
200
210
  @connection = Faraday.new(url: @url) do |f|
@@ -218,7 +228,6 @@ module E11y
218
228
  f.adapter Faraday.default_adapter
219
229
  end
220
230
  end
221
- # rubocop:enable Metrics/MethodLength
222
231
 
223
232
  # Check if buffer should be flushed
224
233
  def flush_if_needed!
@@ -280,22 +289,23 @@ module E11y
280
289
 
281
290
  # Extract labels from event
282
291
  #
292
+ # Uses normalized event_name (e.g., "Events::TestLoki" -> "test.loki") for consistent
293
+ # querying via LogQL. Matches Versioning middleware convention.
294
+ #
283
295
  # @param event_data [Hash] Event data
284
296
  # @return [Hash] Labels for Loki stream
285
297
  def extract_labels(event_data)
286
298
  event_labels = {
287
- event_name: event_data[:event_name].to_s,
299
+ event_name: normalize_event_name_for_labels(event_data[:event_name].to_s),
288
300
  severity: event_data[:severity].to_s
289
301
  }
290
302
 
291
303
  # Merge static and event labels
292
304
  all_labels = @labels.merge(event_labels)
293
305
 
294
- # C04: Apply cardinality protection if enabled (enterprise use case)
295
- # Disabled by default - Loki is a log system, labels are for stream filtering only
296
- if @enable_cardinality_protection && @cardinality_protection
297
- all_labels = @cardinality_protection.filter(all_labels, "loki.stream")
298
- end
306
+ # C04: Cardinality protection for labels only. Labels = event_name + severity (payload
307
+ # stays in log line). Filter by user_uuid via LogQL: | json | user_uuid="xxx"
308
+ all_labels = @cardinality_protection.filter(all_labels, "loki.stream") if @enable_cardinality_protection && @cardinality_protection
299
309
 
300
310
  all_labels.transform_keys(&:to_s)
301
311
  end
@@ -305,7 +315,17 @@ module E11y
305
315
  # @param event_data [Hash] Event data
306
316
  # @return [Array] [timestamp_ns, line]
307
317
  def format_loki_entry(event_data)
308
- timestamp_ns = (event_data[:timestamp] || Time.now).to_f * 1_000_000_000
318
+ # Parse timestamp - can be Time object, ISO8601 string, or nil
319
+ timestamp = event_data[:timestamp]
320
+ timestamp = if timestamp.is_a?(String)
321
+ Time.parse(timestamp)
322
+ elsif timestamp.nil?
323
+ Time.now
324
+ else
325
+ timestamp
326
+ end
327
+
328
+ timestamp_ns = timestamp.to_f * 1_000_000_000
309
329
  line = event_data.to_json
310
330
 
311
331
  [timestamp_ns.to_i.to_s, line]
@@ -323,6 +343,21 @@ module E11y
323
343
  io.string
324
344
  end
325
345
 
346
+ # Normalize event name for Loki labels (matches Versioning middleware convention)
347
+ #
348
+ # @param name [String] Event name (e.g., "Events::TestLoki")
349
+ # @return [String] Normalized name (e.g., "test.loki")
350
+ def normalize_event_name_for_labels(name)
351
+ return name if name.nil? || name.empty?
352
+
353
+ n = name.sub(/^Events::/, "").sub(/V\d+$/, "")
354
+ n.gsub("::", ".")
355
+ .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
356
+ .gsub(/([a-z\d])([A-Z])/, '\1_\2')
357
+ .downcase
358
+ .tr("_", ".")
359
+ end
360
+
326
361
  # Build HTTP headers
327
362
  #
328
363
  # @return [Hash] Headers for Loki request
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ module E11y
4
+ module Adapters
5
+ # Null Adapter — silently discards all events.
6
+ #
7
+ # Designed for use in tests and development environments where you want
8
+ # to suppress all output while still being able to assert that events
9
+ # were tracked (via the `events` reader).
10
+ #
11
+ # @example In tests
12
+ # RSpec.configure do |config|
13
+ # config.before do
14
+ # E11y.configure do |c|
15
+ # c.adapters[:null] = E11y::Adapters::NullAdapter.new
16
+ # end
17
+ # end
18
+ # end
19
+ #
20
+ # @example Asserting events
21
+ # null_adapter = E11y::Adapters::NullAdapter.new
22
+ # E11y.configure { |c| c.adapters[:null] = null_adapter }
23
+ #
24
+ # Events::OrderPaid.track(order_id: "123", amount: 99.99)
25
+ #
26
+ # expect(null_adapter.events.size).to eq(1)
27
+ # expect(null_adapter.events.last[:event_name]).to eq("order.paid")
28
+ class Null < Base
29
+ attr_reader :events
30
+
31
+ # @param config [Hash] Options
32
+ # @option config [Boolean] :store_events (true) When false, truly discards (no retention).
33
+ # Use store_events: false for memory profiling to measure pipeline-only allocations.
34
+ def initialize(config = {})
35
+ super
36
+ @store_events = config.fetch(:store_events, true)
37
+ @events = []
38
+ @mutex = Mutex.new
39
+ end
40
+
41
+ # Accept event. When store_events: true, stores for inspection. When false, truly discards.
42
+ #
43
+ # @param event_data [Hash] Event payload
44
+ # @return [Boolean] always true
45
+ # rubocop:disable Naming/PredicateMethod -- implements Base adapter interface
46
+ def write(event_data)
47
+ @mutex.synchronize { @events << event_data.dup } if @store_events
48
+ true
49
+ end
50
+ # rubocop:enable Naming/PredicateMethod
51
+
52
+ # Accept batch. When store_events: true, stores for inspection. When false, truly discards.
53
+ #
54
+ # @param events [Array<Hash>] Event payloads
55
+ # @return [Boolean] always true
56
+ # rubocop:disable Naming/PredicateMethod -- implements Base adapter interface
57
+ def write_batch(events)
58
+ @mutex.synchronize { @events.concat(events.map(&:dup)) } if @store_events
59
+ true
60
+ end
61
+ # rubocop:enable Naming/PredicateMethod
62
+
63
+ # Clear all stored events (useful between test examples).
64
+ #
65
+ # @return [void]
66
+ def clear!
67
+ @mutex.synchronize { @events.clear }
68
+ end
69
+
70
+ def healthy?
71
+ true
72
+ end
73
+
74
+ def capabilities
75
+ { batching: true, compression: false, async: false, streaming: false, null: true }
76
+ end
77
+ end
78
+
79
+ # Convenience alias matching Quick Start documentation.
80
+ NullAdapter = Null
81
+ end
82
+ end
@@ -0,0 +1,183 @@
1
+ # frozen_string_literal: true
2
+
3
+ # OTLP HTTP adapter — requires Faraday
4
+ begin
5
+ require "faraday"
6
+ rescue LoadError
7
+ raise LoadError, <<~ERROR
8
+ Faraday not available!
9
+
10
+ To use E11y::Adapters::OpenTelemetryCollector, add to your Gemfile:
11
+
12
+ gem 'faraday'
13
+
14
+ Then run: bundle install
15
+ ERROR
16
+ end
17
+
18
+ require "e11y/opentelemetry/semantic_conventions"
19
+
20
+ module E11y
21
+ module Adapters
22
+ # OpenTelemetry Collector adapter (ADR-007 §3, F1)
23
+ #
24
+ # Sends E11y events to OpenTelemetry Collector via OTLP HTTP.
25
+ # No OpenTelemetry SDK required — uses raw HTTP (Faraday).
26
+ #
27
+ # **Use case:** When you want to send logs to OTel Collector without
28
+ # loading the full OTel SDK (e.g. lightweight apps, or OTelLogs already
29
+ # handles in-process; this adapter sends to external Collector).
30
+ #
31
+ # @example Configuration
32
+ # E11y.configure do |config|
33
+ # config.adapters[:otel_collector] = E11y::Adapters::OpenTelemetryCollector.new(
34
+ # endpoint: "http://localhost:4318",
35
+ # service_name: "my-app"
36
+ # )
37
+ # end
38
+ #
39
+ # @see ADR-007 §3 OTel Collector Adapter
40
+ class OpenTelemetryCollector < Base
41
+ SEVERITY_MAPPING = {
42
+ debug: 5, info: 9, success: 9, warn: 13, error: 17, fatal: 21
43
+ }.freeze
44
+
45
+ def initialize(endpoint: nil, service_name: nil, headers: {}, timeout: 10, max_attributes: 50, compress: true, **)
46
+ super(**)
47
+ @endpoint = (endpoint || ENV["OTEL_EXPORTER_OTLP_ENDPOINT"] || "http://localhost:4318").chomp("/")
48
+ @service_name = service_name || E11y.config&.service_name || "e11y"
49
+ @headers = headers
50
+ @timeout = timeout
51
+ @max_attributes = max_attributes
52
+ @compress = compress
53
+ @connection = build_connection
54
+ end
55
+
56
+ def write(event_data)
57
+ payload = build_otlp_payload([event_data])
58
+ body = payload.to_json
59
+ body = compress_body(body) if @compress
60
+
61
+ response = @connection.post("/v1/logs") do |req|
62
+ req.headers["Content-Type"] = "application/json"
63
+ req.headers["Content-Encoding"] = "gzip" if @compress
64
+ req.body = body
65
+ end
66
+ response.success?
67
+ rescue Faraday::Error => e
68
+ warn "[E11y::OpenTelemetryCollector] HTTP error: #{e.message}"
69
+ false
70
+ end
71
+
72
+ def healthy?
73
+ !@connection.nil?
74
+ end
75
+
76
+ def capabilities
77
+ { batching: false, compression: @compress, async: false, streaming: false }
78
+ end
79
+
80
+ private
81
+
82
+ def compress_body(body)
83
+ io = StringIO.new
84
+ gz = Zlib::GzipWriter.new(io)
85
+ gz.write(body)
86
+ gz.close
87
+ io.string
88
+ end
89
+
90
+ def build_connection
91
+ Faraday.new(url: @endpoint, request: { timeout: @timeout }) do |f|
92
+ @headers.each { |k, v| f.headers[k.to_s] = v }
93
+ f.adapter Faraday.default_adapter
94
+ end
95
+ end
96
+
97
+ def build_otlp_payload(events)
98
+ log_records = events.map { |e| to_otel_log_record(e) }
99
+ {
100
+ resourceLogs: [{
101
+ resource: { attributes: resource_attributes },
102
+ scopeLogs: [{
103
+ scope: { name: "e11y", version: E11y::VERSION },
104
+ logRecords: log_records
105
+ }]
106
+ }]
107
+ }
108
+ end
109
+
110
+ def resource_attributes
111
+ [
112
+ { key: "service.name", value: { stringValue: @service_name } },
113
+ { key: "service.version", value: { stringValue: E11y::VERSION } },
114
+ { key: "deployment.environment", value: { stringValue: E11y.config&.environment || ENV["RAILS_ENV"] || "development" } },
115
+ { key: "host.name", value: { stringValue: hostname } },
116
+ { key: "process.pid", value: { intValue: Process.pid.to_s } }
117
+ ]
118
+ end
119
+
120
+ def hostname
121
+ require "socket"
122
+ Socket.gethostname
123
+ rescue StandardError
124
+ ENV["HOSTNAME"] || "unknown"
125
+ end
126
+
127
+ def to_otel_log_record(event)
128
+ ts = event[:timestamp] || Time.now.utc
129
+ ts_nano = (ts.to_f * 1_000_000_000).to_i
130
+ {
131
+ timeUnixNano: ts_nano.to_s,
132
+ observedTimeUnixNano: (Time.now.to_f * 1_000_000_000).to_i.to_s,
133
+ severityNumber: SEVERITY_MAPPING[event[:severity]] || 9,
134
+ severityText: (event[:severity] || :info).to_s.upcase,
135
+ body: { stringValue: event[:event_name] },
136
+ attributes: build_log_attributes(event),
137
+ traceId: encode_hex(event[:trace_id], 32),
138
+ spanId: encode_hex(event[:span_id], 16)
139
+ }.compact
140
+ end
141
+
142
+ def build_log_attributes(event)
143
+ attrs = []
144
+ attrs << { key: "event.name", value: { stringValue: event[:event_name] } }
145
+ attrs << { key: "event.version", value: { stringValue: event[:v].to_s } } if event[:v]
146
+ attrs << { key: "service.name", value: { stringValue: @service_name } }
147
+
148
+ payload = event[:payload] || {}
149
+ payload.each do |key, value|
150
+ break if attrs.size >= @max_attributes
151
+
152
+ otel_key = E11y::OpenTelemetry::SemanticConventions.map_key(event[:event_name], key)
153
+ attrs << encode_attr(otel_key, value)
154
+ end
155
+ attrs
156
+ end
157
+
158
+ def encode_attr(key, value)
159
+ case value
160
+ when String
161
+ { key: key.to_s, value: { stringValue: value } }
162
+ when Integer
163
+ { key: key.to_s, value: { intValue: value.to_s } }
164
+ when Float
165
+ { key: key.to_s, value: { doubleValue: value } }
166
+ when TrueClass, FalseClass
167
+ { key: key.to_s, value: { boolValue: value } }
168
+ else
169
+ { key: key.to_s, value: { stringValue: value.to_s } }
170
+ end
171
+ end
172
+
173
+ def encode_hex(str, expected_len)
174
+ return nil if str.to_s.empty?
175
+
176
+ s = str.to_s.gsub(/[^0-9a-fA-F]/, "")
177
+ return nil if s.length != expected_len
178
+
179
+ s.downcase
180
+ end
181
+ end
182
+ end
183
+ end
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "e11y/opentelemetry/semantic_conventions"
4
+
3
5
  # Check if OpenTelemetry SDK is available
4
6
  begin
5
7
  require "opentelemetry/sdk"
6
8
  require "opentelemetry/logs"
9
+ require "opentelemetry-logs-sdk" # Provides OpenTelemetry::SDK::Logs::LoggerProvider
7
10
  rescue LoadError
8
11
  raise LoadError, <<~ERROR
9
12
  OpenTelemetry SDK not available!
@@ -11,7 +14,8 @@ rescue LoadError
11
14
  To use E11y::Adapters::OTelLogs, add to your Gemfile:
12
15
 
13
16
  gem 'opentelemetry-sdk'
14
- gem 'opentelemetry-logs'
17
+ gem 'opentelemetry-logs-api'
18
+ gem 'opentelemetry-logs-sdk'
15
19
 
16
20
  Then run: bundle install
17
21
  ERROR
@@ -58,6 +62,12 @@ module E11y
58
62
  # @see ADR-007 for OpenTelemetry integration architecture
59
63
  # @see UC-008 for use cases
60
64
  class OTelLogs < Base
65
+ # Struct for test assertions (replaces OpenStruct per Style/OpenStructUse)
66
+ LogRecordStruct = Struct.new(
67
+ :timestamp, :observed_timestamp, :severity_number, :severity_text,
68
+ :body, :attributes, :trace_id, :span_id, :trace_flags
69
+ )
70
+
61
71
  # E11y severity → OTel severity_number mapping
62
72
  # See: https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
63
73
  # Severity numbers: TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21
@@ -70,7 +80,8 @@ module E11y
70
80
  fatal: 21 # FATAL
71
81
  }.freeze
72
82
 
73
- # Default baggage allowlist (safe keys that don't contain PII)
83
+ # Default baggage allowlist kept for reference / backward compat.
84
+ # @deprecated Pass baggage_allowlist: :all (the new default) or an explicit Array.
74
85
  DEFAULT_BAGGAGE_ALLOWLIST = %i[
75
86
  trace_id
76
87
  span_id
@@ -82,24 +93,45 @@ module E11y
82
93
  # Initialize OTel Logs adapter
83
94
  #
84
95
  # @param service_name [String] Service name for OTel (default: from config)
85
- # @param baggage_allowlist [Array<Symbol>] Allowlist of safe baggage keys
86
- # @param max_attributes [Integer] Max attributes per log (cardinality protection)
87
- def initialize(service_name: nil, baggage_allowlist: DEFAULT_BAGGAGE_ALLOWLIST, max_attributes: 50, **)
96
+ # @param baggage_allowlist [Array<Symbol>, :all] Keys to include in OTel attributes.
97
+ # `:all` (default) passes every payload key PII is already stripped upstream by
98
+ # Middleware::PIIFilter before the adapter is called.
99
+ # Pass an explicit Array for stricter filtering (backward compat).
100
+ # @param max_attributes [Integer] Max attributes per log (cardinality limit)
101
+ # @param cardinality_protection [Boolean] Use full 3-layer protection (C04). Default false for
102
+ # logs (preserves user_id, order_id for debugging). Set true for cost-sensitive OTLP backends.
103
+ # @param endpoint [String, nil] OTLP endpoint (e.g. http://localhost:4318/v1/logs).
104
+ # When set, logs are exported to OTel Collector. Default: in-process only.
105
+ def initialize(service_name: nil, baggage_allowlist: :all, max_attributes: 50, cardinality_protection: false, endpoint: nil, **)
88
106
  super(**)
89
107
  @service_name = service_name
90
108
  @baggage_allowlist = baggage_allowlist
91
109
  @max_attributes = max_attributes
110
+ @endpoint = endpoint
111
+ @use_cardinality_protection = cardinality_protection
112
+
113
+ if @use_cardinality_protection
114
+ require "e11y/metrics/cardinality_protection"
115
+ @cardinality_protection = E11y::Metrics::CardinalityProtection.new(
116
+ cardinality_limit: 1000,
117
+ overflow_strategy: :drop
118
+ )
119
+ else
120
+ @cardinality_protection = nil
121
+ end
92
122
 
93
123
  setup_logger_provider
94
124
  end
95
125
 
96
126
  # Write event to OTel Logs API
97
127
  #
128
+ # Uses Logger#on_emit (OTel SDK 0.4+) with keyword arguments.
129
+ #
98
130
  # @param event_data [Hash] Event payload
99
131
  # @return [Boolean] true on success
100
132
  def write(event_data)
101
- log_record = build_log_record(event_data)
102
- @logger.emit_log_record(log_record)
133
+ params = build_log_record_params(event_data)
134
+ @logger.on_emit(**params)
103
135
  true
104
136
  rescue StandardError => e
105
137
  warn "[E11y::OTelLogs] Failed to write event: #{e.message}"
@@ -129,19 +161,73 @@ module E11y
129
161
 
130
162
  # Setup OTel Logger Provider
131
163
  def setup_logger_provider
132
- @logger_provider = OpenTelemetry::SDK::Logs::LoggerProvider.new
164
+ resource = build_resource
165
+ @logger_provider = ::OpenTelemetry::SDK::Logs::LoggerProvider.new(resource: resource)
166
+
167
+ # Add OTLP exporter when endpoint configured (sends to OTel Collector)
168
+ if @endpoint
169
+ require "opentelemetry-exporter-otlp-logs"
170
+ exporter = ::OpenTelemetry::Exporter::OTLP::Logs::LogsExporter.new(endpoint: @endpoint)
171
+ processor = ::OpenTelemetry::SDK::Logs::Export::BatchLogRecordProcessor.new(exporter)
172
+ @logger_provider.add_log_record_processor(processor)
173
+ end
174
+
133
175
  @logger = @logger_provider.logger(
134
176
  name: "e11y",
135
177
  version: E11y::VERSION
136
178
  )
179
+ rescue LoadError => e
180
+ warn "[E11y::OTelLogs] OTLP export requested but opentelemetry-exporter-otlp-logs not available: #{e.message}"
181
+ resource = build_resource
182
+ @logger_provider ||= ::OpenTelemetry::SDK::Logs::LoggerProvider.new(resource: resource)
183
+ @logger = @logger_provider.logger(name: "e11y", version: E11y::VERSION)
184
+ end
185
+
186
+ # Build OTel Resource with full attributes (ADR-007 §7, F5).
187
+ #
188
+ # @return [::OpenTelemetry::SDK::Resources::Resource]
189
+ def build_resource
190
+ attrs = {}
191
+
192
+ # Service (required)
193
+ attrs["service.name"] = @service_name || E11y.config&.service_name || "e11y"
194
+ attrs["service.version"] = E11y::VERSION
195
+
196
+ # Deployment
197
+ attrs["deployment.environment"] = E11y.config&.environment || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
198
+
199
+ # Host
200
+ attrs["host.name"] = hostname
201
+
202
+ # Process
203
+ attrs["process.pid"] = Process.pid
204
+
205
+ # Merge with OTel default (process.runtime, telemetry.sdk) when available
206
+ base = ::OpenTelemetry::SDK::Resources::Resource
207
+ resource = base.create(attrs)
208
+ resource = base.default.merge(resource) if base.respond_to?(:default)
209
+ resource
210
+ rescue StandardError
211
+ # Fallback: minimal resource
212
+ ::OpenTelemetry::SDK::Resources::Resource.create(
213
+ "service.name" => @service_name || "e11y",
214
+ "service.version" => E11y::VERSION
215
+ )
216
+ end
217
+
218
+ def hostname
219
+ require "socket"
220
+ Socket.gethostname
221
+ rescue StandardError
222
+ ENV["HOSTNAME"] || "unknown"
137
223
  end
138
224
 
139
- # Build OTel log record from E11y event
225
+ # Build params for Logger#on_emit from E11y event
140
226
  #
141
227
  # @param event_data [Hash] E11y event payload
142
- # @return [OpenTelemetry::SDK::Logs::LogRecord] OTel log record
143
- def build_log_record(event_data)
144
- OpenTelemetry::SDK::Logs::LogRecord.new(
228
+ # @return [Hash] Keyword args for on_emit
229
+ def build_log_record_params(event_data)
230
+ {
145
231
  timestamp: event_data[:timestamp] || Time.now.utc,
146
232
  observed_timestamp: Time.now.utc,
147
233
  severity_number: map_severity(event_data[:severity]),
@@ -151,7 +237,16 @@ module E11y
151
237
  trace_id: event_data[:trace_id],
152
238
  span_id: event_data[:span_id],
153
239
  trace_flags: nil
154
- )
240
+ }
241
+ end
242
+
243
+ # Build log record struct for testing (same data as build_log_record_params)
244
+ #
245
+ # @param event_data [Hash] E11y event payload
246
+ # @return [LogRecordStruct] Struct with attributes for test assertions
247
+ def build_log_record(event_data)
248
+ params = build_log_record_params(event_data)
249
+ LogRecordStruct.new(**params)
155
250
  end
156
251
 
157
252
  # Map E11y severity to OTel severity
@@ -165,39 +260,57 @@ module E11y
165
260
  # Build OTel attributes from E11y payload
166
261
  #
167
262
  # Applies:
263
+ # - Semantic conventions (ADR-007 §4, F4) — maps known keys to OTel semantic names
168
264
  # - Cardinality protection (C04 Resolution)
169
- # - Baggage PII filtering (C08 Resolution)
265
+ # - Optional baggage allowlist filter (C08 Resolution — pass an Array to enable)
266
+ #
267
+ # By default (`baggage_allowlist: :all`) all payload keys are included.
268
+ # PII fields are stripped upstream by Middleware::PIIFilter before any adapter
269
+ # is called, so no additional filtering is needed at this layer.
170
270
  #
171
271
  # @param event_data [Hash] E11y event payload
172
272
  # @return [Hash] OTel attributes
173
273
  def build_attributes(event_data)
174
274
  attributes = {}
175
275
 
176
- # Add event metadata
276
+ # Add event metadata (low cardinality)
177
277
  attributes["event.name"] = event_data[:event_name]
178
278
  attributes["event.version"] = event_data[:v] if event_data[:v]
179
279
  attributes["service.name"] = @service_name if @service_name
180
280
 
181
- # Add payload (with cardinality protection)
182
281
  payload = event_data[:payload] || {}
183
- payload.each do |key, value|
184
- # C04: Cardinality protection - limit attributes
185
- break if attributes.size >= @max_attributes
186
282
 
187
- # C08: Baggage PII protection - only allowlisted keys
283
+ # C04: Optional cardinality protection (denylist + per-key limits). Off by default for logs.
284
+ if @cardinality_protection
285
+ payload_symbols = payload.transform_keys { |k| k.to_s.to_sym }
286
+ payload = @cardinality_protection.filter(payload_symbols, "otel_logs")
287
+ end
288
+
289
+ # Map payload to OTel semantic keys (F4)
290
+ payload.each do |key, value|
188
291
  next unless baggage_allowed?(key)
189
292
 
190
- attributes["event.#{key}"] = value
293
+ otel_key = E11y::OpenTelemetry::SemanticConventions.map_key(
294
+ event_data[:event_name],
295
+ key
296
+ )
297
+ attributes[otel_key] = value
298
+ break if attributes.size >= @max_attributes
191
299
  end
192
300
 
193
301
  attributes
194
302
  end
195
303
 
196
- # Check if key is allowed in baggage (C08 Resolution)
304
+ # Check if key is allowed in baggage.
305
+ #
306
+ # Returns true when allowlist is :all (default).
307
+ # Returns true only for listed keys when an explicit Array was configured.
197
308
  #
198
309
  # @param key [Symbol, String] Attribute key
199
- # @return [Boolean] true if key is in allowlist
310
+ # @return [Boolean]
200
311
  def baggage_allowed?(key)
312
+ return true if @baggage_allowlist == :all
313
+
201
314
  @baggage_allowlist.include?(key.to_sym)
202
315
  end
203
316
  end