e11y 0.2.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +130 -10
- data/CHANGELOG.md +56 -1
- data/CLAUDE.md +168 -0
- data/CONTRIBUTING.md +640 -0
- data/README.md +134 -702
- data/RELEASE.md +18 -3
- data/Rakefile +108 -29
- data/config/README.md +1 -1
- data/config/loki-local-config.yaml +12 -0
- data/config/otel-collector-config.yaml +44 -0
- data/cucumber.yml +1 -0
- data/docker-compose.yml +18 -2
- data/docs/ADAPTERS.md +76 -0
- data/docs/ADAPTIVE_SAMPLING.md +59 -0
- data/docs/COMPARISON.md +104 -0
- data/docs/CONFIGURATION.md +52 -0
- data/docs/DISTRIBUTED_TRACING.md +44 -0
- data/docs/LIMITATIONS.md +13 -0
- data/docs/METRICS_DSL.md +84 -0
- data/docs/PERFORMANCE.md +60 -0
- data/docs/PII_FILTERING.md +40 -0
- data/docs/PRESETS.md +65 -0
- data/docs/QUICK-START.md +546 -587
- data/docs/RAILS_INTEGRATION.md +29 -0
- data/docs/SCHEMA_VALIDATION.md +63 -0
- data/docs/SLO-PROMQL-ALERTS.md +161 -0
- data/docs/TESTING.md +69 -0
- data/docs/{ADR-001-architecture.md → architecture/ADR-001-architecture.md} +35 -64
- data/docs/{ADR-002-metrics-yabeda.md → architecture/ADR-002-metrics-yabeda.md} +62 -236
- data/docs/{ADR-003-slo-observability.md → architecture/ADR-003-slo-observability.md} +27 -466
- data/docs/{ADR-004-adapter-architecture.md → architecture/ADR-004-adapter-architecture.md} +163 -146
- data/docs/{ADR-005-tracing-context.md → architecture/ADR-005-tracing-context.md} +10 -9
- data/docs/{ADR-006-security-compliance.md → architecture/ADR-006-security-compliance.md} +184 -191
- data/docs/{ADR-007-opentelemetry-integration.md → architecture/ADR-007-opentelemetry-integration.md} +3 -21
- data/docs/{ADR-008-rails-integration.md → architecture/ADR-008-rails-integration.md} +209 -339
- data/docs/{ADR-009-cost-optimization.md → architecture/ADR-009-cost-optimization.md} +45 -54
- data/docs/architecture/ADR-010-developer-experience.md +522 -0
- data/docs/{ADR-011-testing-strategy.md → architecture/ADR-011-testing-strategy.md} +41 -83
- data/docs/{ADR-013-reliability-error-handling.md → architecture/ADR-013-reliability-error-handling.md} +37 -12
- data/docs/{ADR-014-event-driven-slo.md → architecture/ADR-014-event-driven-slo.md} +12 -24
- data/docs/{ADR-015-middleware-order.md → architecture/ADR-015-middleware-order.md} +23 -41
- data/docs/{ADR-016-self-monitoring-slo.md → architecture/ADR-016-self-monitoring-slo.md} +52 -349
- data/docs/{ADR-017-multi-rails-compatibility.md → architecture/ADR-017-multi-rails-compatibility.md} +4 -11
- data/docs/architecture/ADR-018-memory-optimization.md +366 -0
- data/docs/{ADR-INDEX.md → architecture/ADR-INDEX.md} +11 -6
- data/docs/{00-ICP-AND-TIMELINE.md → prd/00-ICP-AND-TIMELINE.md} +6 -6
- data/docs/{01-SCALE-REQUIREMENTS.md → prd/01-SCALE-REQUIREMENTS.md} +6 -6
- data/docs/prd/01-overview-vision.md +19 -14
- data/docs/use_cases/README.md +22 -23
- data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +50 -44
- data/docs/use_cases/UC-002-business-event-tracking.md +26 -95
- data/docs/use_cases/UC-003-event-metrics.md +66 -0
- data/docs/use_cases/UC-004-zero-config-slo-tracking.md +42 -101
- data/docs/use_cases/UC-005-sentry-integration.md +13 -15
- data/docs/use_cases/UC-006-trace-context-management.md +30 -28
- data/docs/use_cases/UC-007-pii-filtering.md +35 -87
- data/docs/use_cases/UC-008-opentelemetry-integration.md +51 -89
- data/docs/use_cases/UC-009-multi-service-tracing.md +4 -4
- data/docs/use_cases/UC-010-background-job-tracking.md +5 -5
- data/docs/use_cases/UC-011-rate-limiting.md +95 -168
- data/docs/use_cases/UC-012-audit-trail.md +21 -46
- data/docs/use_cases/UC-013-high-cardinality-protection.md +29 -167
- data/docs/use_cases/UC-014-adaptive-sampling.md +2 -2
- data/docs/use_cases/UC-015-cost-optimization.md +46 -99
- data/docs/use_cases/UC-016-rails-logger-migration.md +39 -213
- data/docs/use_cases/UC-017-local-development.md +203 -777
- data/docs/use_cases/UC-018-testing-events.md +3 -3
- data/docs/use_cases/UC-019-retention-based-routing.md +53 -106
- data/docs/use_cases/UC-020-event-versioning.md +8 -9
- data/docs/use_cases/UC-021-error-handling-retry-dlq.md +18 -22
- data/docs/use_cases/UC-022-event-registry.md +15 -21
- data/docs/use_cases/backlog.md +119 -87
- data/e11y.gemspec +2 -2
- data/gems/e11y-devtools/README.md +136 -0
- data/gems/e11y-devtools/config/routes.rb +8 -0
- data/gems/e11y-devtools/e11y-devtools.gemspec +25 -0
- data/gems/e11y-devtools/exe/e11y +34 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/server.rb +96 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tool_base.rb +25 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/clear.rb +31 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/errors.rb +35 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/event_detail.rb +33 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/events_by_trace.rb +33 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/interactions.rb +40 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/recent_events.rb +34 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/search.rb +34 -0
- data/gems/e11y-devtools/lib/e11y/devtools/mcp/tools/stats.rb +30 -0
- data/gems/e11y-devtools/lib/e11y/devtools/overlay/assets/overlay.js +115 -0
- data/gems/e11y-devtools/lib/e11y/devtools/overlay/controller.rb +54 -0
- data/gems/e11y-devtools/lib/e11y/devtools/overlay/engine.rb +26 -0
- data/gems/e11y-devtools/lib/e11y/devtools/overlay/middleware.rb +80 -0
- data/gems/e11y-devtools/lib/e11y/devtools/overlay/rails_controller.rb +42 -0
- data/gems/e11y-devtools/lib/e11y/devtools/tui/app.rb +262 -0
- data/gems/e11y-devtools/lib/e11y/devtools/tui/grouping.rb +66 -0
- data/gems/e11y-devtools/lib/e11y/devtools/tui/widgets/event_detail.rb +62 -0
- data/gems/e11y-devtools/lib/e11y/devtools/tui/widgets/event_list.rb +70 -0
- data/gems/e11y-devtools/lib/e11y/devtools/tui/widgets/interaction_list.rb +47 -0
- data/gems/e11y-devtools/lib/e11y/devtools/version.rb +8 -0
- data/gems/e11y-devtools/lib/e11y/devtools.rb +13 -0
- data/gems/e11y-devtools/spec/e11y/devtools/mcp/tools_spec.rb +107 -0
- data/gems/e11y-devtools/spec/e11y/devtools/overlay/controller_spec.rb +58 -0
- data/gems/e11y-devtools/spec/e11y/devtools/overlay/middleware_spec.rb +46 -0
- data/gems/e11y-devtools/spec/e11y/devtools/tui/app_spec.rb +85 -0
- data/gems/e11y-devtools/spec/e11y/devtools/tui/grouping_spec.rb +64 -0
- data/gems/e11y-devtools/spec/spec_helper.rb +5 -0
- data/gems/e11y-devtools/spec/tui/widgets/event_list_spec.rb +44 -0
- data/gems/e11y-devtools/spec/tui/widgets/interaction_list_spec.rb +62 -0
- data/lib/e11y/adapters/audit_encrypted.rb +53 -11
- data/lib/e11y/adapters/base.rb +33 -34
- data/lib/e11y/adapters/dev_log/file_store.rb +143 -0
- data/lib/e11y/adapters/dev_log/query.rb +219 -0
- data/lib/e11y/adapters/dev_log.rb +118 -0
- data/lib/e11y/adapters/file.rb +3 -6
- data/lib/e11y/adapters/in_memory.rb +52 -5
- data/lib/e11y/adapters/in_memory_test.rb +29 -0
- data/lib/e11y/adapters/loki.rb +58 -23
- data/lib/e11y/adapters/null.rb +82 -0
- data/lib/e11y/adapters/opentelemetry_collector.rb +183 -0
- data/lib/e11y/adapters/otel_logs.rb +136 -23
- data/lib/e11y/adapters/sentry.rb +4 -7
- data/lib/e11y/adapters/stdout.rb +73 -7
- data/lib/e11y/adapters/yabeda.rb +153 -29
- data/lib/e11y/buffers/adaptive_buffer.rb +3 -17
- data/lib/e11y/buffers/{request_scoped_buffer.rb → ephemeral_buffer.rb} +72 -58
- data/lib/e11y/buffers/ring_buffer.rb +3 -16
- data/lib/e11y/configuration.rb +272 -0
- data/lib/e11y/console.rb +10 -17
- data/lib/e11y/current.rb +53 -1
- data/lib/e11y/debug/pipeline_inspector.rb +96 -0
- data/lib/e11y/documentation/generator.rb +48 -0
- data/lib/e11y/event/base.rb +176 -82
- data/lib/e11y/event/value_sampling_config.rb +1 -5
- data/lib/e11y/events/rails/database/query.rb +1 -4
- data/lib/e11y/events/rails/job/failed.rb +2 -0
- data/lib/e11y/instruments/active_job.rb +46 -12
- data/lib/e11y/instruments/rails_instrumentation.rb +49 -24
- data/lib/e11y/instruments/sidekiq.rb +137 -31
- data/lib/e11y/linters/base.rb +11 -0
- data/lib/e11y/linters/pii/pii_declaration_linter.rb +120 -0
- data/lib/e11y/linters/slo/config_consistency_linter.rb +76 -0
- data/lib/e11y/linters/slo/explicit_declaration_linter.rb +36 -0
- data/lib/e11y/linters/slo/slo_status_from_linter.rb +41 -0
- data/lib/e11y/logger/bridge.rb +26 -7
- data/lib/e11y/metrics/cardinality_protection.rb +10 -15
- data/lib/e11y/metrics/cardinality_tracker.rb +16 -6
- data/lib/e11y/metrics/registry.rb +3 -5
- data/lib/e11y/metrics/test_backend.rb +62 -0
- data/lib/e11y/metrics.rb +56 -10
- data/lib/e11y/middleware/adapter_resolver.rb +40 -0
- data/lib/e11y/middleware/audit_signing.rb +43 -6
- data/lib/e11y/middleware/baggage_protection.rb +75 -0
- data/lib/e11y/middleware/dev_log_source.rb +24 -0
- data/lib/e11y/middleware/event_slo.rb +23 -9
- data/lib/e11y/middleware/otel_span.rb +23 -0
- data/lib/e11y/middleware/pii_filter.rb +104 -75
- data/lib/e11y/middleware/rate_limiting.rb +54 -27
- data/lib/e11y/middleware/request.rb +70 -23
- data/lib/e11y/middleware/routing.rb +78 -21
- data/lib/e11y/middleware/sampling.rb +66 -17
- data/lib/e11y/middleware/self_monitoring_emit.rb +39 -0
- data/lib/e11y/middleware/trace_context.rb +45 -10
- data/lib/e11y/middleware/track_latency.rb +34 -0
- data/lib/e11y/middleware/validation.rb +7 -16
- data/lib/e11y/middleware/versioning.rb +26 -22
- data/lib/e11y/opentelemetry/semantic_conventions.rb +109 -0
- data/lib/e11y/opentelemetry/span_creator.rb +142 -0
- data/lib/e11y/pii/patterns.rb +12 -1
- data/lib/e11y/pipeline/builder.rb +1 -1
- data/lib/e11y/presets/audit_event.rb +13 -2
- data/lib/e11y/railtie.rb +52 -15
- data/lib/e11y/registry.rb +306 -0
- data/lib/e11y/reliability/circuit_breaker.rb +19 -21
- data/lib/e11y/reliability/dlq/base.rb +71 -0
- data/lib/e11y/reliability/dlq/file_adapter.rb +301 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +63 -34
- data/lib/e11y/reliability/dlq/filter.rb +37 -54
- data/lib/e11y/reliability/retry_handler.rb +26 -29
- data/lib/e11y/reliability/retry_rate_limiter.rb +3 -11
- data/lib/e11y/sampling/error_spike_detector.rb +0 -2
- data/lib/e11y/sampling/load_monitor.rb +5 -9
- data/lib/e11y/sampling/stratified_tracker.rb +18 -0
- data/lib/e11y/self_monitoring/buffer_monitor.rb +2 -0
- data/lib/e11y/self_monitoring/performance_monitor.rb +19 -61
- data/lib/e11y/self_monitoring/reliability_monitor.rb +4 -74
- data/lib/e11y/slo/config_loader.rb +40 -0
- data/lib/e11y/slo/config_validator.rb +58 -0
- data/lib/e11y/slo/dashboard_generator.rb +122 -0
- data/lib/e11y/slo/event_driven.rb +8 -0
- data/lib/e11y/slo/tracker.rb +31 -4
- data/lib/e11y/testing/have_tracked_event_matcher.rb +190 -0
- data/lib/e11y/testing/rspec_matchers.rb +21 -0
- data/lib/e11y/testing/snapshot_matcher.rb +86 -0
- data/lib/e11y/trace_context/sampler.rb +35 -0
- data/lib/e11y/tracing/faraday_middleware.rb +31 -0
- data/lib/e11y/tracing/net_http_patch.rb +33 -0
- data/lib/e11y/tracing/propagator.rb +116 -0
- data/lib/e11y/tracing.rb +47 -0
- data/lib/e11y/version.rb +1 -1
- data/lib/e11y/versioning/version_extractor.rb +32 -0
- data/lib/e11y.rb +141 -265
- data/lib/generators/e11y/event/event_generator.rb +22 -0
- data/lib/generators/e11y/event/templates/event.rb.tt +16 -0
- data/lib/generators/e11y/grafana_dashboard/grafana_dashboard_generator.rb +30 -0
- data/lib/generators/e11y/grafana_dashboard/templates/e11y_dashboard.json +81 -0
- data/lib/generators/e11y/install/install_generator.rb +34 -0
- data/lib/generators/e11y/install/templates/e11y.rb +239 -0
- data/lib/generators/e11y/prometheus_alerts/prometheus_alerts_generator.rb +29 -0
- data/lib/generators/e11y/prometheus_alerts/templates/e11y_alerts.yml +28 -0
- data/lib/tasks/e11y_docs.rake +30 -0
- data/lib/tasks/e11y_events.rake +71 -0
- data/lib/tasks/e11y_lint.rake +91 -0
- data/lib/tasks/e11y_slo.rake +29 -0
- metadata +129 -39
- data/docs/ADR-010-developer-experience.md +0 -2166
- data/docs/API-REFERENCE-L28.md +0 -914
- data/docs/COMPREHENSIVE-CONFIGURATION.md +0 -2366
- data/docs/CONTRIBUTING.md +0 -312
- data/docs/IMPLEMENTATION_NOTES.md +0 -2804
- data/docs/IMPLEMENTATION_PLAN.md +0 -1971
- data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +0 -586
- data/docs/PLAN.md +0 -148
- data/docs/README.md +0 -296
- data/docs/design/00-memory-optimization.md +0 -593
- data/docs/guides/MIGRATION-L27-L28.md +0 -692
- data/docs/guides/PERFORMANCE-BENCHMARKS.md +0 -434
- data/docs/guides/README.md +0 -44
- data/docs/use_cases/UC-003-pattern-based-metrics.md +0 -1627
- data/lib/e11y/adapters/registry.rb +0 -141
- /data/docs/{ADR-012-event-evolution.md → architecture/ADR-012-event-evolution.md} +0 -0
|
@@ -369,12 +369,12 @@ module E11y
|
|
|
369
369
|
end
|
|
370
370
|
|
|
371
371
|
def no_events_message
|
|
372
|
-
|
|
372
|
+
events = E11y.test_adapter.events
|
|
373
373
|
|
|
374
|
-
if
|
|
374
|
+
if events.empty?
|
|
375
375
|
"expected to have tracked #{event_name}, but no events were tracked at all"
|
|
376
376
|
else
|
|
377
|
-
tracked_names =
|
|
377
|
+
tracked_names = events.map { |e| e[:event_name] }.uniq.join(', ')
|
|
378
378
|
"expected to have tracked #{event_name}, but only tracked: #{tracked_names}"
|
|
379
379
|
end
|
|
380
380
|
end
|
|
@@ -495,78 +495,38 @@ end
|
|
|
495
495
|
|
|
496
496
|
```ruby
|
|
497
497
|
# lib/e11y/adapters/in_memory.rb
|
|
498
|
+
# API: write, write_batch (adapter contract), events, find_events, clear!, any_event?
|
|
498
499
|
module E11y
|
|
499
500
|
module Adapters
|
|
500
501
|
class InMemory < Base
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
@mutex
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
def send_batch(events)
|
|
508
|
-
@mutex.synchronize do
|
|
509
|
-
events.each do |event|
|
|
510
|
-
@events << event.dup
|
|
511
|
-
end
|
|
512
|
-
end
|
|
513
|
-
|
|
514
|
-
{ success: true, sent: events.size }
|
|
502
|
+
attr_reader :events
|
|
503
|
+
|
|
504
|
+
def write(event_data)
|
|
505
|
+
@mutex.synchronize { @events << event_data }
|
|
506
|
+
true
|
|
515
507
|
end
|
|
516
|
-
|
|
517
|
-
def
|
|
508
|
+
|
|
509
|
+
def write_batch(events)
|
|
510
|
+
@mutex.synchronize { @events.concat(events) }
|
|
518
511
|
true
|
|
519
512
|
end
|
|
520
|
-
|
|
513
|
+
|
|
521
514
|
# Test helper methods
|
|
522
|
-
def
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
def find_events(pattern)
|
|
527
|
-
@mutex.synchronize do
|
|
528
|
-
@events.select do |event|
|
|
529
|
-
case pattern
|
|
530
|
-
when String
|
|
531
|
-
event[:event_name] == pattern
|
|
532
|
-
when Regexp
|
|
533
|
-
event[:event_name] =~ pattern
|
|
534
|
-
when Class
|
|
535
|
-
event[:event_name] == pattern.event_name
|
|
536
|
-
else
|
|
537
|
-
false
|
|
538
|
-
end
|
|
539
|
-
end
|
|
540
|
-
end
|
|
541
|
-
end
|
|
542
|
-
|
|
543
|
-
def find_event(pattern)
|
|
544
|
-
find_events(pattern).first
|
|
515
|
+
def find_events(pattern) # pattern: String or Regexp
|
|
516
|
+
pattern = Regexp.new(Regexp.escape(pattern)) if pattern.is_a?(String)
|
|
517
|
+
@events.select { |e| e[:event_name].to_s.match?(pattern) }
|
|
545
518
|
end
|
|
546
|
-
|
|
547
|
-
def event_count(
|
|
548
|
-
|
|
549
|
-
find_events(pattern).size
|
|
550
|
-
else
|
|
551
|
-
@mutex.synchronize { @events.size }
|
|
552
|
-
end
|
|
519
|
+
|
|
520
|
+
def event_count(event_name = nil)
|
|
521
|
+
event_name ? @events.count { |e| e[:event_name] == event_name } : @events.size
|
|
553
522
|
end
|
|
554
|
-
|
|
523
|
+
|
|
555
524
|
def clear!
|
|
556
|
-
@
|
|
557
|
-
end
|
|
558
|
-
|
|
559
|
-
def tracked?(event_class_or_pattern)
|
|
560
|
-
find_events(event_class_or_pattern).any?
|
|
561
|
-
end
|
|
562
|
-
|
|
563
|
-
# Pretty print for debugging
|
|
564
|
-
def inspect
|
|
565
|
-
"#<E11y::Adapters::InMemory events=#{@events.size}>"
|
|
525
|
+
@events.clear
|
|
566
526
|
end
|
|
567
|
-
|
|
568
|
-
def
|
|
569
|
-
|
|
527
|
+
|
|
528
|
+
def any_event?(pattern) # true if any events match pattern
|
|
529
|
+
find_events(pattern).any?
|
|
570
530
|
end
|
|
571
531
|
end
|
|
572
532
|
end
|
|
@@ -590,7 +550,7 @@ module E11y
|
|
|
590
550
|
@call_count = 0
|
|
591
551
|
end
|
|
592
552
|
|
|
593
|
-
def
|
|
553
|
+
def write_batch(events)
|
|
594
554
|
@call_count += 1
|
|
595
555
|
|
|
596
556
|
# Simulate delay
|
|
@@ -634,7 +594,7 @@ RSpec.configure do |config|
|
|
|
634
594
|
e11y_config.adapters.register :test, E11y::Adapters::InMemory.new
|
|
635
595
|
|
|
636
596
|
# Disable rate limiting in tests
|
|
637
|
-
e11y_config.
|
|
597
|
+
e11y_config.rate_limiting_enabled = false
|
|
638
598
|
|
|
639
599
|
# Disable sampling (track everything)
|
|
640
600
|
e11y_config.sampling.default_sample_rate = 1.0
|
|
@@ -661,10 +621,10 @@ RSpec.configure do |config|
|
|
|
661
621
|
config.alias_example_group_to :describe_event, type: :event
|
|
662
622
|
end
|
|
663
623
|
|
|
664
|
-
# Convenience method to access test adapter
|
|
624
|
+
# Convenience method to access test adapter (uses config.adapters[:test] or [:memory])
|
|
665
625
|
module E11y
|
|
666
626
|
def self.test_adapter
|
|
667
|
-
|
|
627
|
+
configuration.adapters[:test] || configuration.adapters[:memory]
|
|
668
628
|
end
|
|
669
629
|
end
|
|
670
630
|
```
|
|
@@ -963,7 +923,7 @@ RSpec.describe 'Order flow integration', type: :integration do
|
|
|
963
923
|
.and have_tracked_event(Events::EmailQueued)
|
|
964
924
|
|
|
965
925
|
# Verify event sequence
|
|
966
|
-
events = E11y.test_adapter.
|
|
926
|
+
events = E11y.test_adapter.events
|
|
967
927
|
event_names = events.map { |e| e[:event_name] }
|
|
968
928
|
|
|
969
929
|
expect(event_names).to eq([
|
|
@@ -1050,16 +1010,16 @@ end
|
|
|
1050
1010
|
RSpec.shared_examples 'an E11y adapter' do
|
|
1051
1011
|
let(:adapter) { described_class.new }
|
|
1052
1012
|
|
|
1053
|
-
describe '#
|
|
1013
|
+
describe '#write_batch' do
|
|
1054
1014
|
it 'accepts array of events' do
|
|
1055
1015
|
events = [build(:event)]
|
|
1056
1016
|
|
|
1057
|
-
expect { adapter.
|
|
1017
|
+
expect { adapter.write_batch(events) }.not_to raise_error
|
|
1058
1018
|
end
|
|
1059
1019
|
|
|
1060
1020
|
it 'returns success hash' do
|
|
1061
1021
|
events = [build(:event)]
|
|
1062
|
-
result = adapter.
|
|
1022
|
+
result = adapter.write_batch(events)
|
|
1063
1023
|
|
|
1064
1024
|
expect(result).to be_a(Hash)
|
|
1065
1025
|
expect(result).to have_key(:success)
|
|
@@ -1067,11 +1027,11 @@ RSpec.shared_examples 'an E11y adapter' do
|
|
|
1067
1027
|
end
|
|
1068
1028
|
|
|
1069
1029
|
it 'handles empty batch' do
|
|
1070
|
-
expect { adapter.
|
|
1030
|
+
expect { adapter.write_batch([]) }.not_to raise_error
|
|
1071
1031
|
end
|
|
1072
1032
|
|
|
1073
1033
|
it 'raises on invalid input' do
|
|
1074
|
-
expect { adapter.
|
|
1034
|
+
expect { adapter.write_batch(nil) }.to raise_error(ArgumentError)
|
|
1075
1035
|
end
|
|
1076
1036
|
end
|
|
1077
1037
|
|
|
@@ -1537,8 +1497,8 @@ RSpec.describe 'Rate limiting', type: :integration do
|
|
|
1537
1497
|
|
|
1538
1498
|
before do
|
|
1539
1499
|
E11y.configure do |config|
|
|
1540
|
-
config.
|
|
1541
|
-
config.
|
|
1500
|
+
config.rate_limiting_enabled = true
|
|
1501
|
+
config.rate_limiting_global_limit = 10 # 10 events/sec
|
|
1542
1502
|
end
|
|
1543
1503
|
end
|
|
1544
1504
|
|
|
@@ -1554,7 +1514,7 @@ RSpec.describe 'Rate limiting', type: :integration do
|
|
|
1554
1514
|
end
|
|
1555
1515
|
|
|
1556
1516
|
it 'never drops critical events (bypass rate limit)' do
|
|
1557
|
-
E11y.config.
|
|
1517
|
+
E11y.config.rate_limiting_global_limit = 1 # Extremely low
|
|
1558
1518
|
|
|
1559
1519
|
# Track 10 critical events
|
|
1560
1520
|
10.times { Events::PaymentFailed.track(payment_id: 'p123') }
|
|
@@ -1564,9 +1524,7 @@ RSpec.describe 'Rate limiting', type: :integration do
|
|
|
1564
1524
|
end
|
|
1565
1525
|
|
|
1566
1526
|
it 'applies per-event rate limits' do
|
|
1567
|
-
E11y.config.
|
|
1568
|
-
'Events::DebugQuery' => 5 # Max 5 per second
|
|
1569
|
-
}
|
|
1527
|
+
E11y.config.add_rate_limit_per_event('Events::DebugQuery', limit: 5) # Max 5 per second
|
|
1570
1528
|
|
|
1571
1529
|
10.times { Events::DebugQuery.track(sql: 'SELECT 1') }
|
|
1572
1530
|
|
|
@@ -1659,12 +1617,12 @@ RSpec.configure do |config|
|
|
|
1659
1617
|
next unless E11y.config.pii.strict_mode
|
|
1660
1618
|
|
|
1661
1619
|
# Get all events from all adapters
|
|
1662
|
-
|
|
1663
|
-
adapter.respond_to?(:
|
|
1620
|
+
events = E11y.config.adapters.all.flat_map do |adapter|
|
|
1621
|
+
adapter.respond_to?(:events) ? adapter.events : []
|
|
1664
1622
|
end
|
|
1665
1623
|
|
|
1666
1624
|
# Verify no plain-text PII (if event declares contains_pii)
|
|
1667
|
-
|
|
1625
|
+
events.each do |event|
|
|
1668
1626
|
event_class = E11y::Registry.get(event[:event_name])
|
|
1669
1627
|
|
|
1670
1628
|
if event_class&.contains_pii?
|
|
@@ -1038,11 +1038,11 @@ module E11y
|
|
|
1038
1038
|
class SidekiqErrorHandlingMiddleware
|
|
1039
1039
|
def call(worker, job, queue)
|
|
1040
1040
|
# Save original setting
|
|
1041
|
-
original_fail_on_error = E11y.config.
|
|
1041
|
+
original_fail_on_error = E11y.config.error_handling_fail_on_error
|
|
1042
1042
|
|
|
1043
1043
|
# Disable failing on errors for this job
|
|
1044
1044
|
# Observability should NOT block business logic!
|
|
1045
|
-
E11y.config.
|
|
1045
|
+
E11y.config.error_handling_fail_on_error = false
|
|
1046
1046
|
|
|
1047
1047
|
E11y.logger.debug(
|
|
1048
1048
|
"Sidekiq job starting with fail_on_error=false",
|
|
@@ -1053,7 +1053,7 @@ module E11y
|
|
|
1053
1053
|
yield
|
|
1054
1054
|
ensure
|
|
1055
1055
|
# Restore original setting
|
|
1056
|
-
E11y.config.
|
|
1056
|
+
E11y.config.error_handling_fail_on_error = original_fail_on_error
|
|
1057
1057
|
end
|
|
1058
1058
|
end
|
|
1059
1059
|
end
|
|
@@ -1112,7 +1112,7 @@ module E11y
|
|
|
1112
1112
|
|
|
1113
1113
|
def self.handle_error(error)
|
|
1114
1114
|
# Should we raise or swallow?
|
|
1115
|
-
if E11y.config.
|
|
1115
|
+
if E11y.config.error_handling_fail_on_error
|
|
1116
1116
|
# Web request context: RAISE (fast feedback!)
|
|
1117
1117
|
raise error
|
|
1118
1118
|
else
|
|
@@ -1177,7 +1177,7 @@ class CriticalReportJob < ApplicationJob
|
|
|
1177
1177
|
|
|
1178
1178
|
def perform(report_id)
|
|
1179
1179
|
# Temporarily enable fail_on_error
|
|
1180
|
-
E11y.config.
|
|
1180
|
+
E11y.config.error_handling_fail_on_error = true
|
|
1181
1181
|
|
|
1182
1182
|
# Generate report
|
|
1183
1183
|
report = Report.generate(report_id)
|
|
@@ -1190,7 +1190,7 @@ class CriticalReportJob < ApplicationJob
|
|
|
1190
1190
|
# ↑ If this fails, job SHOULD fail (retry later)
|
|
1191
1191
|
ensure
|
|
1192
1192
|
# Restore default (will be restored by middleware anyway)
|
|
1193
|
-
E11y.config.
|
|
1193
|
+
E11y.config.error_handling_fail_on_error = false
|
|
1194
1194
|
end
|
|
1195
1195
|
end
|
|
1196
1196
|
```
|
|
@@ -1501,7 +1501,7 @@ module E11y
|
|
|
1501
1501
|
# Re-dispatch event through normal pipeline
|
|
1502
1502
|
E11y::Pipeline.dispatch(
|
|
1503
1503
|
entry[:event_data],
|
|
1504
|
-
metadata: entry[:metadata].merge(
|
|
1504
|
+
metadata: entry[:metadata].merge(dlq_replayed: true)
|
|
1505
1505
|
)
|
|
1506
1506
|
|
|
1507
1507
|
# Delete from DLQ after successful replay
|
|
@@ -1665,7 +1665,7 @@ module E11y
|
|
|
1665
1665
|
# Re-dispatch
|
|
1666
1666
|
E11y::Pipeline.dispatch(
|
|
1667
1667
|
entry[:event_data],
|
|
1668
|
-
metadata: entry[:metadata].merge(
|
|
1668
|
+
metadata: entry[:metadata].merge(dlq_replayed: true)
|
|
1669
1669
|
)
|
|
1670
1670
|
|
|
1671
1671
|
# Delete from DLQ
|
|
@@ -1782,6 +1782,31 @@ module E11y
|
|
|
1782
1782
|
end
|
|
1783
1783
|
```
|
|
1784
1784
|
|
|
1785
|
+
### 4.4.1. DLQ Filter: Event DSL (Implemented)
|
|
1786
|
+
|
|
1787
|
+
The DLQ filter uses **Event DSL** with a single `use_dlq` flag:
|
|
1788
|
+
|
|
1789
|
+
```ruby
|
|
1790
|
+
# Audit events (Presets::AuditEvent) have use_dlq true by default
|
|
1791
|
+
class Events::UserDeleted < E11y::Events::BaseAuditEvent
|
|
1792
|
+
# use_dlq true from preset — always saved to DLQ
|
|
1793
|
+
end
|
|
1794
|
+
|
|
1795
|
+
# Explicit opt-in for critical business events
|
|
1796
|
+
class Events::PaymentFailed < E11y::Event::Base
|
|
1797
|
+
use_dlq true
|
|
1798
|
+
end
|
|
1799
|
+
|
|
1800
|
+
# Explicit opt-out for noise
|
|
1801
|
+
class Events::DebugTrace < E11y::Event::Base
|
|
1802
|
+
use_dlq false
|
|
1803
|
+
end
|
|
1804
|
+
|
|
1805
|
+
# Default (nil): severity-based (error/fatal saved) + default_behavior
|
|
1806
|
+
```
|
|
1807
|
+
|
|
1808
|
+
**Priority order:** 1) `use_dlq false` → 2) `use_dlq true` → 3) severity → 4) default.
|
|
1809
|
+
|
|
1785
1810
|
### 4.5. DLQ Configuration
|
|
1786
1811
|
|
|
1787
1812
|
```ruby
|
|
@@ -2562,8 +2587,8 @@ module E11y
|
|
|
2562
2587
|
end
|
|
2563
2588
|
|
|
2564
2589
|
def check_all_adapters
|
|
2565
|
-
E11y
|
|
2566
|
-
circuit_breaker = CircuitBreaker.for(
|
|
2590
|
+
E11y.config.adapters.each do |name, adapter|
|
|
2591
|
+
circuit_breaker = CircuitBreaker.for(name)
|
|
2567
2592
|
|
|
2568
2593
|
# Only check adapters with open/half-open circuits
|
|
2569
2594
|
next if circuit_breaker.healthy?
|
|
@@ -2572,11 +2597,11 @@ module E11y
|
|
|
2572
2597
|
adapter.health_check
|
|
2573
2598
|
|
|
2574
2599
|
E11y::Metrics.increment('e11y.health_check.success', {
|
|
2575
|
-
adapter:
|
|
2600
|
+
adapter: name
|
|
2576
2601
|
})
|
|
2577
2602
|
rescue => error
|
|
2578
2603
|
E11y::Metrics.increment('e11y.health_check.failure', {
|
|
2579
|
-
adapter:
|
|
2604
|
+
adapter: name,
|
|
2580
2605
|
error: error.class.name
|
|
2581
2606
|
})
|
|
2582
2607
|
end
|
|
@@ -752,27 +752,15 @@ app_wide:
|
|
|
752
752
|
### 6.1. Yabeda Metrics Definition
|
|
753
753
|
|
|
754
754
|
```ruby
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
# Availability metric (counter)
|
|
765
|
-
counter :event_result_total,
|
|
766
|
-
comment: "Total count of Event SLO results (success/failure)",
|
|
767
|
-
tags: [:event_class, :slo_name, :slo_status] # slo_status = 'success' | 'failure'
|
|
768
|
-
|
|
769
|
-
# Latency metric (histogram)
|
|
770
|
-
histogram :event_duration_seconds,
|
|
771
|
-
comment: "Event processing duration for SLO (seconds)",
|
|
772
|
-
tags: [:event_class, :slo_name],
|
|
773
|
-
buckets: [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0]
|
|
774
|
-
end
|
|
775
|
-
end
|
|
755
|
+
Yabeda.configure do
|
|
756
|
+
group :e11y_slo do
|
|
757
|
+
counter :event_result_total,
|
|
758
|
+
comment: "Event SLO results (success/failure)",
|
|
759
|
+
tags: [:event_class, :slo_name, :slo_status]
|
|
760
|
+
histogram :event_duration_seconds,
|
|
761
|
+
comment: "Event processing duration (seconds)",
|
|
762
|
+
tags: [:event_class, :slo_name],
|
|
763
|
+
buckets: [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0]
|
|
776
764
|
end
|
|
777
765
|
end
|
|
778
766
|
```
|
|
@@ -814,7 +802,7 @@ module E11y
|
|
|
814
802
|
def self.validate!
|
|
815
803
|
errors = []
|
|
816
804
|
|
|
817
|
-
E11y::Registry.
|
|
805
|
+
E11y::Registry.event_classes.each do |event_class|
|
|
818
806
|
# Check: Has slo declaration?
|
|
819
807
|
has_slo_enabled = event_class.slo_enabled?
|
|
820
808
|
has_slo_disabled = event_class.slo_disabled?
|
|
@@ -850,7 +838,7 @@ module E11y
|
|
|
850
838
|
def self.validate!
|
|
851
839
|
errors = []
|
|
852
840
|
|
|
853
|
-
E11y::Registry.
|
|
841
|
+
E11y::Registry.event_classes.each do |event_class|
|
|
854
842
|
next unless event_class.slo_enabled?
|
|
855
843
|
|
|
856
844
|
# Check: Has slo_status_from block?
|
|
@@ -923,7 +911,7 @@ module E11y
|
|
|
923
911
|
end
|
|
924
912
|
|
|
925
913
|
# Check reverse: Events with slo enabled but NOT in slo.yml
|
|
926
|
-
E11y::Registry.
|
|
914
|
+
E11y::Registry.event_classes.each do |event_class|
|
|
927
915
|
next unless event_class.slo_enabled?
|
|
928
916
|
|
|
929
917
|
slo_name = event_class.slo_config.contributes_to
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
3. [Correct Order](#3-correct-order)
|
|
15
15
|
- 3.1. Pipeline Flow
|
|
16
16
|
- 3.2. Why Each Middleware Needs Original Class Name
|
|
17
|
-
- 3.3. Audit
|
|
17
|
+
- 3.3. Audit Events in Single Pipeline (C01 Resolution) ⚠️ CRITICAL
|
|
18
18
|
- 3.3.1. The Problem: PII Filtering Breaks Audit Trail
|
|
19
19
|
- 3.3.2. Decision: Two Pipeline Configurations
|
|
20
20
|
- 3.3.3. Declaring Audit Events
|
|
@@ -244,17 +244,17 @@ Loki receives:
|
|
|
244
244
|
- [ ] PII rules are configured **per original class** (if differ)
|
|
245
245
|
- [ ] Sampling rules are configured **per original class** (if differ)
|
|
246
246
|
- [ ] Metrics track **both** normalized name and version
|
|
247
|
-
- [ ] Audit events
|
|
247
|
+
- [ ] Audit events skip PII filtering via conditional logic (C01 - see §3.3, single pipeline)
|
|
248
248
|
- [ ] Audit events stored in encrypted adapter (C01 requirement)
|
|
249
249
|
|
|
250
250
|
---
|
|
251
251
|
|
|
252
|
-
## 3.3. Audit
|
|
252
|
+
## 3.3. Audit Events in Single Pipeline (C01 Resolution)
|
|
253
253
|
|
|
254
254
|
> **⚠️ CRITICAL: C01 Conflict Resolution - PII Filtering × Audit Trail Signing**
|
|
255
255
|
> **See:** [CONFLICT-ANALYSIS.md C01](researches/CONFLICT-ANALYSIS.md#c01-pii-filtering--audit-trail-signing) for detailed analysis
|
|
256
256
|
> **Problem:** PII filtering before signing breaks non-repudiation (auditors can't verify original event)
|
|
257
|
-
> **Solution:**
|
|
257
|
+
> **Solution:** Single pipeline for all events. Audit events get conditional skip in PIIFilter via `contains_pii false` in AuditEvent preset (:no_pii = pass-through). No separate pipeline — no need.
|
|
258
258
|
|
|
259
259
|
### 3.3.1. The Problem: PII Filtering Breaks Audit Trail
|
|
260
260
|
|
|
@@ -278,39 +278,29 @@ Storage
|
|
|
278
278
|
- **Audit trail:** Must maintain cryptographic chain of custody
|
|
279
279
|
- **Forensics:** Must be able to reconstruct exact event that occurred
|
|
280
280
|
|
|
281
|
-
### 3.3.2. Decision:
|
|
281
|
+
### 3.3.2. Decision: One Pipeline with Conditional Skip
|
|
282
|
+
|
|
283
|
+
**All events go through a single pipeline.** Audit events get conditional skip in middleware:
|
|
282
284
|
|
|
283
|
-
**Standard Events (Non-Audit):**
|
|
284
285
|
```
|
|
285
286
|
1. TraceContext → Add trace_id, span_id, timestamp
|
|
286
287
|
2. Validation → Schema validation (original class)
|
|
287
|
-
3. PIIFiltering →
|
|
288
|
-
4. RateLimiting →
|
|
289
|
-
5. Sampling →
|
|
288
|
+
3. PIIFiltering → Audit: skip (contains_pii false → :no_pii). Standard: filter PII ✅
|
|
289
|
+
4. RateLimiting → Audit: can skip (event_data[:audit_event]). Standard: rate limit
|
|
290
|
+
5. Sampling → Audit: sample_rate 1.0 (preset). Standard: adaptive
|
|
290
291
|
6. Versioning → Normalize event_name (LAST)
|
|
291
|
-
7. Routing → Route to buffer
|
|
292
|
+
7. Routing → Route to buffer / audit buffer
|
|
292
293
|
```
|
|
293
294
|
|
|
294
|
-
**
|
|
295
|
-
```
|
|
296
|
-
1. TraceContext → Add trace_id, span_id, timestamp
|
|
297
|
-
2. Validation → Schema validation (original class)
|
|
298
|
-
3. AuditSigning → Sign ORIGINAL data (includes PII!) ✅
|
|
299
|
-
4. Versioning → Normalize event_name (LAST)
|
|
300
|
-
5. Routing → Route to audit buffer
|
|
301
|
-
|
|
302
|
-
❌ NO PII filtering for audit events!
|
|
303
|
-
❌ NO rate limiting for audit events!
|
|
304
|
-
❌ NO sampling for audit events!
|
|
305
|
-
```
|
|
295
|
+
**AuditEvent preset:** `contains_pii false` → PIIFilter :no_pii = pass-through, original data preserved for signing.
|
|
306
296
|
|
|
307
297
|
### 3.3.3. Declaring Audit Events
|
|
308
298
|
|
|
309
299
|
**Event Class Flag:**
|
|
310
300
|
```ruby
|
|
311
|
-
# Audit event -
|
|
301
|
+
# Audit event - conditional skip in pipeline
|
|
312
302
|
class Events::PermissionChanged < E11y::Event::Base
|
|
313
|
-
|
|
303
|
+
include E11y::Presets::AuditEvent # audit_event true, contains_pii false
|
|
314
304
|
# Auto-set: retention = E11y.config.audit_retention (configurable!)
|
|
315
305
|
# rate_limiting = false (LOCKED!)
|
|
316
306
|
# sampling = false (LOCKED!)
|
|
@@ -330,9 +320,9 @@ class Events::PermissionChanged < E11y::Event::Base
|
|
|
330
320
|
version 1
|
|
331
321
|
end
|
|
332
322
|
|
|
333
|
-
# Standard event -
|
|
323
|
+
# Standard event - full pipeline
|
|
334
324
|
class Events::PageView < E11y::Event::Base
|
|
335
|
-
audit_event false
|
|
325
|
+
# audit_event false (default)
|
|
336
326
|
|
|
337
327
|
schema do
|
|
338
328
|
required(:user_id).filled(:string)
|
|
@@ -355,21 +345,15 @@ end
|
|
|
355
345
|
```ruby
|
|
356
346
|
# config/initializers/e11y.rb
|
|
357
347
|
E11y.configure do |config|
|
|
358
|
-
#
|
|
348
|
+
# Single pipeline for all events (audit and standard)
|
|
359
349
|
config.pipeline.use E11y::Middleware::TraceContext # 1
|
|
360
350
|
config.pipeline.use E11y::Middleware::Validation # 2
|
|
361
|
-
config.pipeline.use E11y::Middleware::PIIFiltering # 3
|
|
351
|
+
config.pipeline.use E11y::Middleware::PIIFiltering # 3 # Audit: skip (contains_pii false)
|
|
362
352
|
config.pipeline.use E11y::Middleware::RateLimiting # 4
|
|
363
353
|
config.pipeline.use E11y::Middleware::Sampling # 5
|
|
364
|
-
config.pipeline.use E11y::Middleware::
|
|
365
|
-
config.pipeline.use E11y::Middleware::
|
|
366
|
-
|
|
367
|
-
# Audit pipeline override (for audit_event: true)
|
|
368
|
-
config.audit_pipeline.use E11y::Middleware::TraceContext # 1
|
|
369
|
-
config.audit_pipeline.use E11y::Middleware::Validation # 2
|
|
370
|
-
config.audit_pipeline.use E11y::Middleware::AuditSigning # 3 (NEW!)
|
|
371
|
-
config.audit_pipeline.use E11y::Middleware::Versioning # 4
|
|
372
|
-
config.audit_pipeline.use E11y::Middleware::AuditRouting # 5
|
|
354
|
+
config.pipeline.use E11y::Middleware::AuditSigning # 6 # Pass-through for non-audit
|
|
355
|
+
config.pipeline.use E11y::Middleware::Versioning # 7 # Last before Routing
|
|
356
|
+
config.pipeline.use E11y::Middleware::Routing # 8
|
|
373
357
|
|
|
374
358
|
# Audit event configuration
|
|
375
359
|
config.audit_events do
|
|
@@ -610,7 +594,7 @@ end
|
|
|
610
594
|
| Aspect | Pro | Con | Mitigation |
|
|
611
595
|
|--------|-----|-----|------------|
|
|
612
596
|
| **Non-repudiation** | ✅ Signature on original data | ⚠️ PII in audit events | Use encrypted storage adapter |
|
|
613
|
-
| **Legal compliance** | ✅ Meets audit requirements | ⚠️
|
|
597
|
+
| **Legal compliance** | ✅ Meets audit requirements | ⚠️ Conditional logic in middleware | Clear documentation |
|
|
614
598
|
| **PII protection** | ✅ Standard events filtered | ⚠️ Audit events not filtered | Restrict access to audit logs |
|
|
615
599
|
| **Performance** | ✅ No PII filter overhead | ⚠️ Signing + encryption overhead | Audit events are rare (<1%) |
|
|
616
600
|
|
|
@@ -1041,7 +1025,7 @@ end
|
|
|
1041
1025
|
```
|
|
1042
1026
|
|
|
1043
1027
|
**Related Conflicts:**
|
|
1044
|
-
- **C01:** Audit events skip PII filtering (see §3.3)
|
|
1028
|
+
- **C01:** Audit events skip PII filtering via contains_pii false (see §3.3)
|
|
1045
1029
|
- **C08:** Baggage PII protection (see ADR-007)
|
|
1046
1030
|
|
|
1047
1031
|
---
|
|
@@ -1053,8 +1037,6 @@ end
|
|
|
1053
1037
|
- **ADR-012: Event Evolution & Versioning** - Full versioning design
|
|
1054
1038
|
- **ADR-013: Reliability & Error Handling** - DLQ replay considerations
|
|
1055
1039
|
- **UC-012: Audit Trail** - Audit event use cases
|
|
1056
|
-
- **COMPREHENSIVE-CONFIGURATION.md** - Complete configuration examples
|
|
1057
|
-
- **CONFLICT-ANALYSIS.md** - Complete conflict analysis
|
|
1058
1040
|
|
|
1059
1041
|
---
|
|
1060
1042
|
|