e11y 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.rubocop.yml +69 -0
- data/CHANGELOG.md +26 -0
- data/CODE_OF_CONDUCT.md +64 -0
- data/LICENSE.txt +21 -0
- data/README.md +179 -0
- data/Rakefile +37 -0
- data/benchmarks/run_all.rb +33 -0
- data/config/README.md +83 -0
- data/config/loki-local-config.yaml +35 -0
- data/config/prometheus.yml +15 -0
- data/docker-compose.yml +78 -0
- data/docs/00-ICP-AND-TIMELINE.md +483 -0
- data/docs/01-SCALE-REQUIREMENTS.md +858 -0
- data/docs/ADR-001-architecture.md +2617 -0
- data/docs/ADR-002-metrics-yabeda.md +1395 -0
- data/docs/ADR-003-slo-observability.md +3337 -0
- data/docs/ADR-004-adapter-architecture.md +2385 -0
- data/docs/ADR-005-tracing-context.md +1372 -0
- data/docs/ADR-006-security-compliance.md +4143 -0
- data/docs/ADR-007-opentelemetry-integration.md +1385 -0
- data/docs/ADR-008-rails-integration.md +1911 -0
- data/docs/ADR-009-cost-optimization.md +2993 -0
- data/docs/ADR-010-developer-experience.md +2166 -0
- data/docs/ADR-011-testing-strategy.md +1836 -0
- data/docs/ADR-012-event-evolution.md +958 -0
- data/docs/ADR-013-reliability-error-handling.md +2750 -0
- data/docs/ADR-014-event-driven-slo.md +1533 -0
- data/docs/ADR-015-middleware-order.md +1061 -0
- data/docs/ADR-016-self-monitoring-slo.md +1234 -0
- data/docs/API-REFERENCE-L28.md +914 -0
- data/docs/COMPREHENSIVE-CONFIGURATION.md +2366 -0
- data/docs/IMPLEMENTATION_NOTES.md +2804 -0
- data/docs/IMPLEMENTATION_PLAN.md +1971 -0
- data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +586 -0
- data/docs/PLAN.md +148 -0
- data/docs/QUICK-START.md +934 -0
- data/docs/README.md +296 -0
- data/docs/design/00-memory-optimization.md +593 -0
- data/docs/guides/MIGRATION-L27-L28.md +692 -0
- data/docs/guides/PERFORMANCE-BENCHMARKS.md +434 -0
- data/docs/guides/README.md +44 -0
- data/docs/prd/01-overview-vision.md +440 -0
- data/docs/use_cases/README.md +119 -0
- data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +813 -0
- data/docs/use_cases/UC-002-business-event-tracking.md +1953 -0
- data/docs/use_cases/UC-003-pattern-based-metrics.md +1627 -0
- data/docs/use_cases/UC-004-zero-config-slo-tracking.md +728 -0
- data/docs/use_cases/UC-005-sentry-integration.md +759 -0
- data/docs/use_cases/UC-006-trace-context-management.md +905 -0
- data/docs/use_cases/UC-007-pii-filtering.md +2648 -0
- data/docs/use_cases/UC-008-opentelemetry-integration.md +1153 -0
- data/docs/use_cases/UC-009-multi-service-tracing.md +1043 -0
- data/docs/use_cases/UC-010-background-job-tracking.md +1018 -0
- data/docs/use_cases/UC-011-rate-limiting.md +1906 -0
- data/docs/use_cases/UC-012-audit-trail.md +2301 -0
- data/docs/use_cases/UC-013-high-cardinality-protection.md +2127 -0
- data/docs/use_cases/UC-014-adaptive-sampling.md +1940 -0
- data/docs/use_cases/UC-015-cost-optimization.md +735 -0
- data/docs/use_cases/UC-016-rails-logger-migration.md +785 -0
- data/docs/use_cases/UC-017-local-development.md +867 -0
- data/docs/use_cases/UC-018-testing-events.md +1081 -0
- data/docs/use_cases/UC-019-tiered-storage-migration.md +562 -0
- data/docs/use_cases/UC-020-event-versioning.md +708 -0
- data/docs/use_cases/UC-021-error-handling-retry-dlq.md +956 -0
- data/docs/use_cases/UC-022-event-registry.md +648 -0
- data/docs/use_cases/backlog.md +226 -0
- data/e11y.gemspec +76 -0
- data/lib/e11y/adapters/adaptive_batcher.rb +207 -0
- data/lib/e11y/adapters/audit_encrypted.rb +239 -0
- data/lib/e11y/adapters/base.rb +580 -0
- data/lib/e11y/adapters/file.rb +224 -0
- data/lib/e11y/adapters/in_memory.rb +216 -0
- data/lib/e11y/adapters/loki.rb +333 -0
- data/lib/e11y/adapters/otel_logs.rb +203 -0
- data/lib/e11y/adapters/registry.rb +141 -0
- data/lib/e11y/adapters/sentry.rb +230 -0
- data/lib/e11y/adapters/stdout.rb +108 -0
- data/lib/e11y/adapters/yabeda.rb +370 -0
- data/lib/e11y/buffers/adaptive_buffer.rb +339 -0
- data/lib/e11y/buffers/base_buffer.rb +40 -0
- data/lib/e11y/buffers/request_scoped_buffer.rb +246 -0
- data/lib/e11y/buffers/ring_buffer.rb +267 -0
- data/lib/e11y/buffers.rb +14 -0
- data/lib/e11y/console.rb +122 -0
- data/lib/e11y/current.rb +48 -0
- data/lib/e11y/event/base.rb +894 -0
- data/lib/e11y/event/value_sampling_config.rb +84 -0
- data/lib/e11y/events/base_audit_event.rb +43 -0
- data/lib/e11y/events/base_payment_event.rb +33 -0
- data/lib/e11y/events/rails/cache/delete.rb +21 -0
- data/lib/e11y/events/rails/cache/read.rb +23 -0
- data/lib/e11y/events/rails/cache/write.rb +22 -0
- data/lib/e11y/events/rails/database/query.rb +45 -0
- data/lib/e11y/events/rails/http/redirect.rb +21 -0
- data/lib/e11y/events/rails/http/request.rb +26 -0
- data/lib/e11y/events/rails/http/send_file.rb +21 -0
- data/lib/e11y/events/rails/http/start_processing.rb +26 -0
- data/lib/e11y/events/rails/job/completed.rb +22 -0
- data/lib/e11y/events/rails/job/enqueued.rb +22 -0
- data/lib/e11y/events/rails/job/failed.rb +22 -0
- data/lib/e11y/events/rails/job/scheduled.rb +23 -0
- data/lib/e11y/events/rails/job/started.rb +22 -0
- data/lib/e11y/events/rails/log.rb +56 -0
- data/lib/e11y/events/rails/view/render.rb +23 -0
- data/lib/e11y/events.rb +18 -0
- data/lib/e11y/instruments/active_job.rb +201 -0
- data/lib/e11y/instruments/rails_instrumentation.rb +141 -0
- data/lib/e11y/instruments/sidekiq.rb +175 -0
- data/lib/e11y/logger/bridge.rb +205 -0
- data/lib/e11y/metrics/cardinality_protection.rb +172 -0
- data/lib/e11y/metrics/cardinality_tracker.rb +134 -0
- data/lib/e11y/metrics/registry.rb +234 -0
- data/lib/e11y/metrics/relabeling.rb +226 -0
- data/lib/e11y/metrics.rb +102 -0
- data/lib/e11y/middleware/audit_signing.rb +174 -0
- data/lib/e11y/middleware/base.rb +140 -0
- data/lib/e11y/middleware/event_slo.rb +167 -0
- data/lib/e11y/middleware/pii_filter.rb +266 -0
- data/lib/e11y/middleware/pii_filtering.rb +280 -0
- data/lib/e11y/middleware/rate_limiting.rb +214 -0
- data/lib/e11y/middleware/request.rb +163 -0
- data/lib/e11y/middleware/routing.rb +157 -0
- data/lib/e11y/middleware/sampling.rb +254 -0
- data/lib/e11y/middleware/slo.rb +168 -0
- data/lib/e11y/middleware/trace_context.rb +131 -0
- data/lib/e11y/middleware/validation.rb +118 -0
- data/lib/e11y/middleware/versioning.rb +132 -0
- data/lib/e11y/middleware.rb +12 -0
- data/lib/e11y/pii/patterns.rb +90 -0
- data/lib/e11y/pii.rb +13 -0
- data/lib/e11y/pipeline/builder.rb +155 -0
- data/lib/e11y/pipeline/zone_validator.rb +110 -0
- data/lib/e11y/pipeline.rb +12 -0
- data/lib/e11y/presets/audit_event.rb +65 -0
- data/lib/e11y/presets/debug_event.rb +34 -0
- data/lib/e11y/presets/high_value_event.rb +51 -0
- data/lib/e11y/presets.rb +19 -0
- data/lib/e11y/railtie.rb +138 -0
- data/lib/e11y/reliability/circuit_breaker.rb +216 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +277 -0
- data/lib/e11y/reliability/dlq/filter.rb +117 -0
- data/lib/e11y/reliability/retry_handler.rb +207 -0
- data/lib/e11y/reliability/retry_rate_limiter.rb +117 -0
- data/lib/e11y/sampling/error_spike_detector.rb +225 -0
- data/lib/e11y/sampling/load_monitor.rb +161 -0
- data/lib/e11y/sampling/stratified_tracker.rb +92 -0
- data/lib/e11y/sampling/value_extractor.rb +82 -0
- data/lib/e11y/self_monitoring/buffer_monitor.rb +79 -0
- data/lib/e11y/self_monitoring/performance_monitor.rb +97 -0
- data/lib/e11y/self_monitoring/reliability_monitor.rb +146 -0
- data/lib/e11y/slo/event_driven.rb +150 -0
- data/lib/e11y/slo/tracker.rb +119 -0
- data/lib/e11y/version.rb +9 -0
- data/lib/e11y.rb +283 -0
- metadata +452 -0
|
@@ -0,0 +1,1971 @@
|
|
|
1
|
+
# E11y Gem: 6-Level Implementation Plan
|
|
2
|
+
|
|
3
|
+
**Created:** 2026-01-17
|
|
4
|
+
**Updated:** 2026-01-17 (Added Phase 0: Gem Best Practices Research)
|
|
5
|
+
**Based On:** 38 documents analysis (22 UC + 16 ADR), 21 conflicts resolved
|
|
6
|
+
**Target Scale:** Small (1K events/sec) → Medium (10K) → Large (100K)
|
|
7
|
+
**Strategy:** Maximum parallelization + Minimal integration overhead
|
|
8
|
+
**Quality Standard:** Professional Rails gem (Rails 8+ compatibility)
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 📊 EXECUTIVE SUMMARY
|
|
13
|
+
|
|
14
|
+
### Key Metrics
|
|
15
|
+
- **Total Phases:** 6 phases (Phase 0: Gem Setup → Phase 5: Scale)
|
|
16
|
+
- **Parallelizable Streams:** 4-6 developers can work simultaneously
|
|
17
|
+
- **Critical Path:** Phase 0 → Foundation → Core → Rails (phases 0-3 sequential)
|
|
18
|
+
- **Estimated Effort:** ~23-27 weeks (with proper parallelization)
|
|
19
|
+
- **ADR Coverage:** 16 ADRs fully implemented
|
|
20
|
+
- **UC Coverage:** 22 Use Cases fully implemented
|
|
21
|
+
|
|
22
|
+
### Parallelization Strategy
|
|
23
|
+
- **Phase 0 (Gem Setup):** 1-2 devs, Week -1 (BEFORE Phase 1)
|
|
24
|
+
- **Phase 1 (Foundation):** 3 parallel streams
|
|
25
|
+
- **Phase 2 (Core):** 5 parallel streams
|
|
26
|
+
- **Phase 3 (Rails):** 4 parallel streams
|
|
27
|
+
- **Phase 4 (Production):** 6 parallel streams
|
|
28
|
+
- **Phase 5 (Scale):** 3 parallel streams
|
|
29
|
+
|
|
30
|
+
### Integration Points (CRITICAL for Parallel Work)
|
|
31
|
+
1. **Event::Base Interface** (Level 2.1) - Contract for всех components
|
|
32
|
+
2. **Middleware::Base Contract** (Level 2.2) - Pipeline interface
|
|
33
|
+
3. **Adapter::Base Contract** (Level 2.4) - Adapter interface
|
|
34
|
+
4. **Buffer Interface** (Level 2.3) - Buffering contract
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## 🎯 PLAN STRUCTURE
|
|
39
|
+
|
|
40
|
+
### Level 1: PHASES (6 Strategic Phases)
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
PHASE 0: GEM SETUP & BEST PRACTICES (Week -1) 🆕
|
|
44
|
+
├─ Research: Devise, Sidekiq, Puma, Dry-rb, Yabeda, Sentry
|
|
45
|
+
├─ Gem Structure & Conventions
|
|
46
|
+
├─ CI/CD Setup (GitHub Actions, RSpec, Rubocop)
|
|
47
|
+
└─ Documentation Standards (YARD, README, Guides)
|
|
48
|
+
|
|
49
|
+
PHASE 1: FOUNDATION (Weeks 1-4)
|
|
50
|
+
├─ Zero-Allocation Event Base
|
|
51
|
+
├─ Adaptive Buffer (C20)
|
|
52
|
+
└─ Middleware Pipeline
|
|
53
|
+
|
|
54
|
+
PHASE 2: CORE FEATURES (Weeks 5-10)
|
|
55
|
+
├─ PII Filtering & Security (C01, C19)
|
|
56
|
+
├─ Adapter Architecture (C04, C06)
|
|
57
|
+
├─ Metrics & Yabeda (C03, C11)
|
|
58
|
+
└─ Sampling & Cost Optimization (C05, C11)
|
|
59
|
+
|
|
60
|
+
PHASE 3: RAILS INTEGRATION (Weeks 11-14)
|
|
61
|
+
├─ Railtie & Auto-Configuration
|
|
62
|
+
├─ ActiveSupport::Notifications Bridge (Unidirectional!)
|
|
63
|
+
├─ Sidekiq/ActiveJob Integration (C17, C18)
|
|
64
|
+
└─ Rails.logger Migration (UC-016)
|
|
65
|
+
|
|
66
|
+
PHASE 4: PRODUCTION HARDENING (Weeks 15-20)
|
|
67
|
+
├─ Reliability & Error Handling (C02, C06, C18)
|
|
68
|
+
├─ OpenTelemetry Integration (C08)
|
|
69
|
+
├─ Event Evolution & Versioning (C15)
|
|
70
|
+
└─ SLO Tracking & Self-Monitoring (C11, C14)
|
|
71
|
+
|
|
72
|
+
PHASE 5: SCALE & OPTIMIZATION (Weeks 21-26)
|
|
73
|
+
├─ High Cardinality Protection (C13)
|
|
74
|
+
├─ Tiered Storage Migration (UC-019)
|
|
75
|
+
├─ Performance Optimization (1K → 10K → 100K)
|
|
76
|
+
└─ Production Deployment & Documentation
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## 📋 LEVEL 2-6 BREAKDOWN
|
|
82
|
+
|
|
83
|
+
### Legend:
|
|
84
|
+
- 🔴 **CRITICAL** - Blocking dependency for other tasks
|
|
85
|
+
- 🟠 **HIGH** - Important for feature completeness
|
|
86
|
+
- 🟡 **MEDIUM** - Nice-to-have, can be deferred
|
|
87
|
+
- ⚙️ **PARALLELIZABLE** - Can be worked on simultaneously
|
|
88
|
+
- 🔗 **DEPENDS ON** - Explicit dependency
|
|
89
|
+
- ✅ **DoD** - Definition of Done (acceptance criteria)
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## PHASE 0: GEM SETUP & BEST PRACTICES (Week -1) 🆕
|
|
94
|
+
|
|
95
|
+
**Purpose:** Research and setup professional gem structure BEFORE coding
|
|
96
|
+
**Timeline:** Week -1 (1 week, BEFORE Phase 1 starts)
|
|
97
|
+
**Team:** 1-2 developers
|
|
98
|
+
**Critical:** ⚠️ MUST COMPLETE BEFORE PHASE 1
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
### L2.0: Gem Structure & Best Practices Research 🔴
|
|
103
|
+
|
|
104
|
+
**Purpose:** Learn from successful gems, establish conventions
|
|
105
|
+
**Depends On:** None (foundation)
|
|
106
|
+
**Parallelizable:** ⚙️ 1-2 devs (can split research + setup)
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
#### L3.0.1: Best Practices Research
|
|
111
|
+
|
|
112
|
+
**Tasks:**
|
|
113
|
+
|
|
114
|
+
1. **Study Successful Rails Gems (20 hours)**
|
|
115
|
+
- **Devise** (authentication)
|
|
116
|
+
- Controller/view overrides pattern
|
|
117
|
+
- Modular design (strategies, modules)
|
|
118
|
+
- Configuration DSL
|
|
119
|
+
- ✅ DoD: Document 5+ patterns in `docs/research/devise_patterns.md`
|
|
120
|
+
|
|
121
|
+
- **Sidekiq** (background jobs)
|
|
122
|
+
- Middleware pattern
|
|
123
|
+
- Configuration DSL (simple blocks)
|
|
124
|
+
- Redis integration patterns
|
|
125
|
+
- Error handling & retry logic
|
|
126
|
+
- ✅ DoD: Document 5+ patterns in `docs/research/sidekiq_patterns.md`
|
|
127
|
+
|
|
128
|
+
- **Puma** (web server)
|
|
129
|
+
- Configuration DSL
|
|
130
|
+
- Plugin system
|
|
131
|
+
- Thread safety patterns
|
|
132
|
+
- ✅ DoD: Document 5+ patterns in `docs/research/puma_patterns.md`
|
|
133
|
+
|
|
134
|
+
- **Dry-rb gems** (dry-schema, dry-validation, dry-types)
|
|
135
|
+
- Functional design
|
|
136
|
+
- Composition patterns
|
|
137
|
+
- Type system
|
|
138
|
+
- Schema DSL
|
|
139
|
+
- ✅ DoD: Document 5+ patterns in `docs/research/dry_rb_patterns.md`
|
|
140
|
+
|
|
141
|
+
- **Yabeda** (metrics)
|
|
142
|
+
- DSL design (counter, histogram, gauge)
|
|
143
|
+
- Extensibility (adapters, collectors)
|
|
144
|
+
- Rails integration
|
|
145
|
+
- ✅ DoD: Document 5+ patterns in `docs/research/yabeda_patterns.md`
|
|
146
|
+
|
|
147
|
+
- **Sentry-ruby** (error tracking)
|
|
148
|
+
- Rails integration (Railtie)
|
|
149
|
+
- Configuration management
|
|
150
|
+
- Context propagation
|
|
151
|
+
- Async processing
|
|
152
|
+
- ✅ DoD: Document 5+ patterns in `docs/research/sentry_patterns.md`
|
|
153
|
+
|
|
154
|
+
2. **Gem Structure Analysis (8 hours)**
|
|
155
|
+
- Directory structure conventions
|
|
156
|
+
- File naming patterns
|
|
157
|
+
- Module organization (E11y::, E11y::Instruments::, etc.)
|
|
158
|
+
- Autoloading strategy (Zeitwerk best practices)
|
|
159
|
+
- Gemspec best practices
|
|
160
|
+
- ✅ DoD: `docs/research/gem_structure_template.md`
|
|
161
|
+
|
|
162
|
+
3. **Configuration DSL Patterns (8 hours)**
|
|
163
|
+
- Block-based DSL (`E11y.configure { |config| ... }`)
|
|
164
|
+
- Nested configuration (`config.adapters do ... end`)
|
|
165
|
+
- Type validation (dry-types integration?)
|
|
166
|
+
- Environment-specific config
|
|
167
|
+
- Override patterns (Devise-style)
|
|
168
|
+
- ✅ DoD: `docs/research/configuration_dsl_design.md`
|
|
169
|
+
|
|
170
|
+
4. **Testing Strategies (6 hours)**
|
|
171
|
+
- RSpec setup (spec_helper, rails_helper)
|
|
172
|
+
- Test organization (unit, integration, system)
|
|
173
|
+
- Test factories (FactoryBot vs Plain Ruby)
|
|
174
|
+
- Contract tests for public APIs
|
|
175
|
+
- Rails integration testing patterns
|
|
176
|
+
- Load testing setup
|
|
177
|
+
- Coverage requirements (aim for 100%)
|
|
178
|
+
- ✅ DoD: `docs/research/testing_strategy.md`
|
|
179
|
+
|
|
180
|
+
5. **Documentation Standards (6 hours)**
|
|
181
|
+
- YARD documentation (inline docs)
|
|
182
|
+
- README structure (badges, quick start, examples)
|
|
183
|
+
- Guides vs API reference
|
|
184
|
+
- Code examples quality
|
|
185
|
+
- Changelog format (Keep a Changelog)
|
|
186
|
+
- GitHub wiki vs docs/ folder
|
|
187
|
+
- ✅ DoD: `docs/research/documentation_standards.md`
|
|
188
|
+
|
|
189
|
+
6. **CI/CD Best Practices (4 hours)**
|
|
190
|
+
- GitHub Actions setup
|
|
191
|
+
- Matrix testing (Ruby 3.2, 3.3, Rails 8.0+)
|
|
192
|
+
- Code coverage (SimpleCov)
|
|
193
|
+
- Linting (Rubocop, StandardRB)
|
|
194
|
+
- Security scanning (Brakeman, Bundler Audit)
|
|
195
|
+
- Release automation
|
|
196
|
+
- ✅ DoD: `docs/research/ci_cd_setup.md`
|
|
197
|
+
|
|
198
|
+
7. **Gem Release Process (4 hours)**
|
|
199
|
+
- Semantic versioning strategy
|
|
200
|
+
- Changelog generation
|
|
201
|
+
- RubyGems release checklist
|
|
202
|
+
- GitHub releases with notes
|
|
203
|
+
- Deprecation warnings strategy
|
|
204
|
+
- Breaking change communication
|
|
205
|
+
- ✅ DoD: `docs/research/gem_release_process.md`
|
|
206
|
+
|
|
207
|
+
**Verification (L6):**
|
|
208
|
+
- All research documents created and reviewed
|
|
209
|
+
- Team meeting: Present findings (1 hour)
|
|
210
|
+
- Consensus on patterns to use
|
|
211
|
+
|
|
212
|
+
---
|
|
213
|
+
|
|
214
|
+
#### L3.0.2: Project Skeleton Setup
|
|
215
|
+
|
|
216
|
+
**Tasks:**
|
|
217
|
+
|
|
218
|
+
1. **Initialize Gem (1 hour)**
|
|
219
|
+
```bash
|
|
220
|
+
bundle gem e11y --mit --test=rspec --ci=github --linter=rubocop
|
|
221
|
+
```
|
|
222
|
+
- MIT License
|
|
223
|
+
- RSpec testing
|
|
224
|
+
- GitHub Actions CI
|
|
225
|
+
- Rubocop linting
|
|
226
|
+
- ✅ DoD: Gem skeleton created, `git init` done
|
|
227
|
+
|
|
228
|
+
2. **Setup Directory Structure (2 hours)**
|
|
229
|
+
```
|
|
230
|
+
e11y/
|
|
231
|
+
├── lib/
|
|
232
|
+
│ ├── e11y.rb # Main entry point
|
|
233
|
+
│ └── e11y/
|
|
234
|
+
│ ├── version.rb
|
|
235
|
+
│ ├── configuration.rb
|
|
236
|
+
│ ├── event/
|
|
237
|
+
│ │ └── base.rb
|
|
238
|
+
│ ├── middleware/
|
|
239
|
+
│ │ └── base.rb
|
|
240
|
+
│ ├── adapters/
|
|
241
|
+
│ │ └── base.rb
|
|
242
|
+
│ ├── buffers/
|
|
243
|
+
│ │ └── base_buffer.rb
|
|
244
|
+
│ ├── instruments/
|
|
245
|
+
│ │ └── rails_instrumentation.rb
|
|
246
|
+
│ └── railtie.rb # Rails integration
|
|
247
|
+
├── spec/
|
|
248
|
+
│ ├── spec_helper.rb
|
|
249
|
+
│ ├── rails_helper.rb
|
|
250
|
+
│ ├── support/
|
|
251
|
+
│ │ ├── shared_examples/
|
|
252
|
+
│ │ └── helpers/
|
|
253
|
+
│ └── e11y/
|
|
254
|
+
│ ├── event/
|
|
255
|
+
│ ├── middleware/
|
|
256
|
+
│ └── adapters/
|
|
257
|
+
├── benchmarks/
|
|
258
|
+
│ ├── e11y_benchmarks.rb
|
|
259
|
+
│ └── load_tests.rb
|
|
260
|
+
├── docs/
|
|
261
|
+
│ ├── guides/
|
|
262
|
+
│ ├── research/ # Research from L3.0.1
|
|
263
|
+
│ └── (existing ADR, UC files)
|
|
264
|
+
├── bin/
|
|
265
|
+
│ ├── console # IRB with E11y loaded
|
|
266
|
+
│ └── setup # Setup script
|
|
267
|
+
├── .github/
|
|
268
|
+
│ └── workflows/
|
|
269
|
+
│ ├── ci.yml
|
|
270
|
+
│ └── release.yml
|
|
271
|
+
├── docker-compose.yml # Test backends
|
|
272
|
+
├── Gemfile
|
|
273
|
+
├── e11y.gemspec
|
|
274
|
+
├── Rakefile
|
|
275
|
+
├── .rubocop.yml
|
|
276
|
+
├── .gitignore
|
|
277
|
+
└── README.md
|
|
278
|
+
```
|
|
279
|
+
- ✅ DoD: Professional gem structure
|
|
280
|
+
|
|
281
|
+
3. **Setup Zeitwerk Autoloading (1 hour)**
|
|
282
|
+
```ruby
|
|
283
|
+
# lib/e11y.rb
|
|
284
|
+
require 'zeitwerk'
|
|
285
|
+
|
|
286
|
+
loader = Zeitwerk::Loader.for_gem
|
|
287
|
+
loader.setup
|
|
288
|
+
|
|
289
|
+
module E11y
|
|
290
|
+
# Gem entry point
|
|
291
|
+
end
|
|
292
|
+
```
|
|
293
|
+
- Test autoloading works
|
|
294
|
+
- ✅ DoD: `require 'e11y'` loads all files correctly
|
|
295
|
+
|
|
296
|
+
4. **Setup CI/CD - GitHub Actions (2 hours)**
|
|
297
|
+
```yaml
|
|
298
|
+
# .github/workflows/ci.yml
|
|
299
|
+
name: CI
|
|
300
|
+
on: [push, pull_request]
|
|
301
|
+
|
|
302
|
+
jobs:
|
|
303
|
+
test:
|
|
304
|
+
runs-on: ubuntu-latest
|
|
305
|
+
strategy:
|
|
306
|
+
fail-fast: false
|
|
307
|
+
matrix:
|
|
308
|
+
ruby: ['3.2', '3.3']
|
|
309
|
+
rails: ['8.0']
|
|
310
|
+
|
|
311
|
+
steps:
|
|
312
|
+
- uses: actions/checkout@v4
|
|
313
|
+
|
|
314
|
+
- name: Set up Ruby
|
|
315
|
+
uses: ruby/setup-ruby@v1
|
|
316
|
+
with:
|
|
317
|
+
ruby-version: ${{ matrix.ruby }}
|
|
318
|
+
bundler-cache: true
|
|
319
|
+
|
|
320
|
+
- name: Run tests
|
|
321
|
+
run: bundle exec rspec
|
|
322
|
+
|
|
323
|
+
- name: Run linter
|
|
324
|
+
run: bundle exec rubocop
|
|
325
|
+
|
|
326
|
+
- name: Check coverage
|
|
327
|
+
run: |
|
|
328
|
+
bundle exec rspec
|
|
329
|
+
cat coverage/.last_run.json | jq '.result.line'
|
|
330
|
+
```
|
|
331
|
+
- ✅ DoD: CI runs on every commit, badge in README
|
|
332
|
+
|
|
333
|
+
5. **Setup Docker Compose (Test Backends) (2 hours)**
|
|
334
|
+
```yaml
|
|
335
|
+
# docker-compose.yml
|
|
336
|
+
version: '3.8'
|
|
337
|
+
services:
|
|
338
|
+
loki:
|
|
339
|
+
image: grafana/loki:2.9.0
|
|
340
|
+
ports:
|
|
341
|
+
- "3100:3100"
|
|
342
|
+
volumes:
|
|
343
|
+
- ./config/loki-local-config.yaml:/etc/loki/local-config.yaml
|
|
344
|
+
|
|
345
|
+
prometheus:
|
|
346
|
+
image: prom/prometheus:v2.45.0
|
|
347
|
+
ports:
|
|
348
|
+
- "9090:9090"
|
|
349
|
+
|
|
350
|
+
elasticsearch:
|
|
351
|
+
image: elasticsearch:8.9.0
|
|
352
|
+
ports:
|
|
353
|
+
- "9200:9200"
|
|
354
|
+
environment:
|
|
355
|
+
- discovery.type=single-node
|
|
356
|
+
- xpack.security.enabled=false
|
|
357
|
+
|
|
358
|
+
redis:
|
|
359
|
+
image: redis:7-alpine
|
|
360
|
+
ports:
|
|
361
|
+
- "6379:6379"
|
|
362
|
+
```
|
|
363
|
+
- ✅ DoD: `docker-compose up` runs all backends
|
|
364
|
+
|
|
365
|
+
6. **Setup SimpleCov (Code Coverage) (1 hour)**
|
|
366
|
+
```ruby
|
|
367
|
+
# spec/spec_helper.rb
|
|
368
|
+
require 'simplecov'
|
|
369
|
+
|
|
370
|
+
SimpleCov.start do
|
|
371
|
+
add_filter '/spec/'
|
|
372
|
+
add_filter '/benchmarks/'
|
|
373
|
+
|
|
374
|
+
minimum_coverage 100 # Enforce 100% coverage!
|
|
375
|
+
refuse_coverage_drop # Fail if coverage decreases
|
|
376
|
+
end
|
|
377
|
+
```
|
|
378
|
+
- ✅ DoD: Coverage enforced in CI
|
|
379
|
+
|
|
380
|
+
7. **Setup Rubocop (Linting) (1 hour)**
|
|
381
|
+
```yaml
|
|
382
|
+
# .rubocop.yml
|
|
383
|
+
require:
|
|
384
|
+
- rubocop-performance
|
|
385
|
+
- rubocop-rspec
|
|
386
|
+
|
|
387
|
+
AllCops:
|
|
388
|
+
TargetRubyVersion: 3.2
|
|
389
|
+
NewCops: enable
|
|
390
|
+
Exclude:
|
|
391
|
+
- 'vendor/**/*'
|
|
392
|
+
- 'benchmarks/**/*'
|
|
393
|
+
|
|
394
|
+
Style/Documentation:
|
|
395
|
+
Enabled: true # Enforce YARD docs!
|
|
396
|
+
|
|
397
|
+
Metrics/BlockLength:
|
|
398
|
+
Exclude:
|
|
399
|
+
- 'spec/**/*'
|
|
400
|
+
|
|
401
|
+
Metrics/MethodLength:
|
|
402
|
+
Max: 15
|
|
403
|
+
|
|
404
|
+
Metrics/ClassLength:
|
|
405
|
+
Max: 100
|
|
406
|
+
```
|
|
407
|
+
- ✅ DoD: Linting passes in CI
|
|
408
|
+
|
|
409
|
+
8. **Setup Gemspec (1 hour)**
|
|
410
|
+
```ruby
|
|
411
|
+
# e11y.gemspec
|
|
412
|
+
Gem::Specification.new do |spec|
|
|
413
|
+
spec.name = "e11y"
|
|
414
|
+
spec.version = E11y::VERSION
|
|
415
|
+
spec.authors = ["Your Team"]
|
|
416
|
+
spec.email = ["team@example.com"]
|
|
417
|
+
|
|
418
|
+
spec.summary = "Event-driven observability gem for Rails 8+"
|
|
419
|
+
spec.description = "Structured business events with request-scoped buffering, pattern-based metrics, and zero-config SLO tracking"
|
|
420
|
+
spec.homepage = "https://github.com/yourorg/e11y"
|
|
421
|
+
spec.license = "MIT"
|
|
422
|
+
spec.required_ruby_version = ">= 3.2.0"
|
|
423
|
+
|
|
424
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
425
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
|
426
|
+
spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
|
|
427
|
+
|
|
428
|
+
spec.files = Dir["{lib,docs}/**/*", "LICENSE.txt", "README.md", "CHANGELOG.md"]
|
|
429
|
+
spec.require_paths = ["lib"]
|
|
430
|
+
|
|
431
|
+
# Runtime dependencies
|
|
432
|
+
spec.add_dependency "rails", ">= 8.0.0"
|
|
433
|
+
spec.add_dependency "zeitwerk", "~> 2.6"
|
|
434
|
+
spec.add_dependency "dry-schema", "~> 1.13"
|
|
435
|
+
spec.add_dependency "concurrent-ruby", "~> 1.2"
|
|
436
|
+
|
|
437
|
+
# Development dependencies
|
|
438
|
+
spec.add_development_dependency "rspec", "~> 3.12"
|
|
439
|
+
spec.add_development_dependency "rubocop", "~> 1.57"
|
|
440
|
+
spec.add_development_dependency "rubocop-performance", "~> 1.19"
|
|
441
|
+
spec.add_development_dependency "rubocop-rspec", "~> 2.25"
|
|
442
|
+
spec.add_development_dependency "simplecov", "~> 0.22"
|
|
443
|
+
end
|
|
444
|
+
```
|
|
445
|
+
- ✅ DoD: `gem build e11y.gemspec` succeeds
|
|
446
|
+
|
|
447
|
+
**Verification (L6):**
|
|
448
|
+
- Gem builds: `gem build e11y.gemspec`
|
|
449
|
+
- Tests run: `bundle exec rspec` (even if empty)
|
|
450
|
+
- Linting passes: `bundle exec rubocop`
|
|
451
|
+
- CI green: GitHub Actions badge
|
|
452
|
+
- Docker Compose: `docker-compose up` runs all services
|
|
453
|
+
|
|
454
|
+
---
|
|
455
|
+
|
|
456
|
+
#### L3.0.3: Research Documentation Deliverables
|
|
457
|
+
|
|
458
|
+
**Deliverables:**
|
|
459
|
+
|
|
460
|
+
1. **`docs/research/devise_patterns.md`** - Controller overrides, modular design
|
|
461
|
+
2. **`docs/research/sidekiq_patterns.md`** - Middleware, configuration, error handling
|
|
462
|
+
3. **`docs/research/puma_patterns.md`** - Configuration DSL, plugins, thread safety
|
|
463
|
+
4. **`docs/research/dry_rb_patterns.md`** - Functional design, composition, types
|
|
464
|
+
5. **`docs/research/yabeda_patterns.md`** - Metrics DSL, extensibility
|
|
465
|
+
6. **`docs/research/sentry_patterns.md`** - Rails integration, context propagation
|
|
466
|
+
7. **`docs/research/gem_structure_template.md`** - Directory conventions, autoloading
|
|
467
|
+
8. **`docs/research/configuration_dsl_design.md`** - DSL patterns, type validation
|
|
468
|
+
9. **`docs/research/testing_strategy.md`** - RSpec setup, coverage, factories
|
|
469
|
+
10. **`docs/research/documentation_standards.md`** - YARD, README, guides
|
|
470
|
+
11. **`docs/research/ci_cd_setup.md`** - GitHub Actions, matrix testing
|
|
471
|
+
12. **`docs/research/gem_release_process.md`** - Versioning, changelog, deprecation
|
|
472
|
+
|
|
473
|
+
**Review Meeting (2 hours):**
|
|
474
|
+
- Present research findings to team
|
|
475
|
+
- Decide which patterns to adopt
|
|
476
|
+
- Update ADRs if needed (architectural decisions from research)
|
|
477
|
+
|
|
478
|
+
---
|
|
479
|
+
|
|
480
|
+
## PHASE 1: FOUNDATION (Weeks 1-4)
|
|
481
|
+
|
|
482
|
+
### L2.1: Zero-Allocation Event Base 🔴
|
|
483
|
+
|
|
484
|
+
**ADR:** ADR-001 §2 (Zero-Allocation Design)
|
|
485
|
+
**UC:** UC-002 (Business Event Tracking)
|
|
486
|
+
**Depends On:** None (foundation)
|
|
487
|
+
**Parallelizable:** ⚙️ Core team (1 dev)
|
|
488
|
+
|
|
489
|
+
#### L3.1.1: Event::Base Class Implementation
|
|
490
|
+
|
|
491
|
+
**Tasks:**
|
|
492
|
+
1. **Create Event::Base Class Structure**
|
|
493
|
+
- File: `lib/e11y/event/base.rb`
|
|
494
|
+
- Zero-allocation design (class methods only)
|
|
495
|
+
- Schema DSL (dry-schema integration)
|
|
496
|
+
- Severity levels (:debug, :info, :success, :warn, :error, :fatal)
|
|
497
|
+
- Version tracking
|
|
498
|
+
- ✅ DoD: All 22 severity-based tests pass
|
|
499
|
+
|
|
500
|
+
2. **Implement track() Method (Zero-Allocation)**
|
|
501
|
+
- `self.track(**payload)` class method
|
|
502
|
+
- Payload validation (dry-schema)
|
|
503
|
+
- Hash-based event creation (no object instantiation)
|
|
504
|
+
- Return event hash (for testing/debugging)
|
|
505
|
+
- ✅ DoD: <50μs per track() call (benchmark)
|
|
506
|
+
|
|
507
|
+
3. **Configuration DSL (Event-Level)**
|
|
508
|
+
- `severity(value)` class method
|
|
509
|
+
- `schema(&block)` class method
|
|
510
|
+
- `version(value)` class method
|
|
511
|
+
- `adapters(list)` class method (references)
|
|
512
|
+
- Resolution logic (precedence: explicit > preset > base > convention)
|
|
513
|
+
- ✅ DoD: All 15 configuration tests pass
|
|
514
|
+
|
|
515
|
+
4. **Convention-Based Defaults (CONTRADICTION_01 Solution)**
|
|
516
|
+
- Severity from event name (`*Failed` → :error, `*Paid` → :success)
|
|
517
|
+
- Adapter from severity (`:error` → `[:sentry]`, others → `[:loki]`)
|
|
518
|
+
- Sample rate from severity (error: 1.0, success: 0.1, debug: 0.01)
|
|
519
|
+
- Retention from severity (error: 90d, success: 30d, debug: 7d)
|
|
520
|
+
- ✅ DoD: Zero config for 90% of events (only schema required)
|
|
521
|
+
|
|
522
|
+
5. **Base Event Classes & Presets (Inheritance)**
|
|
523
|
+
- `Events::BaseAuditEvent` (audit config)
|
|
524
|
+
- `Events::BasePaymentEvent` (high-value config)
|
|
525
|
+
- `E11y::Presets::HighValueEvent` module
|
|
526
|
+
- `E11y::Presets::DebugEvent` module
|
|
527
|
+
- `E11y::Presets::AuditEvent` module
|
|
528
|
+
- ✅ DoD: 1-5 lines config per event (schema + inheritance)
|
|
529
|
+
|
|
530
|
+
**Verification (L6):**
|
|
531
|
+
- Unit tests: `spec/e11y/event/base_spec.rb` (100+ tests)
|
|
532
|
+
- Benchmark: `<50μs` per track() call (p99)
|
|
533
|
+
- Memory: Zero allocations (verified via allocation_tracer)
|
|
534
|
+
- ADR Compliance: ADR-001 §2 checklist
|
|
535
|
+
- UC Compliance: UC-002 §3 (Event DSL requirements)
|
|
536
|
+
|
|
537
|
+
---
|
|
538
|
+
|
|
539
|
+
### L2.2: Adaptive Buffer (C20 Resolution) 🔴
|
|
540
|
+
|
|
541
|
+
**ADR:** ADR-001 §3.3 (Adaptive Buffer with Memory Limits)
|
|
542
|
+
**UC:** UC-001 (Request-Scoped Debug Buffering)
|
|
543
|
+
**Depends On:** None
|
|
544
|
+
**Parallelizable:** ⚙️ Can run parallel to L2.1
|
|
545
|
+
|
|
546
|
+
#### L3.2.1: RingBuffer (Lock-Free)
|
|
547
|
+
|
|
548
|
+
**Tasks:**
|
|
549
|
+
1. **Implement RingBuffer Class**
|
|
550
|
+
- File: `lib/e11y/buffers/ring_buffer.rb`
|
|
551
|
+
- Fixed capacity SPSC (Single-Producer, Single-Consumer)
|
|
552
|
+
- Atomic operations (`Concurrent::AtomicFixnum`)
|
|
553
|
+
- Backpressure strategies: `:block`, `:drop`, `:throttle`
|
|
554
|
+
- `push(event)`, `pop()`, `flush_all()` methods
|
|
555
|
+
- ✅ DoD: 100K events/sec throughput (benchmark)
|
|
556
|
+
|
|
557
|
+
2. **Write RingBuffer Tests**
|
|
558
|
+
- Thread safety tests (100+ threads)
|
|
559
|
+
- Backpressure tests (overflow scenarios)
|
|
560
|
+
- Flush tests (partial/full)
|
|
561
|
+
- Memory tests (no leaks)
|
|
562
|
+
- ✅ DoD: 100% code coverage, all concurrency tests pass
|
|
563
|
+
|
|
564
|
+
**Verification (L6):**
|
|
565
|
+
- Unit tests: `spec/e11y/buffers/ring_buffer_spec.rb`
|
|
566
|
+
- Concurrency tests: 100+ threads, no data races
|
|
567
|
+
- Benchmark: `100K events/sec` throughput (1KB events)
|
|
568
|
+
- Memory: No memory leaks (valgrind check)
|
|
569
|
+
|
|
570
|
+
#### L3.2.2: AdaptiveBuffer (C20 Resolution)
|
|
571
|
+
|
|
572
|
+
**Tasks:**
|
|
573
|
+
1. **Implement AdaptiveBuffer Class**
|
|
574
|
+
- File: `lib/e11y/buffers/adaptive_buffer.rb`
|
|
575
|
+
- Global memory tracking (`@@total_memory_bytes`)
|
|
576
|
+
- Event size estimation (`estimate_size(event)`)
|
|
577
|
+
- Memory limit enforcement (100MB default)
|
|
578
|
+
- Early flush at 80% threshold
|
|
579
|
+
- Backpressure on memory limit exceeded
|
|
580
|
+
- ✅ DoD: Memory <100MB under 10K events/sec load
|
|
581
|
+
|
|
582
|
+
2. **Memory Size Estimation**
|
|
583
|
+
- `estimate_size(event_hash)` method
|
|
584
|
+
- String size calculation
|
|
585
|
+
- Hash overhead estimation (~40 bytes per key-value)
|
|
586
|
+
- Array overhead estimation
|
|
587
|
+
- ✅ DoD: ±10% accuracy vs actual memory usage
|
|
588
|
+
|
|
589
|
+
3. **Backpressure Strategies**
|
|
590
|
+
- `:block` - Block track() until memory frees up
|
|
591
|
+
- `:drop` - Drop event, increment drop counter
|
|
592
|
+
- `:throttle` - Sleep 1ms before retry (5 retries max)
|
|
593
|
+
- Metrics: `e11y.buffer.events_dropped`, `e11y.buffer.backpressure_triggered`
|
|
594
|
+
- ✅ DoD: No memory exhaustion under 10K+ events/sec
|
|
595
|
+
|
|
596
|
+
**Verification (L6):**
|
|
597
|
+
- Unit tests: `spec/e11y/buffers/adaptive_buffer_spec.rb`
|
|
598
|
+
- Load tests: 10K events/sec for 60 seconds
|
|
599
|
+
- Memory tests: Memory stays <100MB (verified)
|
|
600
|
+
- ADR Compliance: ADR-001 §3.3 checklist
|
|
601
|
+
|
|
602
|
+
#### L3.2.3: Request-Scoped Buffer (UC-001)
|
|
603
|
+
|
|
604
|
+
**Tasks:**
|
|
605
|
+
1. **Implement RequestScopedBuffer**
|
|
606
|
+
- File: `lib/e11y/buffers/request_scoped_buffer.rb`
|
|
607
|
+
- Thread-local storage (`ActiveSupport::CurrentAttributes`)
|
|
608
|
+
- Severity-based buffering (`:debug` events only)
|
|
609
|
+
- Auto-flush on error (severity >= `:error`)
|
|
610
|
+
- Auto-flush on request end
|
|
611
|
+
- ✅ DoD: Debug events flushed only on error
|
|
612
|
+
|
|
613
|
+
2. **Rails Middleware Integration**
|
|
614
|
+
- File: `lib/e11y/middleware/request.rb`
|
|
615
|
+
- Setup request buffer at request start
|
|
616
|
+
- Flush on error (exception caught)
|
|
617
|
+
- Flush on request end (ensure cleanup)
|
|
618
|
+
- Metrics: `e11y.buffer.request_flushes`, `e11y.buffer.debug_events_dropped`
|
|
619
|
+
- ✅ DoD: Zero debug logs in success requests
|
|
620
|
+
|
|
621
|
+
**Verification (L6):**
|
|
622
|
+
- Unit tests: `spec/e11y/buffers/request_scoped_buffer_spec.rb`
|
|
623
|
+
- Integration tests: Rails request simulation
|
|
624
|
+
- UC Compliance: UC-001 §4 (Auto-Flush Logic)
|
|
625
|
+
|
|
626
|
+
---
|
|
627
|
+
|
|
628
|
+
### L2.3: Middleware Pipeline 🔴
|
|
629
|
+
|
|
630
|
+
**ADR:** ADR-001 §2.2 (Middleware Chain), ADR-015 (Middleware Order)
|
|
631
|
+
**UC:** UC-001, UC-007, UC-011
|
|
632
|
+
**Depends On:** L2.1 (Event::Base), L2.2 (Buffers)
|
|
633
|
+
**Parallelizable:** ⚙️ Can run parallel to L2.2
|
|
634
|
+
|
|
635
|
+
#### L3.3.1: Middleware::Base Contract
|
|
636
|
+
|
|
637
|
+
**Tasks:**
|
|
638
|
+
1. **Create Middleware::Base Class**
|
|
639
|
+
- File: `lib/e11y/middleware/base.rb`
|
|
640
|
+
- `call(event_data)` method contract
|
|
641
|
+
- `@app.call(event_data)` chain pattern
|
|
642
|
+
- Middleware zones support (`:pre_processing`, `:security`, `:routing`, `:post_processing`, `:adapters`)
|
|
643
|
+
- `middleware_zone(zone)` class method
|
|
644
|
+
- ✅ DoD: All middlewares inherit from Base
|
|
645
|
+
|
|
646
|
+
2. **Pipeline Builder**
|
|
647
|
+
- File: `lib/e11y/pipeline/builder.rb`
|
|
648
|
+
- `use(middleware_class, *args, **options)` method
|
|
649
|
+
- Zone-based organization
|
|
650
|
+
- Middleware ordering validation
|
|
651
|
+
- Boot-time validation (`validate_zones!`)
|
|
652
|
+
- ✅ DoD: Pipeline builds correctly, zones validated
|
|
653
|
+
|
|
654
|
+
**Verification (L6):**
|
|
655
|
+
- Unit tests: `spec/e11y/middleware/base_spec.rb`
|
|
656
|
+
- Pipeline tests: `spec/e11y/pipeline/builder_spec.rb`
|
|
657
|
+
- ADR Compliance: ADR-015 §3.4 (Middleware Zones)
|
|
658
|
+
|
|
659
|
+
#### L3.3.2: Core Middlewares (Minimal Set)
|
|
660
|
+
|
|
661
|
+
**Tasks:**
|
|
662
|
+
1. **TraceContext Middleware**
|
|
663
|
+
- File: `lib/e11y/middleware/trace_context.rb`
|
|
664
|
+
- Add `trace_id`, `span_id`, `timestamp` (ISO8601)
|
|
665
|
+
- Zone: `:pre_processing`
|
|
666
|
+
- ✅ DoD: All events have trace_id
|
|
667
|
+
|
|
668
|
+
2. **Validation Middleware**
|
|
669
|
+
- File: `lib/e11y/middleware/validation.rb`
|
|
670
|
+
- Schema validation (dry-schema)
|
|
671
|
+
- Original class name validation
|
|
672
|
+
- Zone: `:pre_processing`
|
|
673
|
+
- ✅ DoD: Invalid events rejected with clear error
|
|
674
|
+
|
|
675
|
+
3. **Versioning Middleware (LAST!)**
|
|
676
|
+
- File: `lib/e11y/middleware/versioning.rb`
|
|
677
|
+
- Normalize event_name (`Events::OrderPaidV2` → `Events::OrderPaid`)
|
|
678
|
+
- Add `v: 2` to payload
|
|
679
|
+
- Zone: `:post_processing` (LAST before routing!)
|
|
680
|
+
- ✅ DoD: All business logic uses original class name
|
|
681
|
+
|
|
682
|
+
4. **Routing Middleware**
|
|
683
|
+
- File: `lib/e11y/middleware/routing.rb`
|
|
684
|
+
- Route to buffer (main, request-scoped, audit)
|
|
685
|
+
- Adapter routing based on event config
|
|
686
|
+
- Zone: `:adapters`
|
|
687
|
+
- ✅ DoD: Events routed to correct buffer
|
|
688
|
+
|
|
689
|
+
**Verification (L6):**
|
|
690
|
+
- Unit tests: Each middleware has 100% coverage
|
|
691
|
+
- Integration tests: Full pipeline execution
|
|
692
|
+
- ADR Compliance: ADR-015 §3.1 (Correct Order)
|
|
693
|
+
|
|
694
|
+
---
|
|
695
|
+
|
|
696
|
+
## PHASE 2: CORE FEATURES (Weeks 5-10)
|
|
697
|
+
|
|
698
|
+
### L2.4: PII Filtering & Security (C01, C19) 🔴
|
|
699
|
+
|
|
700
|
+
**ADR:** ADR-006 (Security & Compliance), ADR-015 §3.3 (Audit Pipeline)
|
|
701
|
+
**UC:** UC-007 (PII Filtering), UC-012 (Audit Trail)
|
|
702
|
+
**Depends On:** L2.1, L2.3 (Middleware Pipeline)
|
|
703
|
+
**Parallelizable:** ⚙️ Stream A (1-2 devs)
|
|
704
|
+
|
|
705
|
+
#### L3.4.1: PII Filtering Middleware
|
|
706
|
+
|
|
707
|
+
**Tasks:**
|
|
708
|
+
1. **PIIFiltering Middleware**
|
|
709
|
+
- File: `lib/e11y/middleware/pii_filtering.rb`
|
|
710
|
+
- 3-tier filtering strategy (Tier 1: no PII, Tier 2: hash, Tier 3: mask)
|
|
711
|
+
- Field-level strategies (`:mask`, `:hash`, `:redact`, `:allow`)
|
|
712
|
+
- Pattern-based detection (`/email|password|ssn|card/i`)
|
|
713
|
+
- Zone: `:security` (CRITICAL!)
|
|
714
|
+
- ✅ DoD: All PII fields filtered before adapters
|
|
715
|
+
|
|
716
|
+
2. **Event-Level PII Configuration**
|
|
717
|
+
- `contains_pii(boolean)` class method
|
|
718
|
+
- `pii_filtering(&block)` DSL
|
|
719
|
+
- `masks(*fields)` shortcut
|
|
720
|
+
- `hashes(*fields)` shortcut
|
|
721
|
+
- `allows(*fields)` shortcut
|
|
722
|
+
- ✅ DoD: Per-event PII rules work correctly
|
|
723
|
+
|
|
724
|
+
3. **Global PII Patterns**
|
|
725
|
+
- File: `lib/e11y/pii/patterns.rb`
|
|
726
|
+
- Universal patterns (email, password, ssn, credit_card, ip_address)
|
|
727
|
+
- Custom patterns support
|
|
728
|
+
- ✅ DoD: 95%+ PII detection rate (test dataset)
|
|
729
|
+
|
|
730
|
+
**Verification (L6):**
|
|
731
|
+
- Unit tests: `spec/e11y/middleware/pii_filtering_spec.rb`
|
|
732
|
+
- Integration tests: Full pipeline with PII events
|
|
733
|
+
- UC Compliance: UC-007 §3 (3-Tier Strategy)
|
|
734
|
+
- Security audit: GDPR compliance checklist
|
|
735
|
+
|
|
736
|
+
#### L3.4.2: Audit Pipeline (C01 Resolution)
|
|
737
|
+
|
|
738
|
+
**Tasks:**
|
|
739
|
+
1. **AuditSigning Middleware**
|
|
740
|
+
- File: `lib/e11y/middleware/audit_signing.rb`
|
|
741
|
+
- HMAC-SHA256 signature on original data (pre-PII filtering!)
|
|
742
|
+
- Signature metadata (timestamp, key_id, algorithm)
|
|
743
|
+
- Zone: `:security` (runs INSTEAD of PIIFiltering for audit events)
|
|
744
|
+
- ✅ DoD: Audit events have valid signatures
|
|
745
|
+
|
|
746
|
+
2. **Audit Pipeline Configuration**
|
|
747
|
+
- `audit_event(boolean)` class method on Event::Base
|
|
748
|
+
- Separate pipeline for audit events (skip PII filtering, rate limiting, sampling)
|
|
749
|
+
- Auto-route to encrypted storage adapter
|
|
750
|
+
- ✅ DoD: Audit events use audit pipeline
|
|
751
|
+
|
|
752
|
+
3. **AuditEncryptedAdapter**
|
|
753
|
+
- File: `lib/e11y/adapters/audit_encrypted.rb`
|
|
754
|
+
- AES-256-GCM encryption
|
|
755
|
+
- Signature verification before storage
|
|
756
|
+
- File-based storage with encryption
|
|
757
|
+
- ✅ DoD: Audit events stored encrypted with valid signatures
|
|
758
|
+
|
|
759
|
+
**Verification (L6):**
|
|
760
|
+
- Unit tests: `spec/e11y/middleware/audit_signing_spec.rb`
|
|
761
|
+
- Integration tests: Full audit pipeline
|
|
762
|
+
- Security tests: Signature verification, encryption validation
|
|
763
|
+
- ADR Compliance: ADR-015 §3.3 (C01 Resolution)
|
|
764
|
+
- UC Compliance: UC-012 §3 (Non-Repudiation)
|
|
765
|
+
|
|
766
|
+
#### L3.4.3: Middleware Zones (C19 Resolution)
|
|
767
|
+
|
|
768
|
+
**Tasks:**
|
|
769
|
+
1. **Zone Validation**
|
|
770
|
+
- File: `lib/e11y/pipeline/zone_validator.rb`
|
|
771
|
+
- Boot-time zone order validation
|
|
772
|
+
- Runtime zone violation detection
|
|
773
|
+
- Warning system (dev/staging) vs. error (production)
|
|
774
|
+
- ✅ DoD: Invalid middleware order rejected at boot
|
|
775
|
+
|
|
776
|
+
2. **Custom Middleware Constraints**
|
|
777
|
+
- Zone declaration (`middleware_zone :pre_processing`)
|
|
778
|
+
- `modifies_fields(*fields)` declaration
|
|
779
|
+
- PII bypass detection
|
|
780
|
+
- ✅ DoD: Custom middleware cannot bypass PII filtering
|
|
781
|
+
|
|
782
|
+
**Verification (L6):**
|
|
783
|
+
- Unit tests: `spec/e11y/pipeline/zone_validator_spec.rb`
|
|
784
|
+
- Integration tests: Safe vs. unsafe middleware scenarios
|
|
785
|
+
- ADR Compliance: ADR-015 §3.4 (C19 Resolution)
|
|
786
|
+
|
|
787
|
+
---
|
|
788
|
+
|
|
789
|
+
### L2.5: Adapter Architecture 🟠
|
|
790
|
+
|
|
791
|
+
**ADR:** ADR-004 (Adapter Architecture)
|
|
792
|
+
**UC:** UC-005 (Sentry), UC-008 (OpenTelemetry)
|
|
793
|
+
**Depends On:** L2.1 (Event::Base)
|
|
794
|
+
**Parallelizable:** ⚙️ Stream B (2-3 devs, can work in parallel with Stream A)
|
|
795
|
+
|
|
796
|
+
#### L3.5.1: Adapter::Base Contract
|
|
797
|
+
|
|
798
|
+
**Tasks:**
|
|
799
|
+
1. **Create Adapter::Base Class**
|
|
800
|
+
- File: `lib/e11y/adapters/base.rb`
|
|
801
|
+
- `write(event_data)` method
|
|
802
|
+
- `write_batch(events)` method (default: loop `write`)
|
|
803
|
+
- `healthy?()` method
|
|
804
|
+
- `close()` method
|
|
805
|
+
- `capabilities()` method (returns supported features)
|
|
806
|
+
- ✅ DoD: All adapters inherit from Base
|
|
807
|
+
|
|
808
|
+
2. **Adapter Registry**
|
|
809
|
+
- File: `lib/e11y/adapters/registry.rb`
|
|
810
|
+
- `register(name, adapter_instance)` method
|
|
811
|
+
- `resolve(name)` method
|
|
812
|
+
- Thread-safe registry
|
|
813
|
+
- ✅ DoD: Adapters registered and resolved correctly
|
|
814
|
+
|
|
815
|
+
**Verification (L6):**
|
|
816
|
+
- Unit tests: `spec/e11y/adapters/base_spec.rb`
|
|
817
|
+
- Contract tests: All adapters pass contract tests
|
|
818
|
+
- ADR Compliance: ADR-004 §2 (Base Adapter Contract)
|
|
819
|
+
|
|
820
|
+
#### L3.5.2: Built-In Adapters
|
|
821
|
+
|
|
822
|
+
**Tasks (Parallelizable - Each adapter can be built independently):**
|
|
823
|
+
|
|
824
|
+
1. **StdoutAdapter** (Priority: HIGH, for dev)
|
|
825
|
+
- File: `lib/e11y/adapters/stdout.rb`
|
|
826
|
+
- Pretty-print events to STDOUT
|
|
827
|
+
- Colorization support (severity-based colors)
|
|
828
|
+
- JSON formatting
|
|
829
|
+
- ✅ DoD: Events printed correctly, colors work
|
|
830
|
+
|
|
831
|
+
2. **FileAdapter** (Priority: HIGH, for local dev)
|
|
832
|
+
- File: `lib/e11y/adapters/file.rb`
|
|
833
|
+
- JSONL format
|
|
834
|
+
- Log rotation support (size/time-based)
|
|
835
|
+
- Compression support (gzip)
|
|
836
|
+
- ✅ DoD: Events written to file, rotation works
|
|
837
|
+
|
|
838
|
+
3. **LokiAdapter** (Priority: CRITICAL, main adapter)
|
|
839
|
+
- File: `lib/e11y/adapters/loki.rb`
|
|
840
|
+
- Loki Push API integration
|
|
841
|
+
- Batching (500 events or 5s timeout)
|
|
842
|
+
- Compression (gzip)
|
|
843
|
+
- Label extraction (severity, event_name)
|
|
844
|
+
- ✅ DoD: Events sent to Loki successfully
|
|
845
|
+
|
|
846
|
+
4. **SentryAdapter** (Priority: HIGH, error tracking)
|
|
847
|
+
- File: `lib/e11y/adapters/sentry.rb`
|
|
848
|
+
- Sentry SDK integration
|
|
849
|
+
- Severity mapping (error/fatal → Sentry)
|
|
850
|
+
- Breadcrumb support
|
|
851
|
+
- Context enrichment
|
|
852
|
+
- ✅ DoD: Errors reported to Sentry with context
|
|
853
|
+
|
|
854
|
+
5. **ElasticsearchAdapter** (Priority: MEDIUM, optional)
|
|
855
|
+
- File: `lib/e11y/adapters/elasticsearch.rb`
|
|
856
|
+
- Elasticsearch bulk API
|
|
857
|
+
- Index rotation (daily/weekly)
|
|
858
|
+
- Type mapping
|
|
859
|
+
- ✅ DoD: Events indexed in Elasticsearch
|
|
860
|
+
|
|
861
|
+
6. **InMemoryAdapter** (Priority: HIGH, for tests)
|
|
862
|
+
- File: `lib/e11y/adapters/in_memory.rb`
|
|
863
|
+
- Store events in array
|
|
864
|
+
- Query methods for RSpec assertions
|
|
865
|
+
- ✅ DoD: Test events captured and queryable
|
|
866
|
+
|
|
867
|
+
**Verification (L6):**
|
|
868
|
+
- Unit tests: Each adapter has 100% coverage
|
|
869
|
+
- Integration tests: Real backend tests (Docker Compose)
|
|
870
|
+
- ADR Compliance: ADR-004 §3 (Built-In Adapters)
|
|
871
|
+
|
|
872
|
+
#### L3.5.3: Connection Pooling & Retry
|
|
873
|
+
|
|
874
|
+
**Tasks:**
|
|
875
|
+
1. **ConnectionPool**
|
|
876
|
+
- File: `lib/e11y/adapters/connection_pool.rb`
|
|
877
|
+
- Pool size configuration (default: 5)
|
|
878
|
+
- Checkout/checkin pattern
|
|
879
|
+
- Idle timeout
|
|
880
|
+
- ✅ DoD: Connections reused efficiently
|
|
881
|
+
|
|
882
|
+
2. **RetryHandler (Exponential Backoff + Jitter)**
|
|
883
|
+
- File: `lib/e11y/adapters/retry_handler.rb`
|
|
884
|
+
- Exponential backoff (1s, 2s, 4s, 8s, 16s)
|
|
885
|
+
- Jitter (+/-20%)
|
|
886
|
+
- Retriable errors (network, timeout, 5xx)
|
|
887
|
+
- ✅ DoD: Transient errors retried successfully
|
|
888
|
+
|
|
889
|
+
3. **CircuitBreaker**
|
|
890
|
+
- File: `lib/e11y/adapters/circuit_breaker.rb`
|
|
891
|
+
- Failure threshold (5 failures → open)
|
|
892
|
+
- Half-open state (test 1 request)
|
|
893
|
+
- Timeout (30s before half-open)
|
|
894
|
+
- ✅ DoD: Cascading failures prevented
|
|
895
|
+
|
|
896
|
+
**Verification (L6):**
|
|
897
|
+
- Unit tests: `spec/e11y/adapters/retry_handler_spec.rb`
|
|
898
|
+
- Integration tests: Network failure scenarios
|
|
899
|
+
- ADR Compliance: ADR-004 §5 (Error Handling)
|
|
900
|
+
|
|
901
|
+
#### L3.5.4: Adaptive Batching
|
|
902
|
+
|
|
903
|
+
**Tasks:**
|
|
904
|
+
1. **AdaptiveBatcher**
|
|
905
|
+
- File: `lib/e11y/adapters/adaptive_batcher.rb`
|
|
906
|
+
- Batch size: 500 events (default)
|
|
907
|
+
- Timeout: 5 seconds (default)
|
|
908
|
+
- Flush on batch full or timeout
|
|
909
|
+
- ✅ DoD: Events batched efficiently, latency <5s
|
|
910
|
+
|
|
911
|
+
**Verification (L6):**
|
|
912
|
+
- Unit tests: `spec/e11y/adapters/adaptive_batcher_spec.rb`
|
|
913
|
+
- Performance tests: Batching efficiency vs. latency trade-off
|
|
914
|
+
- ADR Compliance: ADR-004 §6 (Performance & Batching)
|
|
915
|
+
|
|
916
|
+
---
|
|
917
|
+
|
|
918
|
+
### L2.6: Metrics & Yabeda Integration 🟠
|
|
919
|
+
|
|
920
|
+
**ADR:** ADR-002 (Metrics & Yabeda Integration)
|
|
921
|
+
**UC:** UC-003 (Pattern-Based Metrics), UC-013 (High Cardinality Protection)
|
|
922
|
+
**Depends On:** L2.1 (Event::Base), L2.3 (Middleware Pipeline)
|
|
923
|
+
**Parallelizable:** ⚙️ Stream C (1-2 devs, parallel to Stream A & B)
|
|
924
|
+
|
|
925
|
+
#### L3.6.1: Yabeda Integration
|
|
926
|
+
|
|
927
|
+
**Tasks:**
|
|
928
|
+
1. **YabedaIntegration Setup**
|
|
929
|
+
- File: `lib/e11y/metrics/yabeda_integration.rb`
|
|
930
|
+
- Yabeda group `:e11y`
|
|
931
|
+
- Auto-register metrics (counters, histograms, gauges)
|
|
932
|
+
- Metric naming convention (`e11y.events.tracked`, `e11y.events.dropped`)
|
|
933
|
+
- ✅ DoD: Metrics exported to Prometheus
|
|
934
|
+
|
|
935
|
+
2. **Pattern-Based Metrics**
|
|
936
|
+
- File: `lib/e11y/metrics/pattern_matcher.rb`
|
|
937
|
+
- Glob pattern to regex conversion
|
|
938
|
+
- Pattern matching (e.g., `payment.*` matches `payment.succeeded`)
|
|
939
|
+
- Metric configuration DSL
|
|
940
|
+
- ✅ DoD: Patterns match events correctly
|
|
941
|
+
|
|
942
|
+
3. **Metrics Middleware**
|
|
943
|
+
- File: `lib/e11y/middleware/metrics.rb`
|
|
944
|
+
- Pattern matching
|
|
945
|
+
- Label extraction (from payload)
|
|
946
|
+
- Metric update (counter/histogram/gauge)
|
|
947
|
+
- Zone: `:post_processing`
|
|
948
|
+
- ✅ DoD: Metrics updated for matched events
|
|
949
|
+
|
|
950
|
+
**Verification (L6):**
|
|
951
|
+
- Unit tests: `spec/e11y/metrics/yabeda_integration_spec.rb`
|
|
952
|
+
- Integration tests: Full metric collection pipeline
|
|
953
|
+
- UC Compliance: UC-003 §3 (Pattern-Based Metrics)
|
|
954
|
+
- ADR Compliance: ADR-002 §3 (Yabeda Integration)
|
|
955
|
+
|
|
956
|
+
#### L3.6.2: Cardinality Protection
|
|
957
|
+
|
|
958
|
+
**Tasks:**
|
|
959
|
+
1. **CardinalityProtection**
|
|
960
|
+
- File: `lib/e11y/metrics/cardinality_protection.rb`
|
|
961
|
+
- 4-layer defense (Denylist, Allowlist, Per-Metric Limits, Dynamic Actions)
|
|
962
|
+
- `FORBIDDEN_LABELS` constant (user_id, email, ip, etc.)
|
|
963
|
+
- `SAFE_LABELS` constant (severity, env, service)
|
|
964
|
+
- Cardinality tracker (per-metric unique values)
|
|
965
|
+
- ✅ DoD: High-cardinality labels rejected
|
|
966
|
+
|
|
967
|
+
2. **CardinalityTracker**
|
|
968
|
+
- File: `lib/e11y/metrics/cardinality_tracker.rb`
|
|
969
|
+
- Per-metric cardinality tracking
|
|
970
|
+
- Limit enforcement (default: 100 unique values)
|
|
971
|
+
- Metrics: `e11y.metrics.cardinality_exceeded`
|
|
972
|
+
- ✅ DoD: Cardinality explosions prevented
|
|
973
|
+
|
|
974
|
+
3. **Relabeling Rules**
|
|
975
|
+
- File: `lib/e11y/metrics/relabeling.rb`
|
|
976
|
+
- HTTP status code → class (200-299 → 2xx)
|
|
977
|
+
- Path pattern matching (`/users/123` → `/users/:id`)
|
|
978
|
+
- Regex-based relabeling
|
|
979
|
+
- ✅ DoD: High-cardinality labels transformed
|
|
980
|
+
|
|
981
|
+
**Verification (L6):**
|
|
982
|
+
- Unit tests: `spec/e11y/metrics/cardinality_protection_spec.rb`
|
|
983
|
+
- Load tests: High-cardinality scenario (10K+ unique values)
|
|
984
|
+
- UC Compliance: UC-013 §3 (4-Layer Defense)
|
|
985
|
+
- ADR Compliance: ADR-002 §4 (Cardinality Protection)
|
|
986
|
+
|
|
987
|
+
---
|
|
988
|
+
|
|
989
|
+
### L2.7: Sampling & Cost Optimization 🟠
|
|
990
|
+
|
|
991
|
+
**ADR:** ADR-009 (Cost Optimization), UC-014 (Adaptive Sampling)
|
|
992
|
+
**UC:** UC-014 (Adaptive Sampling), UC-015 (Cost Optimization)
|
|
993
|
+
**Depends On:** L2.1, L2.3 (Middleware Pipeline)
|
|
994
|
+
**Parallelizable:** ⚙️ Stream D (1 dev, parallel to A/B/C)
|
|
995
|
+
|
|
996
|
+
#### L3.7.1: Sampling Middleware
|
|
997
|
+
|
|
998
|
+
**Tasks:**
|
|
999
|
+
1. **Sampling Middleware**
|
|
1000
|
+
- File: `lib/e11y/middleware/sampling.rb`
|
|
1001
|
+
- Per-event sample rate (from event config)
|
|
1002
|
+
- Severity-based sampling (error: 1.0, success: 0.1, debug: 0.01)
|
|
1003
|
+
- Pattern-based sampling (e.g., `debug.*` → 0.01)
|
|
1004
|
+
- Zone: `:routing`
|
|
1005
|
+
- ✅ DoD: Events sampled according to config
|
|
1006
|
+
|
|
1007
|
+
2. **Trace-Aware Sampling (C05 Resolution)**
|
|
1008
|
+
- File: `lib/e11y/sampling/trace_aware.rb`
|
|
1009
|
+
- Trace-level sampling decision (not per-event)
|
|
1010
|
+
- Decision cache (TTL: 60s)
|
|
1011
|
+
- W3C Trace Context propagation
|
|
1012
|
+
- ✅ DoD: Distributed traces stay intact (all events sampled or none)
|
|
1013
|
+
|
|
1014
|
+
3. **Stratified Sampling for SLO (C11 Resolution)**
|
|
1015
|
+
- File: `lib/e11y/sampling/stratified.rb`
|
|
1016
|
+
- Sample by severity (error: 100%, success: 10%)
|
|
1017
|
+
- Sampling correction for metrics
|
|
1018
|
+
- ✅ DoD: SLO metrics accurate despite sampling
|
|
1019
|
+
|
|
1020
|
+
**Verification (L6):**
|
|
1021
|
+
- Unit tests: `spec/e11y/middleware/sampling_spec.rb`
|
|
1022
|
+
- Integration tests: Sampling scenarios
|
|
1023
|
+
- ADR Compliance: ADR-009 §3.6 (C05), §3.7 (C11)
|
|
1024
|
+
- UC Compliance: UC-014 §3 (Adaptive Sampling)
|
|
1025
|
+
|
|
1026
|
+
---
|
|
1027
|
+
|
|
1028
|
+
## PHASE 3: RAILS INTEGRATION (Weeks 11-14)
|
|
1029
|
+
|
|
1030
|
+
### L2.8: Railtie & Auto-Configuration 🔴
|
|
1031
|
+
|
|
1032
|
+
**ADR:** ADR-008 (Rails Integration)
|
|
1033
|
+
**UC:** UC-016 (Rails Logger Migration), UC-017 (Local Development)
|
|
1034
|
+
**Depends On:** L2.1-L2.7 (all core features)
|
|
1035
|
+
**Parallelizable:** ⚙️ Stream A (1-2 devs)
|
|
1036
|
+
|
|
1037
|
+
#### L3.8.1: Railtie Implementation
|
|
1038
|
+
|
|
1039
|
+
**Tasks:**
|
|
1040
|
+
1. **E11y::Railtie**
|
|
1041
|
+
- File: `lib/e11y/railtie.rb`
|
|
1042
|
+
- Auto-initialization on Rails boot
|
|
1043
|
+
- Middleware insertion (`E11y::Middleware::Request`)
|
|
1044
|
+
- ActiveSupport::Notifications integration
|
|
1045
|
+
- Sidekiq/ActiveJob hooks
|
|
1046
|
+
- Rails.logger bridge setup
|
|
1047
|
+
- ✅ DoD: E11y auto-configures in Rails app
|
|
1048
|
+
|
|
1049
|
+
2. **Quick Start Configuration**
|
|
1050
|
+
- `E11y.quick_start!` method
|
|
1051
|
+
- Environment detection (development, test, production)
|
|
1052
|
+
- Auto-register default adapters (stdout in dev, loki in prod)
|
|
1053
|
+
- Sensible defaults (sampling, rate limits)
|
|
1054
|
+
- ✅ DoD: Zero-config Rails app with E11y
|
|
1055
|
+
|
|
1056
|
+
**Verification (L6):**
|
|
1057
|
+
- Integration tests: Rails app with E11y auto-configured
|
|
1058
|
+
- UC Compliance: UC-017 §3 (Local Development Setup)
|
|
1059
|
+
- ADR Compliance: ADR-008 §2 (Railtie Implementation)
|
|
1060
|
+
|
|
1061
|
+
#### L3.8.2: Rails Instrumentation (Unidirectional Flow)
|
|
1062
|
+
|
|
1063
|
+
**Design Update (2026-01-17):** Unidirectional flow (ASN → E11y), no reverse publishing.
|
|
1064
|
+
|
|
1065
|
+
**Tasks:**
|
|
1066
|
+
|
|
1067
|
+
1. **Rails Instrumentation Setup**
|
|
1068
|
+
- File: `lib/e11y/instruments/rails_instrumentation.rb`
|
|
1069
|
+
- ONLY subscribe to ActiveSupport::Notifications (ASN → E11y)
|
|
1070
|
+
- Subscribe to `*.action_controller`, `*.active_record`, `*.active_job`
|
|
1071
|
+
- Convert Rails events to E11y events
|
|
1072
|
+
- Selective instrumentation (configure which events to track)
|
|
1073
|
+
- ✅ DoD: Rails internal events tracked as E11y events (unidirectional)
|
|
1074
|
+
|
|
1075
|
+
2. **Built-In Event Classes (Overridable!)**
|
|
1076
|
+
- `Events::Rails::Database::Query` (sql.active_record)
|
|
1077
|
+
- `Events::Rails::Http::Request` (process_action.action_controller)
|
|
1078
|
+
- `Events::Rails::Job::Enqueued` (enqueue.active_job)
|
|
1079
|
+
- `Events::Rails::Job::Started` (perform_start.active_job)
|
|
1080
|
+
- `Events::Rails::Job::Completed` (perform.active_job)
|
|
1081
|
+
- `Events::Rails::Job::Failed` (perform.active_job with exception)
|
|
1082
|
+
- ✅ DoD: All critical Rails events captured
|
|
1083
|
+
|
|
1084
|
+
3. **Devise-Style Overrides Configuration**
|
|
1085
|
+
- Configuration DSL: `event_class_for` method
|
|
1086
|
+
- Example: `event_class_for 'sql.active_record', MyApp::CustomDatabaseQuery`
|
|
1087
|
+
- `ignore_event` method for disabling specific events
|
|
1088
|
+
- ✅ DoD: Event classes can be overridden in config
|
|
1089
|
+
|
|
1090
|
+
**Verification (L6):**
|
|
1091
|
+
- Integration tests: Rails app with E11y tracking Rails events
|
|
1092
|
+
- Override test: Custom event class used instead of default
|
|
1093
|
+
- ADR Compliance: ADR-008 §4 (Rails Instrumentation, Updated)
|
|
1094
|
+
|
|
1095
|
+
---
|
|
1096
|
+
|
|
1097
|
+
### L2.9: Sidekiq/ActiveJob Integration 🟠
|
|
1098
|
+
|
|
1099
|
+
**ADR:** ADR-008 §4 (Sidekiq Integration), ADR-005 §8.3 (C17 Resolution)
|
|
1100
|
+
**UC:** UC-010 (Background Job Tracking)
|
|
1101
|
+
**Depends On:** L2.8 (Railtie)
|
|
1102
|
+
**Parallelizable:** ⚙️ Stream B (1 dev, parallel to Stream A)
|
|
1103
|
+
|
|
1104
|
+
#### L3.9.1: Sidekiq Integration
|
|
1105
|
+
|
|
1106
|
+
**Tasks:**
|
|
1107
|
+
1. **Sidekiq Server Middleware**
|
|
1108
|
+
- File: `lib/e11y/instruments/sidekiq/server_middleware.rb`
|
|
1109
|
+
- Job-scoped buffer setup
|
|
1110
|
+
- Trace context propagation (C17 hybrid model)
|
|
1111
|
+
- Job execution tracking (start, complete, fail)
|
|
1112
|
+
- Auto-flush on job end
|
|
1113
|
+
- ✅ DoD: Job events tracked with trace context
|
|
1114
|
+
|
|
1115
|
+
2. **Sidekiq Client Middleware**
|
|
1116
|
+
- File: `lib/e11y/instruments/sidekiq/client_middleware.rb`
|
|
1117
|
+
- Track job enqueuing
|
|
1118
|
+
- Propagate trace context to job
|
|
1119
|
+
- ✅ DoD: Job enqueue events tracked
|
|
1120
|
+
|
|
1121
|
+
**Verification (L6):**
|
|
1122
|
+
- Integration tests: Sidekiq job execution with E11y
|
|
1123
|
+
- ADR Compliance: ADR-008 §4 (Sidekiq Integration)
|
|
1124
|
+
- UC Compliance: UC-010 §3 (Job-Scoped Buffering)
|
|
1125
|
+
|
|
1126
|
+
#### L3.9.2: ActiveJob Integration
|
|
1127
|
+
|
|
1128
|
+
**Tasks:**
|
|
1129
|
+
1. **ActiveJob Callbacks**
|
|
1130
|
+
- File: `lib/e11y/instruments/active_job/callbacks.rb`
|
|
1131
|
+
- `before_perform` callback (setup buffer)
|
|
1132
|
+
- `after_perform` callback (flush buffer)
|
|
1133
|
+
- `around_perform` callback (exception handling)
|
|
1134
|
+
- ✅ DoD: ActiveJob events tracked
|
|
1135
|
+
|
|
1136
|
+
**Verification (L6):**
|
|
1137
|
+
- Integration tests: ActiveJob with E11y
|
|
1138
|
+
- ADR Compliance: ADR-008 §5 (ActiveJob Integration)
|
|
1139
|
+
|
|
1140
|
+
#### L3.9.3: Hybrid Background Job Tracing (C17 Resolution)
|
|
1141
|
+
|
|
1142
|
+
**Tasks:**
|
|
1143
|
+
1. **Hybrid Tracing Strategy**
|
|
1144
|
+
- File: `lib/e11y/trace/background_job.rb`
|
|
1145
|
+
- New `trace_id` per job
|
|
1146
|
+
- `parent_trace_id` link to originating request
|
|
1147
|
+
- Bounded traces (max 50 events per job)
|
|
1148
|
+
- ✅ DoD: Jobs have own trace_id + link to parent
|
|
1149
|
+
|
|
1150
|
+
**Verification (L6):**
|
|
1151
|
+
- Integration tests: Multi-service job tracing
|
|
1152
|
+
- ADR Compliance: ADR-005 §8.3 (C17 Resolution)
|
|
1153
|
+
|
|
1154
|
+
---
|
|
1155
|
+
|
|
1156
|
+
### L2.10: Rails.logger Migration 🟡
|
|
1157
|
+
|
|
1158
|
+
**ADR:** ADR-008 §6 (Rails.logger Migration)
|
|
1159
|
+
**UC:** UC-016 (Rails Logger Migration)
|
|
1160
|
+
**Depends On:** L2.8 (Railtie)
|
|
1161
|
+
**Parallelizable:** ⚙️ Stream C (1 dev, parallel to A & B)
|
|
1162
|
+
|
|
1163
|
+
#### L3.10.1: Logger Bridge
|
|
1164
|
+
|
|
1165
|
+
**Tasks:**
|
|
1166
|
+
1. **E11y::Logger::Bridge**
|
|
1167
|
+
- File: `lib/e11y/logger/bridge.rb`
|
|
1168
|
+
- Drop-in replacement for `Rails.logger`
|
|
1169
|
+
- Method delegation (debug, info, warn, error, fatal)
|
|
1170
|
+
- Structured logging (convert string to event)
|
|
1171
|
+
- Dual logging (E11y + original logger)
|
|
1172
|
+
- ✅ DoD: `Rails.logger = E11y::Logger::Bridge.new` works
|
|
1173
|
+
|
|
1174
|
+
2. **Structured Event Conversion**
|
|
1175
|
+
- Parse log string to structured event
|
|
1176
|
+
- Extract severity from method (`.debug()` → severity: :debug)
|
|
1177
|
+
- Add context (trace_id, request_id)
|
|
1178
|
+
- ✅ DoD: Log strings converted to structured events
|
|
1179
|
+
|
|
1180
|
+
**Verification (L6):**
|
|
1181
|
+
- Integration tests: Rails app with logger bridge
|
|
1182
|
+
- UC Compliance: UC-016 §3 (Drop-In Replacement)
|
|
1183
|
+
- ADR Compliance: ADR-008 §6 (Logger Bridge)
|
|
1184
|
+
|
|
1185
|
+
---
|
|
1186
|
+
|
|
1187
|
+
## PHASE 4: PRODUCTION HARDENING (Weeks 15-20)
|
|
1188
|
+
|
|
1189
|
+
### L2.11: Reliability & Error Handling 🔴
|
|
1190
|
+
|
|
1191
|
+
**ADR:** ADR-013 (Reliability & Error Handling)
|
|
1192
|
+
**UC:** UC-021 (Error Handling, Retry, DLQ)
|
|
1193
|
+
**Depends On:** L2.5 (Adapters)
|
|
1194
|
+
**Parallelizable:** ⚙️ Stream A (2 devs)
|
|
1195
|
+
|
|
1196
|
+
#### L3.11.1: Dead Letter Queue (DLQ)
|
|
1197
|
+
|
|
1198
|
+
**Tasks:**
|
|
1199
|
+
1. **DLQ Implementation**
|
|
1200
|
+
- File: `lib/e11y/dlq/queue.rb`
|
|
1201
|
+
- File-based storage (JSONL format)
|
|
1202
|
+
- Failed event metadata (error, retry_count, timestamp)
|
|
1203
|
+
- DLQ size limit (1GB default)
|
|
1204
|
+
- ✅ DoD: Failed events stored in DLQ
|
|
1205
|
+
|
|
1206
|
+
2. **DLQ Replay**
|
|
1207
|
+
- File: `lib/e11y/dlq/replayer.rb`
|
|
1208
|
+
- Replay events from DLQ
|
|
1209
|
+
- Schema migration support (V1 → V2)
|
|
1210
|
+
- Batch replay (100 events at a time)
|
|
1211
|
+
- ✅ DoD: DLQ events replayed successfully
|
|
1212
|
+
|
|
1213
|
+
**Verification (L6):**
|
|
1214
|
+
- Unit tests: `spec/e11y/dlq/queue_spec.rb`
|
|
1215
|
+
- Integration tests: Failed adapter → DLQ → Replay
|
|
1216
|
+
- UC Compliance: UC-021 §3 (DLQ & Retry)
|
|
1217
|
+
- ADR Compliance: ADR-013 §4 (DLQ Implementation)
|
|
1218
|
+
|
|
1219
|
+
#### L3.11.2: Rate Limiting (C02, C06 Resolution)
|
|
1220
|
+
|
|
1221
|
+
**Tasks:**
|
|
1222
|
+
1. **RateLimiting Middleware**
|
|
1223
|
+
- File: `lib/e11y/middleware/rate_limiting.rb`
|
|
1224
|
+
- Global rate limit (10K events/sec)
|
|
1225
|
+
- Per-event rate limit (1K events/sec default)
|
|
1226
|
+
- Redis-based (optional, fallback to in-memory)
|
|
1227
|
+
- Zone: `:routing`
|
|
1228
|
+
- ✅ DoD: Rate limits enforced correctly
|
|
1229
|
+
|
|
1230
|
+
2. **Critical Events Bypass (C02 Resolution)**
|
|
1231
|
+
- Critical events bypass rate limiting
|
|
1232
|
+
- Route to DLQ (not dropped)
|
|
1233
|
+
- ✅ DoD: Critical events never dropped
|
|
1234
|
+
|
|
1235
|
+
3. **Retry Rate Limiting (C06 Resolution)**
|
|
1236
|
+
- Separate rate limiter for retries
|
|
1237
|
+
- Staged batching (prevent thundering herd)
|
|
1238
|
+
- ✅ DoD: Retry storms prevented
|
|
1239
|
+
|
|
1240
|
+
**Verification (L6):**
|
|
1241
|
+
- Unit tests: `spec/e11y/middleware/rate_limiting_spec.rb`
|
|
1242
|
+
- Load tests: High-load scenario (10K+ events/sec)
|
|
1243
|
+
- ADR Compliance: ADR-013 §3.5 (C06 Resolution)
|
|
1244
|
+
|
|
1245
|
+
#### L3.11.3: Non-Failing Event Tracking (C18 Resolution)
|
|
1246
|
+
|
|
1247
|
+
**Tasks:**
|
|
1248
|
+
1. **Error Handling in Jobs**
|
|
1249
|
+
- File: `lib/e11y/error_handling/job_wrapper.rb`
|
|
1250
|
+
- Rescue E11y errors
|
|
1251
|
+
- Job continues despite E11y failure
|
|
1252
|
+
- Metrics: `e11y.errors.job_failures`
|
|
1253
|
+
- ✅ DoD: Jobs don't fail due to E11y errors
|
|
1254
|
+
|
|
1255
|
+
**Verification (L6):**
|
|
1256
|
+
- Integration tests: Job with E11y error
|
|
1257
|
+
- ADR Compliance: ADR-013 §3.6 (C18 Resolution)
|
|
1258
|
+
|
|
1259
|
+
---
|
|
1260
|
+
|
|
1261
|
+
### L2.12: OpenTelemetry Integration 🟠
|
|
1262
|
+
|
|
1263
|
+
**ADR:** ADR-007 (OpenTelemetry Integration)
|
|
1264
|
+
**UC:** UC-008 (OpenTelemetry Integration)
|
|
1265
|
+
**Depends On:** L2.5 (Adapters)
|
|
1266
|
+
**Parallelizable:** ⚙️ Stream B (1-2 devs, parallel to Stream A)
|
|
1267
|
+
|
|
1268
|
+
#### L3.12.1: OpenTelemetry Logs Integration
|
|
1269
|
+
|
|
1270
|
+
**Tasks:**
|
|
1271
|
+
1. **OTelLogsAdapter**
|
|
1272
|
+
- File: `lib/e11y/adapters/otel_logs.rb`
|
|
1273
|
+
- OpenTelemetry Logs API integration
|
|
1274
|
+
- Severity mapping (E11y → OTel)
|
|
1275
|
+
- Attributes mapping
|
|
1276
|
+
- ✅ DoD: Events sent to OTel Collector
|
|
1277
|
+
|
|
1278
|
+
2. **Baggage PII Protection (C08 Resolution)**
|
|
1279
|
+
- File: `lib/e11y/trace/baggage_filter.rb`
|
|
1280
|
+
- Baggage allowlist (only safe keys)
|
|
1281
|
+
- PII detection in baggage
|
|
1282
|
+
- Drop PII baggage keys
|
|
1283
|
+
- ✅ DoD: No PII in OTel baggage
|
|
1284
|
+
|
|
1285
|
+
**Verification (L6):**
|
|
1286
|
+
- Integration tests: OTel Collector integration
|
|
1287
|
+
- Security tests: Baggage PII filtering
|
|
1288
|
+
- UC Compliance: UC-008 §3 (OTel Logs)
|
|
1289
|
+
- ADR Compliance: ADR-007 §4 (OTel Integration)
|
|
1290
|
+
|
|
1291
|
+
#### L3.12.2: Universal Cardinality Protection (C04 Resolution)
|
|
1292
|
+
|
|
1293
|
+
**Tasks:**
|
|
1294
|
+
1. **Cardinality Protection for OTel**
|
|
1295
|
+
- Apply cardinality protection to OTel attributes
|
|
1296
|
+
- Prometheus, Loki, OTel all protected
|
|
1297
|
+
- ✅ DoD: Cardinality protection universal
|
|
1298
|
+
|
|
1299
|
+
**Verification (L6):**
|
|
1300
|
+
- Integration tests: High-cardinality OTel attributes
|
|
1301
|
+
- ADR Compliance: ADR-009 §8 (C04 Resolution)
|
|
1302
|
+
|
|
1303
|
+
---
|
|
1304
|
+
|
|
1305
|
+
### L2.13: Event Evolution & Versioning 🟠
|
|
1306
|
+
|
|
1307
|
+
**ADR:** ADR-012 (Event Evolution), ADR-015 (Middleware Order - Versioning LAST!)
|
|
1308
|
+
**UC:** UC-020 (Event Versioning)
|
|
1309
|
+
**Depends On:** L2.1 (Event::Base), L2.11 (DLQ)
|
|
1310
|
+
**Parallelizable:** ⚙️ Stream C (1 dev, parallel to A & B)
|
|
1311
|
+
|
|
1312
|
+
#### L3.13.1: Event Versioning
|
|
1313
|
+
|
|
1314
|
+
**Tasks:**
|
|
1315
|
+
1. **Versioning Support**
|
|
1316
|
+
- `version(value)` class method on Event::Base
|
|
1317
|
+
- Versioned event classes (`Events::OrderPaidV2`)
|
|
1318
|
+
- Middleware normalization (LAST in pipeline!)
|
|
1319
|
+
- ✅ DoD: V1 and V2 events coexist
|
|
1320
|
+
|
|
1321
|
+
2. **Schema Migrations (C15 Resolution)**
|
|
1322
|
+
- File: `lib/e11y/schema/migration.rb`
|
|
1323
|
+
- Migration rules (V1 → V2 transformation)
|
|
1324
|
+
- DLQ replay with migration
|
|
1325
|
+
- ✅ DoD: Old events replayed with new schema
|
|
1326
|
+
|
|
1327
|
+
**Verification (L6):**
|
|
1328
|
+
- Unit tests: `spec/e11y/schema/migration_spec.rb`
|
|
1329
|
+
- Integration tests: V1 → V2 migration
|
|
1330
|
+
- UC Compliance: UC-020 §3 (Schema Evolution)
|
|
1331
|
+
- ADR Compliance: ADR-012 §8 (C15 Resolution)
|
|
1332
|
+
|
|
1333
|
+
---
|
|
1334
|
+
|
|
1335
|
+
### L2.14: SLO Tracking & Self-Monitoring 🟠
|
|
1336
|
+
|
|
1337
|
+
**ADR:** ADR-003 (SLO Tracking), ADR-016 (Self-Monitoring)
|
|
1338
|
+
**UC:** UC-004 (Zero-Config SLO), UC-014 (Event-Driven SLO)
|
|
1339
|
+
**Depends On:** L2.6 (Metrics)
|
|
1340
|
+
**Parallelizable:** ⚙️ Stream D (1 dev, parallel to A/B/C)
|
|
1341
|
+
|
|
1342
|
+
#### L3.14.1: SLO Tracking
|
|
1343
|
+
|
|
1344
|
+
**Tasks:**
|
|
1345
|
+
1. **Zero-Config SLO Tracking**
|
|
1346
|
+
- File: `lib/e11y/slo/tracker.rb`
|
|
1347
|
+
- Auto-detect SLI events (success/error ratio)
|
|
1348
|
+
- Pattern-based SLO definitions
|
|
1349
|
+
- SLO metrics (success rate, error budget)
|
|
1350
|
+
- ✅ DoD: SLO metrics tracked automatically
|
|
1351
|
+
|
|
1352
|
+
2. **Event-Driven SLO**
|
|
1353
|
+
- File: `lib/e11y/slo/event_driven.rb`
|
|
1354
|
+
- SLO events (`Events::SLO::Request`, `Events::SLO::Job`)
|
|
1355
|
+
- SLO aggregation
|
|
1356
|
+
- ✅ DoD: SLO calculated from events
|
|
1357
|
+
|
|
1358
|
+
**Verification (L6):**
|
|
1359
|
+
- Unit tests: `spec/e11y/slo/tracker_spec.rb`
|
|
1360
|
+
- Integration tests: SLO tracking in Rails app
|
|
1361
|
+
- UC Compliance: UC-004 §3 (Zero-Config SLO)
|
|
1362
|
+
- ADR Compliance: ADR-003 §4 (SLO Tracking)
|
|
1363
|
+
|
|
1364
|
+
#### L3.14.2: Self-Monitoring
|
|
1365
|
+
|
|
1366
|
+
**Tasks:**
|
|
1367
|
+
1. **E11y Internal Metrics**
|
|
1368
|
+
- File: `lib/e11y/self_monitoring/metrics.rb`
|
|
1369
|
+
- Internal metrics (events tracked, dropped, latency)
|
|
1370
|
+
- Buffer metrics (size, flushes, overflows)
|
|
1371
|
+
- Adapter metrics (writes, errors, latency)
|
|
1372
|
+
- ✅ DoD: E11y monitors itself
|
|
1373
|
+
|
|
1374
|
+
2. **Internal SLO for E11y**
|
|
1375
|
+
- <1ms p99 latency (track() method)
|
|
1376
|
+
- <0.01% drop rate
|
|
1377
|
+
- 100% adapter success rate (with retries)
|
|
1378
|
+
- ✅ DoD: E11y meets its own SLO
|
|
1379
|
+
|
|
1380
|
+
**Verification (L6):**
|
|
1381
|
+
- Unit tests: `spec/e11y/self_monitoring/metrics_spec.rb`
|
|
1382
|
+
- Load tests: Verify SLO under load
|
|
1383
|
+
- ADR Compliance: ADR-016 §3 (Self-Monitoring)
|
|
1384
|
+
|
|
1385
|
+
---
|
|
1386
|
+
|
|
1387
|
+
## PHASE 5: SCALE & OPTIMIZATION (Weeks 21-26)
|
|
1388
|
+
|
|
1389
|
+
### L2.15: High Cardinality Protection (Full) 🟡
|
|
1390
|
+
|
|
1391
|
+
**ADR:** ADR-002 §4 (Cardinality Protection)
|
|
1392
|
+
**UC:** UC-013 (High Cardinality Protection)
|
|
1393
|
+
**Depends On:** L2.6 (Metrics), L2.12 (OTel)
|
|
1394
|
+
**Parallelizable:** ⚙️ Stream A (1 dev)
|
|
1395
|
+
|
|
1396
|
+
#### L3.15.1: Advanced Cardinality Protection
|
|
1397
|
+
|
|
1398
|
+
**Tasks:**
|
|
1399
|
+
1. **Dynamic Actions**
|
|
1400
|
+
- File: `lib/e11y/metrics/cardinality_dynamic.rb`
|
|
1401
|
+
- Auto-relabeling on cardinality explosion
|
|
1402
|
+
- Alert on limit exceeded
|
|
1403
|
+
- ✅ DoD: Cardinality auto-managed
|
|
1404
|
+
|
|
1405
|
+
**Verification (L6):**
|
|
1406
|
+
- Load tests: High-cardinality scenario
|
|
1407
|
+
- UC Compliance: UC-013 §4 (Dynamic Actions)
|
|
1408
|
+
|
|
1409
|
+
---
|
|
1410
|
+
|
|
1411
|
+
### L2.16: Tiered Storage Migration 🟡
|
|
1412
|
+
|
|
1413
|
+
**ADR:** None (UC-015 only)
|
|
1414
|
+
**UC:** UC-015 (Tiered Storage Migration)
|
|
1415
|
+
**Depends On:** L2.5 (Adapters)
|
|
1416
|
+
**Parallelizable:** ⚙️ Stream B (1 dev, parallel to Stream A)
|
|
1417
|
+
|
|
1418
|
+
#### L3.16.1: Tiered Storage Adapter
|
|
1419
|
+
|
|
1420
|
+
**Tasks:**
|
|
1421
|
+
1. **TieredStorageAdapter**
|
|
1422
|
+
- File: `lib/e11y/adapters/tiered_storage.rb`
|
|
1423
|
+
- Hot tier (7 days, fast SSD)
|
|
1424
|
+
- Warm tier (30 days, slow HDD)
|
|
1425
|
+
- Cold tier (90+ days, S3 Glacier)
|
|
1426
|
+
- Auto-migration based on retention tags
|
|
1427
|
+
- ✅ DoD: Events auto-migrate between tiers
|
|
1428
|
+
|
|
1429
|
+
**Verification (L6):**
|
|
1430
|
+
- Integration tests: Tiered storage simulation
|
|
1431
|
+
- UC Compliance: UC-015 §3 (Tiered Storage)
|
|
1432
|
+
|
|
1433
|
+
---
|
|
1434
|
+
|
|
1435
|
+
### L2.17: Performance Optimization (Scale: 1K → 10K → 100K) 🔴
|
|
1436
|
+
|
|
1437
|
+
**ADR:** ADR-001 §5 (Performance Requirements)
|
|
1438
|
+
**UC:** None (implicit from scale requirements)
|
|
1439
|
+
**Depends On:** All previous phases
|
|
1440
|
+
**Parallelizable:** ❌ Sequential (requires all features complete)
|
|
1441
|
+
|
|
1442
|
+
#### L3.17.1: Performance Benchmarks
|
|
1443
|
+
|
|
1444
|
+
**Tasks:**
|
|
1445
|
+
1. **Benchmark Suite**
|
|
1446
|
+
- File: `benchmarks/e11y_benchmarks.rb`
|
|
1447
|
+
- track() latency (p50, p99, p99.9)
|
|
1448
|
+
- Buffer throughput (events/sec)
|
|
1449
|
+
- Memory usage (MB per 1K events)
|
|
1450
|
+
- Adapter latency
|
|
1451
|
+
- ✅ DoD: Benchmarks meet targets (see below)
|
|
1452
|
+
|
|
1453
|
+
2. **Performance Targets (Small Scale - 1K events/sec)**
|
|
1454
|
+
- track() latency: <50μs (p99)
|
|
1455
|
+
- Buffer throughput: 10K events/sec
|
|
1456
|
+
- Memory: <100MB
|
|
1457
|
+
- CPU overhead: <5%
|
|
1458
|
+
- ✅ DoD: All targets met under 1K events/sec load
|
|
1459
|
+
|
|
1460
|
+
3. **Performance Targets (Medium Scale - 10K events/sec)**
|
|
1461
|
+
- track() latency: <1ms (p99)
|
|
1462
|
+
- Buffer throughput: 50K events/sec
|
|
1463
|
+
- Memory: <500MB
|
|
1464
|
+
- CPU overhead: <10%
|
|
1465
|
+
- ✅ DoD: All targets met under 10K events/sec load
|
|
1466
|
+
|
|
1467
|
+
4. **Performance Targets (Large Scale - 100K events/sec)**
|
|
1468
|
+
- track() latency: <5ms (p99)
|
|
1469
|
+
- Buffer throughput: 200K events/sec
|
|
1470
|
+
- Memory: <2GB
|
|
1471
|
+
- CPU overhead: <15%
|
|
1472
|
+
- ✅ DoD: All targets met under 100K events/sec load
|
|
1473
|
+
|
|
1474
|
+
**Verification (L6):**
|
|
1475
|
+
- Load tests: Sustained load (60 min)
|
|
1476
|
+
- Stress tests: Burst load (10x normal)
|
|
1477
|
+
- Memory profiling: No leaks
|
|
1478
|
+
- ADR Compliance: ADR-001 §5 (Performance Requirements)
|
|
1479
|
+
|
|
1480
|
+
#### L3.17.2: Optimization (If benchmarks fail)
|
|
1481
|
+
|
|
1482
|
+
**Tasks:**
|
|
1483
|
+
1. **Memory Optimization**
|
|
1484
|
+
- Reduce object allocations
|
|
1485
|
+
- Optimize buffer size
|
|
1486
|
+
- Pool reusable objects
|
|
1487
|
+
- ✅ DoD: Memory usage meets targets
|
|
1488
|
+
|
|
1489
|
+
2. **CPU Optimization**
|
|
1490
|
+
- Reduce regex matches
|
|
1491
|
+
- Cache pattern compilation
|
|
1492
|
+
- Optimize JSON serialization
|
|
1493
|
+
- ✅ DoD: CPU overhead meets targets
|
|
1494
|
+
|
|
1495
|
+
3. **I/O Optimization**
|
|
1496
|
+
- Increase batching
|
|
1497
|
+
- Connection pooling tuning
|
|
1498
|
+
- Compression optimization
|
|
1499
|
+
- ✅ DoD: Adapter latency meets targets
|
|
1500
|
+
|
|
1501
|
+
**Verification (L6):**
|
|
1502
|
+
- Re-run benchmarks
|
|
1503
|
+
- Compare before/after optimization
|
|
1504
|
+
- ADR Compliance: ADR-009 §9.2 (Optimization Targets)
|
|
1505
|
+
|
|
1506
|
+
---
|
|
1507
|
+
|
|
1508
|
+
### L2.18: Production Deployment & Documentation 🔴
|
|
1509
|
+
|
|
1510
|
+
**ADR:** ADR-010 (Developer Experience)
|
|
1511
|
+
**UC:** UC-017 (Local Development), UC-018 (Testing)
|
|
1512
|
+
**Depends On:** All previous phases
|
|
1513
|
+
**Parallelizable:** ⚙️ 3 parallel streams (docs, tests, deployment)
|
|
1514
|
+
|
|
1515
|
+
#### L3.18.1: Documentation (Stream A)
|
|
1516
|
+
|
|
1517
|
+
**Tasks:**
|
|
1518
|
+
1. **README.md**
|
|
1519
|
+
- Quick start guide (5 min setup)
|
|
1520
|
+
- Installation instructions
|
|
1521
|
+
- Configuration examples
|
|
1522
|
+
- ✅ DoD: User can setup E11y in <5 min
|
|
1523
|
+
|
|
1524
|
+
2. **API Documentation**
|
|
1525
|
+
- Event::Base API
|
|
1526
|
+
- Middleware API
|
|
1527
|
+
- Adapter API
|
|
1528
|
+
- Configuration DSL
|
|
1529
|
+
- ✅ DoD: All public APIs documented
|
|
1530
|
+
|
|
1531
|
+
3. **Guides**
|
|
1532
|
+
- Migration from Rails.logger
|
|
1533
|
+
- Custom middleware guide
|
|
1534
|
+
- Custom adapter guide
|
|
1535
|
+
- Performance tuning guide
|
|
1536
|
+
- ✅ DoD: All common scenarios documented
|
|
1537
|
+
|
|
1538
|
+
4. **ADR Index**
|
|
1539
|
+
- Index of all 16 ADRs
|
|
1540
|
+
- Cross-references
|
|
1541
|
+
- Decision log
|
|
1542
|
+
- ✅ DoD: Architecture decisions discoverable
|
|
1543
|
+
|
|
1544
|
+
**Verification (L6):**
|
|
1545
|
+
- Documentation review (peer review)
|
|
1546
|
+
- User testing (new user tries setup)
|
|
1547
|
+
- ADR Compliance: ADR-010 §3 (Developer Experience)
|
|
1548
|
+
|
|
1549
|
+
#### L3.18.2: Testing Strategy (Stream B)
|
|
1550
|
+
|
|
1551
|
+
**Tasks:**
|
|
1552
|
+
1. **Unit Tests (100% Coverage)**
|
|
1553
|
+
- RSpec tests for all classes
|
|
1554
|
+
- Edge cases covered
|
|
1555
|
+
- ✅ DoD: 100% code coverage
|
|
1556
|
+
|
|
1557
|
+
2. **Integration Tests**
|
|
1558
|
+
- Full pipeline tests
|
|
1559
|
+
- Rails integration tests
|
|
1560
|
+
- Sidekiq integration tests
|
|
1561
|
+
- Adapter integration tests
|
|
1562
|
+
- ✅ DoD: All integration scenarios covered
|
|
1563
|
+
|
|
1564
|
+
3. **Contract Tests**
|
|
1565
|
+
- Adapter contract tests
|
|
1566
|
+
- Middleware contract tests
|
|
1567
|
+
- ✅ DoD: All contracts validated
|
|
1568
|
+
|
|
1569
|
+
4. **Load Tests**
|
|
1570
|
+
- 1K, 10K, 100K events/sec load tests
|
|
1571
|
+
- Memory leak detection
|
|
1572
|
+
- ✅ DoD: All performance targets met
|
|
1573
|
+
|
|
1574
|
+
**Verification (L6):**
|
|
1575
|
+
- Run full test suite (CI)
|
|
1576
|
+
- ADR Compliance: ADR-011 §3 (Testing Strategy)
|
|
1577
|
+
|
|
1578
|
+
#### L3.18.3: Gem Release (Stream C)
|
|
1579
|
+
|
|
1580
|
+
**Tasks:**
|
|
1581
|
+
1. **Gemspec**
|
|
1582
|
+
- File: `e11y.gemspec`
|
|
1583
|
+
- Dependencies (dry-schema, yabeda, concurrent-ruby)
|
|
1584
|
+
- Version (1.0.0)
|
|
1585
|
+
- License (MIT)
|
|
1586
|
+
- ✅ DoD: Gem builds successfully
|
|
1587
|
+
|
|
1588
|
+
2. **Semantic Versioning**
|
|
1589
|
+
- Version file: `lib/e11y/version.rb`
|
|
1590
|
+
- Changelog: `CHANGELOG.md`
|
|
1591
|
+
- ✅ DoD: Versioning follows SemVer
|
|
1592
|
+
|
|
1593
|
+
3. **RubyGems Release**
|
|
1594
|
+
- Publish to RubyGems.org
|
|
1595
|
+
- GitHub release with notes
|
|
1596
|
+
- ✅ DoD: Gem available on RubyGems
|
|
1597
|
+
|
|
1598
|
+
**Verification (L6):**
|
|
1599
|
+
- Test gem installation (`gem install e11y`)
|
|
1600
|
+
- Test in fresh Rails app
|
|
1601
|
+
- ADR Compliance: ADR-001 §9 (Deployment & Versioning)
|
|
1602
|
+
|
|
1603
|
+
#### L3.18.4: Production Checklist
|
|
1604
|
+
|
|
1605
|
+
**Tasks:**
|
|
1606
|
+
1. **Production Deployment Checklist**
|
|
1607
|
+
- File: `docs/PRODUCTION_CHECKLIST.md`
|
|
1608
|
+
- Security checklist (PII filtering enabled, audit signing configured)
|
|
1609
|
+
- Performance checklist (rate limits, sampling, memory limits)
|
|
1610
|
+
- Reliability checklist (retries, circuit breakers, DLQ)
|
|
1611
|
+
- Monitoring checklist (metrics, alerts, SLO)
|
|
1612
|
+
- ✅ DoD: Production checklist complete
|
|
1613
|
+
|
|
1614
|
+
**Verification (L6):**
|
|
1615
|
+
- Run production checklist on test deployment
|
|
1616
|
+
- All checks pass
|
|
1617
|
+
- Documentation: `docs/researches/final_analysis/PRODUCTION_CHECKLIST.md`
|
|
1618
|
+
|
|
1619
|
+
---
|
|
1620
|
+
|
|
1621
|
+
## 🔗 DEPENDENCY MAP & PARALLELIZATION
|
|
1622
|
+
|
|
1623
|
+
### Critical Path (Sequential):
|
|
1624
|
+
```
|
|
1625
|
+
L2.1 (Event::Base)
|
|
1626
|
+
→ L2.2 (Adaptive Buffer)
|
|
1627
|
+
→ L2.3 (Middleware Pipeline)
|
|
1628
|
+
→ L2.4-L2.7 (Core Features - PARALLEL)
|
|
1629
|
+
→ L2.8-L2.10 (Rails Integration - PARALLEL)
|
|
1630
|
+
→ L2.11-L2.14 (Production Hardening - PARALLEL)
|
|
1631
|
+
→ L2.17 (Performance Optimization)
|
|
1632
|
+
→ L2.18 (Production Deployment)
|
|
1633
|
+
```
|
|
1634
|
+
|
|
1635
|
+
### Parallelization Windows:
|
|
1636
|
+
|
|
1637
|
+
**Window 1 (Phase 1 - Weeks 1-4):**
|
|
1638
|
+
- Stream A: L2.1 (Event::Base) - 1 dev
|
|
1639
|
+
- Stream B: L2.2 (Adaptive Buffer) - 1 dev
|
|
1640
|
+
- Stream C: L2.3 (Middleware Pipeline) - 1 dev
|
|
1641
|
+
- **Total: 3 devs in parallel**
|
|
1642
|
+
|
|
1643
|
+
**Window 2 (Phase 2 - Weeks 5-10):**
|
|
1644
|
+
- Stream A: L2.4 (PII Filtering & Security) - 2 devs
|
|
1645
|
+
- Stream B: L2.5 (Adapter Architecture) - 3 devs (adapters parallelizable!)
|
|
1646
|
+
- Stream C: L2.6 (Metrics & Yabeda) - 2 devs
|
|
1647
|
+
- Stream D: L2.7 (Sampling & Cost Optimization) - 1 dev
|
|
1648
|
+
- **Total: 6 devs in parallel**
|
|
1649
|
+
|
|
1650
|
+
**Window 3 (Phase 3 - Weeks 11-14):**
|
|
1651
|
+
- Stream A: L2.8 (Railtie & Auto-Config) - 2 devs
|
|
1652
|
+
- Stream B: L2.9 (Sidekiq/ActiveJob) - 1 dev
|
|
1653
|
+
- Stream C: L2.10 (Rails.logger Migration) - 1 dev
|
|
1654
|
+
- **Total: 4 devs in parallel**
|
|
1655
|
+
|
|
1656
|
+
**Window 4 (Phase 4 - Weeks 15-20):**
|
|
1657
|
+
- Stream A: L2.11 (Reliability & Error Handling) - 2 devs
|
|
1658
|
+
- Stream B: L2.12 (OpenTelemetry Integration) - 2 devs
|
|
1659
|
+
- Stream C: L2.13 (Event Evolution & Versioning) - 1 dev
|
|
1660
|
+
- Stream D: L2.14 (SLO Tracking & Self-Monitoring) - 1 dev
|
|
1661
|
+
- **Total: 6 devs in parallel**
|
|
1662
|
+
|
|
1663
|
+
**Window 5 (Phase 5 - Weeks 21-26):**
|
|
1664
|
+
- Stream A: L2.15 (High Cardinality Protection) - 1 dev
|
|
1665
|
+
- Stream B: L2.16 (Tiered Storage Migration) - 1 dev
|
|
1666
|
+
- Stream C: L2.17 (Performance Optimization) - 2 devs (sequential after others)
|
|
1667
|
+
- Stream D: L2.18.1 (Documentation) - 1 dev (parallel with optimization)
|
|
1668
|
+
- Stream E: L2.18.2 (Testing) - 1 dev (parallel with optimization)
|
|
1669
|
+
- Stream F: L2.18.3 (Gem Release) - 1 dev (parallel with optimization)
|
|
1670
|
+
- **Total: 6 devs in parallel (optimization blocks release, but docs/tests parallel)**
|
|
1671
|
+
|
|
1672
|
+
---
|
|
1673
|
+
|
|
1674
|
+
## 📊 INTEGRATION CONTRACTS (CRITICAL for Parallel Work)
|
|
1675
|
+
|
|
1676
|
+
### Contract 1: Event::Base Interface
|
|
1677
|
+
```ruby
|
|
1678
|
+
# lib/e11y/event/base.rb
|
|
1679
|
+
module E11y
|
|
1680
|
+
class Event
|
|
1681
|
+
class Base
|
|
1682
|
+
# PUBLIC API (guaranteed stable for all components)
|
|
1683
|
+
|
|
1684
|
+
# Track event (zero-allocation)
|
|
1685
|
+
# @param [Hash] payload Event data
|
|
1686
|
+
# @return [Hash] event_data (for testing/debugging)
|
|
1687
|
+
def self.track(**payload)
|
|
1688
|
+
|
|
1689
|
+
# Configuration DSL
|
|
1690
|
+
def self.severity(value = nil)
|
|
1691
|
+
def self.schema(&block)
|
|
1692
|
+
def self.version(value = nil)
|
|
1693
|
+
def self.adapters(list = nil)
|
|
1694
|
+
|
|
1695
|
+
# Resolution methods (read-only)
|
|
1696
|
+
def self.resolve_severity
|
|
1697
|
+
def self.resolve_adapters
|
|
1698
|
+
def self.resolve_sample_rate
|
|
1699
|
+
def self.resolve_rate_limit
|
|
1700
|
+
end
|
|
1701
|
+
end
|
|
1702
|
+
end
|
|
1703
|
+
```
|
|
1704
|
+
|
|
1705
|
+
**Contract Guarantees:**
|
|
1706
|
+
- `track()` returns `Hash` with keys: `:event_name`, `:payload`, `:severity`, `:version`, `:adapters`
|
|
1707
|
+
- All resolution methods return sensible defaults (never nil)
|
|
1708
|
+
- Schema validation happens inside `track()` (raises `E11y::ValidationError` on invalid payload)
|
|
1709
|
+
|
|
1710
|
+
**Consumers:**
|
|
1711
|
+
- L2.3 (Middleware Pipeline) - calls `track()` internally
|
|
1712
|
+
- L2.4 (PII Filtering) - reads `event_data[:payload]`
|
|
1713
|
+
- L2.6 (Metrics) - reads `event_data[:event_name]`, `event_data[:severity]`
|
|
1714
|
+
- L2.7 (Sampling) - reads `resolve_sample_rate()`
|
|
1715
|
+
|
|
1716
|
+
---
|
|
1717
|
+
|
|
1718
|
+
### Contract 2: Middleware::Base Interface
|
|
1719
|
+
```ruby
|
|
1720
|
+
# lib/e11y/middleware/base.rb
|
|
1721
|
+
module E11y
|
|
1722
|
+
module Middleware
|
|
1723
|
+
class Base
|
|
1724
|
+
# PUBLIC API
|
|
1725
|
+
|
|
1726
|
+
# Initialize middleware with next app in chain
|
|
1727
|
+
# @param [Middleware::Base] app Next middleware in chain
|
|
1728
|
+
def initialize(app)
|
|
1729
|
+
|
|
1730
|
+
# Call middleware (process event)
|
|
1731
|
+
# @param [Hash] event_data Event data hash
|
|
1732
|
+
# @return [Hash] modified event_data (or nil to drop event)
|
|
1733
|
+
def call(event_data)
|
|
1734
|
+
|
|
1735
|
+
# Declare middleware zone
|
|
1736
|
+
# @param [Symbol] zone Zone name (:pre_processing, :security, :routing, :post_processing, :adapters)
|
|
1737
|
+
def self.middleware_zone(zone = nil)
|
|
1738
|
+
end
|
|
1739
|
+
end
|
|
1740
|
+
end
|
|
1741
|
+
```
|
|
1742
|
+
|
|
1743
|
+
**Contract Guarantees:**
|
|
1744
|
+
- `call(event_data)` receives `Hash` with keys: `:event_name`, `:payload`, `:severity`, `:version`, `:trace_id`, `:timestamp`
|
|
1745
|
+
- `call(event_data)` returns modified `Hash` or `nil` (drop event)
|
|
1746
|
+
- Middlewares MUST call `@app.call(event_data)` to continue chain
|
|
1747
|
+
- Middlewares MUST NOT mutate `event_data` without creating new hash (immutability)
|
|
1748
|
+
|
|
1749
|
+
**Consumers:**
|
|
1750
|
+
- L2.4 (PII Filtering Middleware)
|
|
1751
|
+
- L2.6 (Metrics Middleware)
|
|
1752
|
+
- L2.7 (Sampling Middleware)
|
|
1753
|
+
- L2.11 (Rate Limiting Middleware)
|
|
1754
|
+
- L2.13 (Versioning Middleware)
|
|
1755
|
+
|
|
1756
|
+
---
|
|
1757
|
+
|
|
1758
|
+
### Contract 3: Adapter::Base Interface
|
|
1759
|
+
```ruby
|
|
1760
|
+
# lib/e11y/adapters/base.rb
|
|
1761
|
+
module E11y
|
|
1762
|
+
module Adapters
|
|
1763
|
+
class Base
|
|
1764
|
+
# PUBLIC API
|
|
1765
|
+
|
|
1766
|
+
# Write single event
|
|
1767
|
+
# @param [Hash] event_data Event data hash
|
|
1768
|
+
# @return [Boolean] true if successful, false otherwise
|
|
1769
|
+
def write(event_data)
|
|
1770
|
+
|
|
1771
|
+
# Write batch of events (default: loop write)
|
|
1772
|
+
# @param [Array<Hash>] events Array of event data hashes
|
|
1773
|
+
# @return [Integer] number of successfully written events
|
|
1774
|
+
def write_batch(events)
|
|
1775
|
+
|
|
1776
|
+
# Health check
|
|
1777
|
+
# @return [Boolean] true if adapter is healthy
|
|
1778
|
+
def healthy?
|
|
1779
|
+
|
|
1780
|
+
# Close adapter (cleanup resources)
|
|
1781
|
+
# @return [void]
|
|
1782
|
+
def close
|
|
1783
|
+
|
|
1784
|
+
# Adapter capabilities
|
|
1785
|
+
# @return [Hash] capabilities (:batching, :compression, :encryption)
|
|
1786
|
+
def capabilities
|
|
1787
|
+
end
|
|
1788
|
+
end
|
|
1789
|
+
end
|
|
1790
|
+
```
|
|
1791
|
+
|
|
1792
|
+
**Contract Guarantees:**
|
|
1793
|
+
- `write(event_data)` is idempotent (safe to retry)
|
|
1794
|
+
- `write_batch(events)` returns number of successful writes (for metrics)
|
|
1795
|
+
- `healthy?` is fast (<10ms) and non-blocking
|
|
1796
|
+
- Adapters MUST handle retries internally (exponential backoff)
|
|
1797
|
+
|
|
1798
|
+
**Consumers:**
|
|
1799
|
+
- L2.5.2 (Built-In Adapters - Stdout, File, Loki, Sentry, Elasticsearch, InMemory)
|
|
1800
|
+
- L2.11 (Retry Handler, Circuit Breaker)
|
|
1801
|
+
- L2.12 (OTelLogsAdapter)
|
|
1802
|
+
|
|
1803
|
+
---
|
|
1804
|
+
|
|
1805
|
+
### Contract 4: Buffer Interface
|
|
1806
|
+
```ruby
|
|
1807
|
+
# lib/e11y/buffers/base_buffer.rb
|
|
1808
|
+
module E11y
|
|
1809
|
+
module Buffers
|
|
1810
|
+
class BaseBuffer
|
|
1811
|
+
# PUBLIC API
|
|
1812
|
+
|
|
1813
|
+
# Push event to buffer
|
|
1814
|
+
# @param [Hash] event_data Event data hash
|
|
1815
|
+
# @return [Boolean] true if pushed, false if dropped (backpressure)
|
|
1816
|
+
def push(event_data)
|
|
1817
|
+
|
|
1818
|
+
# Pop event from buffer (non-blocking)
|
|
1819
|
+
# @return [Hash, nil] event_data or nil if empty
|
|
1820
|
+
def pop
|
|
1821
|
+
|
|
1822
|
+
# Flush all events from buffer
|
|
1823
|
+
# @return [Array<Hash>] all buffered events
|
|
1824
|
+
def flush_all
|
|
1825
|
+
|
|
1826
|
+
# Buffer size
|
|
1827
|
+
# @return [Integer] number of events in buffer
|
|
1828
|
+
def size
|
|
1829
|
+
|
|
1830
|
+
# Buffer capacity
|
|
1831
|
+
# @return [Integer] maximum number of events buffer can hold
|
|
1832
|
+
def capacity
|
|
1833
|
+
end
|
|
1834
|
+
end
|
|
1835
|
+
end
|
|
1836
|
+
```
|
|
1837
|
+
|
|
1838
|
+
**Contract Guarantees:**
|
|
1839
|
+
- `push()` is thread-safe
|
|
1840
|
+
- `pop()` is non-blocking (returns nil if empty)
|
|
1841
|
+
- `flush_all()` is atomic (all or nothing)
|
|
1842
|
+
- Buffer MUST implement backpressure strategy (block/drop/throttle)
|
|
1843
|
+
|
|
1844
|
+
**Consumers:**
|
|
1845
|
+
- L2.2 (RingBuffer, AdaptiveBuffer, RequestScopedBuffer)
|
|
1846
|
+
- L2.3 (Routing Middleware)
|
|
1847
|
+
- L2.8 (Rails Middleware)
|
|
1848
|
+
|
|
1849
|
+
---
|
|
1850
|
+
|
|
1851
|
+
## 🎯 VERIFICATION STRATEGY (Level 6)
|
|
1852
|
+
|
|
1853
|
+
### Testing Pyramid:
|
|
1854
|
+
|
|
1855
|
+
```
|
|
1856
|
+
▲
|
|
1857
|
+
/ \
|
|
1858
|
+
/ \
|
|
1859
|
+
/ E2E \ 10% - Full Rails app tests (1-2 per UC)
|
|
1860
|
+
/-------\
|
|
1861
|
+
/ Integr. \ 20% - Component integration tests
|
|
1862
|
+
/-----------\
|
|
1863
|
+
/ Unit Tests \ 70% - Unit tests (all methods)
|
|
1864
|
+
/--------------\
|
|
1865
|
+
```
|
|
1866
|
+
|
|
1867
|
+
### Test Categories:
|
|
1868
|
+
|
|
1869
|
+
1. **Unit Tests (70%):**
|
|
1870
|
+
- File: `spec/e11y/**/*_spec.rb`
|
|
1871
|
+
- Coverage: 100% (enforced by SimpleCov)
|
|
1872
|
+
- Run time: <30 seconds (full suite)
|
|
1873
|
+
- ✅ DoD: All classes/methods have unit tests
|
|
1874
|
+
|
|
1875
|
+
2. **Integration Tests (20%):**
|
|
1876
|
+
- File: `spec/integration/**/*_spec.rb`
|
|
1877
|
+
- Coverage: All component interactions
|
|
1878
|
+
- Run time: <2 minutes (full suite)
|
|
1879
|
+
- ✅ DoD: All integration scenarios tested
|
|
1880
|
+
|
|
1881
|
+
3. **E2E Tests (10%):**
|
|
1882
|
+
- File: `spec/e2e/**/*_spec.rb`
|
|
1883
|
+
- Coverage: Critical user journeys (UC-001, UC-002, UC-007, UC-016)
|
|
1884
|
+
- Run time: <5 minutes (full suite)
|
|
1885
|
+
- ✅ DoD: All critical UCs have E2E tests
|
|
1886
|
+
|
|
1887
|
+
4. **Load Tests:**
|
|
1888
|
+
- File: `benchmarks/load_tests.rb`
|
|
1889
|
+
- Scenarios: 1K, 10K, 100K events/sec
|
|
1890
|
+
- Duration: 60 min sustained load
|
|
1891
|
+
- ✅ DoD: All performance targets met
|
|
1892
|
+
|
|
1893
|
+
5. **Contract Tests:**
|
|
1894
|
+
- File: `spec/contracts/**/*_spec.rb`
|
|
1895
|
+
- Coverage: All public APIs (Event::Base, Middleware::Base, Adapter::Base, Buffer)
|
|
1896
|
+
- ✅ DoD: All contracts validated
|
|
1897
|
+
|
|
1898
|
+
---
|
|
1899
|
+
|
|
1900
|
+
## 📋 ADR/UC COMPLIANCE CHECKLIST
|
|
1901
|
+
|
|
1902
|
+
### ADR Coverage (16 ADRs):
|
|
1903
|
+
|
|
1904
|
+
| ADR | Phase | Component | Verification |
|
|
1905
|
+
|-----|-------|-----------|--------------|
|
|
1906
|
+
| ADR-001 | Phase 1 | L2.1, L2.2, L2.3 | Unit tests + benchmarks |
|
|
1907
|
+
| ADR-002 | Phase 2 | L2.6 | Integration tests + Prometheus |
|
|
1908
|
+
| ADR-003 | Phase 4 | L2.14 | SLO metrics validation |
|
|
1909
|
+
| ADR-004 | Phase 2 | L2.5 | Adapter contract tests |
|
|
1910
|
+
| ADR-005 | Phase 3 | L2.9 | Job tracing tests |
|
|
1911
|
+
| ADR-006 | Phase 2 | L2.4 | PII filtering tests + security audit |
|
|
1912
|
+
| ADR-007 | Phase 4 | L2.12 | OTel integration tests |
|
|
1913
|
+
| ADR-008 | Phase 3 | L2.8, L2.9, L2.10 | Rails integration tests |
|
|
1914
|
+
| ADR-009 | Phase 2 | L2.7 | Sampling tests + cost analysis |
|
|
1915
|
+
| ADR-010 | Phase 5 | L2.18 | Documentation review |
|
|
1916
|
+
| ADR-011 | Phase 5 | L2.18.2 | Testing strategy validation |
|
|
1917
|
+
| ADR-012 | Phase 4 | L2.13 | Versioning tests |
|
|
1918
|
+
| ADR-013 | Phase 4 | L2.11 | Reliability tests |
|
|
1919
|
+
| ADR-014 | Phase 4 | L2.14 | Event-driven SLO tests |
|
|
1920
|
+
| ADR-015 | Phase 2 | L2.4 | Pipeline order validation |
|
|
1921
|
+
| ADR-016 | Phase 4 | L2.14 | Self-monitoring metrics |
|
|
1922
|
+
|
|
1923
|
+
### UC Coverage (22 UCs):
|
|
1924
|
+
|
|
1925
|
+
| UC | Phase | Component | Verification |
|
|
1926
|
+
|----|-------|-----------|--------------|
|
|
1927
|
+
| UC-001 | Phase 1 | L2.2 | Request-scoped buffer tests |
|
|
1928
|
+
| UC-002 | Phase 1 | L2.1 | Event tracking tests |
|
|
1929
|
+
| UC-003 | Phase 2 | L2.6 | Pattern-based metrics tests |
|
|
1930
|
+
| UC-004 | Phase 4 | L2.14 | Zero-config SLO tests |
|
|
1931
|
+
| UC-005 | Phase 2 | L2.5 | Sentry adapter tests |
|
|
1932
|
+
| UC-006 | Phase 3 | L2.9 | Trace context tests |
|
|
1933
|
+
| UC-007 | Phase 2 | L2.4 | PII filtering tests |
|
|
1934
|
+
| UC-008 | Phase 4 | L2.12 | OTel integration tests |
|
|
1935
|
+
| UC-009 | Phase 3 | L2.9 | Multi-service tracing tests |
|
|
1936
|
+
| UC-010 | Phase 3 | L2.9 | Background job tracking tests |
|
|
1937
|
+
| UC-011 | Phase 4 | L2.11 | Rate limiting tests |
|
|
1938
|
+
| UC-012 | Phase 2 | L2.4 | Audit trail tests |
|
|
1939
|
+
| UC-013 | Phase 5 | L2.15 | Cardinality protection tests |
|
|
1940
|
+
| UC-014 | Phase 2 | L2.7 | Adaptive sampling tests |
|
|
1941
|
+
| UC-015 | Phase 5 | L2.16 | Tiered storage tests |
|
|
1942
|
+
| UC-016 | Phase 3 | L2.10 | Logger migration tests |
|
|
1943
|
+
| UC-017 | Phase 3 | L2.8 | Local development setup tests |
|
|
1944
|
+
| UC-018 | Phase 5 | L2.18 | Testing helpers validation |
|
|
1945
|
+
| UC-019 | Phase 5 | L2.16 | Tiered storage migration tests |
|
|
1946
|
+
| UC-020 | Phase 4 | L2.13 | Event versioning tests |
|
|
1947
|
+
| UC-021 | Phase 4 | L2.11 | Error handling tests |
|
|
1948
|
+
| UC-022 | Phase 1 | L2.1 | Event registry tests |
|
|
1949
|
+
|
|
1950
|
+
---
|
|
1951
|
+
|
|
1952
|
+
## 🚀 NEXT STEPS
|
|
1953
|
+
|
|
1954
|
+
1. **Review this plan** with team
|
|
1955
|
+
2. **Assign developers** to parallelizable streams
|
|
1956
|
+
3. **Setup project skeleton** (gem structure, CI/CD, Docker Compose for adapters)
|
|
1957
|
+
4. **Start Phase 1** (Foundation) - 3 parallel streams
|
|
1958
|
+
5. **Weekly sync** - integration point validation
|
|
1959
|
+
6. **After Phase 1** - validate contracts, adjust plan if needed
|
|
1960
|
+
7. **Continue** through phases 2-5
|
|
1961
|
+
|
|
1962
|
+
---
|
|
1963
|
+
|
|
1964
|
+
**Status:** ✅ 6-Level Implementation Plan Complete
|
|
1965
|
+
**Last Updated:** 2026-01-17
|
|
1966
|
+
**Estimated Total Effort:** 22-26 weeks (with proper parallelization)
|
|
1967
|
+
**Maximum Parallelism:** 6 developers
|
|
1968
|
+
**Critical Path:** ~14-16 weeks (Foundation → Core → Rails → Performance)
|
|
1969
|
+
|
|
1970
|
+
---
|
|
1971
|
+
|