e11y 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +4 -0
  3. data/.rubocop.yml +69 -0
  4. data/CHANGELOG.md +26 -0
  5. data/CODE_OF_CONDUCT.md +64 -0
  6. data/LICENSE.txt +21 -0
  7. data/README.md +179 -0
  8. data/Rakefile +37 -0
  9. data/benchmarks/run_all.rb +33 -0
  10. data/config/README.md +83 -0
  11. data/config/loki-local-config.yaml +35 -0
  12. data/config/prometheus.yml +15 -0
  13. data/docker-compose.yml +78 -0
  14. data/docs/00-ICP-AND-TIMELINE.md +483 -0
  15. data/docs/01-SCALE-REQUIREMENTS.md +858 -0
  16. data/docs/ADR-001-architecture.md +2617 -0
  17. data/docs/ADR-002-metrics-yabeda.md +1395 -0
  18. data/docs/ADR-003-slo-observability.md +3337 -0
  19. data/docs/ADR-004-adapter-architecture.md +2385 -0
  20. data/docs/ADR-005-tracing-context.md +1372 -0
  21. data/docs/ADR-006-security-compliance.md +4143 -0
  22. data/docs/ADR-007-opentelemetry-integration.md +1385 -0
  23. data/docs/ADR-008-rails-integration.md +1911 -0
  24. data/docs/ADR-009-cost-optimization.md +2993 -0
  25. data/docs/ADR-010-developer-experience.md +2166 -0
  26. data/docs/ADR-011-testing-strategy.md +1836 -0
  27. data/docs/ADR-012-event-evolution.md +958 -0
  28. data/docs/ADR-013-reliability-error-handling.md +2750 -0
  29. data/docs/ADR-014-event-driven-slo.md +1533 -0
  30. data/docs/ADR-015-middleware-order.md +1061 -0
  31. data/docs/ADR-016-self-monitoring-slo.md +1234 -0
  32. data/docs/API-REFERENCE-L28.md +914 -0
  33. data/docs/COMPREHENSIVE-CONFIGURATION.md +2366 -0
  34. data/docs/IMPLEMENTATION_NOTES.md +2804 -0
  35. data/docs/IMPLEMENTATION_PLAN.md +1971 -0
  36. data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +586 -0
  37. data/docs/PLAN.md +148 -0
  38. data/docs/QUICK-START.md +934 -0
  39. data/docs/README.md +296 -0
  40. data/docs/design/00-memory-optimization.md +593 -0
  41. data/docs/guides/MIGRATION-L27-L28.md +692 -0
  42. data/docs/guides/PERFORMANCE-BENCHMARKS.md +434 -0
  43. data/docs/guides/README.md +44 -0
  44. data/docs/prd/01-overview-vision.md +440 -0
  45. data/docs/use_cases/README.md +119 -0
  46. data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +813 -0
  47. data/docs/use_cases/UC-002-business-event-tracking.md +1953 -0
  48. data/docs/use_cases/UC-003-pattern-based-metrics.md +1627 -0
  49. data/docs/use_cases/UC-004-zero-config-slo-tracking.md +728 -0
  50. data/docs/use_cases/UC-005-sentry-integration.md +759 -0
  51. data/docs/use_cases/UC-006-trace-context-management.md +905 -0
  52. data/docs/use_cases/UC-007-pii-filtering.md +2648 -0
  53. data/docs/use_cases/UC-008-opentelemetry-integration.md +1153 -0
  54. data/docs/use_cases/UC-009-multi-service-tracing.md +1043 -0
  55. data/docs/use_cases/UC-010-background-job-tracking.md +1018 -0
  56. data/docs/use_cases/UC-011-rate-limiting.md +1906 -0
  57. data/docs/use_cases/UC-012-audit-trail.md +2301 -0
  58. data/docs/use_cases/UC-013-high-cardinality-protection.md +2127 -0
  59. data/docs/use_cases/UC-014-adaptive-sampling.md +1940 -0
  60. data/docs/use_cases/UC-015-cost-optimization.md +735 -0
  61. data/docs/use_cases/UC-016-rails-logger-migration.md +785 -0
  62. data/docs/use_cases/UC-017-local-development.md +867 -0
  63. data/docs/use_cases/UC-018-testing-events.md +1081 -0
  64. data/docs/use_cases/UC-019-tiered-storage-migration.md +562 -0
  65. data/docs/use_cases/UC-020-event-versioning.md +708 -0
  66. data/docs/use_cases/UC-021-error-handling-retry-dlq.md +956 -0
  67. data/docs/use_cases/UC-022-event-registry.md +648 -0
  68. data/docs/use_cases/backlog.md +226 -0
  69. data/e11y.gemspec +76 -0
  70. data/lib/e11y/adapters/adaptive_batcher.rb +207 -0
  71. data/lib/e11y/adapters/audit_encrypted.rb +239 -0
  72. data/lib/e11y/adapters/base.rb +580 -0
  73. data/lib/e11y/adapters/file.rb +224 -0
  74. data/lib/e11y/adapters/in_memory.rb +216 -0
  75. data/lib/e11y/adapters/loki.rb +333 -0
  76. data/lib/e11y/adapters/otel_logs.rb +203 -0
  77. data/lib/e11y/adapters/registry.rb +141 -0
  78. data/lib/e11y/adapters/sentry.rb +230 -0
  79. data/lib/e11y/adapters/stdout.rb +108 -0
  80. data/lib/e11y/adapters/yabeda.rb +370 -0
  81. data/lib/e11y/buffers/adaptive_buffer.rb +339 -0
  82. data/lib/e11y/buffers/base_buffer.rb +40 -0
  83. data/lib/e11y/buffers/request_scoped_buffer.rb +246 -0
  84. data/lib/e11y/buffers/ring_buffer.rb +267 -0
  85. data/lib/e11y/buffers.rb +14 -0
  86. data/lib/e11y/console.rb +122 -0
  87. data/lib/e11y/current.rb +48 -0
  88. data/lib/e11y/event/base.rb +894 -0
  89. data/lib/e11y/event/value_sampling_config.rb +84 -0
  90. data/lib/e11y/events/base_audit_event.rb +43 -0
  91. data/lib/e11y/events/base_payment_event.rb +33 -0
  92. data/lib/e11y/events/rails/cache/delete.rb +21 -0
  93. data/lib/e11y/events/rails/cache/read.rb +23 -0
  94. data/lib/e11y/events/rails/cache/write.rb +22 -0
  95. data/lib/e11y/events/rails/database/query.rb +45 -0
  96. data/lib/e11y/events/rails/http/redirect.rb +21 -0
  97. data/lib/e11y/events/rails/http/request.rb +26 -0
  98. data/lib/e11y/events/rails/http/send_file.rb +21 -0
  99. data/lib/e11y/events/rails/http/start_processing.rb +26 -0
  100. data/lib/e11y/events/rails/job/completed.rb +22 -0
  101. data/lib/e11y/events/rails/job/enqueued.rb +22 -0
  102. data/lib/e11y/events/rails/job/failed.rb +22 -0
  103. data/lib/e11y/events/rails/job/scheduled.rb +23 -0
  104. data/lib/e11y/events/rails/job/started.rb +22 -0
  105. data/lib/e11y/events/rails/log.rb +56 -0
  106. data/lib/e11y/events/rails/view/render.rb +23 -0
  107. data/lib/e11y/events.rb +18 -0
  108. data/lib/e11y/instruments/active_job.rb +201 -0
  109. data/lib/e11y/instruments/rails_instrumentation.rb +141 -0
  110. data/lib/e11y/instruments/sidekiq.rb +175 -0
  111. data/lib/e11y/logger/bridge.rb +205 -0
  112. data/lib/e11y/metrics/cardinality_protection.rb +172 -0
  113. data/lib/e11y/metrics/cardinality_tracker.rb +134 -0
  114. data/lib/e11y/metrics/registry.rb +234 -0
  115. data/lib/e11y/metrics/relabeling.rb +226 -0
  116. data/lib/e11y/metrics.rb +102 -0
  117. data/lib/e11y/middleware/audit_signing.rb +174 -0
  118. data/lib/e11y/middleware/base.rb +140 -0
  119. data/lib/e11y/middleware/event_slo.rb +167 -0
  120. data/lib/e11y/middleware/pii_filter.rb +266 -0
  121. data/lib/e11y/middleware/pii_filtering.rb +280 -0
  122. data/lib/e11y/middleware/rate_limiting.rb +214 -0
  123. data/lib/e11y/middleware/request.rb +163 -0
  124. data/lib/e11y/middleware/routing.rb +157 -0
  125. data/lib/e11y/middleware/sampling.rb +254 -0
  126. data/lib/e11y/middleware/slo.rb +168 -0
  127. data/lib/e11y/middleware/trace_context.rb +131 -0
  128. data/lib/e11y/middleware/validation.rb +118 -0
  129. data/lib/e11y/middleware/versioning.rb +132 -0
  130. data/lib/e11y/middleware.rb +12 -0
  131. data/lib/e11y/pii/patterns.rb +90 -0
  132. data/lib/e11y/pii.rb +13 -0
  133. data/lib/e11y/pipeline/builder.rb +155 -0
  134. data/lib/e11y/pipeline/zone_validator.rb +110 -0
  135. data/lib/e11y/pipeline.rb +12 -0
  136. data/lib/e11y/presets/audit_event.rb +65 -0
  137. data/lib/e11y/presets/debug_event.rb +34 -0
  138. data/lib/e11y/presets/high_value_event.rb +51 -0
  139. data/lib/e11y/presets.rb +19 -0
  140. data/lib/e11y/railtie.rb +138 -0
  141. data/lib/e11y/reliability/circuit_breaker.rb +216 -0
  142. data/lib/e11y/reliability/dlq/file_storage.rb +277 -0
  143. data/lib/e11y/reliability/dlq/filter.rb +117 -0
  144. data/lib/e11y/reliability/retry_handler.rb +207 -0
  145. data/lib/e11y/reliability/retry_rate_limiter.rb +117 -0
  146. data/lib/e11y/sampling/error_spike_detector.rb +225 -0
  147. data/lib/e11y/sampling/load_monitor.rb +161 -0
  148. data/lib/e11y/sampling/stratified_tracker.rb +92 -0
  149. data/lib/e11y/sampling/value_extractor.rb +82 -0
  150. data/lib/e11y/self_monitoring/buffer_monitor.rb +79 -0
  151. data/lib/e11y/self_monitoring/performance_monitor.rb +97 -0
  152. data/lib/e11y/self_monitoring/reliability_monitor.rb +146 -0
  153. data/lib/e11y/slo/event_driven.rb +150 -0
  154. data/lib/e11y/slo/tracker.rb +119 -0
  155. data/lib/e11y/version.rb +9 -0
  156. data/lib/e11y.rb +283 -0
  157. metadata +452 -0
@@ -0,0 +1,239 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "openssl"
4
+ require "json"
5
+ require "fileutils"
6
+ require "base64"
7
+
8
+ module E11y
9
+ module Adapters
10
+ # Audit Encrypted Adapter - AES-256-GCM encrypted storage for audit events
11
+ #
12
+ # Stores audit events with encryption at rest for compliance requirements.
13
+ # Each event is individually encrypted with AES-256-GCM.
14
+ #
15
+ # **Security:**
16
+ # - AES-256-GCM authenticated encryption
17
+ # - Per-event nonce (never reused)
18
+ # - Authentication tag validation
19
+ # - Separate encryption key from signing key
20
+ #
21
+ # @example Configuration
22
+ # E11y.configure do |config|
23
+ # config.adapter :audit_encrypted do |a|
24
+ # a.storage_path = Rails.root.join('log', 'audit')
25
+ # a.encryption_key = ENV['E11Y_AUDIT_ENCRYPTION_KEY']
26
+ # end
27
+ # end
28
+ #
29
+ # @see ADR-006 §4.0 Audit Trail Security
30
+ # @see UC-012 Audit Trail
31
+ class AuditEncrypted < Base
32
+ # AES-256-GCM cipher
33
+ CIPHER = "aes-256-gcm"
34
+
35
+ # Encryption key (256 bits = 32 bytes)
36
+ # Must be set via ENV or configuration
37
+ attr_accessor :encryption_key
38
+
39
+ # Storage path for encrypted audit logs
40
+ attr_accessor :storage_path
41
+
42
+ # Initialize adapter
43
+ #
44
+ # @param config [Hash] Configuration options
45
+ def initialize(config = {})
46
+ @encryption_key = config[:encryption_key] || default_encryption_key
47
+ @storage_path = config[:storage_path] || default_storage_path
48
+
49
+ super
50
+
51
+ ensure_storage_directory!
52
+ end
53
+
54
+ # Write encrypted audit event
55
+ #
56
+ # @param event_data [Hash] Event data with signature
57
+ # @return [Boolean] true on success, false on failure
58
+ def write(event_data)
59
+ # 1. Encrypt event data
60
+ encrypted = encrypt_event(event_data)
61
+
62
+ # 2. Write to storage
63
+ write_to_storage(encrypted)
64
+ true
65
+ rescue StandardError => e
66
+ warn "AuditEncrypted adapter error: #{e.message}"
67
+ false
68
+ end
69
+
70
+ # Adapter capabilities
71
+ #
72
+ # @return [Hash] Capability flags
73
+ def capabilities
74
+ {
75
+ batching: false,
76
+ compression: false,
77
+ async: false,
78
+ streaming: false
79
+ }
80
+ end
81
+
82
+ # Read and decrypt audit event (for verification)
83
+ #
84
+ # @param event_id [String] Event ID
85
+ # @return [Hash] Decrypted event data
86
+ def read(event_id)
87
+ encrypted_data = read_from_storage(event_id)
88
+ decrypt_event(encrypted_data)
89
+ end
90
+
91
+ private
92
+
93
+ # Encrypt event data with AES-256-GCM
94
+ #
95
+ # @param event_data [Hash] Event data
96
+ # @return [Hash] Encrypted data with nonce and tag
97
+ def encrypt_event(event_data)
98
+ cipher = OpenSSL::Cipher.new(CIPHER)
99
+ cipher.encrypt
100
+ cipher.key = encryption_key_bytes
101
+
102
+ # Generate random nonce (never reuse!)
103
+ nonce = cipher.random_iv
104
+
105
+ # Serialize event data
106
+ plaintext = JSON.generate(event_data)
107
+
108
+ # Encrypt
109
+ ciphertext = cipher.update(plaintext) + cipher.final
110
+
111
+ # Get authentication tag
112
+ auth_tag = cipher.auth_tag
113
+
114
+ {
115
+ encrypted_data: Base64.strict_encode64(ciphertext),
116
+ nonce: Base64.strict_encode64(nonce),
117
+ auth_tag: Base64.strict_encode64(auth_tag),
118
+ event_name: event_data[:event_name],
119
+ timestamp: event_data[:timestamp],
120
+ cipher: CIPHER
121
+ }
122
+ end
123
+
124
+ # Decrypt event data
125
+ #
126
+ # @param encrypted [Hash] Encrypted data with nonce and tag
127
+ # @return [Hash] Decrypted event data
128
+ def decrypt_event(encrypted)
129
+ cipher = OpenSSL::Cipher.new(CIPHER)
130
+ cipher.decrypt
131
+ cipher.key = encryption_key_bytes
132
+ cipher.iv = Base64.strict_decode64(encrypted[:nonce])
133
+ cipher.auth_tag = Base64.strict_decode64(encrypted[:auth_tag])
134
+
135
+ ciphertext = Base64.strict_decode64(encrypted[:encrypted_data])
136
+ plaintext = cipher.update(ciphertext) + cipher.final
137
+
138
+ JSON.parse(plaintext, symbolize_names: true)
139
+ end
140
+
141
+ # Write encrypted data to storage
142
+ #
143
+ # @param encrypted [Hash] Encrypted data
144
+ # @return [void]
145
+ def write_to_storage(encrypted)
146
+ # Generate filename with timestamp for sorting
147
+ timestamp = Time.now.utc.strftime("%Y%m%d_%H%M%S_%6N")
148
+ event_name = encrypted[:event_name].to_s.gsub("::", "_")
149
+ filename = "#{timestamp}_#{event_name}.enc"
150
+
151
+ filepath = ::File.join(storage_path, filename)
152
+
153
+ # Write atomically
154
+ ::File.write(filepath, JSON.generate(encrypted))
155
+ end
156
+
157
+ # Read encrypted data from storage
158
+ #
159
+ # @param event_id [String] Event ID (filename)
160
+ # @return [Hash] Encrypted data
161
+ def read_from_storage(event_id)
162
+ filepath = ::File.join(storage_path, event_id)
163
+ data = ::File.read(filepath)
164
+ JSON.parse(data, symbolize_names: true)
165
+ end
166
+
167
+ # Validate configuration
168
+ #
169
+ # @raise [E11y::Error] if configuration invalid
170
+ # @return [void]
171
+ def validate_config!
172
+ # Allow nil key for development (will use default)
173
+ return if encryption_key.nil? && !production?
174
+
175
+ if encryption_key && encryption_key.bytesize != 32
176
+ raise E11y::Error, "Audit encryption key must be 32 bytes (256 bits), got #{encryption_key.bytesize}"
177
+ end
178
+
179
+ return unless storage_path.nil? || storage_path.empty?
180
+
181
+ raise E11y::Error, "Audit storage path must be set"
182
+ end
183
+
184
+ # Check if running in production
185
+ #
186
+ # @return [Boolean]
187
+ def production?
188
+ defined?(Rails) && Rails.env.production?
189
+ end
190
+
191
+ # Ensure storage directory exists
192
+ #
193
+ # @return [void]
194
+ def ensure_storage_directory!
195
+ FileUtils.mkdir_p(storage_path)
196
+ end
197
+
198
+ # Get encryption key as bytes
199
+ #
200
+ # @return [String] Encryption key bytes
201
+ def encryption_key_bytes
202
+ @encryption_key_bytes ||= if encryption_key.bytesize == 32
203
+ encryption_key
204
+ else
205
+ # Hex-decode if provided as hex string
206
+ [encryption_key].pack("H*")
207
+ end
208
+ end
209
+
210
+ # Default encryption key (development only)
211
+ #
212
+ # @return [String] Encryption key
213
+ def default_encryption_key
214
+ key = ENV.fetch("E11Y_AUDIT_ENCRYPTION_KEY") do
215
+ if defined?(Rails) && Rails.env.production?
216
+ raise E11y::Error, "E11Y_AUDIT_ENCRYPTION_KEY must be set in production"
217
+ end
218
+
219
+ # Development fallback
220
+ OpenSSL::Random.random_bytes(32)
221
+ end
222
+
223
+ # Ensure 32 bytes
224
+ key.bytesize == 32 ? key : [key].pack("H*")
225
+ end
226
+
227
+ # Default storage path
228
+ #
229
+ # @return [String] Storage path
230
+ def default_storage_path
231
+ if defined?(Rails)
232
+ Rails.root.join("log", "audit").to_s
233
+ else
234
+ ::File.join(Dir.pwd, "log", "audit")
235
+ end
236
+ end
237
+ end
238
+ end
239
+ end