e11y 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.rubocop.yml +69 -0
- data/CHANGELOG.md +26 -0
- data/CODE_OF_CONDUCT.md +64 -0
- data/LICENSE.txt +21 -0
- data/README.md +179 -0
- data/Rakefile +37 -0
- data/benchmarks/run_all.rb +33 -0
- data/config/README.md +83 -0
- data/config/loki-local-config.yaml +35 -0
- data/config/prometheus.yml +15 -0
- data/docker-compose.yml +78 -0
- data/docs/00-ICP-AND-TIMELINE.md +483 -0
- data/docs/01-SCALE-REQUIREMENTS.md +858 -0
- data/docs/ADR-001-architecture.md +2617 -0
- data/docs/ADR-002-metrics-yabeda.md +1395 -0
- data/docs/ADR-003-slo-observability.md +3337 -0
- data/docs/ADR-004-adapter-architecture.md +2385 -0
- data/docs/ADR-005-tracing-context.md +1372 -0
- data/docs/ADR-006-security-compliance.md +4143 -0
- data/docs/ADR-007-opentelemetry-integration.md +1385 -0
- data/docs/ADR-008-rails-integration.md +1911 -0
- data/docs/ADR-009-cost-optimization.md +2993 -0
- data/docs/ADR-010-developer-experience.md +2166 -0
- data/docs/ADR-011-testing-strategy.md +1836 -0
- data/docs/ADR-012-event-evolution.md +958 -0
- data/docs/ADR-013-reliability-error-handling.md +2750 -0
- data/docs/ADR-014-event-driven-slo.md +1533 -0
- data/docs/ADR-015-middleware-order.md +1061 -0
- data/docs/ADR-016-self-monitoring-slo.md +1234 -0
- data/docs/API-REFERENCE-L28.md +914 -0
- data/docs/COMPREHENSIVE-CONFIGURATION.md +2366 -0
- data/docs/IMPLEMENTATION_NOTES.md +2804 -0
- data/docs/IMPLEMENTATION_PLAN.md +1971 -0
- data/docs/IMPLEMENTATION_PLAN_ARCHITECTURE.md +586 -0
- data/docs/PLAN.md +148 -0
- data/docs/QUICK-START.md +934 -0
- data/docs/README.md +296 -0
- data/docs/design/00-memory-optimization.md +593 -0
- data/docs/guides/MIGRATION-L27-L28.md +692 -0
- data/docs/guides/PERFORMANCE-BENCHMARKS.md +434 -0
- data/docs/guides/README.md +44 -0
- data/docs/prd/01-overview-vision.md +440 -0
- data/docs/use_cases/README.md +119 -0
- data/docs/use_cases/UC-001-request-scoped-debug-buffering.md +813 -0
- data/docs/use_cases/UC-002-business-event-tracking.md +1953 -0
- data/docs/use_cases/UC-003-pattern-based-metrics.md +1627 -0
- data/docs/use_cases/UC-004-zero-config-slo-tracking.md +728 -0
- data/docs/use_cases/UC-005-sentry-integration.md +759 -0
- data/docs/use_cases/UC-006-trace-context-management.md +905 -0
- data/docs/use_cases/UC-007-pii-filtering.md +2648 -0
- data/docs/use_cases/UC-008-opentelemetry-integration.md +1153 -0
- data/docs/use_cases/UC-009-multi-service-tracing.md +1043 -0
- data/docs/use_cases/UC-010-background-job-tracking.md +1018 -0
- data/docs/use_cases/UC-011-rate-limiting.md +1906 -0
- data/docs/use_cases/UC-012-audit-trail.md +2301 -0
- data/docs/use_cases/UC-013-high-cardinality-protection.md +2127 -0
- data/docs/use_cases/UC-014-adaptive-sampling.md +1940 -0
- data/docs/use_cases/UC-015-cost-optimization.md +735 -0
- data/docs/use_cases/UC-016-rails-logger-migration.md +785 -0
- data/docs/use_cases/UC-017-local-development.md +867 -0
- data/docs/use_cases/UC-018-testing-events.md +1081 -0
- data/docs/use_cases/UC-019-tiered-storage-migration.md +562 -0
- data/docs/use_cases/UC-020-event-versioning.md +708 -0
- data/docs/use_cases/UC-021-error-handling-retry-dlq.md +956 -0
- data/docs/use_cases/UC-022-event-registry.md +648 -0
- data/docs/use_cases/backlog.md +226 -0
- data/e11y.gemspec +76 -0
- data/lib/e11y/adapters/adaptive_batcher.rb +207 -0
- data/lib/e11y/adapters/audit_encrypted.rb +239 -0
- data/lib/e11y/adapters/base.rb +580 -0
- data/lib/e11y/adapters/file.rb +224 -0
- data/lib/e11y/adapters/in_memory.rb +216 -0
- data/lib/e11y/adapters/loki.rb +333 -0
- data/lib/e11y/adapters/otel_logs.rb +203 -0
- data/lib/e11y/adapters/registry.rb +141 -0
- data/lib/e11y/adapters/sentry.rb +230 -0
- data/lib/e11y/adapters/stdout.rb +108 -0
- data/lib/e11y/adapters/yabeda.rb +370 -0
- data/lib/e11y/buffers/adaptive_buffer.rb +339 -0
- data/lib/e11y/buffers/base_buffer.rb +40 -0
- data/lib/e11y/buffers/request_scoped_buffer.rb +246 -0
- data/lib/e11y/buffers/ring_buffer.rb +267 -0
- data/lib/e11y/buffers.rb +14 -0
- data/lib/e11y/console.rb +122 -0
- data/lib/e11y/current.rb +48 -0
- data/lib/e11y/event/base.rb +894 -0
- data/lib/e11y/event/value_sampling_config.rb +84 -0
- data/lib/e11y/events/base_audit_event.rb +43 -0
- data/lib/e11y/events/base_payment_event.rb +33 -0
- data/lib/e11y/events/rails/cache/delete.rb +21 -0
- data/lib/e11y/events/rails/cache/read.rb +23 -0
- data/lib/e11y/events/rails/cache/write.rb +22 -0
- data/lib/e11y/events/rails/database/query.rb +45 -0
- data/lib/e11y/events/rails/http/redirect.rb +21 -0
- data/lib/e11y/events/rails/http/request.rb +26 -0
- data/lib/e11y/events/rails/http/send_file.rb +21 -0
- data/lib/e11y/events/rails/http/start_processing.rb +26 -0
- data/lib/e11y/events/rails/job/completed.rb +22 -0
- data/lib/e11y/events/rails/job/enqueued.rb +22 -0
- data/lib/e11y/events/rails/job/failed.rb +22 -0
- data/lib/e11y/events/rails/job/scheduled.rb +23 -0
- data/lib/e11y/events/rails/job/started.rb +22 -0
- data/lib/e11y/events/rails/log.rb +56 -0
- data/lib/e11y/events/rails/view/render.rb +23 -0
- data/lib/e11y/events.rb +18 -0
- data/lib/e11y/instruments/active_job.rb +201 -0
- data/lib/e11y/instruments/rails_instrumentation.rb +141 -0
- data/lib/e11y/instruments/sidekiq.rb +175 -0
- data/lib/e11y/logger/bridge.rb +205 -0
- data/lib/e11y/metrics/cardinality_protection.rb +172 -0
- data/lib/e11y/metrics/cardinality_tracker.rb +134 -0
- data/lib/e11y/metrics/registry.rb +234 -0
- data/lib/e11y/metrics/relabeling.rb +226 -0
- data/lib/e11y/metrics.rb +102 -0
- data/lib/e11y/middleware/audit_signing.rb +174 -0
- data/lib/e11y/middleware/base.rb +140 -0
- data/lib/e11y/middleware/event_slo.rb +167 -0
- data/lib/e11y/middleware/pii_filter.rb +266 -0
- data/lib/e11y/middleware/pii_filtering.rb +280 -0
- data/lib/e11y/middleware/rate_limiting.rb +214 -0
- data/lib/e11y/middleware/request.rb +163 -0
- data/lib/e11y/middleware/routing.rb +157 -0
- data/lib/e11y/middleware/sampling.rb +254 -0
- data/lib/e11y/middleware/slo.rb +168 -0
- data/lib/e11y/middleware/trace_context.rb +131 -0
- data/lib/e11y/middleware/validation.rb +118 -0
- data/lib/e11y/middleware/versioning.rb +132 -0
- data/lib/e11y/middleware.rb +12 -0
- data/lib/e11y/pii/patterns.rb +90 -0
- data/lib/e11y/pii.rb +13 -0
- data/lib/e11y/pipeline/builder.rb +155 -0
- data/lib/e11y/pipeline/zone_validator.rb +110 -0
- data/lib/e11y/pipeline.rb +12 -0
- data/lib/e11y/presets/audit_event.rb +65 -0
- data/lib/e11y/presets/debug_event.rb +34 -0
- data/lib/e11y/presets/high_value_event.rb +51 -0
- data/lib/e11y/presets.rb +19 -0
- data/lib/e11y/railtie.rb +138 -0
- data/lib/e11y/reliability/circuit_breaker.rb +216 -0
- data/lib/e11y/reliability/dlq/file_storage.rb +277 -0
- data/lib/e11y/reliability/dlq/filter.rb +117 -0
- data/lib/e11y/reliability/retry_handler.rb +207 -0
- data/lib/e11y/reliability/retry_rate_limiter.rb +117 -0
- data/lib/e11y/sampling/error_spike_detector.rb +225 -0
- data/lib/e11y/sampling/load_monitor.rb +161 -0
- data/lib/e11y/sampling/stratified_tracker.rb +92 -0
- data/lib/e11y/sampling/value_extractor.rb +82 -0
- data/lib/e11y/self_monitoring/buffer_monitor.rb +79 -0
- data/lib/e11y/self_monitoring/performance_monitor.rb +97 -0
- data/lib/e11y/self_monitoring/reliability_monitor.rb +146 -0
- data/lib/e11y/slo/event_driven.rb +150 -0
- data/lib/e11y/slo/tracker.rb +119 -0
- data/lib/e11y/version.rb +9 -0
- data/lib/e11y.rb +283 -0
- metadata +452 -0
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "json"
|
|
5
|
+
require "fileutils"
|
|
6
|
+
require "base64"
|
|
7
|
+
|
|
8
|
+
module E11y
|
|
9
|
+
module Adapters
|
|
10
|
+
# Audit Encrypted Adapter - AES-256-GCM encrypted storage for audit events
|
|
11
|
+
#
|
|
12
|
+
# Stores audit events with encryption at rest for compliance requirements.
|
|
13
|
+
# Each event is individually encrypted with AES-256-GCM.
|
|
14
|
+
#
|
|
15
|
+
# **Security:**
|
|
16
|
+
# - AES-256-GCM authenticated encryption
|
|
17
|
+
# - Per-event nonce (never reused)
|
|
18
|
+
# - Authentication tag validation
|
|
19
|
+
# - Separate encryption key from signing key
|
|
20
|
+
#
|
|
21
|
+
# @example Configuration
|
|
22
|
+
# E11y.configure do |config|
|
|
23
|
+
# config.adapter :audit_encrypted do |a|
|
|
24
|
+
# a.storage_path = Rails.root.join('log', 'audit')
|
|
25
|
+
# a.encryption_key = ENV['E11Y_AUDIT_ENCRYPTION_KEY']
|
|
26
|
+
# end
|
|
27
|
+
# end
|
|
28
|
+
#
|
|
29
|
+
# @see ADR-006 §4.0 Audit Trail Security
|
|
30
|
+
# @see UC-012 Audit Trail
|
|
31
|
+
class AuditEncrypted < Base
|
|
32
|
+
# AES-256-GCM cipher
|
|
33
|
+
CIPHER = "aes-256-gcm"
|
|
34
|
+
|
|
35
|
+
# Encryption key (256 bits = 32 bytes)
|
|
36
|
+
# Must be set via ENV or configuration
|
|
37
|
+
attr_accessor :encryption_key
|
|
38
|
+
|
|
39
|
+
# Storage path for encrypted audit logs
|
|
40
|
+
attr_accessor :storage_path
|
|
41
|
+
|
|
42
|
+
# Initialize adapter
|
|
43
|
+
#
|
|
44
|
+
# @param config [Hash] Configuration options
|
|
45
|
+
def initialize(config = {})
|
|
46
|
+
@encryption_key = config[:encryption_key] || default_encryption_key
|
|
47
|
+
@storage_path = config[:storage_path] || default_storage_path
|
|
48
|
+
|
|
49
|
+
super
|
|
50
|
+
|
|
51
|
+
ensure_storage_directory!
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Write encrypted audit event
|
|
55
|
+
#
|
|
56
|
+
# @param event_data [Hash] Event data with signature
|
|
57
|
+
# @return [Boolean] true on success, false on failure
|
|
58
|
+
def write(event_data)
|
|
59
|
+
# 1. Encrypt event data
|
|
60
|
+
encrypted = encrypt_event(event_data)
|
|
61
|
+
|
|
62
|
+
# 2. Write to storage
|
|
63
|
+
write_to_storage(encrypted)
|
|
64
|
+
true
|
|
65
|
+
rescue StandardError => e
|
|
66
|
+
warn "AuditEncrypted adapter error: #{e.message}"
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Adapter capabilities
|
|
71
|
+
#
|
|
72
|
+
# @return [Hash] Capability flags
|
|
73
|
+
def capabilities
|
|
74
|
+
{
|
|
75
|
+
batching: false,
|
|
76
|
+
compression: false,
|
|
77
|
+
async: false,
|
|
78
|
+
streaming: false
|
|
79
|
+
}
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Read and decrypt audit event (for verification)
|
|
83
|
+
#
|
|
84
|
+
# @param event_id [String] Event ID
|
|
85
|
+
# @return [Hash] Decrypted event data
|
|
86
|
+
def read(event_id)
|
|
87
|
+
encrypted_data = read_from_storage(event_id)
|
|
88
|
+
decrypt_event(encrypted_data)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
# Encrypt event data with AES-256-GCM
|
|
94
|
+
#
|
|
95
|
+
# @param event_data [Hash] Event data
|
|
96
|
+
# @return [Hash] Encrypted data with nonce and tag
|
|
97
|
+
def encrypt_event(event_data)
|
|
98
|
+
cipher = OpenSSL::Cipher.new(CIPHER)
|
|
99
|
+
cipher.encrypt
|
|
100
|
+
cipher.key = encryption_key_bytes
|
|
101
|
+
|
|
102
|
+
# Generate random nonce (never reuse!)
|
|
103
|
+
nonce = cipher.random_iv
|
|
104
|
+
|
|
105
|
+
# Serialize event data
|
|
106
|
+
plaintext = JSON.generate(event_data)
|
|
107
|
+
|
|
108
|
+
# Encrypt
|
|
109
|
+
ciphertext = cipher.update(plaintext) + cipher.final
|
|
110
|
+
|
|
111
|
+
# Get authentication tag
|
|
112
|
+
auth_tag = cipher.auth_tag
|
|
113
|
+
|
|
114
|
+
{
|
|
115
|
+
encrypted_data: Base64.strict_encode64(ciphertext),
|
|
116
|
+
nonce: Base64.strict_encode64(nonce),
|
|
117
|
+
auth_tag: Base64.strict_encode64(auth_tag),
|
|
118
|
+
event_name: event_data[:event_name],
|
|
119
|
+
timestamp: event_data[:timestamp],
|
|
120
|
+
cipher: CIPHER
|
|
121
|
+
}
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Decrypt event data
|
|
125
|
+
#
|
|
126
|
+
# @param encrypted [Hash] Encrypted data with nonce and tag
|
|
127
|
+
# @return [Hash] Decrypted event data
|
|
128
|
+
def decrypt_event(encrypted)
|
|
129
|
+
cipher = OpenSSL::Cipher.new(CIPHER)
|
|
130
|
+
cipher.decrypt
|
|
131
|
+
cipher.key = encryption_key_bytes
|
|
132
|
+
cipher.iv = Base64.strict_decode64(encrypted[:nonce])
|
|
133
|
+
cipher.auth_tag = Base64.strict_decode64(encrypted[:auth_tag])
|
|
134
|
+
|
|
135
|
+
ciphertext = Base64.strict_decode64(encrypted[:encrypted_data])
|
|
136
|
+
plaintext = cipher.update(ciphertext) + cipher.final
|
|
137
|
+
|
|
138
|
+
JSON.parse(plaintext, symbolize_names: true)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Write encrypted data to storage
|
|
142
|
+
#
|
|
143
|
+
# @param encrypted [Hash] Encrypted data
|
|
144
|
+
# @return [void]
|
|
145
|
+
def write_to_storage(encrypted)
|
|
146
|
+
# Generate filename with timestamp for sorting
|
|
147
|
+
timestamp = Time.now.utc.strftime("%Y%m%d_%H%M%S_%6N")
|
|
148
|
+
event_name = encrypted[:event_name].to_s.gsub("::", "_")
|
|
149
|
+
filename = "#{timestamp}_#{event_name}.enc"
|
|
150
|
+
|
|
151
|
+
filepath = ::File.join(storage_path, filename)
|
|
152
|
+
|
|
153
|
+
# Write atomically
|
|
154
|
+
::File.write(filepath, JSON.generate(encrypted))
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Read encrypted data from storage
|
|
158
|
+
#
|
|
159
|
+
# @param event_id [String] Event ID (filename)
|
|
160
|
+
# @return [Hash] Encrypted data
|
|
161
|
+
def read_from_storage(event_id)
|
|
162
|
+
filepath = ::File.join(storage_path, event_id)
|
|
163
|
+
data = ::File.read(filepath)
|
|
164
|
+
JSON.parse(data, symbolize_names: true)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Validate configuration
|
|
168
|
+
#
|
|
169
|
+
# @raise [E11y::Error] if configuration invalid
|
|
170
|
+
# @return [void]
|
|
171
|
+
def validate_config!
|
|
172
|
+
# Allow nil key for development (will use default)
|
|
173
|
+
return if encryption_key.nil? && !production?
|
|
174
|
+
|
|
175
|
+
if encryption_key && encryption_key.bytesize != 32
|
|
176
|
+
raise E11y::Error, "Audit encryption key must be 32 bytes (256 bits), got #{encryption_key.bytesize}"
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
return unless storage_path.nil? || storage_path.empty?
|
|
180
|
+
|
|
181
|
+
raise E11y::Error, "Audit storage path must be set"
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Check if running in production
|
|
185
|
+
#
|
|
186
|
+
# @return [Boolean]
|
|
187
|
+
def production?
|
|
188
|
+
defined?(Rails) && Rails.env.production?
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Ensure storage directory exists
|
|
192
|
+
#
|
|
193
|
+
# @return [void]
|
|
194
|
+
def ensure_storage_directory!
|
|
195
|
+
FileUtils.mkdir_p(storage_path)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# Get encryption key as bytes
|
|
199
|
+
#
|
|
200
|
+
# @return [String] Encryption key bytes
|
|
201
|
+
def encryption_key_bytes
|
|
202
|
+
@encryption_key_bytes ||= if encryption_key.bytesize == 32
|
|
203
|
+
encryption_key
|
|
204
|
+
else
|
|
205
|
+
# Hex-decode if provided as hex string
|
|
206
|
+
[encryption_key].pack("H*")
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Default encryption key (development only)
|
|
211
|
+
#
|
|
212
|
+
# @return [String] Encryption key
|
|
213
|
+
def default_encryption_key
|
|
214
|
+
key = ENV.fetch("E11Y_AUDIT_ENCRYPTION_KEY") do
|
|
215
|
+
if defined?(Rails) && Rails.env.production?
|
|
216
|
+
raise E11y::Error, "E11Y_AUDIT_ENCRYPTION_KEY must be set in production"
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Development fallback
|
|
220
|
+
OpenSSL::Random.random_bytes(32)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Ensure 32 bytes
|
|
224
|
+
key.bytesize == 32 ? key : [key].pack("H*")
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Default storage path
|
|
228
|
+
#
|
|
229
|
+
# @return [String] Storage path
|
|
230
|
+
def default_storage_path
|
|
231
|
+
if defined?(Rails)
|
|
232
|
+
Rails.root.join("log", "audit").to_s
|
|
233
|
+
else
|
|
234
|
+
::File.join(Dir.pwd, "log", "audit")
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
end
|