e11y 0.2.0 → 1.1.0

data/lib/e11y/reliability/dlq/file_adapter.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "securerandom"
+require_relative "base"
+
+module E11y
+  module Reliability
+    module DLQ
+      # File-based Dead Letter Queue storage.
+      #
+      # Stores failed events to a JSONL file for later analysis/replay.
+      # Each line is a JSON object representing a failed event with metadata.
+      #
+      # @example Usage
+      #   dlq = FileAdapter.new(file_path: "log/e11y_dlq.jsonl")
+      #   dlq.save(event_data, metadata: { error: "Timeout", retry_count: 3 })
+      #
+      # @see ADR-013 §4 (Dead Letter Queue)
+      # @see UC-021 §3 (DLQ File Storage)
+      # rubocop:disable Metrics/ClassLength
+      # DLQ file storage is a cohesive unit handling event persistence, rotation, and querying
+      class FileAdapter < Base
+        # @param file_path [String] Path to DLQ file (default: log/e11y_dlq.jsonl)
+        # @param max_file_size_mb [Integer] Maximum file size in MB before rotation (default: 100)
+        # @param retention_days [Integer] Days to retain DLQ files (default: 30)
+        def initialize(file_path: nil, max_file_size_mb: 100, retention_days: 30)
+          super()
+          @file_path = file_path || default_file_path
+          @max_file_size_bytes = max_file_size_mb * 1024 * 1024
+          @retention_days = retention_days
+          @mutex = Mutex.new
+
+          ensure_directory_exists
+        end
+
+        # Save failed event to DLQ.
+        #
+        # @param event_data [Hash] Event data
+        # @param metadata [Hash] Failure metadata (error, retry_count, adapter, etc.)
+        # @return [String] Event ID (UUID)
+        # DLQ save requires building entry, writing, rotation, cleanup, and metrics
+        def save(event_data, metadata: {})
+          event_id = SecureRandom.uuid
+          timestamp = Time.now.utc
+
+          dlq_entry = {
+            id: event_id,
+            timestamp: timestamp.iso8601(3),
+            event_name: event_data[:event_name],
+            event_data: event_data,
+            metadata: metadata.merge(
+              failed_at: timestamp.iso8601(3),
+              retry_count: metadata[:retry_count] || 0,
+              error_message: metadata[:error]&.message,
+              error_class: metadata[:error]&.class&.name
+            )
+          }
+
+          write_entry(dlq_entry)
+          rotate_if_needed
+          cleanup_old_files
+
+          increment_metric("e11y.dlq.saved", event_name: event_data[:event_name])
+
+          event_id
+        end
+
+        # List DLQ entries with optional filters.
+        #
+        # @param limit [Integer] Maximum entries to return
+        # @param offset [Integer] Number of entries to skip
+        # @param filters [Hash] Filter options (event_name, after, before)
+        # @return [Array<Hash>] Array of DLQ entries
+        # rubocop:disable Metrics/AbcSize
+        # DLQ listing requires file iteration, pagination, multiple filters, and error handling
+        def list(limit: 100, offset: 0, filters: {})
+          entries = []
+
+          return entries unless File.exist?(@file_path)
+
+          File.foreach(@file_path).with_index do |line, index|
+            next if index < offset
+            break if entries.size >= limit
+
+            entry = JSON.parse(line, symbolize_names: true)
+
+            # Apply filters
+            next if filters[:event_name] && entry[:event_name] != filters[:event_name]
+            next if filters[:after] && Time.parse(entry[:timestamp]) < filters[:after]
+            next if filters[:before] && Time.parse(entry[:timestamp]) > filters[:before]
+
+            entries << entry
+          end
+
+          entries
+        rescue JSON::ParserError => e
+          # Log parsing error but don't crash
+          increment_metric("e11y.dlq.parse_error", error: e.class.name)
+          entries
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # Get DLQ statistics.
+        #
+        # @return [Hash] Statistics (total_entries, file_size_mb, oldest_entry, newest_entry)
+        # rubocop:disable Metrics/AbcSize
+        # DLQ stats requires reading file size, counting entries, extracting timestamps, and error handling
+        def stats
+          return default_stats unless File.exist?(@file_path)
+
+          file_size_bytes = File.size(@file_path)
+          total_entries = File.foreach(@file_path).count
+
+          oldest_entry = nil
+          newest_entry = nil
+
+          # Read first and last line for oldest/newest timestamps
+          File.foreach(@file_path).with_index do |line, index|
+            entry = JSON.parse(line, symbolize_names: true)
+            oldest_entry = entry[:timestamp] if index.zero?
+            newest_entry = entry[:timestamp]
+          end
+
+          {
+            total_entries: total_entries,
+            file_size_mb: (file_size_bytes / 1024.0 / 1024.0).round(2),
+            oldest_entry: oldest_entry,
+            newest_entry: newest_entry,
+            file_path: @file_path
+          }
+        rescue StandardError => e
+          increment_metric("e11y.dlq.stats_error", error: e.class.name)
+          default_stats
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # Replay single event from DLQ.
+        #
+        # Re-dispatches the stored event_data to all registered adapters so the
+        # event reaches every adapter regardless of original routing configuration.
+        #
+        # NOTE: Delivers directly to adapters, bypassing the middleware pipeline.
+        # This means middleware (PII filtering, rate limiting, routing) is NOT applied.
+        # Trade-off: ensures event reaches all registered adapters regardless of routing rules.
+        #
+        # @param event_id [String] Event ID to replay
+        # @return [Boolean] true if replayed successfully
+        def replay(event_id)
+          entry = find_entry(event_id)
+          return false unless entry
+
+          # Reconstruct event_data from stored entry (keys are symbolized after JSON parse)
+          event_data = entry[:event_data]
+
+          # Deliver directly to all registered adapters, bypassing routing.
+          # This ensures the replayed event reaches every adapter (including
+          # adapters added after the original event was stored in the DLQ).
+          E11y.configuration.adapters.each_value do |adapter|
+            adapter.write(event_data)
+          rescue StandardError => e
+            E11y.logger.error("DLQ replay write failed: #{e.message}")
+          end
+
+          increment_metric("e11y.dlq.replayed", event_name: entry[:event_name])
+          true
+        rescue StandardError => e
+          increment_metric("e11y.dlq.replay_failed", error: e.class.name)
+          false
+        end
+
+        # Delete entry from DLQ.
+        #
+        # Rewrites the JSONL file excluding the line whose :id matches event_id.
+        # Returns true if an entry was found and removed, false otherwise.
+        #
+        # @param event_id [String] Event ID to delete
+        # @return [Boolean] true if deleted
+        # delete is an action method returning boolean status, not a predicate query
+        # rubocop:disable Metrics/AbcSize -- file rewrite with mutex, filters, error handling
+        def delete(event_id)
+          return false unless File.exist?(@file_path)
+
+          deleted = false
+          @mutex.synchronize do
+            all_lines = File.readlines(@file_path).map(&:chomp).reject(&:empty?)
+            original_count = all_lines.length
+
+            remaining = all_lines.reject do |line|
+              entry = JSON.parse(line, symbolize_names: true)
+              entry[:id].to_s == event_id.to_s
+            end
+
+            if remaining.length < original_count
+              content = remaining.join("\n")
+              content += "\n" unless content.empty? || content.end_with?("\n")
+              File.write(@file_path, content)
+              deleted = true
+            end
+          end
+          deleted
+        rescue StandardError => e
+          E11y.logger.error("DLQ delete failed for #{event_id}: #{e.message}")
+          false
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        private
+
+        # Get default file path (log/e11y_dlq.jsonl).
+        def default_file_path
+          ::Rails.root.join("log", "e11y_dlq.jsonl").to_s
+        end
+
+        # Ensure log directory exists.
+        def ensure_directory_exists
+          dir = File.dirname(@file_path)
+          FileUtils.mkdir_p(dir)
+        end
+
+        # Write DLQ entry to file (thread-safe).
+        def write_entry(entry)
+          @mutex.synchronize do
+            File.open(@file_path, "a") do |f|
+              f.flock(File::LOCK_EX)
+              f.puts(JSON.generate(entry))
+            end
+          end
+        end
+
+        # Rotate file if size exceeds max_file_size.
+        def rotate_if_needed
+          return unless File.exist?(@file_path)
+          return if File.size(@file_path) < @max_file_size_bytes
+
+          @mutex.synchronize do
+            # Rotate: log/e11y_dlq.jsonl → log/e11y_dlq.2026-01-20T12:34:56Z.jsonl
+            timestamp = Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+            rotated_path = @file_path.sub(/\.jsonl$/, ".#{timestamp}.jsonl")
+
+            FileUtils.mv(@file_path, rotated_path)
+
+            increment_metric("e11y.dlq.rotated", new_file: rotated_path)
+          end
+        end
+
+        # Cleanup old rotated files.
+        def cleanup_old_files
+          dir = File.dirname(@file_path)
+          base_name = File.basename(@file_path, ".jsonl")
+
+          # Find all rotated files: e11y_dlq.*.jsonl
+          pattern = File.join(dir, "#{base_name}.*.jsonl")
+
+          Dir.glob(pattern).each do |file|
+            next unless File.file?(file)
+
+            file_age_days = (Time.now - File.mtime(file)) / 86_400
+
+            if file_age_days > @retention_days
+              File.delete(file)
+              increment_metric("e11y.dlq.cleaned_up", file: file)
+            end
+          end
+        end
+
+        # Find DLQ entry by ID.
+        def find_entry(event_id)
+          return nil unless File.exist?(@file_path)
+
+          File.foreach(@file_path) do |line|
+            entry = JSON.parse(line, symbolize_names: true)
+            return entry if entry[:id] == event_id
+          end
+
+          nil
+        rescue JSON::ParserError
+          nil
+        end
+
+        # Default stats when file doesn't exist.
+        def default_stats
+          {
+            total_entries: 0,
+            file_size_mb: 0.0,
+            oldest_entry: nil,
+            newest_entry: nil,
+            file_path: @file_path
+          }
+        end
+
+        # Increment DLQ metric (no-op when E11y::Metrics backend not configured).
+        def increment_metric(metric_name, tags = {})
+          E11y::Metrics.increment(metric_name, **tags)
+        end
+      end
+      # rubocop:enable Metrics/ClassLength
+    end
+  end
+end

data/lib/e11y/reliability/dlq/file_storage.rb

@@ -38,7 +38,6 @@ module E11y
         # @param event_data [Hash] Event data
         # @param metadata [Hash] Failure metadata (error, retry_count, adapter, etc.)
         # @return [String] Event ID (UUID)
-        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         # DLQ save requires building entry, writing, rotation, cleanup, and metrics
         def save(event_data, metadata: {})
           event_id = SecureRandom.uuid
@@ -61,11 +60,11 @@ module E11y
           rotate_if_needed
           cleanup_old_files
 
-          increment_metric("e11y.dlq.saved", event_name: event_data[:event_name])
+          E11y::Metrics.increment("e11y.dlq.saved", event_name: event_data[:event_name])
+          update_dlq_size_gauge
 
           event_id
         end
-        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
         # List DLQ entries with optional filters.
         #
@@ -73,7 +72,7 @@ module E11y
         # @param offset [Integer] Number of entries to skip
         # @param filters [Hash] Filter options (event_name, after, before)
         # @return [Array<Hash>] Array of DLQ entries
-        # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        # rubocop:disable Metrics/AbcSize
         # DLQ listing requires file iteration, pagination, multiple filters, and error handling
         def list(limit: 100, offset: 0, filters: {})
           entries = []
@@ -97,15 +96,14 @@ module E11y
           entries
         rescue JSON::ParserError => e
           # Log parsing error but don't crash
-          increment_metric("e11y.dlq.parse_error", error: e.class.name)
+          E11y::Metrics.increment("e11y.dlq.parse_error", error: e.class.name)
           entries
         end
-        # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        # rubocop:enable Metrics/AbcSize
 
         # Get DLQ statistics.
         #
         # @return [Hash] Statistics (total_entries, file_size_mb, oldest_entry, newest_entry)
-        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         # DLQ stats requires reading file size, counting entries, extracting timestamps, and error handling
         def stats
           return default_stats unless File.exist?(@file_path)
@@ -130,29 +128,32 @@ module E11y
             newest_entry: newest_entry,
             file_path: @file_path
           }
-        rescue StandardError => e
-          increment_metric("e11y.dlq.stats_error", error: e.class.name)
+        rescue StandardError
           default_stats
         end
-        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
         # Replay single event from DLQ.
         #
+        # Re-dispatches event through E11y pipeline so it reaches adapters.
+        #
         # @param event_id [String] Event ID to replay
         # @return [Boolean] true if replayed successfully
         def replay(event_id)
           entry = find_entry(event_id)
           return false unless entry
 
-          # Re-dispatch event through E11y pipeline
-          # TODO: Implement E11y::Pipeline.dispatch
-          # E11y::Pipeline.dispatch(entry[:event_data], metadata: entry[:metadata].merge(replayed: true))
+          event_data = entry[:event_data]
+          return false unless event_data
 
-          # For now, just mark as replayed
-          increment_metric("e11y.dlq.replayed", event_name: entry[:event_name])
+          # F-004/C07: Mark as DLQ-replayed so PIIFilter skips (avoid double-hashing)
+          event_data = event_data.dup
+          event_data[:dlq_replayed] = true
+
+          E11y.config.built_pipeline.call(event_data)
+          E11y::Metrics.increment("e11y.dlq.replayed", event_name: entry[:event_name])
           true
         rescue StandardError => e
-          increment_metric("e11y.dlq.replay_failed", error: e.class.name)
+          E11y::Metrics.increment("e11y.dlq.replay_failed", error: e.class.name)
           false
         end
 
@@ -177,23 +178,58 @@ module E11y
 
         # Delete entry from DLQ.
         #
-        # Note: This is a simplified implementation.
-        # In production, consider using a database or append-only log with tombstones.
+        # Rewrites file excluding the entry. For large files this is expensive.
         #
         # @param event_id [String] Event ID to delete
         # @return [Boolean] true if deleted
-        # rubocop:disable Naming/PredicateMethod
         # delete is an action method returning boolean status, not a predicate query
-        def delete(_event_id)
-          # TODO: Implement deletion (requires rewriting file)
-          # For JSONL, deletion is expensive (requires full file rewrite)
-          # Consider marking as deleted instead or using database
+        def delete(event_id)
+          return false unless File.exist?(@file_path)
+
+          entries, found = read_entries_excluding(event_id)
+          return false unless found
+
+          rewrite_file_with(entries)
+          update_dlq_size_gauge
+          true
+        rescue StandardError
           false
         end
-        # rubocop:enable Naming/PredicateMethod
 
         private
 
+        def update_dlq_size_gauge
+          return unless defined?(E11y::Metrics) && E11y::Metrics.respond_to?(:gauge)
+
+          count = stats[:total_entries]
+          E11y::Metrics.gauge(:e11y_dlq_size, count)
+        rescue StandardError
+          # Non-fatal: gauge update must not break DLQ operations
+        end
+
+        def read_entries_excluding(event_id)
+          entries = []
+          found = false
+          File.foreach(@file_path) do |line|
+            entry = JSON.parse(line, symbolize_names: true)
+            if entry[:id] == event_id
+              found = true
+            else
+              entries << entry
+            end
+          end
+          [entries, found]
+        end
+
+        def rewrite_file_with(entries)
+          @mutex.synchronize do
+            File.open(@file_path, "w") do |f|
+              f.flock(File::LOCK_EX)
+              entries.each { |e| f.puts(JSON.generate(e)) }
+            end
+          end
+        end
+
         # Get default file path (log/e11y_dlq.jsonl).
         def default_file_path
           ::Rails.root.join("log", "e11y_dlq.jsonl").to_s
@@ -226,8 +262,6 @@ module E11y
             rotated_path = @file_path.sub(/\.jsonl$/, ".#{timestamp}.jsonl")
 
             FileUtils.mv(@file_path, rotated_path)
-
-            increment_metric("e11y.dlq.rotated", new_file: rotated_path)
           end
         end
 
@@ -244,10 +278,7 @@ module E11y
 
             file_age_days = (Time.now - File.mtime(file)) / 86_400
 
-            if file_age_days > @retention_days
-              File.delete(file)
-              increment_metric("e11y.dlq.cleaned_up", file: file)
-            end
+            File.delete(file) if file_age_days > @retention_days
           end
         end
 
@@ -277,10 +308,8 @@ module E11y
         end
 
         # Increment DLQ metric.
-        def increment_metric(metric_name, tags = {})
-          # TODO: Integrate with Yabeda metrics
-          # E11y::Metrics.increment(metric_name, tags)
-        end
+        #
+        # Normalizes metric_name like "e11y.dlq.saved" to :e11y_dlq_saved_total.
       end
       # rubocop:enable Metrics/ClassLength
     end

data/lib/e11y/reliability/dlq/filter.rb

@@ -5,91 +5,79 @@ module E11y
     module DLQ
       # DLQ Filter determines which failed events should be saved to DLQ.
       #
-      # Supports:
-      # - Always save patterns (e.g., payment.*, audit.*)
-      # - Always discard patterns (e.g., debug.*, test.*)
-      # - Severity-based filtering (e.g., always save :error, :fatal)
+      # Uses Event DSL (use_dlq) when event class is registered.
+      # Audit events (Presets::AuditEvent) have use_dlq true by default.
       #
-      # @example Configuration
-      #   filter = Filter.new(
-      #     always_save_patterns: [/^payment\./, /^audit\./],
-      #     always_discard_patterns: [/^debug\./, /^test\./],
-      #     save_severities: [:error, :fatal]
-      #   )
+      # Priority order:
+      # 1. Event class use_dlq == false → discard
+      # 2. Event class use_dlq == true → save
+      # 3. Severity-based (save_severities)
+      # 4. Default behavior
       #
-      #   filter.should_save?(event_data)  # => true/false
+      # @example Event DSL
+      #   class Events::AuditLogin < E11y::Events::BaseAuditEvent
+      #     # use_dlq true from preset
+      #   end
+      #
+      #   class Events::DebugTrace < E11y::Event::Base
+      #     use_dlq false
+      #   end
       #
       # @see ADR-013 §4.3 (DLQ Filter)
       # @see UC-021 §3.2 (DLQ Filter Configuration)
       class Filter
-        # @param always_save_patterns [Array<Regexp>] Event patterns to always save
-        # @param always_discard_patterns [Array<Regexp>] Event patterns to always discard
         # @param save_severities [Array<Symbol>] Severities to always save (:error, :fatal)
-        # @param default_behavior [Symbol] Default behavior when no rule matches (:save or :discard)
+        # @param default_behavior [Symbol] Default when no Event DSL rule (:save or :discard)
         def initialize(
-          always_save_patterns: [],
-          always_discard_patterns: [],
           save_severities: %i[error fatal],
           default_behavior: :save
         )
-          @always_save_patterns = always_save_patterns
-          @always_discard_patterns = always_discard_patterns
           @save_severities = save_severities
           @default_behavior = default_behavior
         end
 
         # Check if event should be saved to DLQ.
         #
-        # Priority order:
-        # 1. Always discard patterns (highest priority)
-        # 2. Always save patterns
-        # 3. Severity-based rules
-        # 4. Default behavior
-        #
         # @param event_data [Hash] Event data
+        # @param error [StandardError, nil] The error that caused the DLQ save (optional)
         # @return [Boolean] true if event should be saved to DLQ
         # rubocop:disable Metrics/MethodLength
-        # DLQ filter requires 4-priority decision tree with metrics tracking for each branch
-        def should_save?(event_data)
-          event_name = event_data[:event_name].to_s
+        def should_save?(event_data, _error = nil)
+          event_class = resolve_event_class(event_data[:event_name])
           severity = event_data[:severity]
 
-          # Priority 1: Always discard (highest priority)
-          if matches_patterns?(event_name, @always_discard_patterns)
-            increment_metric("e11y.dlq.filter.discarded", reason: "always_discard_pattern")
+          # Priority 1: Event DSL use_dlq == false
+          if event_class.respond_to?(:use_dlq) && event_class.use_dlq == false
+            increment_filter_metric("discarded", "use_dlq")
             return false
           end
 
-          # Priority 2: Always save
-          if matches_patterns?(event_name, @always_save_patterns)
-            increment_metric("e11y.dlq.filter.saved", reason: "always_save_pattern")
+          # Priority 2: Event DSL use_dlq == true
+          if event_class.respond_to?(:use_dlq) && event_class.use_dlq == true
+            increment_filter_metric("saved", "use_dlq")
             return true
           end
 
           # Priority 3: Severity-based
           if @save_severities.include?(severity)
-            increment_metric("e11y.dlq.filter.saved", reason: "severity")
+            increment_filter_metric("saved", "severity")
             return true
           end
 
           # Priority 4: Default behavior
           if @default_behavior == :save
-            increment_metric("e11y.dlq.filter.saved", reason: "default")
+            increment_filter_metric("saved", "default")
             true
           else
-            increment_metric("e11y.dlq.filter.discarded", reason: "default")
+            increment_filter_metric("discarded", "default")
             false
           end
         end
         # rubocop:enable Metrics/MethodLength
 
-        # Get filter statistics.
-        #
         # @return [Hash] Filter configuration stats
         def stats
           {
-            always_save_patterns: @always_save_patterns.map(&:inspect),
-            always_discard_patterns: @always_discard_patterns.map(&:inspect),
             save_severities: @save_severities,
             default_behavior: @default_behavior
           }
@@ -97,22 +85,17 @@ module E11y
 
         private
 
-        # Check if event name matches any of the patterns.
-        #
-        # @param event_name [String] Event name
-        # @param patterns [Array<Regexp>] Patterns to match
-        # @return [Boolean] true if event matches any pattern
-        def matches_patterns?(event_name, patterns)
-          patterns.any? { |pattern| pattern.match?(event_name) }
+        def resolve_event_class(event_name)
+          return nil unless event_name
+          return nil unless defined?(E11y::Registry) && E11y::Registry.respond_to?(:find)
+
+          E11y::Registry.find(event_name.to_s)
         end
 
-        # Increment DLQ filter metric.
-        #
-        # @param metric_name [String] Metric name
-        # @param tags [Hash] Additional tags
-        def increment_metric(metric_name, tags = {})
-          # TODO: Integrate with Yabeda metrics
-          # E11y::Metrics.increment(metric_name, tags)
+        def increment_filter_metric(action, reason)
+          return unless defined?(E11y::Metrics) && E11y::Metrics.respond_to?(:increment)
+
+          E11y::Metrics.increment(:e11y_dlq_filter_decisions_total, { action: action, reason: reason })
         end
       end
     end