e11y 0.1.0

data/lib/e11y/buffers/adaptive_buffer.rb

@@ -0,0 +1,339 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module E11y
+  module Buffers
+    # Adaptive buffer with global memory tracking and backpressure
+    #
+    # Key features:
+    # - Global memory limit (100MB default)
+    # - Per-adapter buffering (Hash-based)
+    # - Event size estimation (±10% accuracy)
+    # - Early flush at 80% threshold
+    # - Backpressure strategies (:block, :drop)
+    # - Thread-safe flush operations
+    #
+    # Architecture (C20 Resolution):
+    # - Tracks memory usage across ALL buffers globally
+    # - Enforces strict memory limits to prevent OOM
+    # - Adaptive behavior: flush early when approaching limit
+    # - Backpressure: block or drop events when limit exceeded
+    #
+    # References:
+    # - ADR-001 §3.3.2 (Adaptive Buffer with Memory Limits)
+    # - UC-001 (Request-Scoped Debug Buffering)
+    # - UC-014 (Adaptive Sampling)
+    #
+    # @example Basic usage
+    #   buffer = E11y::Buffers::AdaptiveBuffer.new
+    #   buffer.add_event({ event_name: "test", payload: {...}, adapters: [:logs] })
+    #   events = buffer.flush # Returns all events
+    #
+    # @example With memory limit
+    #   buffer = E11y::Buffers::AdaptiveBuffer.new(memory_limit_mb: 50)
+    #   buffer.add_event(event) # May trigger backpressure if limit exceeded
+    #
+    # @see ADR-001 §3.3.2
+    # rubocop:disable Metrics/ClassLength
+    class AdaptiveBuffer
+      # Default memory limit (100 MB)
+      DEFAULT_MEMORY_LIMIT_MB = 100
+
+      # Early flush threshold (80%)
+      EARLY_FLUSH_THRESHOLD = 0.8
+
+      # Available backpressure strategies
+      BACKPRESSURE_STRATEGIES = %i[block drop].freeze
+
+      # Default backpressure strategy
+      DEFAULT_BACKPRESSURE_STRATEGY = :drop
+
+      # Maximum block time for :block strategy (seconds)
+      DEFAULT_MAX_BLOCK_TIME = 1.0
+
+      attr_reader :memory_limit_bytes
+
+      # Initialize adaptive buffer
+      #
+      # @param memory_limit_mb [Integer] Memory limit in megabytes (default: 100)
+      # @param backpressure_strategy [Symbol] Strategy when limit exceeded (:block, :drop)
+      # @param max_block_time [Float] Max wait time for :block strategy (seconds)
+      #
+      # @raise [ArgumentError] if memory_limit_mb <= 0 or invalid strategy
+      #
+      # @example
+      #   buffer = AdaptiveBuffer.new(memory_limit_mb: 50, backpressure_strategy: :block)
+      def initialize(memory_limit_mb: DEFAULT_MEMORY_LIMIT_MB,
+                     backpressure_strategy: DEFAULT_BACKPRESSURE_STRATEGY,
+                     max_block_time: DEFAULT_MAX_BLOCK_TIME)
+        raise ArgumentError, "memory_limit_mb must be > 0" if memory_limit_mb <= 0
+
+        unless BACKPRESSURE_STRATEGIES.include?(backpressure_strategy)
+          raise ArgumentError,
+                "backpressure_strategy must be one of #{BACKPRESSURE_STRATEGIES.inspect}"
+        end
+
+        @memory_limit_bytes = memory_limit_mb * 1024 * 1024
+        @memory_warning_threshold = (@memory_limit_bytes * EARLY_FLUSH_THRESHOLD).to_i
+        @backpressure_strategy = backpressure_strategy
+        @max_block_time = max_block_time
+
+        # Per-adapter buffers (Hash: adapter_key => Array<event_hash>)
+        @buffers = {}
+
+        # Global memory tracking (atomic for thread-safety)
+        @total_memory_bytes = Concurrent::AtomicFixnum.new(0)
+
+        # Flush mutex (synchronize flush operations)
+        @flush_mutex = Mutex.new
+
+        # Early flush callback (optional, for integration with flush worker)
+        @early_flush_callback = nil
+      end
+
+      # Add event to buffer with memory tracking
+      #
+      # This is the main entry point for buffering events.
+      # Tracks memory usage, enforces limits, triggers early flush if needed.
+      #
+      # Behavior on memory limit exceeded:
+      # - :block - Waits up to max_block_time for space, then drops
+      # - :drop - Immediately drops event
+      #
+      # @param event_data [Hash] Event hash to buffer
+      #   - Required keys: :event_name, :payload, :adapters (Array<Symbol>)
+      # @return [Boolean] true if event was added, false if dropped
+      #
+      # @example
+      #   success = buffer.add_event({
+      #     event_name: "UserSignup",
+      #     payload: { user_id: 123 },
+      #     adapters: [:logs, :errors_tracker]
+      #   })
+      #   # => true (or false if dropped due to memory limit)
+      def add_event(event_data)
+        event_size = estimate_size(event_data)
+        current_memory = @total_memory_bytes.value
+
+        # Check memory limit
+        return handle_memory_exhaustion(event_data, event_size) if current_memory + event_size > @memory_limit_bytes
+
+        # Add to appropriate adapter buffers
+        adapters = event_data[:adapters] || [:default]
+        adapters.each do |adapter_key|
+          @buffers[adapter_key] ||= []
+          @buffers[adapter_key] << event_data
+        end
+
+        # Update memory tracking atomically
+        @total_memory_bytes.update { |v| v + event_size }
+
+        # Warning threshold - trigger early flush (AFTER adding event to get accurate memory)
+        trigger_early_flush if @total_memory_bytes.value > @memory_warning_threshold
+
+        true
+      end
+
+      # Flush all buffers and return events
+      #
+      # Thread-safe operation (uses Mutex).
+      # Returns events grouped by adapter, updates memory tracking.
+      #
+      # @return [Hash] Events grouped by adapter key
+      #   - Format: { adapter_key => [event1, event2, ...] }
+      #
+      # @example
+      #   events = buffer.flush
+      #   # => { logs: [...], errors_tracker: [...] }
+      def flush
+        @flush_mutex.synchronize do
+          events_by_adapter = {}
+          memory_freed = 0
+
+          @buffers.each do |adapter_key, events|
+            events_by_adapter[adapter_key] = events.dup
+
+            # Calculate memory freed
+            events.each { |event| memory_freed += estimate_size(event) }
+
+            # Clear buffer
+            events.clear
+          end
+
+          # Update memory tracking atomically
+          @total_memory_bytes.update { |v| [v - memory_freed, 0].max }
+
+          events_by_adapter
+        end
+      end
+
+      # Estimate memory size of event (C20 requirement: ±10% accuracy)
+      #
+      # Estimates memory footprint including:
+      # - Payload JSON size
+      # - Ruby object overhead (~200 bytes per Hash)
+      # - String overhead (~40 bytes per String key)
+      #
+      # @param event_data [Hash] Event hash
+      # @return [Integer] Estimated size in bytes
+      #
+      # @example
+      #   size = buffer.estimate_size({ event_name: "test", payload: { id: 1 } })
+      #   # => ~250 bytes (payload + overhead)
+      def estimate_size(event_data)
+        # Payload size (deep inspection of strings)
+        payload_size = calculate_payload_size(event_data[:payload])
+
+        # Ruby object overhead
+        base_overhead = 200 # Hash object (~200 bytes)
+        string_overhead = event_data.keys.size * 40 # String keys (~40 bytes each)
+
+        payload_size + base_overhead + string_overhead
+      end
+
+      # Get memory statistics for monitoring
+      #
+      # @return [Hash] Memory stats
+      #   - :current_bytes - Current memory usage
+      #   - :limit_bytes - Memory limit
+      #   - :utilization - Percentage (0-100)
+      #   - :buffer_counts - Events per adapter
+      #
+      # @example
+      #   stats = buffer.memory_stats
+      #   # => { current_bytes: 1024000, limit_bytes: 104857600, utilization: 0.98, ... }
+      def memory_stats
+        {
+          current_bytes: @total_memory_bytes.value,
+          limit_bytes: @memory_limit_bytes,
+          utilization: (@total_memory_bytes.value.to_f / @memory_limit_bytes * 100).round(2),
+          buffer_counts: @buffers.transform_values(&:size),
+          warning_threshold: @memory_warning_threshold
+        }
+      end
+
+      # Check if buffer is empty
+      #
+      # @return [Boolean] true if no events buffered
+      def empty?
+        @buffers.values.all?(&:empty?)
+      end
+
+      # Get total number of buffered events
+      #
+      # @return [Integer] Total events across all adapters
+      def size
+        @buffers.values.sum(&:size)
+      end
+
+      # Register early flush callback
+      #
+      # Called when buffer reaches 80% memory threshold.
+      # Used for integration with background flush worker.
+      #
+      # @param block [Proc] Callback to invoke on early flush
+      # @return [void]
+      #
+      # @example
+      #   buffer.on_early_flush { FlushWorker.trigger_immediate_flush }
+      def on_early_flush(&block)
+        @early_flush_callback = block
+      end
+
+      private
+
+      # Calculate payload size recursively
+      #
+      # @param obj [Object] Payload object
+      # @return [Integer] Size in bytes
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def calculate_payload_size(obj)
+        case obj
+        when String
+          obj.bytesize + 40 # String content + overhead
+        when Hash
+          obj.sum do |k, v|
+            k.to_s.bytesize + 40 + calculate_payload_size(v)
+          end
+        when Array
+          obj.sum { |v| calculate_payload_size(v) } + 40
+        when Numeric
+          8 # Numbers are typically 8 bytes
+        else
+          100 # Default for unknown types
+        end
+      rescue StandardError
+        500 # Fallback for errors
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength
+
+      # Handle memory exhaustion according to strategy
+      #
+      # @param event_data [Hash] Event that caused exhaustion
+      # @param event_size [Integer] Size of event
+      # @return [Boolean] true if event was eventually added, false if dropped
+      # rubocop:disable Metrics/MethodLength
+      def handle_memory_exhaustion(event_data, event_size)
+        case @backpressure_strategy
+        when :block
+          # Block event ingestion until space available
+          wait_start = Time.now
+
+          loop do
+            # Check if space available after flush
+            current_memory = @total_memory_bytes.value
+            break if current_memory + event_size <= @memory_limit_bytes
+
+            # Check timeout
+            if Time.now - wait_start > @max_block_time
+              # Timeout exceeded - drop event
+              increment_metric("e11y.buffer.memory_exhaustion.dropped")
+              return false
+            end
+
+            # Brief sleep to avoid busy-wait
+            sleep 0.01 # 10ms
+          end
+
+          # Space available - retry add
+          increment_metric("e11y.buffer.memory_exhaustion.blocked")
+          add_event(event_data)
+
+        when :drop
+          # Drop new event
+          increment_metric("e11y.buffer.memory_exhaustion.dropped")
+          false
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # Trigger early flush (80% threshold reached)
+      #
+      # Invokes registered callback if available.
+      # Used to notify flush worker to flush immediately.
+      #
+      # @return [void]
+      def trigger_early_flush
+        return unless @early_flush_callback
+
+        @early_flush_callback.call
+      rescue StandardError => e
+        # Silently ignore callback errors (don't break event tracking)
+        warn "E11y: Early flush callback failed: #{e.message}"
+      end
+
+      # Increment metric (placeholder for Phase 3: Metrics)
+      #
+      # TODO Phase 3: Replace with actual Yabeda metrics
+      #
+      # @param metric_name [String] Metric to increment
+      # @return [void]
+      def increment_metric(metric_name)
+        # Placeholder - will be implemented in Phase 3
+        # Yabeda.e11y.buffer_memory_exhaustion.increment(strategy: @backpressure_strategy)
+      end
+      # rubocop:enable Metrics/ClassLength
+    end
+  end
+end

data/lib/e11y/buffers/base_buffer.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module E11y
+  module Buffers
+    # Base class for all E11y buffers
+    #
+    # @abstract Subclass and implement buffer-specific logic
+    #
+    # @example Ring Buffer
+    #   class RingBuffer < E11y::Buffers::BaseBuffer
+    #     def initialize(capacity:)
+    #       # Lock-free SPSC implementation
+    #     end
+    #   end
+    class BaseBuffer
+      # Push event to buffer
+      #
+      # @param event [Event::Base] event to buffer
+      # @return [Boolean] true if buffered, false if full
+      def push(_event)
+        raise NotImplementedError, "#{self.class}#push must be implemented"
+      end
+
+      # Flush all buffered events
+      #
+      # @yield [event] passes each event to block
+      # @return [Integer] number of flushed events
+      def flush
+        raise NotImplementedError, "#{self.class}#flush must be implemented"
+      end
+
+      # Get current buffer size
+      #
+      # @return [Integer] number of events in buffer
+      def size
+        raise NotImplementedError, "#{self.class}#size must be implemented"
+      end
+    end
+  end
+end

data/lib/e11y/buffers/request_scoped_buffer.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+module E11y
+  module Buffers
+    # Request-scoped buffer using thread-local storage for debug event buffering.
+    #
+    # This buffer stores debug events in thread-local storage during request processing.
+    # Events are flushed only when an error occurs, keeping logs clean during successful requests.
+    #
+    # Uses Thread.current for thread-local storage (compatible with any Ruby app, not just Rails).
+    # In Rails apps, this works seamlessly with Rack middleware thread model.
+    #
+    # @example Basic usage
+    #   # In Rails middleware
+    #   RequestScopedBuffer.initialize!
+    #
+    #   # Track debug events (buffered)
+    #   RequestScopedBuffer.add_event({ event_name: "debug", severity: :debug })
+    #
+    #   # On error - flush all buffered events
+    #   RequestScopedBuffer.flush_on_error
+    #
+    #   # On success - discard buffered events
+    #   RequestScopedBuffer.discard
+    #
+    # @see UC-001 Request-Scoped Debug Buffering
+    class RequestScopedBuffer
+      # Thread-local storage keys
+      THREAD_KEY_BUFFER = :e11y_request_buffer
+      THREAD_KEY_REQUEST_ID = :e11y_request_id
+      THREAD_KEY_ERROR_OCCURRED = :e11y_error_occurred
+      THREAD_KEY_BUFFER_LIMIT = :e11y_buffer_limit
+
+      # Default buffer limit per request
+      DEFAULT_BUFFER_LIMIT = 100
+
+      class << self
+        # Initialize request-scoped buffer
+        #
+        # @param request_id [String, nil] Optional request ID
+        # @param buffer_limit [Integer] Max events to buffer (default: 100)
+        # @return [void]
+        #
+        # @example
+        #   RequestScopedBuffer.initialize!(request_id: "req-123", buffer_limit: 200)
+        def initialize!(request_id: nil, buffer_limit: DEFAULT_BUFFER_LIMIT)
+          Thread.current[THREAD_KEY_BUFFER] = []
+          Thread.current[THREAD_KEY_REQUEST_ID] = request_id || generate_request_id
+          Thread.current[THREAD_KEY_ERROR_OCCURRED] = false
+          Thread.current[THREAD_KEY_BUFFER_LIMIT] = buffer_limit
+        end
+
+        # Add event to request-scoped buffer
+        #
+        # Only buffers :debug severity events. Other severities return false.
+        # Triggers auto-flush if error severity detected.
+        #
+        # @param event_data [Hash] Event hash with :severity, :event_name, :payload
+        # @return [Boolean] true if buffered, false if not buffered
+        #
+        # @example
+        #   # Debug event - buffered
+        #   RequestScopedBuffer.add_event({ event_name: "test", severity: :debug })
+        #   # => true
+        #
+        #   # Error event - not buffered, triggers flush
+        #   RequestScopedBuffer.add_event({ event_name: "error", severity: :error })
+        #   # => false (and flushes buffer)
+        # rubocop:disable Metrics/MethodLength, Naming/PredicateMethod
+        def add_event(event_data)
+          return false unless active? # Not in request scope
+
+          severity = event_data[:severity]
+
+          # Trigger flush on error severity
+          if error_severity?(severity)
+            Thread.current[THREAD_KEY_ERROR_OCCURRED] = true
+            flush_on_error
+            return false # Error events not buffered
+          end
+
+          # Only buffer debug events
+          return false unless severity == :debug
+
+          current_buffer = buffer
+          return false if current_buffer.nil?
+
+          # Check buffer limit
+          if current_buffer.size >= buffer_limit
+            increment_metric("e11y.request_buffer.overflow")
+            return false # Buffer full, drop event
+          end
+
+          current_buffer << event_data
+          increment_metric("e11y.request_buffer.events_buffered")
+          true
+        end
+        # rubocop:enable Metrics/MethodLength, Naming/PredicateMethod
+
+        # Flush buffered events on error
+        #
+        # Sends all buffered debug events to the main buffer/adapters.
+        # Events keep their original :debug severity.
+        #
+        # @param target [Symbol, nil] Optional target adapter
+        # @return [Integer] Number of events flushed
+        #
+        # @example
+        #   # In rescue block
+        #   rescue StandardError => e
+        #     RequestScopedBuffer.flush_on_error
+        #     raise
+        #   end
+        def flush_on_error(target: nil)
+          current_buffer = buffer
+          return 0 if current_buffer.nil? || current_buffer.empty?
+
+          flushed_count = current_buffer.size
+
+          # Flush events to main buffer/adapters
+          current_buffer.each do |event_data|
+            # TODO: Send to E11y::Collector.collect(event_data) when available
+            # For now, placeholder
+            flush_event(event_data, target: target)
+          end
+
+          current_buffer.clear
+          increment_metric("e11y.request_buffer.flushed_on_error", tags: { events: flushed_count })
+          flushed_count
+        end
+
+        # Discard buffered events (on successful request)
+        #
+        # @return [Integer] Number of events discarded
+        #
+        # @example
+        #   # In middleware ensure block (success path)
+        #   unless RequestScopedBuffer.error_occurred?
+        #     RequestScopedBuffer.discard
+        #   end
+        def discard
+          current_buffer = buffer
+          return 0 if current_buffer.nil? || current_buffer.empty?
+
+          discarded_count = current_buffer.size
+          current_buffer.clear
+          increment_metric("e11y.request_buffer.discarded", tags: { events: discarded_count })
+          discarded_count
+        end
+
+        # Check if request scope is active
+        #
+        # @return [Boolean] true if buffer initialized
+        def active?
+          !buffer.nil?
+        end
+
+        # Check if error occurred during request
+        #
+        # @return [Boolean] true if error severity detected
+        def error_occurred?
+          Thread.current[THREAD_KEY_ERROR_OCCURRED] == true
+        end
+
+        # Get current buffer size
+        #
+        # @return [Integer] Number of buffered events
+        def size
+          buffer&.size || 0
+        end
+
+        # Get current buffer
+        #
+        # @return [Array, nil] Buffer array or nil if not initialized
+        def buffer
+          Thread.current[THREAD_KEY_BUFFER]
+        end
+
+        # Get current request ID
+        #
+        # @return [String, nil] Request ID or nil if not initialized
+        def request_id
+          Thread.current[THREAD_KEY_REQUEST_ID]
+        end
+
+        # Reset request scope (cleanup)
+        #
+        # @return [void]
+        def reset_all
+          Thread.current[THREAD_KEY_BUFFER] = nil
+          Thread.current[THREAD_KEY_REQUEST_ID] = nil
+          Thread.current[THREAD_KEY_ERROR_OCCURRED] = nil
+          Thread.current[THREAD_KEY_BUFFER_LIMIT] = nil
+        end
+
+        private
+
+        # Get buffer limit (with fallback)
+        #
+        # @return [Integer] Buffer limit
+        def buffer_limit
+          Thread.current[THREAD_KEY_BUFFER_LIMIT] || DEFAULT_BUFFER_LIMIT
+        end
+
+        # Check if severity is error-level
+        #
+        # @param severity [Symbol] Event severity
+        # @return [Boolean] true if :error or :fatal
+        def error_severity?(severity)
+          %i[error fatal].include?(severity)
+        end
+
+        # Generate unique request ID
+        #
+        # @return [String] UUID request ID
+        def generate_request_id
+          require "securerandom"
+          SecureRandom.uuid
+        end
+
+        # Flush single event to adapters
+        #
+        # @param event_data [Hash] Event to flush
+        # @param target [Symbol, nil] Optional target adapter (not yet implemented)
+        # @return [void]
+        def flush_event(_event_data, target: nil) # rubocop:disable Lint/UnusedMethodArgument
+          # Placeholder for E11y::Collector integration
+          # Will be implemented when Collector/Adapter classes are available
+
+          # For now, just increment metric
+          increment_metric("e11y.request_buffer.event_flushed")
+        end
+
+        # Increment metric (placeholder)
+        #
+        # @param metric_name [String] Metric name
+        # @param tags [Hash] Optional tags
+        # @return [void]
+        def increment_metric(metric_name, tags: {})
+          # Placeholder for Yabeda integration
+          # Will be implemented in Phase 1 L2.4 (Metrics)
+        end
+      end
+    end
+  end
+end