e11y 0.2.0 → 1.1.0

data/lib/e11y/reliability/retry_handler.rb

@@ -50,12 +50,15 @@ module E11y
       # @option config [Float] :max_delay_ms Maximum delay in milliseconds (default: 5000)
       # @option config [Float] :jitter_factor Jitter factor (0.0-1.0, default: 0.1)
       # @option config [Boolean] :fail_on_error Raise error after max retries (default: true)
-      def initialize(config: {})
+      # @param rate_limiter [RetryRateLimiter, nil] Optional rate limiter for thundering herd prevention (C06)
+      # @param retry_rate_limiter [RetryRateLimiter, nil] Alias for rate_limiter (backward compatibility)
+      def initialize(config: {}, rate_limiter: nil, retry_rate_limiter: nil)
         @max_attempts = config[:max_attempts] || 3
         @base_delay_ms = config[:base_delay_ms] || 100.0
         @max_delay_ms = config[:max_delay_ms] || 5000.0
         @jitter_factor = config[:jitter_factor] || 0.1
         @fail_on_error = config.fetch(:fail_on_error, true)
+        @rate_limiter = rate_limiter || retry_rate_limiter
       end
 
       # Execute block with retry logic.
@@ -98,6 +101,22 @@ module E11y
             delay_ms = calculate_backoff_delay(attempt)
             on_retry_attempt(adapter, event, e, attempt, delay_ms)
 
+            # C06: Thundering herd prevention — check rate limiter before sleeping
+            if @rate_limiter && !@rate_limiter.allow?(adapter.class.name, event)
+              # Rate limit exceeded: stop retrying to prevent thundering herd
+              # With :delay strategy, sleep first to spread out retry load
+              on_limit = @rate_limiter.instance_variable_get(:@on_limit_exceeded)
+              if on_limit == :delay
+                window_sec = @rate_limiter.instance_variable_get(:@window)
+                jitter = rand(0..(delay_ms * 0.2))
+                sleep(((window_sec * 1000) + jitter) / 1000.0)
+              end
+
+              raise RetryExhaustedError.new(e, retry_count: attempt) if @fail_on_error
+
+              return nil
+            end
+
             # Sleep with backoff
             sleep(delay_ms / 1000.0)
           end
@@ -147,17 +166,17 @@ module E11y
 
       # Handle successful execution.
       def on_success(adapter, _event, attempt)
-        increment_metric("e11y.retry.success", adapter: adapter.class.name, attempts: attempt)
+        E11y::Metrics.increment("e11y.retry.success", adapter: adapter.class.name, attempts: attempt)
 
         # Log if retry was needed
         return unless attempt > 1
 
-        increment_metric("e11y.retry.recovered", adapter: adapter.class.name, attempts: attempt)
+        E11y::Metrics.increment("e11y.retry.recovered", adapter: adapter.class.name, attempts: attempt)
       end
 
       # Handle permanent failure (non-retriable error).
       def on_permanent_failure(adapter, _event, error, attempt)
-        increment_metric(
+        E11y::Metrics.increment(
           "e11y.retry.permanent_failure",
           adapter: adapter.class.name,
           error: error.class.name,
@@ -167,7 +186,7 @@ module E11y
 
       # Handle max retries exhausted (all attempts failed).
       def on_max_retries_exhausted(adapter, _event, error, attempt)
-        increment_metric(
+        E11y::Metrics.increment(
           "e11y.retry.exhausted",
           adapter: adapter.class.name,
           error: error.class.name,
@@ -176,35 +195,13 @@ module E11y
       end
 
       # Handle retry attempt.
-      def on_retry_attempt(adapter, _event, error, attempt, delay_ms)
-        increment_metric(
+      def on_retry_attempt(adapter, _event, error, attempt, _delay_ms)
+        E11y::Metrics.increment(
           "e11y.retry.attempt",
           adapter: adapter.class.name,
           error: error.class.name,
           attempt: attempt
         )
-
-        # Track backoff delay histogram
-        track_histogram("e11y.retry.backoff_delay_ms", delay_ms, adapter: adapter.class.name)
-      end
-
-      # Increment retry metric.
-      #
-      # @param metric_name [String] Metric name
-      # @param tags [Hash] Additional tags
-      def increment_metric(metric_name, tags = {})
-        # TODO: Integrate with Yabeda metrics
-        # E11y::Metrics.increment(metric_name, tags)
-      end
-
-      # Track histogram metric.
-      #
-      # @param metric_name [String] Metric name
-      # @param value [Numeric] Value to track
-      # @param tags [Hash] Additional tags
-      def track_histogram(metric_name, value, tags = {})
-        # TODO: Integrate with Yabeda metrics
-        # E11y::Metrics.histogram(metric_name, value, tags)
       end
     end
   end

data/lib/e11y/reliability/retry_rate_limiter.rb

@@ -46,7 +46,6 @@ module E11y
             false
           else
             @retry_counts[adapter_name] << Time.now
-            increment_metric("e11y.retry_rate_limiter.allowed", adapter: adapter_name)
             true
           end
         end
@@ -93,25 +92,18 @@ module E11y
 
       # Handle limit exceeded based on configured strategy.
       def on_limit_exceeded(adapter_name, _event_data)
-        increment_metric("e11y.retry_rate_limiter.exceeded", adapter: adapter_name)
+        E11y::Metrics.increment(:e11y_retry_rate_limiter_total, adapter: adapter_name, event: "exceeded", delay_sec: "")
 
         case @on_limit_exceeded
         when :delay
           # Calculate delay with jitter
           delay_sec = @window + rand((-@jitter_range * @window)..(@jitter_range * @window))
-          increment_metric("e11y.retry_rate_limiter.delayed", adapter: adapter_name, delay_sec: delay_sec)
+          E11y::Metrics.increment(:e11y_retry_rate_limiter_total, adapter: adapter_name, event: "delayed", delay_sec: delay_sec.round(1).to_s)
           # Caller should sleep(delay_sec) before retry
         when :dlq
-          # Caller should save to DLQ instead of retrying
-          increment_metric("e11y.retry_rate_limiter.dlq", adapter: adapter_name)
+          E11y::Metrics.increment(:e11y_retry_rate_limiter_total, adapter: adapter_name, event: "dlq", delay_sec: "")
         end
       end
-
-      # Increment retry rate limiter metric.
-      def increment_metric(metric_name, tags = {})
-        # TODO: Integrate with Yabeda metrics
-        # E11y::Metrics.increment(metric_name, tags)
-      end
     end
   end
 end

data/lib/e11y/sampling/error_spike_detector.rb

@@ -66,7 +66,6 @@ module E11y
       # Check if currently in error spike state
       #
       # @return [Boolean] true if error spike detected
-      # rubocop:disable Metrics/MethodLength
       # Error spike detection requires checking active spike, expiration, and new spike detection
       def error_spike?
         @mutex.synchronize do
@@ -94,7 +93,6 @@ module E11y
           false
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Record an event for error rate tracking
       #

data/lib/e11y/sampling/load_monitor.rb

@@ -92,21 +92,17 @@ module E11y
       def load_level
         rate = current_rate
 
-        # Check thresholds in descending order
-        # rubocop:disable Lint/DuplicateBranch
-        # Values between normal and high thresholds intentionally mapped to :high
+        # Check thresholds in descending order.
+        # rate <= normal → :normal; rate > normal and < high → :high; etc.
         if rate >= @thresholds[:overload]
           :overload
         elsif rate >= @thresholds[:very_high]
           :very_high
-        elsif rate >= @thresholds[:high]
-          :high
-        elsif rate >= @thresholds[:normal]
-          :high # Between normal and high threshold
+        elsif rate >= @thresholds[:high] || rate > @thresholds[:normal]
+          :high # rate > normal (includes rate >= high)
         else
-          :normal
+          :normal # rate <= normal (inclusive of exact threshold)
         end
-        # rubocop:enable Lint/DuplicateBranch
       end
 
       # Get recommended sample rate for current load

data/lib/e11y/sampling/stratified_tracker.rb

@@ -1,7 +1,25 @@
 # frozen_string_literal: true
 
+# Stratified sampling for SLO accuracy (FEAT-4851, C11 Resolution).
+# Tracks sampling statistics per severity stratum for sampling correction.
 module E11y
+  # Stratified sampling module — provides StratifiedTracker for SLO correction.
   module Sampling
+    # Module-level singleton for StratifiedTracker (C11 Resolution).
+    # Used by Sampling middleware and EventSlo for sampling correction.
+    #
+    # @return [StratifiedTracker]
+    def self.stratified_tracker
+      @stratified_tracker ||= StratifiedTracker.new
+    end
+
+    # Reset stratified tracker (for testing).
+    # @return [void]
+    def self.reset_stratified_tracker!
+      @stratified_tracker&.reset!
+      @stratified_tracker = nil
+    end
+
     # Stratified Sampling Tracker for SLO accuracy (FEAT-4851, C11 Resolution)
     #
     # Tracks sampling statistics per severity stratum to enable sampling correction

data/lib/e11y/self_monitoring/buffer_monitor.rb

@@ -4,6 +4,8 @@ require "e11y/metrics"
 
 module E11y
   module SelfMonitoring
+    # NOTE: Not wired to buffer components. Methods are API for future use.
+    #
     # Buffer monitoring for E11y internal operations.
     #
     # Tracks buffer metrics:

data/lib/e11y/self_monitoring/performance_monitor.rb

@@ -6,45 +6,34 @@ module E11y
   module SelfMonitoring
     # Performance monitoring for E11y internal operations.
     #
-    # Tracks latency metrics for:
-    # - Event tracking (E11y.track)
-    # - Middleware execution
-    # - Adapter writes
-    # - Buffer flushes
+    # Tracks:
+    # - Event.track() latency (via TrackLatency middleware)
+    # - Adapter send latency (used by Base adapter)
     #
     # @see ADR-016 §3.1 (Performance Metrics)
     # @example
-    #   E11y::SelfMonitoring::PerformanceMonitor.track_latency(0.5, event_class: 'OrderCreated', severity: :info)
+    #   E11y::SelfMonitoring::PerformanceMonitor.track_latency(0.5, event_class: 'Events::OrderPaid', severity: 'info', result: :success)
+    #   E11y::SelfMonitoring::PerformanceMonitor.track_adapter_latency('E11y::Adapters::Loki', 42)
     module PerformanceMonitor
-      # Track E11y.track() latency.
-      #
-      # @param duration_ms [Numeric] Duration in milliseconds
-      # @param event_class [String] Event class name
-      # @param severity [Symbol] Event severity
-      # @return [void]
-      def self.track_latency(duration_ms, event_class:, severity:)
-        E11y::Metrics.histogram(
-          :e11y_track_duration_seconds,
-          duration_ms / 1000.0,
-          {
-            event_class: event_class,
-            severity: severity
-          },
-          buckets: [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1] # 0.1ms to 100ms
-        )
-      end
+      TRACK_BUCKETS = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1].freeze # 0.1ms to 100ms
+      ADAPTER_BUCKETS = [0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0].freeze # 1ms to 5s
+      RESULT_SUCCESS = "success"
+      RESULT_DROPPED = "dropped"
 
-      # Track middleware execution time.
+      # Track Event.track() pipeline latency (from entry to exit).
       #
-      # @param middleware_name [String] Middleware class name
       # @param duration_ms [Numeric] Duration in milliseconds
+      # @param event_class [String] Event class name (e.g. 'Events::OrderPaid')
+      # @param severity [String] Severity (e.g. 'info', 'error')
+      # @param result [Symbol] :success or :dropped
       # @return [void]
-      def self.track_middleware_latency(middleware_name, duration_ms)
+      def self.track_latency(duration_ms, event_class:, severity:, result:)
+        result_str = result == :success ? RESULT_SUCCESS : RESULT_DROPPED
         E11y::Metrics.histogram(
-          :e11y_middleware_duration_seconds,
+          :e11y_track_duration_seconds,
           duration_ms / 1000.0,
-          { middleware: middleware_name },
-          buckets: [0.00001, 0.0001, 0.0005, 0.001, 0.005] # 0.01ms to 5ms
+          { event_class: event_class, severity: severity, result: result_str },
+          buckets: TRACK_BUCKETS
         )
       end
 
@@ -58,40 +47,9 @@ module E11y
           :e11y_adapter_send_duration_seconds,
           duration_ms / 1000.0,
           { adapter: adapter_name },
-          buckets: [0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0] # 1ms to 5s
+          buckets: ADAPTER_BUCKETS
         )
       end
-
-      # Track buffer flush latency.
-      #
-      # @param duration_ms [Numeric] Duration in milliseconds
-      # @param event_count [Integer] Number of events flushed
-      # @return [void]
-      def self.track_flush_latency(duration_ms, event_count)
-        E11y::Metrics.histogram(
-          :e11y_buffer_flush_duration_seconds,
-          duration_ms / 1000.0,
-          { event_count_bucket: bucket_event_count(event_count) },
-          buckets: [0.001, 0.01, 0.05, 0.1, 0.5, 1.0]
-        )
-      end
-
-      # Convert event count to a low-cardinality bucket label.
-      #
-      # @param count [Integer] Event count
-      # @return [String] Bucket label
-      # @api private
-      def self.bucket_event_count(count)
-        case count
-        when 0..10 then "1-10"
-        when 11..50 then "11-50"
-        when 51..100 then "51-100"
-        when 101..500 then "101-500"
-        else "500+"
-        end
-      end
-
-      private_class_method :bucket_event_count
     end
   end
 end

data/lib/e11y/self_monitoring/reliability_monitor.rb

@@ -6,61 +6,12 @@ module E11y
   module SelfMonitoring
     # Reliability monitoring for E11y internal operations.
     #
-    # Tracks success/failure rates for:
-    # - Event tracking
-    # - Adapter writes
-    # - Buffer operations
-    # - DLQ saves
+    # Tracks success/failure rates for adapter writes and circuit breaker state.
     #
     # @see ADR-016 §3.2 (Reliability Metrics)
     # @example
-    #   E11y::SelfMonitoring::ReliabilityMonitor.track_event_success(event_type: 'order.created')
+    #   E11y::SelfMonitoring::ReliabilityMonitor.track_adapter_success(adapter_name: 'E11y::Adapters::Loki')
     module ReliabilityMonitor
-      # Track successful event tracking.
-      #
-      # @param event_type [String] Event type/name
-      # @return [void]
-      def self.track_event_success(event_type:)
-        E11y::Metrics.increment(
-          :e11y_events_tracked_total,
-          {
-            event_type: event_type,
-            status: "success"
-          }
-        )
-      end
-
-      # Track failed event tracking.
-      #
-      # @param event_type [String] Event type/name
-      # @param reason [String] Failure reason (e.g., 'validation_error', 'adapter_error')
-      # @return [void]
-      def self.track_event_failure(event_type:, reason:)
-        E11y::Metrics.increment(
-          :e11y_events_tracked_total,
-          {
-            event_type: event_type,
-            status: "failure",
-            reason: reason
-          }
-        )
-      end
-
-      # Track dropped event (rate limited, sampled out, etc).
-      #
-      # @param event_type [String] Event type/name
-      # @param reason [String] Drop reason (e.g., 'rate_limited', 'sampled_out')
-      # @return [void]
-      def self.track_event_dropped(event_type:, reason:)
-        E11y::Metrics.increment(
-          :e11y_events_dropped_total,
-          {
-            event_type: event_type,
-            reason: reason
-          }
-        )
-      end
-
       # Track adapter write success.
       #
       # @param adapter_name [String] Adapter class name
@@ -70,7 +21,8 @@ module E11y
           :e11y_adapter_writes_total,
           {
             adapter: adapter_name,
-            status: "success"
+            status: "success",
+            error_class: "" # Prometheus requires consistent label signature with track_adapter_failure
           }
         )
       end
@@ -91,28 +43,6 @@ module E11y
         )
       end
 
-      # Track DLQ save operation.
-      #
-      # @param reason [String] Reason for DLQ save (e.g., 'adapter_error', 'rate_limited')
-      # @return [void]
-      def self.track_dlq_save(reason:)
-        E11y::Metrics.increment(
-          :e11y_dlq_saves_total,
-          { reason: reason }
-        )
-      end
-
-      # Track DLQ replay operation.
-      #
-      # @param status [String] Replay status ('success' or 'failure')
-      # @return [void]
-      def self.track_dlq_replay(status:)
-        E11y::Metrics.increment(
-          :e11y_dlq_replays_total,
-          { status: status }
-        )
-      end
-
       # Track circuit breaker state change.
       #
       # @param adapter_name [String] Adapter class name

data/lib/e11y/slo/config_loader.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module E11y
+  module SLO
+    # Loads SLO configuration from slo.yml files in configurable search paths.
+    class ConfigLoader
+      class << self
+        def load(search_paths: default_search_paths)
+          search_paths.each do |base|
+            path = File.join(base.to_s, "slo.yml")
+            next unless File.file?(path)
+
+            content = File.read(path)
+            return YAML.safe_load(content) || {}
+          end
+          nil
+        end
+
+        # Returns true when e11y_self_monitoring.enabled is true in slo.yml.
+        #
+        # @return [Boolean]
+        def self_monitoring_enabled?
+          config = load
+          return false if config.nil?
+
+          config.dig("e11y_self_monitoring", "enabled") == true
+        end
+
+        private
+
+        def default_search_paths
+          base = defined?(Rails) ? Rails.root.to_s : Dir.pwd
+          [File.join(base, "config"), File.join(base, "config", "e11y"), Dir.pwd]
+        end
+      end
+    end
+  end
+end

data/lib/e11y/slo/config_validator.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module E11y
+  module SLO
+    # Validates slo.yml schema: version, endpoints (controller/pattern), app_wide.aggregated_slo.
+    class ConfigValidator
+      class << self
+        def validate(config)
+          return ["Config is nil or empty"] if config.nil? || config.empty?
+
+          errors = []
+          errors.concat(validate_version(config))
+          errors.concat(validate_endpoints(config["endpoints"]))
+          errors.concat(validate_app_wide(config["app_wide"]))
+          errors.concat(validate_e11y_self_monitoring(config["e11y_self_monitoring"]))
+          errors
+        end
+
+        private
+
+        def validate_version(config)
+          return ["Missing required key: version"] unless config.key?("version")
+
+          []
+        end
+
+        def validate_endpoints(endpoints)
+          return [] if endpoints.nil? || endpoints.empty?
+
+          errors = []
+          endpoints.each_with_index do |ep, i|
+            errors << "endpoints[#{i}]: missing controller or pattern" if ep["controller"].to_s.empty? && ep["pattern"].to_s.empty?
+          end
+          errors
+        end
+
+        def validate_app_wide(app_wide)
+          return [] if app_wide.nil?
+
+          agg = app_wide["aggregated_slo"]
+          return [] if agg.nil? || !agg["enabled"]
+
+          errors = []
+          errors << "app_wide.aggregated_slo: strategy required when enabled" if agg["strategy"].to_s.empty?
+          errors << "app_wide.aggregated_slo: components required" if agg["components"].to_a.empty?
+          errors
+        end
+
+        def validate_e11y_self_monitoring(e11y_self_monitoring)
+          return [] if e11y_self_monitoring.nil? || !e11y_self_monitoring["enabled"]
+
+          # When enabled, targets structure is optional; no validation errors for now
+          []
+        end
+      end
+    end
+  end
+end

data/lib/e11y/slo/dashboard_generator.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "json"
+
+module E11y
+  module SLO
+    # Generates Grafana dashboard JSON from SLO config (endpoints, event SLOs, app-wide).
+    class DashboardGenerator
+      class << self
+        def generate(config)
+          return "{}" if config.nil? || config.empty?
+
+          panels = []
+          panels.concat(build_endpoint_panels(config["endpoints"]))
+          panels.concat(build_app_wide_panels(config["app_wide"]))
+          panels.concat(build_event_slo_panels(config["custom_slos"])) if config["custom_slos"]
+          panels.concat(build_self_monitoring_panels(config["e11y_self_monitoring"]))
+
+          dashboard = {
+            title: "E11y SLO Dashboard",
+            panels: panels,
+            schemaVersion: 38,
+            version: 1
+          }
+          JSON.pretty_generate(dashboard)
+        end
+
+        private
+
+        def build_endpoint_panels(endpoints)
+          return [] if endpoints.to_a.empty?
+
+          [{
+            id: 1,
+            title: "HTTP Availability (Per-Endpoint)",
+            type: "timeseries",
+            targets: [{
+              expr: "sum(rate(e11y_slo_http_requests_total{status=~\"2..|3..\"}[30d])) " \
+                    "by (controller, action) / sum(rate(e11y_slo_http_requests_total[30d])) by (controller, action)",
+              legendFormat: '{{controller}}#{{action}}'
+            }]
+          }]
+        end
+
+        def build_self_monitoring_panels(e11y_self_monitoring)
+          return [] if e11y_self_monitoring.nil? || !e11y_self_monitoring["enabled"]
+
+          [{
+            id: 200,
+            title: "E11y Self-Monitoring Reliability",
+            type: "timeseries",
+            targets: [{
+              expr: 'sum(rate(e11y_e11y_events_tracked_total{result="success"}[30d])) / sum(rate(e11y_e11y_events_tracked_total[30d]))',
+              legendFormat: "reliability"
+            }]
+          }]
+        end
+
+        def build_event_slo_panels(custom_slos)
+          custom_slos.map.with_index do |slo, i|
+            name = slo["name"] || "event_slo_#{i}"
+            {
+              id: 10 + i,
+              title: "Event SLO: #{name}",
+              type: "timeseries",
+              targets: [{
+                expr: "sum(rate(e11y_slo_event_result_total{slo_name=\"#{name}\",slo_status=\"success\"}[30d])) / " \
+                      "sum(rate(e11y_slo_event_result_total{slo_name=\"#{name}\"}[30d]))",
+                legendFormat: "success_rate"
+              }]
+            }
+          end
+        end
+
+        def build_app_wide_panels(app_wide)
+          return [] if app_wide.nil?
+
+          agg = app_wide["aggregated_slo"]
+          return [] if agg.nil? || !agg["enabled"]
+
+          components = agg["components"] || []
+          return [] if components.empty?
+
+          window = agg["window"] || "30d"
+
+          expr = case agg["strategy"].to_s
+                 when "min"
+                   build_min_expr(components, window)
+                 else
+                   build_weighted_expr(components, window)
+                 end
+
+          [{
+            id: 100,
+            title: "App-Wide Aggregated SLO",
+            type: "timeseries",
+            targets: [{ expr: expr, legendFormat: "aggregated" }],
+            fieldConfig: { defaults: { min: 0.99, max: 1.0 } }
+          }]
+        end
+
+        def build_weighted_expr(components, window)
+          parts = components.map do |c|
+            weight = c["weight"] || (1.0 / components.size)
+            metric = (c["metric"] || "").gsub(/\[\d+d\]/, "[#{window}]")
+            metric = metric.strip
+            "(#{weight} * (#{metric}))"
+          end
+          parts.join(" + ")
+        end
+
+        def build_min_expr(components, window)
+          parts = components.map do |c|
+            metric = (c["metric"] || "").gsub(/\[\d+d\]/, "[#{window}]")
+            metric.strip
+          end
+          "min(#{parts.join(', ')})"
+        end
+      end
+    end
+  end
+end

data/lib/e11y/slo/event_driven.rb

@@ -144,6 +144,14 @@ module E11y
         def slo_config
           @slo_config
         end
+
+        def slo_enabled?
+          slo_config&.enabled? == true
+        end
+
+        def slo_disabled?
+          slo_config ? !slo_config.enabled? : false
+        end
       end
     end
   end