decision_agent 0.1.3 → 0.1.6

data/lib/decision_agent/monitoring/metrics_collector.rb

@@ -1,5 +1,12 @@
 require "monitor"
 require "time"
+require_relative "storage/memory_adapter"
+
+begin
+  require_relative "storage/activerecord_adapter"
+rescue LoadError, NameError
+  # ActiveRecord adapter not available
+end
 
 module DecisionAgent
   module Monitoring
@@ -7,11 +14,16 @@ module DecisionAgent
     class MetricsCollector
       include MonitorMixin
 
-      attr_reader :metrics, :window_size
+      attr_reader :metrics, :window_size, :storage_adapter
 
-      def initialize(window_size: 3600)
+      def initialize(window_size: 3600, storage: :auto, cleanup_threshold: 100)
         super()
         @window_size = window_size # Default: 1 hour window
+        @cleanup_threshold = cleanup_threshold # Cleanup every N records
+        @cleanup_counter = 0
+        @storage_adapter = initialize_storage_adapter(storage, window_size)
+
+        # Legacy in-memory metrics for backward compatibility with observers
         @metrics = {
           decisions: [],
           evaluations: [],
@@ -35,8 +47,20 @@ module DecisionAgent
             evaluator_names: decision.evaluations.map(&:evaluator_name).uniq
           }
 
+          # Store in-memory for observers (backward compatibility)
           @metrics[:decisions] << metric
-          cleanup_old_metrics!
+          maybe_cleanup_old_metrics!
+
+          # Persist to storage adapter
+          @storage_adapter.record_decision(
+            decision.decision,
+            context.to_h,
+            confidence: decision.confidence,
+            evaluations_count: decision.evaluations.size,
+            duration_ms: duration_ms,
+            status: determine_decision_status(decision)
+          )
+
           notify_observers(:decision, metric)
           metric
         end
@@ -52,8 +76,18 @@ module DecisionAgent
             evaluator_name: evaluation.evaluator_name
           }
 
+          # Store in-memory for observers (backward compatibility)
           @metrics[:evaluations] << metric
-          cleanup_old_metrics!
+          maybe_cleanup_old_metrics!
+
+          # Persist to storage adapter
+          @storage_adapter.record_evaluation(
+            evaluation.evaluator_name,
+            score: evaluation.weight,
+            success: evaluation.weight.positive?,
+            details: { decision: evaluation.decision }
+          )
+
           notify_observers(:evaluation, metric)
           metric
         end
@@ -70,8 +104,18 @@ module DecisionAgent
             metadata: metadata
           }
 
+          # Store in-memory for observers (backward compatibility)
           @metrics[:performance] << metric
-          cleanup_old_metrics!
+          maybe_cleanup_old_metrics!
+
+          # Persist to storage adapter
+          @storage_adapter.record_performance(
+            operation,
+            duration_ms: duration_ms,
+            status: success ? "success" : "failure",
+            metadata: metadata
+          )
+
           notify_observers(:performance, metric)
           metric
         end
@@ -87,8 +131,19 @@ module DecisionAgent
             context: context
           }
 
+          # Store in-memory for observers (backward compatibility)
           @metrics[:errors] << metric
-          cleanup_old_metrics!
+          maybe_cleanup_old_metrics!
+
+          # Persist to storage adapter
+          @storage_adapter.record_error(
+            error.class.name,
+            message: error.message,
+            stack_trace: error.backtrace,
+            severity: determine_error_severity(error),
+            context: context
+          )
+
           notify_observers(:error, metric)
           metric
         end
@@ -97,6 +152,18 @@ module DecisionAgent
       # Get aggregated statistics
       def statistics(time_range: nil)
         synchronize do
+          # Use in-memory metrics for MemoryAdapter (to maintain backward compatibility)
+          # Only delegate to ActiveRecordAdapter for persistent storage
+          use_storage = time_range &&
+                        @storage_adapter.respond_to?(:statistics) &&
+                        !@storage_adapter.is_a?(Storage::MemoryAdapter)
+
+          if use_storage
+            stats = @storage_adapter.statistics(time_range: time_range)
+            return stats.merge(timestamp: Time.now.utc, storage: @storage_adapter.class.name) if stats
+          end
+
+          # Use in-memory metrics
           range_start = time_range ? Time.now.utc - time_range : nil
 
           decisions = filter_by_time(@metrics[:decisions], range_start)
@@ -115,7 +182,8 @@ module DecisionAgent
             evaluations: compute_evaluation_stats(evaluations),
             performance: compute_performance_stats(performance),
             errors: compute_error_stats(errors),
-            timestamp: Time.now.utc
+            timestamp: Time.now.utc,
+            storage: "memory (fallback)"
           }
         end
       end
@@ -123,6 +191,17 @@ module DecisionAgent
       # Get time-series data for graphing
       def time_series(metric_type:, bucket_size: 60, time_range: 3600)
         synchronize do
+          # Use in-memory metrics for MemoryAdapter (to maintain backward compatibility)
+          # Only delegate to ActiveRecordAdapter for persistent storage
+          use_storage = @storage_adapter.respond_to?(:time_series) &&
+                        !@storage_adapter.is_a?(Storage::MemoryAdapter)
+
+          if use_storage
+            series = @storage_adapter.time_series(metric_type, bucket_size: bucket_size, time_range: time_range)
+            return series if series && series[:timestamps]
+          end
+
+          # Use in-memory metrics
           data = @metrics[metric_type] || []
           range_start = Time.now.utc - time_range
 
@@ -156,22 +235,100 @@ module DecisionAgent
       def clear!
         synchronize do
           @metrics.each_value(&:clear)
+          # Also clear storage adapter if using MemoryAdapter
+          if @storage_adapter.is_a?(Storage::MemoryAdapter)
+            # Clear all by using a very large time period (100 years in seconds)
+            @storage_adapter.cleanup(older_than: 100 * 365 * 24 * 60 * 60)
+          end
         end
       end
 
       # Get current metrics count
       def metrics_count
         synchronize do
+          # Use in-memory metrics for MemoryAdapter (to maintain backward compatibility)
+          # Only delegate to ActiveRecordAdapter for persistent storage
+          use_storage = @storage_adapter.respond_to?(:metrics_count) &&
+                        !@storage_adapter.is_a?(Storage::MemoryAdapter)
+
+          return @storage_adapter.metrics_count if use_storage
+
+          # Use in-memory
           @metrics.transform_values(&:size)
         end
       end
 
+      # Cleanup old metrics from persistent storage
+      def cleanup_old_metrics_from_storage(older_than:)
+        synchronize do
+          return 0 unless @storage_adapter.respond_to?(:cleanup)
+
+          @storage_adapter.cleanup(older_than: older_than)
+        end
+      end
+
       private
 
       def freeze_config
         @window_size.freeze
       end
 
+      def initialize_storage_adapter(storage_option, window_size)
+        case storage_option
+        when :auto
+          # Auto-detect: prefer ActiveRecord if available
+          if defined?(DecisionAgent::Monitoring::Storage::ActiveRecordAdapter) &&
+             DecisionAgent::Monitoring::Storage::ActiveRecordAdapter.available?
+            DecisionAgent::Monitoring::Storage::ActiveRecordAdapter.new
+          else
+            DecisionAgent::Monitoring::Storage::MemoryAdapter.new(window_size: window_size)
+          end
+        when :activerecord, :database
+          unless defined?(DecisionAgent::Monitoring::Storage::ActiveRecordAdapter)
+            raise "ActiveRecord adapter not available. Install models or use :memory storage."
+          end
+
+          DecisionAgent::Monitoring::Storage::ActiveRecordAdapter.new
+        when :memory
+          DecisionAgent::Monitoring::Storage::MemoryAdapter.new(window_size: window_size)
+        when Symbol
+          raise ArgumentError, "Unknown storage option: #{storage_option}. Use :auto, :activerecord, or :memory"
+        else
+          # Custom adapter instance provided
+          storage_option
+        end
+      end
+
+      def determine_decision_status(decision)
+        return "success" if decision.confidence >= 0.7
+        return "failure" if decision.confidence < 0.3
+
+        "success" # Default for medium confidence
+      end
+
+      def determine_error_severity(error)
+        case error
+        when ArgumentError, TypeError
+          "medium"
+        when StandardError
+          "low"
+        when Exception
+          "critical"
+        else
+          "low"
+        end
+      end
+
+      # Conditionally cleanup old metrics based on counter
+      # This reduces O(n) array scans from every record to every N records
+      def maybe_cleanup_old_metrics!
+        @cleanup_counter += 1
+        return unless @cleanup_counter >= @cleanup_threshold
+
+        @cleanup_counter = 0
+        cleanup_old_metrics!
+      end
+
       def cleanup_old_metrics!
         cutoff_time = Time.now.utc - @window_size
 

data/lib/decision_agent/monitoring/storage/activerecord_adapter.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require_relative "base_adapter"
+
+module DecisionAgent
+  module Monitoring
+    module Storage
+      # ActiveRecord adapter for persistent database storage
+      class ActiveRecordAdapter < BaseAdapter
+        def initialize
+          super
+          validate_models!
+        end
+
+        def record_decision(decision, context, confidence: nil, evaluations_count: 0, duration_ms: nil, status: nil)
+          ::DecisionLog.create!(
+            decision: decision,
+            context: context.to_json,
+            confidence: confidence,
+            evaluations_count: evaluations_count,
+            duration_ms: duration_ms,
+            status: status
+          )
+        rescue StandardError => e
+          warn "Failed to record decision to database: #{e.message}"
+        end
+
+        def record_evaluation(evaluator_name, score: nil, success: nil, duration_ms: nil, details: {})
+          ::EvaluationMetric.create!(
+            evaluator_name: evaluator_name,
+            score: score,
+            success: success,
+            duration_ms: duration_ms,
+            details: details.to_json
+          )
+        rescue StandardError => e
+          warn "Failed to record evaluation to database: #{e.message}"
+        end
+
+        def record_performance(operation, duration_ms: nil, status: nil, metadata: {})
+          ::PerformanceMetric.create!(
+            operation: operation,
+            duration_ms: duration_ms,
+            status: status,
+            metadata: metadata.to_json
+          )
+        rescue StandardError => e
+          warn "Failed to record performance to database: #{e.message}"
+        end
+
+        def record_error(error_type, message: nil, stack_trace: nil, severity: nil, context: {})
+          ::ErrorMetric.create!(
+            error_type: error_type,
+            message: message,
+            stack_trace: stack_trace&.to_json,
+            severity: severity,
+            context: context.to_json
+          )
+        rescue StandardError => e
+          warn "Failed to record error to database: #{e.message}"
+        end
+
+        def statistics(time_range: 3600)
+          decisions = ::DecisionLog.recent(time_range)
+          evaluations = ::EvaluationMetric.recent(time_range)
+          performance = ::PerformanceMetric.recent(time_range)
+          errors = ::ErrorMetric.recent(time_range)
+
+          {
+            decisions: decision_statistics(decisions, time_range),
+            evaluations: evaluation_statistics(evaluations),
+            performance: performance_statistics(performance, time_range),
+            errors: error_statistics(errors)
+          }
+        rescue StandardError => e
+          warn "Failed to retrieve statistics from database: #{e.message}"
+          default_statistics
+        end
+
+        def time_series(metric_type, bucket_size: 60, time_range: 3600)
+          case metric_type
+          when :decisions
+            decisions_time_series(bucket_size, time_range)
+          when :evaluations
+            evaluations_time_series(bucket_size, time_range)
+          when :performance
+            performance_time_series(bucket_size, time_range)
+          when :errors
+            errors_time_series(bucket_size, time_range)
+          else
+            { data: [], timestamps: [] }
+          end
+        rescue StandardError => e
+          warn "Failed to retrieve time series from database: #{e.message}"
+          { data: [], timestamps: [] }
+        end
+
+        def metrics_count
+          {
+            decisions: ::DecisionLog.count,
+            evaluations: ::EvaluationMetric.count,
+            performance: ::PerformanceMetric.count,
+            errors: ::ErrorMetric.count
+          }
+        rescue StandardError => e
+          warn "Failed to get metrics count from database: #{e.message}"
+          { decisions: 0, evaluations: 0, performance: 0, errors: 0 }
+        end
+
+        def cleanup(older_than:)
+          cutoff_time = Time.now - older_than
+          count = 0
+
+          count += ::DecisionLog.where("created_at < ?", cutoff_time).delete_all
+          count += ::EvaluationMetric.where("created_at < ?", cutoff_time).delete_all
+          count += ::PerformanceMetric.where("created_at < ?", cutoff_time).delete_all
+          count += ::ErrorMetric.where("created_at < ?", cutoff_time).delete_all
+
+          count
+        rescue StandardError => e
+          warn "Failed to cleanup old metrics from database: #{e.message}"
+          0
+        end
+
+        def self.available?
+          defined?(ActiveRecord) &&
+            defined?(::DecisionLog) &&
+            defined?(::EvaluationMetric) &&
+            defined?(::PerformanceMetric) &&
+            defined?(::ErrorMetric)
+        end
+
+        private
+
+        def decision_statistics(decisions, time_range)
+          {
+            total: decisions.count,
+            by_decision: decisions.group(:decision).count,
+            average_confidence: decisions.where.not(confidence: nil).average(:confidence).to_f,
+            success_rate: ::DecisionLog.success_rate(time_range: time_range)
+          }
+        end
+
+        def evaluation_statistics(evaluations)
+          {
+            total: evaluations.count,
+            by_evaluator: evaluations.group(:evaluator_name).count,
+            average_score: evaluations.where.not(score: nil).average(:score).to_f,
+            success_rate_by_evaluator: evaluations.successful.group(:evaluator_name).count
+          }
+        end
+
+        def performance_statistics(performance, time_range)
+          {
+            total: performance.count,
+            average_duration_ms: performance.average_duration(time_range: time_range),
+            p50: performance.p50(time_range: time_range),
+            p95: performance.p95(time_range: time_range),
+            p99: performance.p99(time_range: time_range),
+            success_rate: performance.success_rate(time_range: time_range)
+          }
+        end
+
+        def error_statistics(errors)
+          {
+            total: errors.count,
+            by_type: errors.group(:error_type).count,
+            by_severity: errors.group(:severity).count,
+            critical_count: errors.critical.count
+          }
+        end
+
+        def validate_models!
+          required_models = %w[DecisionLog EvaluationMetric PerformanceMetric ErrorMetric]
+          missing_models = required_models.reject { |model| Object.const_defined?(model) }
+
+          return if missing_models.empty?
+
+          raise "Missing required models: #{missing_models.join(', ')}. " \
+                "Run 'rails generate decision_agent:install --monitoring' to create them."
+        end
+
+        def decisions_time_series(bucket_size, time_range)
+          counts = ::DecisionLog.recent(time_range)
+                                .group(time_bucket_sql(:created_at, bucket_size))
+                                .count
+
+          format_time_series(counts)
+        end
+
+        def evaluations_time_series(bucket_size, time_range)
+          counts = ::EvaluationMetric.recent(time_range)
+                                     .group(time_bucket_sql(:created_at, bucket_size))
+                                     .count
+
+          format_time_series(counts)
+        end
+
+        def performance_time_series(bucket_size, time_range)
+          durations = ::PerformanceMetric.recent(time_range)
+                                         .where.not(duration_ms: nil)
+                                         .group(time_bucket_sql(:created_at, bucket_size))
+                                         .average(:duration_ms)
+
+          format_time_series(durations)
+        end
+
+        def errors_time_series(bucket_size, time_range)
+          counts = ::ErrorMetric.recent(time_range)
+                                .group(time_bucket_sql(:created_at, bucket_size))
+                                .count
+
+          format_time_series(counts)
+        end
+
+        def time_bucket_sql(column, bucket_size)
+          adapter = ActiveRecord::Base.connection.adapter_name.downcase
+
+          case adapter
+          when /postgres/
+            "(EXTRACT(EPOCH FROM #{column})::bigint / #{bucket_size}) * #{bucket_size}"
+          when /mysql/
+            "(UNIX_TIMESTAMP(#{column}) DIV #{bucket_size}) * #{bucket_size}"
+          when /sqlite/
+            "(CAST(strftime('%s', #{column}) AS INTEGER) / #{bucket_size}) * #{bucket_size}"
+          else
+            # Fallback: use group by timestamp truncated to bucket
+            column.to_s
+          end
+        end
+
+        def format_time_series(data)
+          timestamps = data.keys.sort
+          values = timestamps.map { |ts| data[ts] }
+
+          {
+            timestamps: timestamps.map { |ts| Time.at(ts).iso8601 },
+            data: values.map(&:to_f)
+          }
+        end
+
+        def default_statistics
+          {
+            decisions: { total: 0, by_decision: {}, average_confidence: 0.0, success_rate: 0.0 },
+            evaluations: { total: 0, by_evaluator: {}, average_score: 0.0, success_rate_by_evaluator: {} },
+            performance: { total: 0, average_duration_ms: 0.0, p50: 0.0, p95: 0.0, p99: 0.0, success_rate: 0.0 },
+            errors: { total: 0, by_type: {}, by_severity: {}, critical_count: 0 }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/decision_agent/monitoring/storage/base_adapter.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module DecisionAgent
+  module Monitoring
+    module Storage
+      # Base adapter interface for metrics storage
+      # Subclasses must implement all abstract methods
+      class BaseAdapter
+        # Record a decision
+        # @param decision [String] The decision made
+        # @param context [Hash] Decision context
+        # @param confidence [Float, nil] Confidence score (0-1)
+        # @param evaluations_count [Integer] Number of evaluations
+        # @param duration_ms [Float, nil] Decision duration in milliseconds
+        # @param status [String, nil] Decision status (success, failure, error)
+        # @return [void]
+        def record_decision(decision, context, confidence: nil, evaluations_count: 0, duration_ms: nil, status: nil)
+          raise NotImplementedError, "#{self.class} must implement #record_decision"
+        end
+
+        # Record an evaluation
+        # @param evaluator_name [String] Name of the evaluator
+        # @param score [Float, nil] Evaluation score
+        # @param success [Boolean, nil] Whether evaluation succeeded
+        # @param duration_ms [Float, nil] Evaluation duration
+        # @param details [Hash] Additional details
+        # @return [void]
+        def record_evaluation(evaluator_name, score: nil, success: nil, duration_ms: nil, details: {})
+          raise NotImplementedError, "#{self.class} must implement #record_evaluation"
+        end
+
+        # Record a performance metric
+        # @param operation [String] Operation name
+        # @param duration_ms [Float, nil] Duration in milliseconds
+        # @param status [String, nil] Status (success, failure, error)
+        # @param metadata [Hash] Additional metadata
+        # @return [void]
+        def record_performance(operation, duration_ms: nil, status: nil, metadata: {})
+          raise NotImplementedError, "#{self.class} must implement #record_performance"
+        end
+
+        # Record an error
+        # @param error_type [String] Type of error
+        # @param message [String, nil] Error message
+        # @param stack_trace [Array, nil] Stack trace
+        # @param severity [String, nil] Error severity (low, medium, high, critical)
+        # @param context [Hash] Error context
+        # @return [void]
+        def record_error(error_type, message: nil, stack_trace: nil, severity: nil, context: {})
+          raise NotImplementedError, "#{self.class} must implement #record_error"
+        end
+
+        # Get statistics for a time range
+        # @param time_range [Integer] Time range in seconds
+        # @return [Hash] Statistics summary
+        def statistics(time_range: 3600)
+          raise NotImplementedError, "#{self.class} must implement #statistics"
+        end
+
+        # Get time series data
+        # @param metric_type [Symbol] Type of metric (:decisions, :evaluations, :performance, :errors)
+        # @param bucket_size [Integer] Bucket size in seconds
+        # @param time_range [Integer] Time range in seconds
+        # @return [Hash] Time series data
+        def time_series(metric_type, bucket_size: 60, time_range: 3600)
+          raise NotImplementedError, "#{self.class} must implement #time_series"
+        end
+
+        # Get count of metrics stored
+        # @return [Hash] Count by metric type
+        def metrics_count
+          raise NotImplementedError, "#{self.class} must implement #metrics_count"
+        end
+
+        # Clean up old metrics
+        # @param older_than [Integer] Remove metrics older than this many seconds
+        # @return [Integer] Number of metrics removed
+        def cleanup(older_than:)
+          raise NotImplementedError, "#{self.class} must implement #cleanup"
+        end
+
+        # Check if adapter is available (dependencies installed)
+        # @return [Boolean]
+        def self.available?
+          raise NotImplementedError, "#{self} must implement .available?"
+        end
+      end
+    end
+  end
+end