decision_agent 0.1.2 → 0.1.4

data/lib/decision_agent/monitoring/prometheus_exporter.rb

@@ -0,0 +1,247 @@
+require "monitor"
+
+module DecisionAgent
+  module Monitoring
+    # Prometheus-compatible metrics exporter
+    class PrometheusExporter
+      include MonitorMixin
+
+      CONTENT_TYPE = "text/plain; version=0.0.4".freeze
+
+      def initialize(metrics_collector:, namespace: "decision_agent")
+        super()
+        @metrics_collector = metrics_collector
+        @namespace = namespace
+        @custom_metrics = {}
+        freeze_config
+      end
+
+      # Export metrics in Prometheus format
+      def export
+        synchronize do
+          lines = []
+
+          # Add header
+          lines << "# DecisionAgent Metrics Export"
+          lines << "# Timestamp: #{Time.now.utc.iso8601}"
+          lines << ""
+
+          # Decision metrics
+          lines.concat(export_decision_metrics)
+
+          # Performance metrics
+          lines.concat(export_performance_metrics)
+
+          # Error metrics
+          lines.concat(export_error_metrics)
+
+          # Custom KPI metrics
+          lines.concat(export_custom_metrics)
+
+          # System info
+          lines.concat(export_system_metrics)
+
+          lines.join("\n")
+        end
+      end
+
+      # Register custom KPI
+      def register_kpi(name:, value:, labels: {}, help: nil)
+        synchronize do
+          metric_name = sanitize_name(name)
+          @custom_metrics[metric_name] = {
+            value: value,
+            labels: labels,
+            help: help || "Custom KPI: #{name}",
+            timestamp: Time.now.utc
+          }
+        end
+      end
+
+      # Get metrics in hash format
+      def metrics_hash
+        synchronize do
+          stats = @metrics_collector.statistics
+
+          {
+            decisions: {
+              total: counter_metric("decisions_total", stats.dig(:decisions, :total) || 0),
+              avg_confidence: gauge_metric("decision_confidence_avg", stats.dig(:decisions, :avg_confidence) || 0),
+              avg_duration_ms: gauge_metric("decision_duration_ms_avg", stats.dig(:decisions, :avg_duration_ms) || 0)
+            },
+            performance: {
+              success_rate: gauge_metric("success_rate", stats.dig(:performance, :success_rate) || 0),
+              avg_duration_ms: gauge_metric("operation_duration_ms_avg",
+                                            stats.dig(:performance, :avg_duration_ms) || 0),
+              p95_duration_ms: gauge_metric("operation_duration_ms_p95",
+                                            stats.dig(:performance, :p95_duration_ms) || 0),
+              p99_duration_ms: gauge_metric("operation_duration_ms_p99", stats.dig(:performance, :p99_duration_ms) || 0)
+            },
+            errors: {
+              total: counter_metric("errors_total", stats.dig(:errors, :total) || 0)
+            },
+            system: {
+              version: info_metric("version", DecisionAgent::VERSION)
+            }
+          }
+        end
+      end
+
+      private
+
+      def freeze_config
+        @namespace.freeze
+      end
+
+      def export_decision_metrics
+        stats = @metrics_collector.statistics
+        lines = []
+
+        # Total decisions
+        lines << "# HELP #{metric_name('decisions_total')} Total number of decisions made"
+        lines << "# TYPE #{metric_name('decisions_total')} counter"
+        lines << "#{metric_name('decisions_total')} #{stats.dig(:decisions, :total) || 0}"
+        lines << ""
+
+        # Average confidence
+        lines << "# HELP #{metric_name('decision_confidence_avg')} Average decision confidence"
+        lines << "# TYPE #{metric_name('decision_confidence_avg')} gauge"
+        lines << "#{metric_name('decision_confidence_avg')} #{stats.dig(:decisions, :avg_confidence) || 0}"
+        lines << ""
+
+        # Decision distribution
+        if stats.dig(:decisions, :decision_distribution)
+          lines << "# HELP #{metric_name('decisions_by_type')} Decisions grouped by type"
+          lines << "# TYPE #{metric_name('decisions_by_type')} counter"
+          stats[:decisions][:decision_distribution].each do |decision, count|
+            lines << "#{metric_name('decisions_by_type')}{decision=\"#{decision}\"} #{count}"
+          end
+          lines << ""
+        end
+
+        # Average duration
+        if stats.dig(:decisions, :avg_duration_ms)
+          lines << "# HELP #{metric_name('decision_duration_ms_avg')} Average decision duration in milliseconds"
+          lines << "# TYPE #{metric_name('decision_duration_ms_avg')} gauge"
+          lines << "#{metric_name('decision_duration_ms_avg')} #{stats[:decisions][:avg_duration_ms]}"
+          lines << ""
+        end
+
+        lines
+      end
+
+      def export_performance_metrics
+        stats = @metrics_collector.statistics
+        lines = []
+
+        # Success rate
+        lines << "# HELP #{metric_name('success_rate')} Operation success rate (0-1)"
+        lines << "# TYPE #{metric_name('success_rate')} gauge"
+        lines << "#{metric_name('success_rate')} #{stats.dig(:performance, :success_rate) || 0}"
+        lines << ""
+
+        # Duration metrics
+        if stats.dig(:performance, :avg_duration_ms)
+          lines << "# HELP #{metric_name('operation_duration_ms')} Operation duration in milliseconds"
+          lines << "# TYPE #{metric_name('operation_duration_ms')} summary"
+          lines << "#{metric_name('operation_duration_ms')}{quantile=\"0.5\"} #{stats[:performance][:avg_duration_ms]}"
+          lines << "#{metric_name('operation_duration_ms')}{quantile=\"0.95\"} #{stats[:performance][:p95_duration_ms]}"
+          lines << "#{metric_name('operation_duration_ms')}{quantile=\"0.99\"} #{stats[:performance][:p99_duration_ms]}"
+          lines << "#{metric_name('operation_duration_ms_sum')} #{stats[:performance][:avg_duration_ms] * stats[:performance][:total_operations]}"
+          lines << "#{metric_name('operation_duration_ms_count')} #{stats[:performance][:total_operations]}"
+          lines << ""
+        end
+
+        lines
+      end
+
+      def export_error_metrics
+        stats = @metrics_collector.statistics
+        lines = []
+
+        # Total errors
+        lines << "# HELP #{metric_name('errors_total')} Total number of errors"
+        lines << "# TYPE #{metric_name('errors_total')} counter"
+        lines << "#{metric_name('errors_total')} #{stats.dig(:errors, :total) || 0}"
+        lines << ""
+
+        # Errors by type
+        if stats.dig(:errors, :by_type)
+          lines << "# HELP #{metric_name('errors_by_type')} Errors grouped by type"
+          lines << "# TYPE #{metric_name('errors_by_type')} counter"
+          stats[:errors][:by_type].each do |error_type, count|
+            lines << "#{metric_name('errors_by_type')}{error=\"#{sanitize_label(error_type)}\"} #{count}"
+          end
+          lines << ""
+        end
+
+        lines
+      end
+
+      def export_custom_metrics
+        lines = []
+
+        @custom_metrics.each do |name, metric|
+          full_name = metric_name(name)
+          lines << "# HELP #{full_name} #{metric[:help]}"
+          lines << "# TYPE #{full_name} gauge"
+
+          if metric[:labels].empty?
+            lines << "#{full_name} #{metric[:value]}"
+          else
+            label_str = metric[:labels].map { |k, v| "#{k}=\"#{sanitize_label(v)}\"" }.join(",")
+            lines << "#{full_name}{#{label_str}} #{metric[:value]}"
+          end
+          lines << ""
+        end
+
+        lines
+      end
+
+      def export_system_metrics
+        lines = []
+
+        # Version info
+        lines << "# HELP #{metric_name('info')} DecisionAgent version info"
+        lines << "# TYPE #{metric_name('info')} gauge"
+        lines << "#{metric_name('info')}{version=\"#{DecisionAgent::VERSION}\"} 1"
+        lines << ""
+
+        # Metrics count
+        counts = @metrics_collector.metrics_count
+        lines << "# HELP #{metric_name('metrics_stored')} Number of metrics stored in memory"
+        lines << "# TYPE #{metric_name('metrics_stored')} gauge"
+        counts.each do |type, count|
+          lines << "#{metric_name('metrics_stored')}{type=\"#{type}\"} #{count}"
+        end
+        lines << ""
+
+        lines
+      end
+
+      def metric_name(name)
+        "#{@namespace}_#{sanitize_name(name)}"
+      end
+
+      def sanitize_name(name)
+        name.to_s.gsub(/[^a-zA-Z0-9_]/, "_")
+      end
+
+      def sanitize_label(value)
+        value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
+      end
+
+      def counter_metric(name, value)
+        { name: name, type: "counter", value: value }
+      end
+
+      def gauge_metric(name, value)
+        { name: name, type: "gauge", value: value }
+      end
+
+      def info_metric(name, value)
+        { name: name, type: "info", value: value }
+      end
+    end
+  end
+end

data/lib/decision_agent/monitoring/storage/activerecord_adapter.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require_relative "base_adapter"
+
+module DecisionAgent
+  module Monitoring
+    module Storage
+      # ActiveRecord adapter for persistent database storage
+      class ActiveRecordAdapter < BaseAdapter
+        def initialize
+          super
+          validate_models!
+        end
+
+        def record_decision(decision, context, confidence: nil, evaluations_count: 0, duration_ms: nil, status: nil)
+          ::DecisionLog.create!(
+            decision: decision,
+            context: context.to_json,
+            confidence: confidence,
+            evaluations_count: evaluations_count,
+            duration_ms: duration_ms,
+            status: status
+          )
+        rescue StandardError => e
+          warn "Failed to record decision to database: #{e.message}"
+        end
+
+        def record_evaluation(evaluator_name, score: nil, success: nil, duration_ms: nil, details: {})
+          ::EvaluationMetric.create!(
+            evaluator_name: evaluator_name,
+            score: score,
+            success: success,
+            duration_ms: duration_ms,
+            details: details.to_json
+          )
+        rescue StandardError => e
+          warn "Failed to record evaluation to database: #{e.message}"
+        end
+
+        def record_performance(operation, duration_ms: nil, status: nil, metadata: {})
+          ::PerformanceMetric.create!(
+            operation: operation,
+            duration_ms: duration_ms,
+            status: status,
+            metadata: metadata.to_json
+          )
+        rescue StandardError => e
+          warn "Failed to record performance to database: #{e.message}"
+        end
+
+        def record_error(error_type, message: nil, stack_trace: nil, severity: nil, context: {})
+          ::ErrorMetric.create!(
+            error_type: error_type,
+            message: message,
+            stack_trace: stack_trace&.to_json,
+            severity: severity,
+            context: context.to_json
+          )
+        rescue StandardError => e
+          warn "Failed to record error to database: #{e.message}"
+        end
+
+        def statistics(time_range: 3600)
+          decisions = ::DecisionLog.recent(time_range)
+          evaluations = ::EvaluationMetric.recent(time_range)
+          performance = ::PerformanceMetric.recent(time_range)
+          errors = ::ErrorMetric.recent(time_range)
+
+          {
+            decisions: decision_statistics(decisions, time_range),
+            evaluations: evaluation_statistics(evaluations),
+            performance: performance_statistics(performance, time_range),
+            errors: error_statistics(errors)
+          }
+        rescue StandardError => e
+          warn "Failed to retrieve statistics from database: #{e.message}"
+          default_statistics
+        end
+
+        def time_series(metric_type, bucket_size: 60, time_range: 3600)
+          case metric_type
+          when :decisions
+            decisions_time_series(bucket_size, time_range)
+          when :evaluations
+            evaluations_time_series(bucket_size, time_range)
+          when :performance
+            performance_time_series(bucket_size, time_range)
+          when :errors
+            errors_time_series(bucket_size, time_range)
+          else
+            { data: [], timestamps: [] }
+          end
+        rescue StandardError => e
+          warn "Failed to retrieve time series from database: #{e.message}"
+          { data: [], timestamps: [] }
+        end
+
+        def metrics_count
+          {
+            decisions: ::DecisionLog.count,
+            evaluations: ::EvaluationMetric.count,
+            performance: ::PerformanceMetric.count,
+            errors: ::ErrorMetric.count
+          }
+        rescue StandardError => e
+          warn "Failed to get metrics count from database: #{e.message}"
+          { decisions: 0, evaluations: 0, performance: 0, errors: 0 }
+        end
+
+        def cleanup(older_than:)
+          cutoff_time = Time.now - older_than
+          count = 0
+
+          count += ::DecisionLog.where("created_at < ?", cutoff_time).delete_all
+          count += ::EvaluationMetric.where("created_at < ?", cutoff_time).delete_all
+          count += ::PerformanceMetric.where("created_at < ?", cutoff_time).delete_all
+          count += ::ErrorMetric.where("created_at < ?", cutoff_time).delete_all
+
+          count
+        rescue StandardError => e
+          warn "Failed to cleanup old metrics from database: #{e.message}"
+          0
+        end
+
+        def self.available?
+          defined?(ActiveRecord) &&
+            defined?(::DecisionLog) &&
+            defined?(::EvaluationMetric) &&
+            defined?(::PerformanceMetric) &&
+            defined?(::ErrorMetric)
+        end
+
+        private
+
+        def decision_statistics(decisions, time_range)
+          {
+            total: decisions.count,
+            by_decision: decisions.group(:decision).count,
+            average_confidence: decisions.where.not(confidence: nil).average(:confidence).to_f,
+            success_rate: ::DecisionLog.success_rate(time_range: time_range)
+          }
+        end
+
+        def evaluation_statistics(evaluations)
+          {
+            total: evaluations.count,
+            by_evaluator: evaluations.group(:evaluator_name).count,
+            average_score: evaluations.where.not(score: nil).average(:score).to_f,
+            success_rate_by_evaluator: evaluations.successful.group(:evaluator_name).count
+          }
+        end
+
+        def performance_statistics(performance, time_range)
+          {
+            total: performance.count,
+            average_duration_ms: performance.average_duration(time_range: time_range),
+            p50: performance.p50(time_range: time_range),
+            p95: performance.p95(time_range: time_range),
+            p99: performance.p99(time_range: time_range),
+            success_rate: performance.success_rate(time_range: time_range)
+          }
+        end
+
+        def error_statistics(errors)
+          {
+            total: errors.count,
+            by_type: errors.group(:error_type).count,
+            by_severity: errors.group(:severity).count,
+            critical_count: errors.critical.count
+          }
+        end
+
+        def validate_models!
+          required_models = %w[DecisionLog EvaluationMetric PerformanceMetric ErrorMetric]
+          missing_models = required_models.reject { |model| Object.const_defined?(model) }
+
+          return if missing_models.empty?
+
+          raise "Missing required models: #{missing_models.join(', ')}. " \
+                "Run 'rails generate decision_agent:install --monitoring' to create them."
+        end
+
+        def decisions_time_series(bucket_size, time_range)
+          counts = ::DecisionLog.recent(time_range)
+                                .group(time_bucket_sql(:created_at, bucket_size))
+                                .count
+
+          format_time_series(counts)
+        end
+
+        def evaluations_time_series(bucket_size, time_range)
+          counts = ::EvaluationMetric.recent(time_range)
+                                     .group(time_bucket_sql(:created_at, bucket_size))
+                                     .count
+
+          format_time_series(counts)
+        end
+
+        def performance_time_series(bucket_size, time_range)
+          durations = ::PerformanceMetric.recent(time_range)
+                                         .where.not(duration_ms: nil)
+                                         .group(time_bucket_sql(:created_at, bucket_size))
+                                         .average(:duration_ms)
+
+          format_time_series(durations)
+        end
+
+        def errors_time_series(bucket_size, time_range)
+          counts = ::ErrorMetric.recent(time_range)
+                                .group(time_bucket_sql(:created_at, bucket_size))
+                                .count
+
+          format_time_series(counts)
+        end
+
+        def time_bucket_sql(column, bucket_size)
+          adapter = ActiveRecord::Base.connection.adapter_name.downcase
+
+          case adapter
+          when /postgres/
+            "(EXTRACT(EPOCH FROM #{column})::bigint / #{bucket_size}) * #{bucket_size}"
+          when /mysql/
+            "(UNIX_TIMESTAMP(#{column}) DIV #{bucket_size}) * #{bucket_size}"
+          when /sqlite/
+            "(CAST(strftime('%s', #{column}) AS INTEGER) / #{bucket_size}) * #{bucket_size}"
+          else
+            # Fallback: use group by timestamp truncated to bucket
+            column.to_s
+          end
+        end
+
+        def format_time_series(data)
+          timestamps = data.keys.sort
+          values = timestamps.map { |ts| data[ts] }
+
+          {
+            timestamps: timestamps.map { |ts| Time.at(ts).iso8601 },
+            data: values.map(&:to_f)
+          }
+        end
+
+        def default_statistics
+          {
+            decisions: { total: 0, by_decision: {}, average_confidence: 0.0, success_rate: 0.0 },
+            evaluations: { total: 0, by_evaluator: {}, average_score: 0.0, success_rate_by_evaluator: {} },
+            performance: { total: 0, average_duration_ms: 0.0, p50: 0.0, p95: 0.0, p99: 0.0, success_rate: 0.0 },
+            errors: { total: 0, by_type: {}, by_severity: {}, critical_count: 0 }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/decision_agent/monitoring/storage/base_adapter.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module DecisionAgent
+  module Monitoring
+    module Storage
+      # Base adapter interface for metrics storage
+      # Subclasses must implement all abstract methods
+      class BaseAdapter
+        # Record a decision
+        # @param decision [String] The decision made
+        # @param context [Hash] Decision context
+        # @param confidence [Float, nil] Confidence score (0-1)
+        # @param evaluations_count [Integer] Number of evaluations
+        # @param duration_ms [Float, nil] Decision duration in milliseconds
+        # @param status [String, nil] Decision status (success, failure, error)
+        # @return [void]
+        def record_decision(decision, context, confidence: nil, evaluations_count: 0, duration_ms: nil, status: nil)
+          raise NotImplementedError, "#{self.class} must implement #record_decision"
+        end
+
+        # Record an evaluation
+        # @param evaluator_name [String] Name of the evaluator
+        # @param score [Float, nil] Evaluation score
+        # @param success [Boolean, nil] Whether evaluation succeeded
+        # @param duration_ms [Float, nil] Evaluation duration
+        # @param details [Hash] Additional details
+        # @return [void]
+        def record_evaluation(evaluator_name, score: nil, success: nil, duration_ms: nil, details: {})
+          raise NotImplementedError, "#{self.class} must implement #record_evaluation"
+        end
+
+        # Record a performance metric
+        # @param operation [String] Operation name
+        # @param duration_ms [Float, nil] Duration in milliseconds
+        # @param status [String, nil] Status (success, failure, error)
+        # @param metadata [Hash] Additional metadata
+        # @return [void]
+        def record_performance(operation, duration_ms: nil, status: nil, metadata: {})
+          raise NotImplementedError, "#{self.class} must implement #record_performance"
+        end
+
+        # Record an error
+        # @param error_type [String] Type of error
+        # @param message [String, nil] Error message
+        # @param stack_trace [Array, nil] Stack trace
+        # @param severity [String, nil] Error severity (low, medium, high, critical)
+        # @param context [Hash] Error context
+        # @return [void]
+        def record_error(error_type, message: nil, stack_trace: nil, severity: nil, context: {})
+          raise NotImplementedError, "#{self.class} must implement #record_error"
+        end
+
+        # Get statistics for a time range
+        # @param time_range [Integer] Time range in seconds
+        # @return [Hash] Statistics summary
+        def statistics(time_range: 3600)
+          raise NotImplementedError, "#{self.class} must implement #statistics"
+        end
+
+        # Get time series data
+        # @param metric_type [Symbol] Type of metric (:decisions, :evaluations, :performance, :errors)
+        # @param bucket_size [Integer] Bucket size in seconds
+        # @param time_range [Integer] Time range in seconds
+        # @return [Hash] Time series data
+        def time_series(metric_type, bucket_size: 60, time_range: 3600)
+          raise NotImplementedError, "#{self.class} must implement #time_series"
+        end
+
+        # Get count of metrics stored
+        # @return [Hash] Count by metric type
+        def metrics_count
+          raise NotImplementedError, "#{self.class} must implement #metrics_count"
+        end
+
+        # Clean up old metrics
+        # @param older_than [Integer] Remove metrics older than this many seconds
+        # @return [Integer] Number of metrics removed
+        def cleanup(older_than:)
+          raise NotImplementedError, "#{self.class} must implement #cleanup"
+        end
+
+        # Check if adapter is available (dependencies installed)
+        # @return [Boolean]
+        def self.available?
+          raise NotImplementedError, "#{self} must implement .available?"
+        end
+      end
+    end
+  end
+end