decision_agent 0.1.2 → 0.1.4

data/spec/monitoring/prometheus_exporter_spec.rb

@@ -0,0 +1,242 @@
+require "spec_helper"
+require "decision_agent/monitoring/metrics_collector"
+require "decision_agent/monitoring/prometheus_exporter"
+
+RSpec.describe DecisionAgent::Monitoring::PrometheusExporter do
+  let(:collector) { DecisionAgent::Monitoring::MetricsCollector.new(storage: :memory) }
+  let(:exporter) { described_class.new(metrics_collector: collector, namespace: "test") }
+
+  let(:decision) do
+    double(
+      "Decision",
+      decision: "approve",
+      confidence: 0.85,
+      evaluations: [double("Evaluation", evaluator_name: "test_evaluator")]
+    )
+  end
+  let(:context) { double("Context", to_h: { user: "test" }) }
+
+  describe "#initialize" do
+    it "initializes with metrics collector" do
+      expect(exporter).to be_a(described_class)
+    end
+
+    it "uses default namespace" do
+      exporter = described_class.new(metrics_collector: collector)
+      output = exporter.export
+      expect(output).to include("decision_agent_")
+    end
+
+    it "uses custom namespace" do
+      output = exporter.export
+      expect(output).to include("test_")
+    end
+  end
+
+  describe "#export" do
+    before do
+      # Record some metrics
+      3.times { collector.record_decision(decision, context, duration_ms: 10.0) }
+      collector.record_performance(operation: "decide", duration_ms: 15.0, success: true)
+      collector.record_error(StandardError.new("Test error"))
+    end
+
+    it "exports in Prometheus text format" do
+      output = exporter.export
+
+      expect(output).to be_a(String)
+      expect(output).to include("# DecisionAgent Metrics Export")
+    end
+
+    it "includes decision metrics" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_decisions_total")
+      expect(output).to include("# TYPE test_decisions_total counter")
+      expect(output).to include("test_decisions_total 3")
+    end
+
+    it "includes confidence metrics" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_decision_confidence_avg")
+      expect(output).to include("# TYPE test_decision_confidence_avg gauge")
+      expect(output).to include("test_decision_confidence_avg 0.85")
+    end
+
+    it "includes performance metrics" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_success_rate")
+      expect(output).to include("# TYPE test_success_rate gauge")
+    end
+
+    it "includes error metrics" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_errors_total")
+      expect(output).to include("# TYPE test_errors_total counter")
+      expect(output).to include("test_errors_total 1")
+    end
+
+    it "includes system info" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_info")
+      expect(output).to include("# TYPE test_info gauge")
+      expect(output).to include("version=\"#{DecisionAgent::VERSION}\"")
+    end
+
+    it "includes decision distribution" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_decisions_by_type")
+      expect(output).to include("test_decisions_by_type{decision=\"approve\"} 3")
+    end
+
+    it "includes error distribution by type" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_errors_by_type")
+      expect(output).to include("test_errors_by_type{error=\"StandardError\"} 1")
+    end
+
+    it "includes metrics count" do
+      output = exporter.export
+
+      expect(output).to include("# HELP test_metrics_stored")
+      expect(output).to include("test_metrics_stored{type=\"decisions\"} 3")
+      expect(output).to include("test_metrics_stored{type=\"errors\"} 1")
+    end
+  end
+
+  describe "#register_kpi" do
+    it "registers a custom KPI" do
+      exporter.register_kpi(
+        name: "custom_metric",
+        value: 42.5,
+        help: "A custom metric"
+      )
+
+      output = exporter.export
+      expect(output).to include("# HELP test_custom_metric A custom metric")
+      expect(output).to include("# TYPE test_custom_metric gauge")
+      expect(output).to include("test_custom_metric 42.5")
+    end
+
+    it "registers KPI with labels" do
+      exporter.register_kpi(
+        name: "requests",
+        value: 100,
+        labels: { endpoint: "/api/v1", method: "GET" }
+      )
+
+      output = exporter.export
+      expect(output).to include("test_requests{endpoint=\"/api/v1\",method=\"GET\"} 100")
+    end
+
+    it "sanitizes metric names" do
+      exporter.register_kpi(name: "my-custom.metric!", value: 10)
+
+      output = exporter.export
+      expect(output).to include("test_my_custom_metric_")
+    end
+
+    it "escapes label values" do
+      exporter.register_kpi(
+        name: "metric",
+        value: 1,
+        labels: { message: 'Contains "quotes"' }
+      )
+
+      output = exporter.export
+      expect(output).to include('message="Contains \"quotes\""')
+    end
+  end
+
+  describe "#metrics_hash" do
+    before do
+      collector.record_decision(decision, context, duration_ms: 10.0)
+    end
+
+    it "returns metrics as hash" do
+      metrics = exporter.metrics_hash
+
+      expect(metrics).to be_a(Hash)
+      expect(metrics).to have_key(:decisions)
+      expect(metrics).to have_key(:performance)
+      expect(metrics).to have_key(:errors)
+      expect(metrics).to have_key(:system)
+    end
+
+    it "includes metric types" do
+      metrics = exporter.metrics_hash
+
+      expect(metrics[:decisions][:total][:type]).to eq("counter")
+      expect(metrics[:decisions][:avg_confidence][:type]).to eq("gauge")
+    end
+
+    it "includes metric values" do
+      metrics = exporter.metrics_hash
+
+      expect(metrics[:decisions][:total][:value]).to eq(1)
+      expect(metrics[:decisions][:avg_confidence][:value]).to eq(0.85)
+    end
+  end
+
+  describe "thread safety" do
+    it "handles concurrent KPI registration" do
+      threads = 10.times.map do |i|
+        Thread.new do
+          10.times do |j|
+            exporter.register_kpi(
+              name: "metric_#{i}_#{j}",
+              value: (i * 10) + j
+            )
+          end
+        end
+      end
+
+      expect { threads.each(&:join) }.not_to raise_error
+    end
+
+    it "handles concurrent exports" do
+      threads = 5.times.map do
+        Thread.new do
+          10.times { exporter.export }
+        end
+      end
+
+      expect { threads.each(&:join) }.not_to raise_error
+    end
+  end
+
+  describe "performance metrics export" do
+    before do
+      5.times do |i|
+        collector.record_performance(
+          operation: "decide",
+          duration_ms: (i + 1) * 10.0,
+          success: true
+        )
+      end
+    end
+
+    it "exports summary metrics" do
+      output = exporter.export
+
+      expect(output).to include("# TYPE test_operation_duration_ms summary")
+      expect(output).to include("test_operation_duration_ms{quantile=\"0.5\"}")
+      expect(output).to include("test_operation_duration_ms{quantile=\"0.95\"}")
+      expect(output).to include("test_operation_duration_ms{quantile=\"0.99\"}")
+      expect(output).to include("test_operation_duration_ms_sum")
+      expect(output).to include("test_operation_duration_ms_count")
+    end
+  end
+
+  describe "content type" do
+    it "defines Prometheus content type" do
+      expect(described_class::CONTENT_TYPE).to eq("text/plain; version=0.0.4")
+    end
+  end
+end

data/spec/monitoring/storage/activerecord_adapter_spec.rb

@@ -0,0 +1,346 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "active_record"
+require "decision_agent/monitoring/storage/activerecord_adapter"
+
+RSpec.describe DecisionAgent::Monitoring::Storage::ActiveRecordAdapter do
+  # Setup in-memory SQLite database for testing
+  before(:all) do
+    ActiveRecord::Base.establish_connection(
+      adapter: "sqlite3",
+      database: ":memory:"
+    )
+
+    # Create tables
+    ActiveRecord::Schema.define do
+      create_table :decision_logs, force: true do |t|
+        t.string :decision, null: false
+        t.float :confidence
+        t.integer :evaluations_count, default: 0
+        t.float :duration_ms
+        t.string :status
+        t.text :context
+        t.text :metadata
+        t.timestamps
+      end
+
+      create_table :evaluation_metrics, force: true do |t|
+        t.references :decision_log, foreign_key: true
+        t.string :evaluator_name, null: false
+        t.float :score
+        t.boolean :success
+        t.float :duration_ms
+        t.text :details
+        t.timestamps
+      end
+
+      create_table :performance_metrics, force: true do |t|
+        t.string :operation, null: false
+        t.float :duration_ms
+        t.string :status
+        t.text :metadata
+        t.timestamps
+      end
+
+      create_table :error_metrics, force: true do |t|
+        t.string :error_type, null: false
+        t.text :message
+        t.text :stack_trace
+        t.string :severity
+        t.text :context
+        t.timestamps
+      end
+    end
+
+    # Define models
+    # rubocop:disable Lint/ConstantDefinitionInBlock
+    class DecisionLog < ActiveRecord::Base
+      has_many :evaluation_metrics, dependent: :destroy
+
+      scope :recent, ->(time_range) { where("created_at >= ?", Time.now - time_range) }
+
+      def self.success_rate(time_range: 3600)
+        total = recent(time_range).where.not(status: nil).count
+        return 0.0 if total.zero?
+
+        recent(time_range).where(status: "success").count.to_f / total
+      end
+
+      def parsed_context
+        JSON.parse(context, symbolize_names: true)
+      rescue StandardError
+        {}
+      end
+    end
+
+    class EvaluationMetric < ActiveRecord::Base
+      belongs_to :decision_log, optional: true
+
+      scope :recent, ->(time_range) { where("created_at >= ?", Time.now - time_range) }
+      scope :successful, -> { where(success: true) }
+
+      def parsed_details
+        JSON.parse(details, symbolize_names: true)
+      rescue StandardError
+        {}
+      end
+    end
+
+    class PerformanceMetric < ActiveRecord::Base
+      scope :recent, ->(time_range) { where("created_at >= ?", Time.now - time_range) }
+
+      def self.average_duration(time_range: 3600)
+        recent(time_range).average(:duration_ms).to_f
+      end
+
+      def self.p50(time_range: 3600)
+        percentile(0.50, time_range: time_range)
+      end
+
+      def self.p95(time_range: 3600)
+        percentile(0.95, time_range: time_range)
+      end
+
+      def self.p99(time_range: 3600)
+        percentile(0.99, time_range: time_range)
+      end
+
+      def self.percentile(pct, time_range: 3600)
+        durations = recent(time_range).where.not(duration_ms: nil).order(:duration_ms).pluck(:duration_ms)
+        return 0.0 if durations.empty?
+
+        durations[(durations.length * pct).ceil - 1].to_f
+      end
+
+      def self.success_rate(time_range: 3600)
+        total = recent(time_range).where.not(status: nil).count
+        return 0.0 if total.zero?
+
+        recent(time_range).where(status: "success").count.to_f / total
+      end
+    end
+
+    class ErrorMetric < ActiveRecord::Base
+      scope :recent, ->(time_range) { where("created_at >= ?", Time.now - time_range) }
+      scope :critical, -> { where(severity: "critical") }
+
+      def parsed_context
+        JSON.parse(context, symbolize_names: true)
+      rescue StandardError
+        {}
+      end
+    end
+    # rubocop:enable Lint/ConstantDefinitionInBlock
+  end
+
+  before do
+    DecisionLog.delete_all
+    EvaluationMetric.delete_all
+    PerformanceMetric.delete_all
+    ErrorMetric.delete_all
+  end
+
+  let(:adapter) { described_class.new }
+
+  describe ".available?" do
+    it "returns true when ActiveRecord and models are defined" do
+      expect(described_class.available?).to be_truthy
+    end
+  end
+
+  describe "#record_decision" do
+    it "creates a decision log record" do
+      expect do
+        adapter.record_decision(
+          "approve_payment",
+          { user_id: 123, amount: 500 },
+          confidence: 0.85,
+          evaluations_count: 3,
+          duration_ms: 45.5,
+          status: "success"
+        )
+      end.to change(DecisionLog, :count).by(1)
+
+      log = DecisionLog.last
+      expect(log.decision).to eq("approve_payment")
+      expect(log.confidence).to eq(0.85)
+      expect(log.evaluations_count).to eq(3)
+      expect(log.duration_ms).to eq(45.5)
+      expect(log.status).to eq("success")
+      expect(log.parsed_context).to eq(user_id: 123, amount: 500)
+    end
+  end
+
+  describe "#record_evaluation" do
+    it "creates an evaluation metric record" do
+      expect do
+        adapter.record_evaluation(
+          "FraudDetector",
+          score: 0.92,
+          success: true,
+          duration_ms: 12.3,
+          details: { risk_level: "low" }
+        )
+      end.to change(EvaluationMetric, :count).by(1)
+
+      metric = EvaluationMetric.last
+      expect(metric.evaluator_name).to eq("FraudDetector")
+      expect(metric.score).to eq(0.92)
+      expect(metric.success).to be true
+      expect(metric.duration_ms).to eq(12.3)
+      expect(metric.parsed_details).to eq(risk_level: "low")
+    end
+  end
+
+  describe "#record_performance" do
+    it "creates a performance metric record" do
+      expect do
+        adapter.record_performance(
+          "api_call",
+          duration_ms: 250.5,
+          status: "success",
+          metadata: { endpoint: "/api/v1/users" }
+        )
+      end.to change(PerformanceMetric, :count).by(1)
+
+      metric = PerformanceMetric.last
+      expect(metric.operation).to eq("api_call")
+      expect(metric.duration_ms).to eq(250.5)
+      expect(metric.status).to eq("success")
+    end
+  end
+
+  describe "#record_error" do
+    it "creates an error metric record" do
+      expect do
+        adapter.record_error(
+          "RuntimeError",
+          message: "Something went wrong",
+          stack_trace: ["line 1", "line 2"],
+          severity: "critical",
+          context: { user_id: 456 }
+        )
+      end.to change(ErrorMetric, :count).by(1)
+
+      error = ErrorMetric.last
+      expect(error.error_type).to eq("RuntimeError")
+      expect(error.message).to eq("Something went wrong")
+      expect(error.severity).to eq("critical")
+      expect(error.parsed_context).to eq(user_id: 456)
+    end
+  end
+
+  describe "#statistics" do
+    before do
+      # Create test data
+      3.times do |i|
+        adapter.record_decision(
+          "decision_#{i}",
+          { index: i },
+          confidence: 0.5 + (i * 0.1),
+          evaluations_count: 2,
+          duration_ms: 100 + (i * 10),
+          status: "success"
+        )
+      end
+
+      2.times do |i|
+        adapter.record_evaluation(
+          "Evaluator#{i}",
+          score: 0.8,
+          success: true,
+          duration_ms: 50
+        )
+      end
+
+      4.times do |i|
+        adapter.record_performance(
+          "operation_#{i}",
+          duration_ms: 100 + (i * 50),
+          status: i.even? ? "success" : "failure"
+        )
+      end
+
+      adapter.record_error("TestError", severity: "critical")
+    end
+
+    it "returns comprehensive statistics" do
+      stats = adapter.statistics(time_range: 3600)
+
+      expect(stats[:decisions][:total]).to eq(3)
+      expect(stats[:decisions][:average_confidence]).to be_within(0.01).of(0.6)
+      expect(stats[:evaluations][:total]).to eq(2)
+      expect(stats[:performance][:total]).to eq(4)
+      expect(stats[:errors][:total]).to eq(1)
+      expect(stats[:errors][:critical_count]).to eq(1)
+    end
+  end
+
+  describe "#time_series" do
+    before do
+      # Create metrics at different times
+      [10, 70, 130].each do |seconds_ago|
+        travel_back = Time.now - seconds_ago
+        DecisionLog.create!(
+          decision: "test",
+          confidence: 0.8,
+          created_at: travel_back
+        )
+      end
+    end
+
+    it "returns time series data grouped by buckets" do
+      series = adapter.time_series(:decisions, bucket_size: 60, time_range: 200)
+
+      expect(series[:timestamps]).to be_an(Array)
+      expect(series[:data]).to be_an(Array)
+      expect(series[:data].sum).to eq(3)
+    end
+  end
+
+  describe "#metrics_count" do
+    before do
+      adapter.record_decision("test", {}, confidence: 0.8)
+      adapter.record_evaluation("TestEval", score: 0.9)
+      adapter.record_performance("test_op", duration_ms: 100)
+      adapter.record_error("TestError")
+    end
+
+    it "returns count of all metric types" do
+      counts = adapter.metrics_count
+
+      expect(counts[:decisions]).to eq(1)
+      expect(counts[:evaluations]).to eq(1)
+      expect(counts[:performance]).to eq(1)
+      expect(counts[:errors]).to eq(1)
+    end
+  end
+
+  describe "#cleanup" do
+    before do
+      # Create old metrics
+      old_time = Time.now - 8.days
+      DecisionLog.create!(decision: "old", confidence: 0.8, created_at: old_time)
+      EvaluationMetric.create!(evaluator_name: "old", created_at: old_time)
+      PerformanceMetric.create!(operation: "old", created_at: old_time)
+      ErrorMetric.create!(error_type: "old", created_at: old_time)
+
+      # Create recent metrics
+      adapter.record_decision("recent", {}, confidence: 0.8)
+      adapter.record_evaluation("recent", score: 0.9)
+      adapter.record_performance("recent", duration_ms: 100)
+      adapter.record_error("recent")
+    end
+
+    it "removes old metrics and keeps recent ones" do
+      count = adapter.cleanup(older_than: 7.days.to_i)
+
+      expect(count).to eq(4) # 4 old metrics removed
+      expect(DecisionLog.count).to eq(1)
+      expect(EvaluationMetric.count).to eq(1)
+      expect(PerformanceMetric.count).to eq(1)
+      expect(ErrorMetric.count).to eq(1)
+    end
+  end
+end