decision_agent 0.1.1 → 0.1.3

data/spec/monitoring/metrics_collector_spec.rb

@@ -0,0 +1,281 @@
+require "spec_helper"
+require "decision_agent/monitoring/metrics_collector"
+
+RSpec.describe DecisionAgent::Monitoring::MetricsCollector do
+  let(:collector) { described_class.new(window_size: 60) }
+  let(:decision) do
+    double(
+      "Decision",
+      decision: "approve",
+      confidence: 0.85,
+      evaluations: [
+        double("Evaluation", evaluator_name: "test_evaluator")
+      ]
+    )
+  end
+  let(:context) { double("Context", to_h: { user: "test" }) }
+
+  describe "#initialize" do
+    it "initializes with default window size" do
+      collector = described_class.new
+      expect(collector.window_size).to eq(3600)
+    end
+
+    it "initializes with custom window size" do
+      expect(collector.window_size).to eq(60)
+    end
+
+    it "initializes empty metrics" do
+      counts = collector.metrics_count
+      expect(counts[:decisions]).to eq(0)
+      expect(counts[:evaluations]).to eq(0)
+      expect(counts[:performance]).to eq(0)
+      expect(counts[:errors]).to eq(0)
+    end
+  end
+
+  describe "#record_decision" do
+    it "records a decision metric" do
+      metric = collector.record_decision(decision, context, duration_ms: 10.5)
+
+      expect(metric[:decision]).to eq("approve")
+      expect(metric[:confidence]).to eq(0.85)
+      expect(metric[:duration_ms]).to eq(10.5)
+      expect(metric[:context_size]).to eq(1)
+      expect(metric[:evaluations_count]).to eq(1)
+      expect(metric[:evaluator_names]).to eq(["test_evaluator"])
+    end
+
+    it "increments decision count" do
+      expect do
+        collector.record_decision(decision, context)
+      end.to change { collector.metrics_count[:decisions] }.by(1)
+    end
+
+    it "notifies observers" do
+      observed = []
+      collector.add_observer do |type, metric|
+        observed << [type, metric]
+      end
+
+      collector.record_decision(decision, context)
+
+      expect(observed.size).to eq(1)
+      expect(observed[0][0]).to eq(:decision)
+      expect(observed[0][1][:decision]).to eq("approve")
+    end
+  end
+
+  describe "#record_evaluation" do
+    let(:evaluation) do
+      double(
+        "Evaluation",
+        decision: "approve",
+        weight: 0.9,
+        evaluator_name: "test_evaluator"
+      )
+    end
+
+    it "records an evaluation metric" do
+      metric = collector.record_evaluation(evaluation)
+
+      expect(metric[:decision]).to eq("approve")
+      expect(metric[:weight]).to eq(0.9)
+      expect(metric[:evaluator_name]).to eq("test_evaluator")
+    end
+
+    it "increments evaluation count" do
+      expect do
+        collector.record_evaluation(evaluation)
+      end.to change { collector.metrics_count[:evaluations] }.by(1)
+    end
+  end
+
+  describe "#record_performance" do
+    it "records performance metrics" do
+      metric = collector.record_performance(
+        operation: "decide",
+        duration_ms: 25.5,
+        success: true,
+        metadata: { evaluators: 2 }
+      )
+
+      expect(metric[:operation]).to eq("decide")
+      expect(metric[:duration_ms]).to eq(25.5)
+      expect(metric[:success]).to be true
+      expect(metric[:metadata]).to eq({ evaluators: 2 })
+    end
+
+    it "records failed operations" do
+      metric = collector.record_performance(
+        operation: "decide",
+        duration_ms: 10.0,
+        success: false
+      )
+
+      expect(metric[:success]).to be false
+    end
+  end
+
+  describe "#record_error" do
+    let(:error) { StandardError.new("Test error") }
+
+    it "records error metrics" do
+      metric = collector.record_error(error, context: { user_id: 123 })
+
+      expect(metric[:error_class]).to eq("StandardError")
+      expect(metric[:error_message]).to eq("Test error")
+      expect(metric[:context]).to eq({ user_id: 123 })
+    end
+
+    it "increments error count" do
+      expect do
+        collector.record_error(error)
+      end.to change { collector.metrics_count[:errors] }.by(1)
+    end
+  end
+
+  describe "#statistics" do
+    before do
+      # Record some metrics
+      5.times do |i|
+        collector.record_decision(decision, context, duration_ms: (i + 1) * 10)
+      end
+
+      2.times do
+        collector.record_performance(operation: "decide", duration_ms: 15.0, success: true)
+      end
+      collector.record_performance(operation: "decide", duration_ms: 20.0, success: false)
+
+      collector.record_error(StandardError.new("Error 1"))
+    end
+
+    it "returns summary statistics" do
+      stats = collector.statistics
+
+      expect(stats[:summary][:total_decisions]).to eq(5)
+      expect(stats[:summary][:total_evaluations]).to eq(0)
+      expect(stats[:summary][:total_errors]).to eq(1)
+    end
+
+    it "computes decision statistics" do
+      stats = collector.statistics
+
+      expect(stats[:decisions][:total]).to eq(5)
+      expect(stats[:decisions][:avg_confidence]).to eq(0.85)
+      expect(stats[:decisions][:min_confidence]).to eq(0.85)
+      expect(stats[:decisions][:max_confidence]).to eq(0.85)
+      expect(stats[:decisions][:avg_duration_ms]).to be_within(0.1).of(30.0)
+    end
+
+    it "computes performance statistics" do
+      stats = collector.statistics
+
+      expect(stats[:performance][:total_operations]).to eq(3)
+      expect(stats[:performance][:successful]).to eq(2)
+      expect(stats[:performance][:failed]).to eq(1)
+      expect(stats[:performance][:success_rate]).to be_within(0.01).of(0.6667)
+    end
+
+    it "computes error statistics" do
+      stats = collector.statistics
+
+      expect(stats[:errors][:total]).to eq(1)
+      expect(stats[:errors][:by_type]["StandardError"]).to eq(1)
+    end
+
+    it "filters by time range" do
+      stats = collector.statistics(time_range: 30)
+      expect(stats[:summary][:time_range]).to eq("Last 30s")
+    end
+  end
+
+  describe "#time_series" do
+    before do
+      10.times do
+        collector.record_decision(decision, context)
+        sleep 0.01 # Small delay to ensure different buckets
+      end
+    end
+
+    it "returns time series data" do
+      series = collector.time_series(metric_type: :decisions, bucket_size: 1, time_range: 60)
+
+      expect(series).to be_an(Array)
+      expect(series.first).to have_key(:timestamp)
+      expect(series.first).to have_key(:count)
+      expect(series.first).to have_key(:metrics)
+    end
+
+    it "buckets metrics by time" do
+      series = collector.time_series(metric_type: :decisions, bucket_size: 60, time_range: 3600)
+
+      total_count = series.sum { |s| s[:count] }
+      expect(total_count).to eq(10)
+    end
+  end
+
+  describe "#clear!" do
+    before do
+      collector.record_decision(decision, context)
+      collector.record_error(StandardError.new("Test"))
+    end
+
+    it "clears all metrics" do
+      collector.clear!
+
+      counts = collector.metrics_count
+      expect(counts[:decisions]).to eq(0)
+      expect(counts[:errors]).to eq(0)
+    end
+  end
+
+  describe "thread safety" do
+    it "handles concurrent writes safely" do
+      threads = 10.times.map do
+        Thread.new do
+          10.times do
+            collector.record_decision(decision, context)
+          end
+        end
+      end
+
+      threads.each(&:join)
+
+      expect(collector.metrics_count[:decisions]).to eq(100)
+    end
+
+    it "handles concurrent reads and writes" do
+      writer = Thread.new do
+        50.times do
+          collector.record_decision(decision, context)
+          sleep 0.001
+        end
+      end
+
+      reader = Thread.new do
+        50.times do
+          collector.statistics
+          sleep 0.001
+        end
+      end
+
+      expect { writer.join && reader.join }.not_to raise_error
+    end
+  end
+
+  describe "metric cleanup" do
+    it "removes old metrics outside window" do
+      collector = described_class.new(window_size: 1)
+
+      collector.record_decision(decision, context)
+      expect(collector.metrics_count[:decisions]).to eq(1)
+
+      sleep 1.5
+
+      collector.record_decision(decision, context)
+      # Old metric should be cleaned up
+      expect(collector.metrics_count[:decisions]).to eq(1)
+    end
+  end
+end

data/spec/monitoring/monitored_agent_spec.rb

@@ -0,0 +1,222 @@
+require "spec_helper"
+require "decision_agent/monitoring/metrics_collector"
+require "decision_agent/monitoring/monitored_agent"
+
+RSpec.describe DecisionAgent::Monitoring::MonitoredAgent do
+  let(:collector) { DecisionAgent::Monitoring::MetricsCollector.new }
+  let(:evaluator) do
+    double(
+      "Evaluator",
+      evaluate: DecisionAgent::Evaluation.new(
+        decision: "approve",
+        weight: 0.9,
+        reason: "Test reason",
+        evaluator_name: "test_evaluator"
+      )
+    )
+  end
+  let(:agent) { DecisionAgent::Agent.new(evaluators: [evaluator]) }
+  let(:monitored_agent) { described_class.new(agent: agent, metrics_collector: collector) }
+
+  describe "#initialize" do
+    it "wraps an agent with metrics collection" do
+      expect(monitored_agent.agent).to eq(agent)
+      expect(monitored_agent.metrics_collector).to eq(collector)
+    end
+  end
+
+  describe "#decide" do
+    let(:context) { { amount: 1000 } }
+
+    it "makes a decision and records metrics" do
+      result = monitored_agent.decide(context: context)
+
+      expect(result).to be_a(DecisionAgent::Decision)
+      expect(result.decision).to eq("approve")
+      expect(collector.metrics_count[:decisions]).to eq(1)
+      expect(collector.metrics_count[:evaluations]).to eq(1)
+      expect(collector.metrics_count[:performance]).to eq(1)
+    end
+
+    it "records decision metrics with duration" do
+      monitored_agent.decide(context: context)
+
+      stats = collector.statistics
+      expect(stats[:decisions][:total]).to eq(1)
+      expect(stats[:decisions][:avg_duration_ms]).to be > 0
+    end
+
+    it "records evaluation metrics" do
+      monitored_agent.decide(context: context)
+
+      stats = collector.statistics
+      expect(stats[:evaluations][:total]).to eq(1)
+      expect(stats[:evaluations][:evaluator_distribution]["test_evaluator"]).to eq(1)
+    end
+
+    it "records performance metrics as successful" do
+      monitored_agent.decide(context: context)
+
+      stats = collector.statistics
+      expect(stats[:performance][:total_operations]).to eq(1)
+      expect(stats[:performance][:successful]).to eq(1)
+      expect(stats[:performance][:success_rate]).to eq(1.0)
+    end
+
+    it "includes metadata in performance metrics" do
+      monitored_agent.decide(context: context)
+
+      collector.statistics
+      perf_metric = collector.instance_variable_get(:@metrics)[:performance].first
+
+      expect(perf_metric[:metadata][:evaluators_count]).to eq(1)
+      expect(perf_metric[:metadata][:decision]).to eq("approve")
+      expect(perf_metric[:metadata][:confidence]).to be_a(Float)
+    end
+
+    context "when decision fails" do
+      before do
+        allow(agent).to receive(:decide).and_raise(StandardError.new("Test error"))
+      end
+
+      it "records error metrics" do
+        expect do
+          monitored_agent.decide(context: context)
+        end.to raise_error(StandardError, "Test error")
+
+        expect(collector.metrics_count[:errors]).to eq(1)
+      end
+
+      it "records failed performance metrics" do
+        expect do
+          monitored_agent.decide(context: context)
+        end.to raise_error(StandardError)
+
+        stats = collector.statistics
+        expect(stats[:performance][:total_operations]).to eq(1)
+        expect(stats[:performance][:failed]).to eq(1)
+        expect(stats[:performance][:success_rate]).to eq(0.0)
+      end
+
+      it "includes error details in metrics" do
+        expect do
+          monitored_agent.decide(context: context)
+        end.to raise_error(StandardError)
+
+        error_metric = collector.instance_variable_get(:@metrics)[:errors].first
+        expect(error_metric[:error_class]).to eq("StandardError")
+        expect(error_metric[:error_message]).to eq("Test error")
+        expect(error_metric[:context]).to eq(context)
+      end
+
+      it "re-raises the error" do
+        expect do
+          monitored_agent.decide(context: context)
+        end.to raise_error(StandardError, "Test error")
+      end
+    end
+
+    it "handles Context objects" do
+      ctx = DecisionAgent::Context.new(context)
+      result = monitored_agent.decide(context: ctx)
+
+      expect(result).to be_a(DecisionAgent::Decision)
+      expect(collector.metrics_count[:decisions]).to eq(1)
+    end
+
+    it "handles hash contexts" do
+      result = monitored_agent.decide(context: context)
+
+      expect(result).to be_a(DecisionAgent::Decision)
+      expect(collector.metrics_count[:decisions]).to eq(1)
+    end
+
+    it "measures decision duration accurately" do
+      # Mock agent to introduce delay
+      allow(agent).to receive(:decide) do |*args|
+        sleep 0.01 # 10ms delay
+        evaluator.evaluate(args.first)
+        DecisionAgent::Decision.new(
+          decision: "approve",
+          confidence: 0.9,
+          explanations: ["Test"],
+          evaluations: [evaluator.evaluate(args.first)],
+          audit_payload: {}
+        )
+      end
+
+      monitored_agent.decide(context: context)
+
+      stats = collector.statistics
+      expect(stats[:decisions][:avg_duration_ms]).to be >= 10
+    end
+  end
+
+  describe "method delegation" do
+    it "delegates methods to wrapped agent" do
+      expect(monitored_agent.evaluators).to eq(agent.evaluators)
+      expect(monitored_agent.scoring_strategy).to eq(agent.scoring_strategy)
+      expect(monitored_agent.audit_adapter).to eq(agent.audit_adapter)
+    end
+
+    it "responds to agent methods" do
+      expect(monitored_agent).to respond_to(:evaluators)
+      expect(monitored_agent).to respond_to(:scoring_strategy)
+      expect(monitored_agent).to respond_to(:audit_adapter)
+    end
+  end
+
+  describe "thread safety" do
+    it "handles concurrent decisions safely" do
+      # Materialize let variables before creating threads
+      test_context = { amount: 1000 }
+      test_monitored_agent = monitored_agent
+      test_collector = collector
+
+      threads = 10.times.map do
+        Thread.new do
+          10.times do
+            test_monitored_agent.decide(context: test_context)
+          end
+        end
+      end
+
+      threads.each(&:join)
+
+      expect(test_collector.metrics_count[:decisions]).to eq(100)
+      expect(test_collector.metrics_count[:evaluations]).to eq(100)
+      expect(test_collector.metrics_count[:performance]).to eq(100)
+    end
+  end
+
+  describe "integration test" do
+    it "provides comprehensive metrics for multiple decisions" do
+      contexts = [
+        { amount: 500 },
+        { amount: 1500 },
+        { amount: 2000 }
+      ]
+
+      contexts.each do |ctx|
+        monitored_agent.decide(context: ctx)
+      end
+
+      stats = collector.statistics
+
+      # Summary
+      expect(stats[:summary][:total_decisions]).to eq(3)
+      expect(stats[:summary][:total_evaluations]).to eq(3)
+      expect(stats[:summary][:total_errors]).to eq(0)
+
+      # Decision stats
+      expect(stats[:decisions][:total]).to eq(3)
+      expect(stats[:decisions][:avg_confidence]).to be > 0
+      expect(stats[:decisions][:decision_distribution]["approve"]).to eq(3)
+
+      # Performance stats
+      expect(stats[:performance][:total_operations]).to eq(3)
+      expect(stats[:performance][:success_rate]).to eq(1.0)
+      expect(stats[:performance][:avg_duration_ms]).to be > 0
+    end
+  end
+end