decision_agent 0.1.3 → 0.1.6

data/lib/generators/decision_agent/install/templates/evaluation_metric.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class EvaluationMetric < ApplicationRecord
+  belongs_to :decision_log, optional: true
+
+  validates :evaluator_name, presence: true
+  validates :score, numericality: true, allow_nil: true
+
+  scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
+  scope :by_evaluator, ->(evaluator) { where(evaluator_name: evaluator) }
+  scope :successful, -> { where(success: true) }
+  scope :failed, -> { where(success: false) }
+
+  # Aggregation helpers
+  def self.average_score_by_evaluator(time_range: 3600)
+    recent(time_range)
+      .where.not(score: nil)
+      .group(:evaluator_name)
+      .average(:score)
+  end
+
+  def self.success_rate_by_evaluator(time_range: 3600)
+    recent(time_range)
+      .where.not(success: nil)
+      .group(:evaluator_name)
+      .select("evaluator_name, AVG(CASE WHEN success THEN 1.0 ELSE 0.0 END) as success_rate")
+  end
+
+  def self.count_by_evaluator(time_range: 3600)
+    recent(time_range)
+      .group(:evaluator_name)
+      .count
+  end
+
+  # Parse JSON details field
+  def parsed_details
+    return {} if details.nil?
+
+    JSON.parse(details, symbolize_names: true)
+  rescue JSON::ParserError
+    {}
+  end
+end

data/lib/generators/decision_agent/install/templates/monitoring_migration.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+class CreateDecisionAgentMonitoringTables < ActiveRecord::Migration[7.0]
+  # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+  def change
+    # Decision logs table
+    create_table :decision_logs do |t|
+      t.string :decision, null: false
+      t.float :confidence
+      t.integer :evaluations_count, default: 0
+      t.float :duration_ms
+      t.string :status # success, failure, error
+      t.text :context # JSON
+      t.text :metadata # JSON
+
+      t.timestamps
+    end
+
+    add_index :decision_logs, :decision
+    add_index :decision_logs, :status
+    add_index :decision_logs, :confidence
+    add_index :decision_logs, :created_at
+    add_index :decision_logs, %i[decision created_at]
+    add_index :decision_logs, %i[status created_at]
+
+    # Evaluation metrics table
+    create_table :evaluation_metrics do |t|
+      t.references :decision_log, foreign_key: true, index: true
+      t.string :evaluator_name, null: false
+      t.float :score
+      t.boolean :success
+      t.float :duration_ms
+      t.text :details # JSON
+
+      t.timestamps
+    end
+
+    add_index :evaluation_metrics, :evaluator_name
+    add_index :evaluation_metrics, :success
+    add_index :evaluation_metrics, :created_at
+    add_index :evaluation_metrics, %i[evaluator_name created_at]
+    add_index :evaluation_metrics, %i[evaluator_name success]
+
+    # Performance metrics table
+    create_table :performance_metrics do |t|
+      t.string :operation, null: false
+      t.float :duration_ms
+      t.string :status # success, failure, error
+      t.text :metadata # JSON
+
+      t.timestamps
+    end
+
+    add_index :performance_metrics, :operation
+    add_index :performance_metrics, :status
+    add_index :performance_metrics, :duration_ms
+    add_index :performance_metrics, :created_at
+    add_index :performance_metrics, %i[operation created_at]
+    add_index :performance_metrics, %i[status created_at]
+
+    # Error metrics table
+    create_table :error_metrics do |t|
+      t.string :error_type, null: false
+      t.text :message
+      t.text :stack_trace # JSON array
+      t.string :severity # low, medium, high, critical
+      t.text :context # JSON
+
+      t.timestamps
+    end
+
+    add_index :error_metrics, :error_type
+    add_index :error_metrics, :severity
+    add_index :error_metrics, :created_at
+    add_index :error_metrics, %i[error_type created_at]
+    add_index :error_metrics, %i[severity created_at]
+
+    # PostgreSQL-specific optimizations (optional)
+    return unless adapter_name == "PostgreSQL"
+
+    # Partial indexes for active records (recent data)
+    execute <<-SQL
+        CREATE INDEX index_decision_logs_on_recent
+        ON decision_logs (created_at DESC)
+        WHERE created_at >= NOW() - INTERVAL '7 days';
+    SQL
+
+    execute <<-SQL
+        CREATE INDEX index_performance_metrics_on_recent
+        ON performance_metrics (created_at DESC)
+        WHERE created_at >= NOW() - INTERVAL '7 days';
+    SQL
+
+    execute <<-SQL
+        CREATE INDEX index_error_metrics_on_recent_critical
+        ON error_metrics (created_at DESC)
+        WHERE severity IN ('high', 'critical') AND created_at >= NOW() - INTERVAL '7 days';
+    SQL
+
+    # Consider table partitioning for large-scale deployments
+    # Example: Partition by month for decision_logs
+    # This is commented out by default - enable if needed
+    # execute <<-SQL
+    #   CREATE TABLE decision_logs_partitioned (LIKE decision_logs INCLUDING ALL)
+    #   PARTITION BY RANGE (created_at);
+    # SQL
+  end
+  # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+end

data/lib/generators/decision_agent/install/templates/performance_metric.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+class PerformanceMetric < ApplicationRecord
+  validates :operation, presence: true
+  validates :duration_ms, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+  validates :status, inclusion: { in: %w[success failure error] }, allow_nil: true
+
+  scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
+  scope :by_operation, ->(operation) { where(operation: operation) }
+  scope :successful, -> { where(status: "success") }
+  scope :failed, -> { where(status: "failure") }
+  scope :with_errors, -> { where(status: "error") }
+  scope :slow, ->(threshold_ms = 1000) { where("duration_ms > ?", threshold_ms) }
+
+  # Performance statistics
+  def self.average_duration(time_range: 3600)
+    recent(time_range).where.not(duration_ms: nil).average(:duration_ms).to_f
+  end
+
+  def self.percentile(pct, time_range: 3600)
+    durations = recent(time_range).where.not(duration_ms: nil).order(:duration_ms).pluck(:duration_ms)
+    return 0.0 if durations.empty?
+
+    index = ((durations.length - 1) * pct).ceil
+    durations[index].to_f
+  end
+
+  def self.p50(time_range: 3600)
+    percentile(0.50, time_range: time_range)
+  end
+
+  def self.p95(time_range: 3600)
+    percentile(0.95, time_range: time_range)
+  end
+
+  def self.p99(time_range: 3600)
+    percentile(0.99, time_range: time_range)
+  end
+
+  def self.max_duration(time_range: 3600)
+    recent(time_range).maximum(:duration_ms).to_f
+  end
+
+  def self.min_duration(time_range: 3600)
+    recent(time_range).minimum(:duration_ms).to_f
+  end
+
+  def self.success_rate(time_range: 3600)
+    total = recent(time_range).where.not(status: nil).count
+    return 0.0 if total.zero?
+
+    successful_count = recent(time_range).successful.count
+    successful_count.to_f / total
+  end
+
+  def self.count_by_operation(time_range: 3600)
+    recent(time_range).group(:operation).count
+  end
+
+  # Time series aggregation
+  def self.average_duration_by_time(bucket_size: 60, time_range: 3600)
+    recent(time_range)
+      .where.not(duration_ms: nil)
+      .group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
+      .average(:duration_ms)
+  end
+
+  # Parse JSON metadata field
+  def parsed_metadata
+    return {} if metadata.nil?
+
+    JSON.parse(metadata, symbolize_names: true)
+  rescue JSON::ParserError
+    {}
+  end
+end

data/lib/generators/decision_agent/install/templates/rule_version.rb

@@ -33,7 +33,7 @@ class RuleVersion < ApplicationRecord
       self.class.where(rule_id: rule_id, status: "active")
           .where.not(id: id)
           .find_each do |v|
-        v.update!(status: "archived")
+            v.update!(status: "archived")
       end
 
       # Activate this version

data/spec/ab_testing/ab_test_assignment_spec.rb

@@ -0,0 +1,253 @@
+require "spec_helper"
+require "decision_agent/ab_testing/ab_test_assignment"
+
+RSpec.describe DecisionAgent::ABTesting::ABTestAssignment do
+  describe "#initialize" do
+    it "creates an assignment with required fields" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1"
+      )
+
+      expect(assignment.ab_test_id).to eq("test_1")
+      expect(assignment.variant).to eq(:champion)
+      expect(assignment.version_id).to eq("v1")
+      expect(assignment.timestamp).to be_a(Time)
+    end
+
+    it "accepts optional user_id" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        user_id: "user_123"
+      )
+
+      expect(assignment.user_id).to eq("user_123")
+    end
+
+    it "accepts optional timestamp" do
+      custom_time = Time.new(2024, 1, 1, 12, 0, 0, "+00:00")
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        timestamp: custom_time
+      )
+
+      expect(assignment.timestamp).to eq(custom_time)
+    end
+
+    it "accepts optional decision_result and confidence" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        decision_result: "approve",
+        confidence: 0.95
+      )
+
+      expect(assignment.decision_result).to eq("approve")
+      expect(assignment.confidence).to eq(0.95)
+    end
+
+    it "accepts optional context" do
+      context = { user_type: "premium", region: "us" }
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        context: context
+      )
+
+      expect(assignment.context).to eq(context)
+    end
+
+    it "defaults context to empty hash" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1"
+      )
+
+      expect(assignment.context).to eq({})
+    end
+
+    it "accepts optional id" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        id: "assign_123"
+      )
+
+      expect(assignment.id).to eq("assign_123")
+    end
+
+    it "raises error if ab_test_id is nil" do
+      expect do
+        described_class.new(
+          ab_test_id: nil,
+          variant: :champion,
+          version_id: "v1"
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /AB test ID is required/)
+    end
+
+    it "raises error if variant is nil" do
+      expect do
+        described_class.new(
+          ab_test_id: "test_1",
+          variant: nil,
+          version_id: "v1"
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /Variant is required/)
+    end
+
+    it "raises error if version_id is nil" do
+      expect do
+        described_class.new(
+          ab_test_id: "test_1",
+          variant: :champion,
+          version_id: nil
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /Version ID is required/)
+    end
+
+    it "raises error if variant is not :champion or :challenger" do
+      expect do
+        described_class.new(
+          ab_test_id: "test_1",
+          variant: :invalid,
+          version_id: "v1"
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /Variant must be :champion or :challenger/)
+    end
+
+    it "raises error if confidence is negative" do
+      expect do
+        described_class.new(
+          ab_test_id: "test_1",
+          variant: :champion,
+          version_id: "v1",
+          confidence: -0.1
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /Confidence must be between 0 and 1/)
+    end
+
+    it "raises error if confidence is greater than 1" do
+      expect do
+        described_class.new(
+          ab_test_id: "test_1",
+          variant: :champion,
+          version_id: "v1",
+          confidence: 1.5
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /Confidence must be between 0 and 1/)
+    end
+
+    it "accepts confidence of 0" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        confidence: 0.0
+      )
+
+      expect(assignment.confidence).to eq(0.0)
+    end
+
+    it "accepts confidence of 1" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        confidence: 1.0
+      )
+
+      expect(assignment.confidence).to eq(1.0)
+    end
+
+    it "accepts challenger variant" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :challenger,
+        version_id: "v2"
+      )
+
+      expect(assignment.variant).to eq(:challenger)
+    end
+  end
+
+  describe "#record_decision" do
+    let(:assignment) do
+      described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1"
+      )
+    end
+
+    it "updates decision_result and confidence" do
+      assignment.record_decision("approve", 0.95)
+
+      expect(assignment.decision_result).to eq("approve")
+      expect(assignment.confidence).to eq(0.95)
+    end
+
+    it "can update multiple times" do
+      assignment.record_decision("approve", 0.95)
+      assignment.record_decision("reject", 0.85)
+
+      expect(assignment.decision_result).to eq("reject")
+      expect(assignment.confidence).to eq(0.85)
+    end
+  end
+
+  describe "#to_h" do
+    it "converts assignment to hash with all fields" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1",
+        id: "assign_123",
+        user_id: "user_456",
+        decision_result: "approve",
+        confidence: 0.95,
+        context: { region: "us" },
+        timestamp: Time.new(2024, 1, 1, 12, 0, 0, "+00:00")
+      )
+
+      hash = assignment.to_h
+
+      expect(hash).to eq({
+                           id: "assign_123",
+                           ab_test_id: "test_1",
+                           user_id: "user_456",
+                           variant: :champion,
+                           version_id: "v1",
+                           timestamp: Time.new(2024, 1, 1, 12, 0, 0, "+00:00"),
+                           decision_result: "approve",
+                           confidence: 0.95,
+                           context: { region: "us" }
+                         })
+    end
+
+    it "includes nil values in hash" do
+      assignment = described_class.new(
+        ab_test_id: "test_1",
+        variant: :champion,
+        version_id: "v1"
+      )
+
+      hash = assignment.to_h
+
+      expect(hash[:id]).to be_nil
+      expect(hash[:user_id]).to be_nil
+      expect(hash[:decision_result]).to be_nil
+      expect(hash[:confidence]).to be_nil
+      expect(hash[:context]).to eq({})
+    end
+  end
+end