decision_agent 0.1.2 → 0.1.4

data/lib/generators/decision_agent/install/templates/performance_metric.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+class PerformanceMetric < ApplicationRecord
+  validates :operation, presence: true
+  validates :duration_ms, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+  validates :status, inclusion: { in: %w[success failure error] }, allow_nil: true
+
+  scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
+  scope :by_operation, ->(operation) { where(operation: operation) }
+  scope :successful, -> { where(status: "success") }
+  scope :failed, -> { where(status: "failure") }
+  scope :with_errors, -> { where(status: "error") }
+  scope :slow, ->(threshold_ms = 1000) { where("duration_ms > ?", threshold_ms) }
+
+  # Performance statistics
+  def self.average_duration(time_range: 3600)
+    recent(time_range).where.not(duration_ms: nil).average(:duration_ms).to_f
+  end
+
+  def self.percentile(pct, time_range: 3600)
+    durations = recent(time_range).where.not(duration_ms: nil).order(:duration_ms).pluck(:duration_ms)
+    return 0.0 if durations.empty?
+
+    index = ((durations.length - 1) * pct).ceil
+    durations[index].to_f
+  end
+
+  def self.p50(time_range: 3600)
+    percentile(0.50, time_range: time_range)
+  end
+
+  def self.p95(time_range: 3600)
+    percentile(0.95, time_range: time_range)
+  end
+
+  def self.p99(time_range: 3600)
+    percentile(0.99, time_range: time_range)
+  end
+
+  def self.max_duration(time_range: 3600)
+    recent(time_range).maximum(:duration_ms).to_f
+  end
+
+  def self.min_duration(time_range: 3600)
+    recent(time_range).minimum(:duration_ms).to_f
+  end
+
+  def self.success_rate(time_range: 3600)
+    total = recent(time_range).where.not(status: nil).count
+    return 0.0 if total.zero?
+
+    successful_count = recent(time_range).successful.count
+    successful_count.to_f / total
+  end
+
+  def self.count_by_operation(time_range: 3600)
+    recent(time_range).group(:operation).count
+  end
+
+  # Time series aggregation
+  def self.average_duration_by_time(bucket_size: 60, time_range: 3600)
+    recent(time_range)
+      .where.not(duration_ms: nil)
+      .group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
+      .average(:duration_ms)
+  end
+
+  # Parse JSON metadata field
+  def parsed_metadata
+    return {} if metadata.nil?
+
+    JSON.parse(metadata, symbolize_names: true)
+  rescue JSON::ParserError
+    {}
+  end
+end

data/lib/generators/decision_agent/install/templates/rule.rb

@@ -5,12 +5,12 @@ class Rule < ApplicationRecord
   validates :ruleset, presence: true
   validates :status, inclusion: { in: %w[active inactive archived] }
 
-  scope :active, -> { where(status: 'active') }
+  scope :active, -> { where(status: "active") }
   scope :by_ruleset, ->(ruleset) { where(ruleset: ruleset) }
 
   # Get the active version for this rule
   def active_version
-    rule_versions.find_by(status: 'active')
+    rule_versions.find_by(status: "active")
   end
 
   # Get all versions ordered by version number
@@ -19,7 +19,7 @@ class Rule < ApplicationRecord
   end
 
   # Create a new version
-  def create_version(content:, created_by: 'system', changelog: nil)
+  def create_version(content:, created_by: "system", changelog: nil)
     DecisionAgent::Versioning::VersionManager.new.save_version(
       rule_id: rule_id,
       rule_content: content,

data/lib/generators/decision_agent/install/templates/rule_version.rb

@@ -7,7 +7,7 @@ class RuleVersion < ApplicationRecord
   validates :status, inclusion: { in: %w[draft active archived] }
   validates :created_by, presence: true
 
-  scope :active, -> { where(status: 'active') }
+  scope :active, -> { where(status: "active") }
   scope :for_rule, ->(rule_id) { where(rule_id: rule_id).order(version_number: :desc) }
   scope :latest, -> { order(version_number: :desc).limit(1) }
 
@@ -29,12 +29,15 @@ class RuleVersion < ApplicationRecord
   def activate!
     transaction do
       # Deactivate all other versions for this rule
-      self.class.where(rule_id: rule_id, status: 'active')
-                .where.not(id: id)
-                .update_all(status: 'archived')
+      # Use update! instead of update_all to trigger validations
+      self.class.where(rule_id: rule_id, status: "active")
+          .where.not(id: id)
+          .find_each do |v|
+        v.update!(status: "archived")
+      end
 
       # Activate this version
-      update!(status: 'active')
+      update!(status: "active")
     end
   end
 
@@ -51,9 +54,12 @@ class RuleVersion < ApplicationRecord
   def set_next_version_number
     return if version_number.present?
 
+    # Use pessimistic locking to prevent race conditions when calculating version numbers
+    # Lock the last version record to ensure only one thread can read and increment at a time
     last_version = self.class.where(rule_id: rule_id)
-                             .order(version_number: :desc)
-                             .first
+                       .order(version_number: :desc)
+                       .lock
+                       .first
 
     self.version_number = last_version ? last_version.version_number + 1 : 1
   end

data/spec/ab_testing/ab_test_manager_spec.rb

@@ -0,0 +1,330 @@
+require "spec_helper"
+require "decision_agent/ab_testing/ab_test_manager"
+require "decision_agent/ab_testing/storage/memory_adapter"
+require "decision_agent/versioning/file_storage_adapter"
+
+RSpec.describe DecisionAgent::ABTesting::ABTestManager do
+  let(:version_manager) do
+    DecisionAgent::Versioning::VersionManager.new(
+      adapter: DecisionAgent::Versioning::FileStorageAdapter.new(storage_path: "/tmp/spec_ab_test_versions")
+    )
+  end
+
+  let(:storage_adapter) { DecisionAgent::ABTesting::Storage::MemoryAdapter.new }
+  let(:manager) { described_class.new(storage_adapter: storage_adapter, version_manager: version_manager) }
+
+  before do
+    # Create test versions
+    @champion = version_manager.save_version(
+      rule_id: "test_rule",
+      rule_content: { rules: [{ decision: "approve", weight: 1.0 }] },
+      created_by: "spec"
+    )
+
+    @challenger = version_manager.save_version(
+      rule_id: "test_rule",
+      rule_content: { rules: [{ decision: "reject", weight: 1.0 }] },
+      created_by: "spec"
+    )
+  end
+
+  after do
+    FileUtils.rm_rf("/tmp/spec_ab_test_versions")
+  end
+
+  describe "#create_test" do
+    it "creates a new A/B test" do
+      test = manager.create_test(
+        name: "Test A vs B",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id]
+      )
+
+      expect(test).to be_a(DecisionAgent::ABTesting::ABTest)
+      expect(test.name).to eq("Test A vs B")
+      expect(test.id).not_to be_nil
+    end
+
+    it "validates that champion version exists" do
+      expect do
+        manager.create_test(
+          name: "Test",
+          champion_version_id: "nonexistent",
+          challenger_version_id: @challenger[:id]
+        )
+      end.to raise_error(DecisionAgent::ABTesting::VersionNotFoundError, /Champion/)
+    end
+
+    it "validates that challenger version exists" do
+      expect do
+        manager.create_test(
+          name: "Test",
+          champion_version_id: @champion[:id],
+          challenger_version_id: "nonexistent"
+        )
+      end.to raise_error(DecisionAgent::ABTesting::VersionNotFoundError, /Challenger/)
+    end
+
+    it "accepts custom traffic split" do
+      test = manager.create_test(
+        name: "Custom Split",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        traffic_split: { champion: 70, challenger: 30 }
+      )
+
+      expect(test.traffic_split).to eq({ champion: 70, challenger: 30 })
+    end
+  end
+
+  describe "#get_test" do
+    it "retrieves a test by ID" do
+      created_test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id]
+      )
+
+      retrieved_test = manager.get_test(created_test.id)
+
+      expect(retrieved_test).not_to be_nil
+      expect(retrieved_test.id).to eq(created_test.id)
+      expect(retrieved_test.name).to eq("Test")
+    end
+
+    it "returns nil for nonexistent test" do
+      test = manager.get_test(99_999)
+      expect(test).to be_nil
+    end
+  end
+
+  describe "#assign_variant" do
+    let(:test) do
+      manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+    end
+
+    before do
+      manager.start_test(test.id)
+    end
+
+    it "assigns a variant and returns assignment details" do
+      assignment = manager.assign_variant(test_id: test.id, user_id: "user_123")
+
+      expect(assignment[:test_id]).to eq(test.id)
+      expect(%i[champion challenger]).to include(assignment[:variant])
+      expect([@champion[:id], @challenger[:id]]).to include(assignment[:version_id])
+      expect(assignment[:assignment_id]).not_to be_nil
+    end
+
+    it "assigns same variant to same user" do
+      user_id = "consistent_user"
+      assignment1 = manager.assign_variant(test_id: test.id, user_id: user_id)
+      assignment2 = manager.assign_variant(test_id: test.id, user_id: user_id)
+
+      expect(assignment1[:variant]).to eq(assignment2[:variant])
+    end
+
+    it "raises error for nonexistent test" do
+      expect do
+        manager.assign_variant(test_id: 99_999)
+      end.to raise_error(DecisionAgent::ABTesting::TestNotFoundError)
+    end
+  end
+
+  describe "#record_decision" do
+    let(:test) do
+      test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+      test
+    end
+
+    it "records decision result for an assignment" do
+      assignment = manager.assign_variant(test_id: test.id)
+
+      expect do
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: 0.95
+        )
+      end.not_to raise_error
+    end
+  end
+
+  describe "#get_results" do
+    let(:test) do
+      test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+      test
+    end
+
+    it "returns results with statistics" do
+      # Create some assignments and record decisions
+      10.times do |i|
+        assignment = manager.assign_variant(test_id: test.id, user_id: "user_#{i}")
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: 0.8 + (rand * 0.2)
+        )
+      end
+
+      results = manager.get_results(test.id)
+
+      expect(results[:test]).to be_a(Hash)
+      expect(results[:champion]).to be_a(Hash)
+      expect(results[:challenger]).to be_a(Hash)
+      expect(results[:comparison]).to be_a(Hash)
+      expect(results[:total_assignments]).to eq(10)
+    end
+
+    it "handles tests with no assignments" do
+      results = manager.get_results(test.id)
+
+      expect(results[:total_assignments]).to eq(0)
+      expect(results[:champion][:decisions_recorded]).to eq(0)
+      expect(results[:challenger][:decisions_recorded]).to eq(0)
+    end
+  end
+
+  describe "#active_tests" do
+    it "returns only running tests" do
+      test1 = manager.create_test(
+        name: "Running Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test1.id)
+
+      manager.create_test(
+        name: "Scheduled Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+
+      active = manager.active_tests
+
+      expect(active.size).to eq(1)
+      expect(active.first.id).to eq(test1.id)
+    end
+
+    it "caches active tests for performance" do
+      test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+
+      # First call
+      manager.active_tests
+
+      # Expect storage adapter not to be called again (cached)
+      expect(storage_adapter).not_to receive(:list_tests)
+      manager.active_tests
+    end
+  end
+
+  describe "test lifecycle" do
+    let(:test) do
+      manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+    end
+
+    it "starts a scheduled test" do
+      manager.start_test(test.id)
+      updated_test = manager.get_test(test.id)
+
+      expect(updated_test.status).to eq("running")
+    end
+
+    it "completes a running test" do
+      manager.start_test(test.id)
+      manager.complete_test(test.id)
+      updated_test = manager.get_test(test.id)
+
+      expect(updated_test.status).to eq("completed")
+    end
+
+    it "cancels a test" do
+      manager.cancel_test(test.id)
+      updated_test = manager.get_test(test.id)
+
+      expect(updated_test.status).to eq("cancelled")
+    end
+  end
+
+  describe "statistical analysis" do
+    let(:test) do
+      test = manager.create_test(
+        name: "Statistical Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        traffic_split: { champion: 50, challenger: 50 },
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+      test
+    end
+
+    it "calculates improvement percentage" do
+      # Create assignments with different confidence levels
+      50.times do |i|
+        assignment = manager.assign_variant(test_id: test.id, user_id: "user_#{i}")
+
+        # Champion: avg 0.7, Challenger: avg 0.9
+        confidence = assignment[:variant] == :champion ? 0.7 : 0.9
+
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: confidence
+        )
+      end
+
+      results = manager.get_results(test.id)
+
+      # Challenger should have higher avg confidence (0.9 vs 0.7)
+      expect(results[:comparison][:improvement_percentage]).to be > 0
+      expect(%w[champion challenger inconclusive]).to include(results[:comparison][:winner])
+    end
+
+    it "indicates insufficient data when sample is too small" do
+      # Create only a few assignments
+      5.times do |i|
+        assignment = manager.assign_variant(test_id: test.id, user_id: "user_#{i}")
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: 0.8
+        )
+      end
+
+      results = manager.get_results(test.id)
+
+      expect(results[:comparison][:statistical_significance]).to eq("not_significant")
+    end
+  end
+end