decision_agent 0.1.3 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +84 -233
  3. data/lib/decision_agent/ab_testing/ab_test.rb +197 -0
  4. data/lib/decision_agent/ab_testing/ab_test_assignment.rb +76 -0
  5. data/lib/decision_agent/ab_testing/ab_test_manager.rb +317 -0
  6. data/lib/decision_agent/ab_testing/ab_testing_agent.rb +188 -0
  7. data/lib/decision_agent/ab_testing/storage/activerecord_adapter.rb +155 -0
  8. data/lib/decision_agent/ab_testing/storage/adapter.rb +67 -0
  9. data/lib/decision_agent/ab_testing/storage/memory_adapter.rb +116 -0
  10. data/lib/decision_agent/agent.rb +5 -3
  11. data/lib/decision_agent/auth/access_audit_logger.rb +122 -0
  12. data/lib/decision_agent/auth/authenticator.rb +127 -0
  13. data/lib/decision_agent/auth/password_reset_manager.rb +57 -0
  14. data/lib/decision_agent/auth/password_reset_token.rb +33 -0
  15. data/lib/decision_agent/auth/permission.rb +29 -0
  16. data/lib/decision_agent/auth/permission_checker.rb +43 -0
  17. data/lib/decision_agent/auth/rbac_adapter.rb +278 -0
  18. data/lib/decision_agent/auth/rbac_config.rb +51 -0
  19. data/lib/decision_agent/auth/role.rb +56 -0
  20. data/lib/decision_agent/auth/session.rb +33 -0
  21. data/lib/decision_agent/auth/session_manager.rb +57 -0
  22. data/lib/decision_agent/auth/user.rb +70 -0
  23. data/lib/decision_agent/context.rb +24 -4
  24. data/lib/decision_agent/decision.rb +10 -3
  25. data/lib/decision_agent/dsl/condition_evaluator.rb +378 -1
  26. data/lib/decision_agent/dsl/schema_validator.rb +8 -1
  27. data/lib/decision_agent/errors.rb +38 -0
  28. data/lib/decision_agent/evaluation.rb +10 -3
  29. data/lib/decision_agent/evaluation_validator.rb +8 -13
  30. data/lib/decision_agent/monitoring/dashboard_server.rb +1 -0
  31. data/lib/decision_agent/monitoring/metrics_collector.rb +164 -7
  32. data/lib/decision_agent/monitoring/storage/activerecord_adapter.rb +253 -0
  33. data/lib/decision_agent/monitoring/storage/base_adapter.rb +90 -0
  34. data/lib/decision_agent/monitoring/storage/memory_adapter.rb +222 -0
  35. data/lib/decision_agent/testing/batch_test_importer.rb +373 -0
  36. data/lib/decision_agent/testing/batch_test_runner.rb +244 -0
  37. data/lib/decision_agent/testing/test_coverage_analyzer.rb +191 -0
  38. data/lib/decision_agent/testing/test_result_comparator.rb +235 -0
  39. data/lib/decision_agent/testing/test_scenario.rb +42 -0
  40. data/lib/decision_agent/version.rb +10 -1
  41. data/lib/decision_agent/versioning/activerecord_adapter.rb +1 -1
  42. data/lib/decision_agent/versioning/file_storage_adapter.rb +96 -28
  43. data/lib/decision_agent/web/middleware/auth_middleware.rb +45 -0
  44. data/lib/decision_agent/web/middleware/permission_middleware.rb +94 -0
  45. data/lib/decision_agent/web/public/app.js +184 -29
  46. data/lib/decision_agent/web/public/batch_testing.html +640 -0
  47. data/lib/decision_agent/web/public/index.html +37 -9
  48. data/lib/decision_agent/web/public/login.html +298 -0
  49. data/lib/decision_agent/web/public/users.html +679 -0
  50. data/lib/decision_agent/web/server.rb +873 -7
  51. data/lib/decision_agent.rb +59 -0
  52. data/lib/generators/decision_agent/install/install_generator.rb +37 -0
  53. data/lib/generators/decision_agent/install/templates/ab_test_assignment_model.rb +45 -0
  54. data/lib/generators/decision_agent/install/templates/ab_test_model.rb +54 -0
  55. data/lib/generators/decision_agent/install/templates/ab_testing_migration.rb +43 -0
  56. data/lib/generators/decision_agent/install/templates/ab_testing_tasks.rake +189 -0
  57. data/lib/generators/decision_agent/install/templates/decision_agent_tasks.rake +114 -0
  58. data/lib/generators/decision_agent/install/templates/decision_log.rb +57 -0
  59. data/lib/generators/decision_agent/install/templates/error_metric.rb +53 -0
  60. data/lib/generators/decision_agent/install/templates/evaluation_metric.rb +43 -0
  61. data/lib/generators/decision_agent/install/templates/monitoring_migration.rb +109 -0
  62. data/lib/generators/decision_agent/install/templates/performance_metric.rb +76 -0
  63. data/lib/generators/decision_agent/install/templates/rule_version.rb +1 -1
  64. data/spec/ab_testing/ab_test_assignment_spec.rb +253 -0
  65. data/spec/ab_testing/ab_test_manager_spec.rb +612 -0
  66. data/spec/ab_testing/ab_test_spec.rb +270 -0
  67. data/spec/ab_testing/ab_testing_agent_spec.rb +481 -0
  68. data/spec/ab_testing/storage/adapter_spec.rb +64 -0
  69. data/spec/ab_testing/storage/memory_adapter_spec.rb +485 -0
  70. data/spec/advanced_operators_spec.rb +1003 -0
  71. data/spec/agent_spec.rb +40 -0
  72. data/spec/audit_adapters_spec.rb +18 -0
  73. data/spec/auth/access_audit_logger_spec.rb +394 -0
  74. data/spec/auth/authenticator_spec.rb +112 -0
  75. data/spec/auth/password_reset_spec.rb +294 -0
  76. data/spec/auth/permission_checker_spec.rb +207 -0
  77. data/spec/auth/permission_spec.rb +73 -0
  78. data/spec/auth/rbac_adapter_spec.rb +550 -0
  79. data/spec/auth/rbac_config_spec.rb +82 -0
  80. data/spec/auth/role_spec.rb +51 -0
  81. data/spec/auth/session_manager_spec.rb +172 -0
  82. data/spec/auth/session_spec.rb +112 -0
  83. data/spec/auth/user_spec.rb +130 -0
  84. data/spec/context_spec.rb +43 -0
  85. data/spec/decision_agent_spec.rb +96 -0
  86. data/spec/decision_spec.rb +423 -0
  87. data/spec/dsl/condition_evaluator_spec.rb +774 -0
  88. data/spec/evaluation_spec.rb +364 -0
  89. data/spec/evaluation_validator_spec.rb +165 -0
  90. data/spec/examples.txt +1542 -548
  91. data/spec/issue_verification_spec.rb +95 -21
  92. data/spec/monitoring/metrics_collector_spec.rb +221 -3
  93. data/spec/monitoring/monitored_agent_spec.rb +1 -1
  94. data/spec/monitoring/prometheus_exporter_spec.rb +1 -1
  95. data/spec/monitoring/storage/activerecord_adapter_spec.rb +498 -0
  96. data/spec/monitoring/storage/base_adapter_spec.rb +61 -0
  97. data/spec/monitoring/storage/memory_adapter_spec.rb +247 -0
  98. data/spec/performance_optimizations_spec.rb +486 -0
  99. data/spec/spec_helper.rb +23 -0
  100. data/spec/testing/batch_test_importer_spec.rb +693 -0
  101. data/spec/testing/batch_test_runner_spec.rb +307 -0
  102. data/spec/testing/test_coverage_analyzer_spec.rb +292 -0
  103. data/spec/testing/test_result_comparator_spec.rb +392 -0
  104. data/spec/testing/test_scenario_spec.rb +113 -0
  105. data/spec/versioning/adapter_spec.rb +156 -0
  106. data/spec/versioning_spec.rb +253 -0
  107. data/spec/web/middleware/auth_middleware_spec.rb +133 -0
  108. data/spec/web/middleware/permission_middleware_spec.rb +247 -0
  109. data/spec/web_ui_rack_spec.rb +1705 -0
  110. metadata +123 -6
@@ -36,5 +36,64 @@ require_relative "decision_agent/monitoring/alert_manager"
36
36
  require_relative "decision_agent/monitoring/monitored_agent"
37
37
  # dashboard_server has additional dependencies (faye/websocket) - require it explicitly when needed
38
38
 
39
+ require_relative "decision_agent/ab_testing/ab_test"
40
+ require_relative "decision_agent/ab_testing/ab_test_assignment"
41
+ require_relative "decision_agent/ab_testing/ab_test_manager"
42
+ require_relative "decision_agent/ab_testing/ab_testing_agent"
43
+ require_relative "decision_agent/ab_testing/storage/adapter"
44
+ require_relative "decision_agent/ab_testing/storage/memory_adapter"
45
+
46
+ require_relative "decision_agent/testing/test_scenario"
47
+ require_relative "decision_agent/testing/batch_test_importer"
48
+ require_relative "decision_agent/testing/batch_test_runner"
49
+ require_relative "decision_agent/testing/test_result_comparator"
50
+ require_relative "decision_agent/testing/test_coverage_analyzer"
51
+
52
+ require_relative "decision_agent/auth/user"
53
+ require_relative "decision_agent/auth/role"
54
+ require_relative "decision_agent/auth/permission"
55
+ require_relative "decision_agent/auth/session"
56
+ require_relative "decision_agent/auth/session_manager"
57
+ require_relative "decision_agent/auth/password_reset_token"
58
+ require_relative "decision_agent/auth/password_reset_manager"
59
+ require_relative "decision_agent/auth/authenticator"
60
+ require_relative "decision_agent/auth/rbac_adapter"
61
+ require_relative "decision_agent/auth/rbac_config"
62
+ require_relative "decision_agent/auth/permission_checker"
63
+ require_relative "decision_agent/auth/access_audit_logger"
64
+
39
65
  module DecisionAgent
66
+ # Global RBAC configuration
67
+ @rbac_config = Auth::RbacConfig.new
68
+
69
+ class << self
70
+ attr_reader :rbac_config
71
+
72
+ # Configure RBAC adapter
73
+ # @param adapter_type [Symbol] :default, :devise_cancan, :pundit, or :custom
74
+ # @param options [Hash] Options for the adapter
75
+ # @yield [RbacConfig] Configuration block
76
+ # @example
77
+ # DecisionAgent.configure_rbac(:devise_cancan, ability_class: Ability)
78
+ # @example
79
+ # DecisionAgent.configure_rbac(:custom) do |config|
80
+ # config.adapter = MyCustomAdapter.new
81
+ # end
82
+ def configure_rbac(adapter_type = nil, **options)
83
+ if block_given?
84
+ yield @rbac_config
85
+ elsif adapter_type
86
+ @rbac_config.use(adapter_type, **options)
87
+ end
88
+ @rbac_config
89
+ end
90
+
91
+ # Get the configured permission checker
92
+ def permission_checker
93
+ @permission_checker ||= Auth::PermissionChecker.new(adapter: @rbac_config.adapter)
94
+ end
95
+
96
+ # Set a custom permission checker
97
+ attr_writer :permission_checker
98
+ end
40
99
  end
@@ -10,6 +10,14 @@ module DecisionAgent
10
10
 
11
11
  desc "Installs DecisionAgent models and migrations for Rails"
12
12
 
13
+ class_option :monitoring, type: :boolean,
14
+ default: false,
15
+ desc: "Install monitoring tables and models for persistent metrics storage"
16
+
17
+ class_option :ab_testing, type: :boolean,
18
+ default: false,
19
+ desc: "Install A/B testing tables and models for variant testing"
20
+
13
21
  def self.next_migration_number(dirname)
14
22
  next_migration_number = current_migration_number(dirname) + 1
15
23
  ActiveRecord::Migration.next_migration_number(next_migration_number)
@@ -19,11 +27,40 @@ module DecisionAgent
19
27
  migration_template "migration.rb",
20
28
  "db/migrate/create_decision_agent_tables.rb",
21
29
  migration_version: migration_version
30
+
31
+ if options[:monitoring]
32
+ migration_template "monitoring_migration.rb",
33
+ "db/migrate/create_decision_agent_monitoring_tables.rb",
34
+ migration_version: migration_version
35
+ end
36
+
37
+ return unless options[:ab_testing]
38
+
39
+ migration_template "ab_testing_migration.rb",
40
+ "db/migrate/create_decision_agent_ab_testing_tables.rb",
41
+ migration_version: migration_version
22
42
  end
23
43
 
24
44
  def copy_models
25
45
  copy_file "rule.rb", "app/models/rule.rb"
26
46
  copy_file "rule_version.rb", "app/models/rule_version.rb"
47
+
48
+ if options[:monitoring]
49
+ copy_file "decision_log.rb", "app/models/decision_log.rb"
50
+ copy_file "evaluation_metric.rb", "app/models/evaluation_metric.rb"
51
+ copy_file "performance_metric.rb", "app/models/performance_metric.rb"
52
+ copy_file "error_metric.rb", "app/models/error_metric.rb"
53
+ end
54
+
55
+ return unless options[:ab_testing]
56
+
57
+ copy_file "ab_test_model.rb", "app/models/ab_test_model.rb"
58
+ copy_file "ab_test_assignment_model.rb", "app/models/ab_test_assignment_model.rb"
59
+ end
60
+
61
+ def copy_rake_tasks
62
+ copy_file "decision_agent_tasks.rake", "lib/tasks/decision_agent.rake" if options[:monitoring]
63
+ copy_file "ab_testing_tasks.rake", "lib/tasks/ab_testing.rake" if options[:ab_testing]
27
64
  end
28
65
 
29
66
  def show_readme
@@ -0,0 +1,45 @@
1
+ # A/B Test Assignment model for decision_agent gem
2
+ # Tracks individual variant assignments and their results
3
+ class ABTestAssignmentModel < ActiveRecord::Base
4
+ belongs_to :ab_test_model
5
+
6
+ validates :variant, presence: true, inclusion: { in: %w[champion challenger] }
7
+ validates :version_id, presence: true
8
+ validates :timestamp, presence: true
9
+
10
+ before_validation :set_defaults
11
+
12
+ # Scopes
13
+ scope :champion, -> { where(variant: "champion") }
14
+ scope :challenger, -> { where(variant: "challenger") }
15
+ scope :with_decisions, -> { where.not(decision_result: nil) }
16
+ scope :recent, -> { order(timestamp: :desc) }
17
+ scope :for_user, ->(user_id) { where(user_id: user_id) }
18
+
19
+ serialize :context, JSON
20
+
21
+ # Check if decision has been recorded
22
+ def decision_recorded?
23
+ decision_result.present?
24
+ end
25
+
26
+ # Get the test this assignment belongs to
27
+ def test
28
+ ab_test_model
29
+ end
30
+
31
+ # Record decision result
32
+ def record_decision!(decision, confidence_score)
33
+ update!(
34
+ decision_result: decision,
35
+ confidence: confidence_score
36
+ )
37
+ end
38
+
39
+ private
40
+
41
+ def set_defaults
42
+ self.timestamp ||= Time.current
43
+ self.context ||= {}
44
+ end
45
+ end
@@ -0,0 +1,54 @@
1
+ # A/B Test model for decision_agent gem
2
+ # Stores A/B test configurations
3
+ class ABTestModel < ActiveRecord::Base
4
+ has_many :ab_test_assignment_models, dependent: :destroy
5
+
6
+ validates :name, presence: true
7
+ validates :champion_version_id, presence: true
8
+ validates :challenger_version_id, presence: true
9
+ validates :status, presence: true, inclusion: { in: %w[scheduled running completed cancelled] }
10
+
11
+ serialize :traffic_split, JSON
12
+
13
+ before_validation :set_defaults
14
+
15
+ # Scopes
16
+ scope :active, -> { where(status: "running") }
17
+ scope :scheduled, -> { where(status: "scheduled") }
18
+ scope :completed, -> { where(status: "completed") }
19
+ scope :running_or_scheduled, -> { where(status: %w[running scheduled]) }
20
+
21
+ # Check if test is currently running
22
+ def running?
23
+ status == "running" &&
24
+ (start_date.nil? || start_date <= Time.current) &&
25
+ (end_date.nil? || end_date > Time.current)
26
+ end
27
+
28
+ # Get statistics for this test
29
+ def statistics
30
+ {
31
+ total_assignments: ab_test_assignment_models.count,
32
+ champion_count: ab_test_assignment_models.where(variant: "champion").count,
33
+ challenger_count: ab_test_assignment_models.where(variant: "challenger").count,
34
+ with_decisions: ab_test_assignment_models.where.not(decision_result: nil).count,
35
+ avg_confidence: ab_test_assignment_models.where.not(confidence: nil).average(:confidence)&.to_f
36
+ }
37
+ end
38
+
39
+ # Get assignments by variant
40
+ def champion_assignments
41
+ ab_test_assignment_models.where(variant: "champion")
42
+ end
43
+
44
+ def challenger_assignments
45
+ ab_test_assignment_models.where(variant: "challenger")
46
+ end
47
+
48
+ private
49
+
50
+ def set_defaults
51
+ self.traffic_split ||= { champion: 90, challenger: 10 }
52
+ self.status ||= "scheduled"
53
+ end
54
+ end
@@ -0,0 +1,43 @@
1
+ class CreateDecisionAgentABTestingTables < ActiveRecord::Migration[7.0]
2
+ def change
3
+ # A/B Tests table
4
+ create_table :ab_test_models do |t|
5
+ t.string :name, null: false
6
+ t.string :champion_version_id, null: false
7
+ t.string :challenger_version_id, null: false
8
+ t.text :traffic_split, null: false # JSON: { champion: 90, challenger: 10 }
9
+ t.datetime :start_date
10
+ t.datetime :end_date
11
+ t.string :status, null: false, default: "scheduled" # scheduled, running, completed, cancelled
12
+ t.timestamps
13
+ end
14
+
15
+ add_index :ab_test_models, :status
16
+ add_index :ab_test_models, :start_date
17
+ add_index :ab_test_models, %i[status start_date], name: "index_ab_tests_on_status_and_start_date"
18
+
19
+ # A/B Test Assignments table
20
+ create_table :ab_test_assignment_models do |t|
21
+ t.references :ab_test_model, null: false, foreign_key: true, index: true
22
+ t.string :user_id # Optional: for consistent assignment to same users
23
+ t.string :variant, null: false # "champion" or "challenger"
24
+ t.string :version_id, null: false
25
+ t.datetime :timestamp, null: false, default: -> { "CURRENT_TIMESTAMP" }
26
+
27
+ # Decision results (populated after decision is made)
28
+ t.string :decision_result
29
+ t.float :confidence
30
+ t.text :context # JSON: additional context
31
+
32
+ t.timestamps
33
+ end
34
+
35
+ add_index :ab_test_assignment_models, :user_id
36
+ add_index :ab_test_assignment_models, :variant
37
+ add_index :ab_test_assignment_models, :timestamp
38
+ add_index :ab_test_assignment_models, %i[ab_test_model_id variant], name: "index_assignments_on_test_and_variant"
39
+
40
+ # Optional: Index for querying assignments with decisions
41
+ add_index :ab_test_assignment_models, :decision_result, where: "decision_result IS NOT NULL", name: "index_assignments_with_decisions"
42
+ end
43
+ end
@@ -0,0 +1,189 @@
1
+ # rubocop:disable Metrics/BlockLength
2
+ namespace :decision_agent do
3
+ namespace :ab_testing do
4
+ desc "List all A/B tests"
5
+ task list: :environment do
6
+ require "decision_agent/ab_testing/ab_test_manager"
7
+
8
+ manager = DecisionAgent::ABTesting::ABTestManager.new(
9
+ storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
10
+ )
11
+
12
+ tests = manager.list_tests
13
+ puts "\n📊 A/B Tests (Total: #{tests.size})\n"
14
+ puts "=" * 80
15
+
16
+ if tests.empty?
17
+ puts "No A/B tests found."
18
+ else
19
+ tests.each do |test|
20
+ puts "\nID: #{test.id}"
21
+ puts "Name: #{test.name}"
22
+ puts "Status: #{test.status}"
23
+ puts "Champion Version: #{test.champion_version_id}"
24
+ puts "Challenger Version: #{test.challenger_version_id}"
25
+ puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
26
+ puts "Start Date: #{test.start_date}"
27
+ puts "End Date: #{test.end_date || 'N/A'}"
28
+ puts "-" * 80
29
+ end
30
+ end
31
+ end
32
+
33
+ desc "Show A/B test results - Usage: rake decision_agent:ab_testing:results[test_id]"
34
+ task :results, [:test_id] => :environment do |_t, args|
35
+ require "decision_agent/ab_testing/ab_test_manager"
36
+
37
+ test_id = args[:test_id]
38
+ raise "Test ID is required. Usage: rake decision_agent:ab_testing:results[123]" unless test_id
39
+
40
+ manager = DecisionAgent::ABTesting::ABTestManager.new(
41
+ storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
42
+ )
43
+
44
+ results = manager.get_results(test_id)
45
+
46
+ puts "\n📊 A/B Test Results"
47
+ puts "=" * 80
48
+ puts "\nTest: #{results[:test][:name]}"
49
+ puts "Status: #{results[:test][:status]}"
50
+ puts "Total Assignments: #{results[:total_assignments]}"
51
+ puts ""
52
+
53
+ puts "🏆 Champion (Version #{results[:test][:champion_version_id]})"
54
+ puts " Assignments: #{results[:champion][:total_assignments]}"
55
+ puts " Decisions Recorded: #{results[:champion][:decisions_recorded]}"
56
+ if results[:champion][:avg_confidence]
57
+ puts " Avg Confidence: #{results[:champion][:avg_confidence]}"
58
+ puts " Min/Max Confidence: #{results[:champion][:min_confidence]} / #{results[:champion][:max_confidence]}"
59
+ end
60
+ puts ""
61
+
62
+ puts "🆕 Challenger (Version #{results[:test][:challenger_version_id]})"
63
+ puts " Assignments: #{results[:challenger][:total_assignments]}"
64
+ puts " Decisions Recorded: #{results[:challenger][:decisions_recorded]}"
65
+ if results[:challenger][:avg_confidence]
66
+ puts " Avg Confidence: #{results[:challenger][:avg_confidence]}"
67
+ puts " Min/Max Confidence: #{results[:challenger][:min_confidence]} / #{results[:challenger][:max_confidence]}"
68
+ end
69
+ puts ""
70
+
71
+ if results[:comparison][:statistical_significance] == "insufficient_data"
72
+ puts "⚠️ Insufficient data for statistical comparison"
73
+ else
74
+ puts "📈 Statistical Comparison"
75
+ puts " Improvement: #{results[:comparison][:improvement_percentage]}%"
76
+ puts " Winner: #{results[:comparison][:winner]}"
77
+ puts " Statistical Significance: #{results[:comparison][:statistical_significance]}"
78
+ puts " Confidence Level: #{(results[:comparison][:confidence_level] * 100).round(0)}%"
79
+ puts ""
80
+ puts "💡 Recommendation: #{results[:comparison][:recommendation]}"
81
+ end
82
+
83
+ puts "=" * 80
84
+ end
85
+
86
+ desc "Start an A/B test - Usage: rake decision_agent:ab_testing:start[test_id]"
87
+ task :start, [:test_id] => :environment do |_t, args|
88
+ require "decision_agent/ab_testing/ab_test_manager"
89
+
90
+ test_id = args[:test_id]
91
+ raise "Test ID is required. Usage: rake decision_agent:ab_testing:start[123]" unless test_id
92
+
93
+ manager = DecisionAgent::ABTesting::ABTestManager.new(
94
+ storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
95
+ )
96
+
97
+ manager.start_test(test_id)
98
+ puts "✅ A/B test #{test_id} started successfully!"
99
+ end
100
+
101
+ desc "Complete an A/B test - Usage: rake decision_agent:ab_testing:complete[test_id]"
102
+ task :complete, [:test_id] => :environment do |_t, args|
103
+ require "decision_agent/ab_testing/ab_test_manager"
104
+
105
+ test_id = args[:test_id]
106
+ raise "Test ID is required. Usage: rake decision_agent:ab_testing:complete[123]" unless test_id
107
+
108
+ manager = DecisionAgent::ABTesting::ABTestManager.new(
109
+ storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
110
+ )
111
+
112
+ manager.complete_test(test_id)
113
+ puts "✅ A/B test #{test_id} completed successfully!"
114
+ end
115
+
116
+ desc "Cancel an A/B test - Usage: rake decision_agent:ab_testing:cancel[test_id]"
117
+ task :cancel, [:test_id] => :environment do |_t, args|
118
+ require "decision_agent/ab_testing/ab_test_manager"
119
+
120
+ test_id = args[:test_id]
121
+ raise "Test ID is required. Usage: rake decision_agent:ab_testing:cancel[123]" unless test_id
122
+
123
+ manager = DecisionAgent::ABTesting::ABTestManager.new(
124
+ storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
125
+ )
126
+
127
+ manager.cancel_test(test_id)
128
+ puts "✅ A/B test #{test_id} cancelled successfully!"
129
+ end
130
+
131
+ desc "Create a new A/B test - Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
132
+ task :create, %i[name champion_id challenger_id split] => :environment do |_t, args|
133
+ require "decision_agent/ab_testing/ab_test_manager"
134
+
135
+ name = args[:name]
136
+ champion_id = args[:champion_id]
137
+ challenger_id = args[:challenger_id]
138
+ split = args[:split] || "90,10"
139
+
140
+ unless name && champion_id && challenger_id
141
+ raise "Missing arguments. Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
142
+ end
143
+
144
+ champion_pct, challenger_pct = split.split(",").map(&:to_i)
145
+
146
+ manager = DecisionAgent::ABTesting::ABTestManager.new(
147
+ storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
148
+ )
149
+
150
+ test = manager.create_test(
151
+ name: name,
152
+ champion_version_id: champion_id,
153
+ challenger_version_id: challenger_id,
154
+ traffic_split: { champion: champion_pct, challenger: challenger_pct }
155
+ )
156
+
157
+ puts "✅ A/B test created successfully!"
158
+ puts " ID: #{test.id}"
159
+ puts " Name: #{test.name}"
160
+ puts " Status: #{test.status}"
161
+ end
162
+
163
+ desc "Show active A/B tests"
164
+ task active: :environment do
165
+ require "decision_agent/ab_testing/ab_test_manager"
166
+
167
+ manager = DecisionAgent::ABTesting::ABTestManager.new(
168
+ storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
169
+ )
170
+
171
+ tests = manager.active_tests
172
+ puts "\n🔄 Active A/B Tests (Total: #{tests.size})\n"
173
+ puts "=" * 80
174
+
175
+ if tests.empty?
176
+ puts "No active A/B tests found."
177
+ else
178
+ tests.each do |test|
179
+ puts "\nID: #{test.id}"
180
+ puts "Name: #{test.name}"
181
+ puts "Champion vs Challenger: #{test.champion_version_id} vs #{test.challenger_version_id}"
182
+ puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
183
+ puts "-" * 80
184
+ end
185
+ end
186
+ end
187
+ end
188
+ end
189
+ # rubocop:enable Metrics/BlockLength
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ namespace :decision_agent do
4
+ namespace :monitoring do
5
+ desc "Cleanup old monitoring metrics (default: older than 30 days)"
6
+ task cleanup: :environment do
7
+ older_than = ENV.fetch("OLDER_THAN", 30 * 24 * 3600).to_i # Default: 30 days
8
+
9
+ puts "Cleaning up metrics older than #{older_than / 86_400} days..."
10
+
11
+ count = 0
12
+ count += DecisionLog.where("created_at < ?", Time.now - older_than).delete_all
13
+ count += EvaluationMetric.where("created_at < ?", Time.now - older_than).delete_all
14
+ count += PerformanceMetric.where("created_at < ?", Time.now - older_than).delete_all
15
+ count += ErrorMetric.where("created_at < ?", Time.now - older_than).delete_all
16
+
17
+ puts "Deleted #{count} old metric records"
18
+ end
19
+
20
+ desc "Show monitoring statistics"
21
+ task stats: :environment do
22
+ time_range = ENV.fetch("TIME_RANGE", 3600).to_i # Default: 1 hour
23
+
24
+ puts "\n=== Decision Agent Monitoring Statistics ==="
25
+ puts "Time range: Last #{time_range / 3600.0} hours\n\n"
26
+
27
+ decisions = DecisionLog.recent(time_range)
28
+ evaluations = EvaluationMetric.recent(time_range)
29
+ performance = PerformanceMetric.recent(time_range)
30
+ errors = ErrorMetric.recent(time_range)
31
+
32
+ puts "Decisions:"
33
+ puts " Total: #{decisions.count}"
34
+ puts " Average confidence: #{decisions.average(:confidence)&.round(4) || 'N/A'}"
35
+ puts " Success rate: #{(DecisionLog.success_rate(time_range: time_range) * 100).round(2)}%"
36
+ puts " By decision: #{decisions.group(:decision).count}"
37
+
38
+ puts "\nEvaluations:"
39
+ puts " Total: #{evaluations.count}"
40
+ puts " By evaluator: #{evaluations.group(:evaluator_name).count}"
41
+
42
+ puts "\nPerformance:"
43
+ puts " Total operations: #{performance.count}"
44
+ puts " Average duration: #{performance.average_duration(time_range: time_range)&.round(2) || 'N/A'} ms"
45
+ puts " P95 latency: #{performance.p95(time_range: time_range).round(2)} ms"
46
+ puts " P99 latency: #{performance.p99(time_range: time_range).round(2)} ms"
47
+ puts " Success rate: #{(performance.success_rate(time_range: time_range) * 100).round(2)}%"
48
+
49
+ puts "\nErrors:"
50
+ puts " Total: #{errors.count}"
51
+ puts " By type: #{errors.group(:error_type).count}"
52
+ puts " By severity: #{errors.group(:severity).count}"
53
+ puts " Critical: #{errors.critical.count}"
54
+
55
+ puts "\n=== End of Statistics ===\n"
56
+ end
57
+
58
+ desc "Archive old metrics to JSON file"
59
+ task :archive, [:output_file] => :environment do |_t, args|
60
+ output_file = args[:output_file] || "metrics_archive_#{Time.now.to_i}.json"
61
+ older_than = ENV.fetch("OLDER_THAN", 30 * 24 * 3600).to_i
62
+ cutoff_time = Time.now - older_than
63
+
64
+ puts "Archiving metrics older than #{older_than / 86_400} days to #{output_file}..."
65
+
66
+ archive_data = {
67
+ archived_at: Time.now.iso8601,
68
+ cutoff_time: cutoff_time.iso8601,
69
+ decisions: DecisionLog.where("created_at < ?", cutoff_time).map do |d|
70
+ {
71
+ decision: d.decision,
72
+ confidence: d.confidence,
73
+ context: d.parsed_context,
74
+ status: d.status,
75
+ created_at: d.created_at.iso8601
76
+ }
77
+ end,
78
+ evaluations: EvaluationMetric.where("created_at < ?", cutoff_time).map do |e|
79
+ {
80
+ evaluator_name: e.evaluator_name,
81
+ score: e.score,
82
+ success: e.success,
83
+ details: e.parsed_details,
84
+ created_at: e.created_at.iso8601
85
+ }
86
+ end,
87
+ performance: PerformanceMetric.where("created_at < ?", cutoff_time).map do |p|
88
+ {
89
+ operation: p.operation,
90
+ duration_ms: p.duration_ms,
91
+ status: p.status,
92
+ metadata: p.parsed_metadata,
93
+ created_at: p.created_at.iso8601
94
+ }
95
+ end,
96
+ errors: ErrorMetric.where("created_at < ?", cutoff_time).map do |e|
97
+ {
98
+ error_type: e.error_type,
99
+ message: e.message,
100
+ severity: e.severity,
101
+ context: e.parsed_context,
102
+ created_at: e.created_at.iso8601
103
+ }
104
+ end
105
+ }
106
+
107
+ File.write(output_file, JSON.pretty_generate(archive_data))
108
+
109
+ total = archive_data.values.sum { |v| v.is_a?(Array) ? v.size : 0 }
110
+ puts "Archived #{total} metrics to #{output_file}"
111
+ puts "Run 'rake decision_agent:monitoring:cleanup' to delete these records from the database"
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ class DecisionLog < ApplicationRecord
4
+ has_many :evaluation_metrics, dependent: :destroy
5
+
6
+ validates :decision, presence: true
7
+ validates :confidence, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
8
+ validates :status, inclusion: { in: %w[success failure error] }, allow_nil: true
9
+
10
+ scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
11
+ scope :successful, -> { where(status: "success") }
12
+ scope :failed, -> { where(status: "failure") }
13
+ scope :with_errors, -> { where(status: "error") }
14
+ scope :by_decision, ->(decision) { where(decision: decision) }
15
+ scope :low_confidence, ->(threshold = 0.5) { where("confidence < ?", threshold) }
16
+ scope :high_confidence, ->(threshold = 0.8) { where("confidence >= ?", threshold) }
17
+
18
+ # Time series aggregation helpers
19
+ def self.count_by_time_bucket(bucket_size: 60, time_range: 3600)
20
+ recent(time_range)
21
+ .group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
22
+ .count
23
+ end
24
+
25
+ def self.average_confidence_by_time(bucket_size: 60, time_range: 3600)
26
+ recent(time_range)
27
+ .where.not(confidence: nil)
28
+ .group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
29
+ .average(:confidence)
30
+ end
31
+
32
+ def self.success_rate(time_range: 3600)
33
+ total = recent(time_range).where.not(status: nil).count
34
+ return 0.0 if total.zero?
35
+
36
+ successful_count = recent(time_range).successful.count
37
+ successful_count.to_f / total
38
+ end
39
+
40
+ # Parse JSON context field
41
+ def parsed_context
42
+ return {} if context.nil?
43
+
44
+ JSON.parse(context, symbolize_names: true)
45
+ rescue JSON::ParserError
46
+ {}
47
+ end
48
+
49
+ # Parse JSON metadata field
50
+ def parsed_metadata
51
+ return {} if metadata.nil?
52
+
53
+ JSON.parse(metadata, symbolize_names: true)
54
+ rescue JSON::ParserError
55
+ {}
56
+ end
57
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ class ErrorMetric < ApplicationRecord
4
+ validates :error_type, presence: true
5
+ validates :severity, inclusion: { in: %w[low medium high critical] }, allow_nil: true
6
+
7
+ scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
8
+ scope :by_type, ->(error_type) { where(error_type: error_type) }
9
+ scope :by_severity, ->(severity) { where(severity: severity) }
10
+ scope :critical, -> { where(severity: "critical") }
11
+ scope :high_severity, -> { where(severity: %w[high critical]) }
12
+
13
+ # Aggregation helpers
14
+ def self.count_by_type(time_range: 3600)
15
+ recent(time_range).group(:error_type).count
16
+ end
17
+
18
+ def self.count_by_severity(time_range: 3600)
19
+ recent(time_range).group(:severity).count
20
+ end
21
+
22
+ def self.error_rate(time_range: 3600, total_operations: nil)
23
+ error_count = recent(time_range).count
24
+ return 0.0 if total_operations.nil? || total_operations.zero?
25
+
26
+ error_count.to_f / total_operations
27
+ end
28
+
29
+ # Time series aggregation
30
+ def self.count_by_time_bucket(bucket_size: 60, time_range: 3600)
31
+ recent(time_range)
32
+ .group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
33
+ .count
34
+ end
35
+
36
+ # Parse JSON context field
37
+ def parsed_context
38
+ return {} if context.nil?
39
+
40
+ JSON.parse(context, symbolize_names: true)
41
+ rescue JSON::ParserError
42
+ {}
43
+ end
44
+
45
+ # Parse JSON stack_trace field
46
+ def parsed_stack_trace
47
+ return [] if stack_trace.nil?
48
+
49
+ JSON.parse(stack_trace)
50
+ rescue JSON::ParserError
51
+ []
52
+ end
53
+ end