decision_agent 0.1.3 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +84 -233
- data/lib/decision_agent/ab_testing/ab_test.rb +197 -0
- data/lib/decision_agent/ab_testing/ab_test_assignment.rb +76 -0
- data/lib/decision_agent/ab_testing/ab_test_manager.rb +317 -0
- data/lib/decision_agent/ab_testing/ab_testing_agent.rb +188 -0
- data/lib/decision_agent/ab_testing/storage/activerecord_adapter.rb +155 -0
- data/lib/decision_agent/ab_testing/storage/adapter.rb +67 -0
- data/lib/decision_agent/ab_testing/storage/memory_adapter.rb +116 -0
- data/lib/decision_agent/agent.rb +5 -3
- data/lib/decision_agent/auth/access_audit_logger.rb +122 -0
- data/lib/decision_agent/auth/authenticator.rb +127 -0
- data/lib/decision_agent/auth/password_reset_manager.rb +57 -0
- data/lib/decision_agent/auth/password_reset_token.rb +33 -0
- data/lib/decision_agent/auth/permission.rb +29 -0
- data/lib/decision_agent/auth/permission_checker.rb +43 -0
- data/lib/decision_agent/auth/rbac_adapter.rb +278 -0
- data/lib/decision_agent/auth/rbac_config.rb +51 -0
- data/lib/decision_agent/auth/role.rb +56 -0
- data/lib/decision_agent/auth/session.rb +33 -0
- data/lib/decision_agent/auth/session_manager.rb +57 -0
- data/lib/decision_agent/auth/user.rb +70 -0
- data/lib/decision_agent/context.rb +24 -4
- data/lib/decision_agent/decision.rb +10 -3
- data/lib/decision_agent/dsl/condition_evaluator.rb +378 -1
- data/lib/decision_agent/dsl/schema_validator.rb +8 -1
- data/lib/decision_agent/errors.rb +38 -0
- data/lib/decision_agent/evaluation.rb +10 -3
- data/lib/decision_agent/evaluation_validator.rb +8 -13
- data/lib/decision_agent/monitoring/dashboard_server.rb +1 -0
- data/lib/decision_agent/monitoring/metrics_collector.rb +164 -7
- data/lib/decision_agent/monitoring/storage/activerecord_adapter.rb +253 -0
- data/lib/decision_agent/monitoring/storage/base_adapter.rb +90 -0
- data/lib/decision_agent/monitoring/storage/memory_adapter.rb +222 -0
- data/lib/decision_agent/testing/batch_test_importer.rb +373 -0
- data/lib/decision_agent/testing/batch_test_runner.rb +244 -0
- data/lib/decision_agent/testing/test_coverage_analyzer.rb +191 -0
- data/lib/decision_agent/testing/test_result_comparator.rb +235 -0
- data/lib/decision_agent/testing/test_scenario.rb +42 -0
- data/lib/decision_agent/version.rb +10 -1
- data/lib/decision_agent/versioning/activerecord_adapter.rb +1 -1
- data/lib/decision_agent/versioning/file_storage_adapter.rb +96 -28
- data/lib/decision_agent/web/middleware/auth_middleware.rb +45 -0
- data/lib/decision_agent/web/middleware/permission_middleware.rb +94 -0
- data/lib/decision_agent/web/public/app.js +184 -29
- data/lib/decision_agent/web/public/batch_testing.html +640 -0
- data/lib/decision_agent/web/public/index.html +37 -9
- data/lib/decision_agent/web/public/login.html +298 -0
- data/lib/decision_agent/web/public/users.html +679 -0
- data/lib/decision_agent/web/server.rb +873 -7
- data/lib/decision_agent.rb +59 -0
- data/lib/generators/decision_agent/install/install_generator.rb +37 -0
- data/lib/generators/decision_agent/install/templates/ab_test_assignment_model.rb +45 -0
- data/lib/generators/decision_agent/install/templates/ab_test_model.rb +54 -0
- data/lib/generators/decision_agent/install/templates/ab_testing_migration.rb +43 -0
- data/lib/generators/decision_agent/install/templates/ab_testing_tasks.rake +189 -0
- data/lib/generators/decision_agent/install/templates/decision_agent_tasks.rake +114 -0
- data/lib/generators/decision_agent/install/templates/decision_log.rb +57 -0
- data/lib/generators/decision_agent/install/templates/error_metric.rb +53 -0
- data/lib/generators/decision_agent/install/templates/evaluation_metric.rb +43 -0
- data/lib/generators/decision_agent/install/templates/monitoring_migration.rb +109 -0
- data/lib/generators/decision_agent/install/templates/performance_metric.rb +76 -0
- data/lib/generators/decision_agent/install/templates/rule_version.rb +1 -1
- data/spec/ab_testing/ab_test_assignment_spec.rb +253 -0
- data/spec/ab_testing/ab_test_manager_spec.rb +612 -0
- data/spec/ab_testing/ab_test_spec.rb +270 -0
- data/spec/ab_testing/ab_testing_agent_spec.rb +481 -0
- data/spec/ab_testing/storage/adapter_spec.rb +64 -0
- data/spec/ab_testing/storage/memory_adapter_spec.rb +485 -0
- data/spec/advanced_operators_spec.rb +1003 -0
- data/spec/agent_spec.rb +40 -0
- data/spec/audit_adapters_spec.rb +18 -0
- data/spec/auth/access_audit_logger_spec.rb +394 -0
- data/spec/auth/authenticator_spec.rb +112 -0
- data/spec/auth/password_reset_spec.rb +294 -0
- data/spec/auth/permission_checker_spec.rb +207 -0
- data/spec/auth/permission_spec.rb +73 -0
- data/spec/auth/rbac_adapter_spec.rb +550 -0
- data/spec/auth/rbac_config_spec.rb +82 -0
- data/spec/auth/role_spec.rb +51 -0
- data/spec/auth/session_manager_spec.rb +172 -0
- data/spec/auth/session_spec.rb +112 -0
- data/spec/auth/user_spec.rb +130 -0
- data/spec/context_spec.rb +43 -0
- data/spec/decision_agent_spec.rb +96 -0
- data/spec/decision_spec.rb +423 -0
- data/spec/dsl/condition_evaluator_spec.rb +774 -0
- data/spec/evaluation_spec.rb +364 -0
- data/spec/evaluation_validator_spec.rb +165 -0
- data/spec/examples.txt +1542 -548
- data/spec/issue_verification_spec.rb +95 -21
- data/spec/monitoring/metrics_collector_spec.rb +221 -3
- data/spec/monitoring/monitored_agent_spec.rb +1 -1
- data/spec/monitoring/prometheus_exporter_spec.rb +1 -1
- data/spec/monitoring/storage/activerecord_adapter_spec.rb +498 -0
- data/spec/monitoring/storage/base_adapter_spec.rb +61 -0
- data/spec/monitoring/storage/memory_adapter_spec.rb +247 -0
- data/spec/performance_optimizations_spec.rb +486 -0
- data/spec/spec_helper.rb +23 -0
- data/spec/testing/batch_test_importer_spec.rb +693 -0
- data/spec/testing/batch_test_runner_spec.rb +307 -0
- data/spec/testing/test_coverage_analyzer_spec.rb +292 -0
- data/spec/testing/test_result_comparator_spec.rb +392 -0
- data/spec/testing/test_scenario_spec.rb +113 -0
- data/spec/versioning/adapter_spec.rb +156 -0
- data/spec/versioning_spec.rb +253 -0
- data/spec/web/middleware/auth_middleware_spec.rb +133 -0
- data/spec/web/middleware/permission_middleware_spec.rb +247 -0
- data/spec/web_ui_rack_spec.rb +1705 -0
- metadata +123 -6
data/lib/decision_agent.rb
CHANGED
|
@@ -36,5 +36,64 @@ require_relative "decision_agent/monitoring/alert_manager"
|
|
|
36
36
|
require_relative "decision_agent/monitoring/monitored_agent"
|
|
37
37
|
# dashboard_server has additional dependencies (faye/websocket) - require it explicitly when needed
|
|
38
38
|
|
|
39
|
+
require_relative "decision_agent/ab_testing/ab_test"
|
|
40
|
+
require_relative "decision_agent/ab_testing/ab_test_assignment"
|
|
41
|
+
require_relative "decision_agent/ab_testing/ab_test_manager"
|
|
42
|
+
require_relative "decision_agent/ab_testing/ab_testing_agent"
|
|
43
|
+
require_relative "decision_agent/ab_testing/storage/adapter"
|
|
44
|
+
require_relative "decision_agent/ab_testing/storage/memory_adapter"
|
|
45
|
+
|
|
46
|
+
require_relative "decision_agent/testing/test_scenario"
|
|
47
|
+
require_relative "decision_agent/testing/batch_test_importer"
|
|
48
|
+
require_relative "decision_agent/testing/batch_test_runner"
|
|
49
|
+
require_relative "decision_agent/testing/test_result_comparator"
|
|
50
|
+
require_relative "decision_agent/testing/test_coverage_analyzer"
|
|
51
|
+
|
|
52
|
+
require_relative "decision_agent/auth/user"
|
|
53
|
+
require_relative "decision_agent/auth/role"
|
|
54
|
+
require_relative "decision_agent/auth/permission"
|
|
55
|
+
require_relative "decision_agent/auth/session"
|
|
56
|
+
require_relative "decision_agent/auth/session_manager"
|
|
57
|
+
require_relative "decision_agent/auth/password_reset_token"
|
|
58
|
+
require_relative "decision_agent/auth/password_reset_manager"
|
|
59
|
+
require_relative "decision_agent/auth/authenticator"
|
|
60
|
+
require_relative "decision_agent/auth/rbac_adapter"
|
|
61
|
+
require_relative "decision_agent/auth/rbac_config"
|
|
62
|
+
require_relative "decision_agent/auth/permission_checker"
|
|
63
|
+
require_relative "decision_agent/auth/access_audit_logger"
|
|
64
|
+
|
|
39
65
|
module DecisionAgent
|
|
66
|
+
# Global RBAC configuration
|
|
67
|
+
@rbac_config = Auth::RbacConfig.new
|
|
68
|
+
|
|
69
|
+
class << self
|
|
70
|
+
attr_reader :rbac_config
|
|
71
|
+
|
|
72
|
+
# Configure RBAC adapter
|
|
73
|
+
# @param adapter_type [Symbol] :default, :devise_cancan, :pundit, or :custom
|
|
74
|
+
# @param options [Hash] Options for the adapter
|
|
75
|
+
# @yield [RbacConfig] Configuration block
|
|
76
|
+
# @example
|
|
77
|
+
# DecisionAgent.configure_rbac(:devise_cancan, ability_class: Ability)
|
|
78
|
+
# @example
|
|
79
|
+
# DecisionAgent.configure_rbac(:custom) do |config|
|
|
80
|
+
# config.adapter = MyCustomAdapter.new
|
|
81
|
+
# end
|
|
82
|
+
def configure_rbac(adapter_type = nil, **options)
|
|
83
|
+
if block_given?
|
|
84
|
+
yield @rbac_config
|
|
85
|
+
elsif adapter_type
|
|
86
|
+
@rbac_config.use(adapter_type, **options)
|
|
87
|
+
end
|
|
88
|
+
@rbac_config
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Get the configured permission checker
|
|
92
|
+
def permission_checker
|
|
93
|
+
@permission_checker ||= Auth::PermissionChecker.new(adapter: @rbac_config.adapter)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Set a custom permission checker
|
|
97
|
+
attr_writer :permission_checker
|
|
98
|
+
end
|
|
40
99
|
end
|
|
@@ -10,6 +10,14 @@ module DecisionAgent
|
|
|
10
10
|
|
|
11
11
|
desc "Installs DecisionAgent models and migrations for Rails"
|
|
12
12
|
|
|
13
|
+
class_option :monitoring, type: :boolean,
|
|
14
|
+
default: false,
|
|
15
|
+
desc: "Install monitoring tables and models for persistent metrics storage"
|
|
16
|
+
|
|
17
|
+
class_option :ab_testing, type: :boolean,
|
|
18
|
+
default: false,
|
|
19
|
+
desc: "Install A/B testing tables and models for variant testing"
|
|
20
|
+
|
|
13
21
|
def self.next_migration_number(dirname)
|
|
14
22
|
next_migration_number = current_migration_number(dirname) + 1
|
|
15
23
|
ActiveRecord::Migration.next_migration_number(next_migration_number)
|
|
@@ -19,11 +27,40 @@ module DecisionAgent
|
|
|
19
27
|
migration_template "migration.rb",
|
|
20
28
|
"db/migrate/create_decision_agent_tables.rb",
|
|
21
29
|
migration_version: migration_version
|
|
30
|
+
|
|
31
|
+
if options[:monitoring]
|
|
32
|
+
migration_template "monitoring_migration.rb",
|
|
33
|
+
"db/migrate/create_decision_agent_monitoring_tables.rb",
|
|
34
|
+
migration_version: migration_version
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
return unless options[:ab_testing]
|
|
38
|
+
|
|
39
|
+
migration_template "ab_testing_migration.rb",
|
|
40
|
+
"db/migrate/create_decision_agent_ab_testing_tables.rb",
|
|
41
|
+
migration_version: migration_version
|
|
22
42
|
end
|
|
23
43
|
|
|
24
44
|
def copy_models
|
|
25
45
|
copy_file "rule.rb", "app/models/rule.rb"
|
|
26
46
|
copy_file "rule_version.rb", "app/models/rule_version.rb"
|
|
47
|
+
|
|
48
|
+
if options[:monitoring]
|
|
49
|
+
copy_file "decision_log.rb", "app/models/decision_log.rb"
|
|
50
|
+
copy_file "evaluation_metric.rb", "app/models/evaluation_metric.rb"
|
|
51
|
+
copy_file "performance_metric.rb", "app/models/performance_metric.rb"
|
|
52
|
+
copy_file "error_metric.rb", "app/models/error_metric.rb"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
return unless options[:ab_testing]
|
|
56
|
+
|
|
57
|
+
copy_file "ab_test_model.rb", "app/models/ab_test_model.rb"
|
|
58
|
+
copy_file "ab_test_assignment_model.rb", "app/models/ab_test_assignment_model.rb"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def copy_rake_tasks
|
|
62
|
+
copy_file "decision_agent_tasks.rake", "lib/tasks/decision_agent.rake" if options[:monitoring]
|
|
63
|
+
copy_file "ab_testing_tasks.rake", "lib/tasks/ab_testing.rake" if options[:ab_testing]
|
|
27
64
|
end
|
|
28
65
|
|
|
29
66
|
def show_readme
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# A/B Test Assignment model for decision_agent gem
|
|
2
|
+
# Tracks individual variant assignments and their results
|
|
3
|
+
class ABTestAssignmentModel < ActiveRecord::Base
|
|
4
|
+
belongs_to :ab_test_model
|
|
5
|
+
|
|
6
|
+
validates :variant, presence: true, inclusion: { in: %w[champion challenger] }
|
|
7
|
+
validates :version_id, presence: true
|
|
8
|
+
validates :timestamp, presence: true
|
|
9
|
+
|
|
10
|
+
before_validation :set_defaults
|
|
11
|
+
|
|
12
|
+
# Scopes
|
|
13
|
+
scope :champion, -> { where(variant: "champion") }
|
|
14
|
+
scope :challenger, -> { where(variant: "challenger") }
|
|
15
|
+
scope :with_decisions, -> { where.not(decision_result: nil) }
|
|
16
|
+
scope :recent, -> { order(timestamp: :desc) }
|
|
17
|
+
scope :for_user, ->(user_id) { where(user_id: user_id) }
|
|
18
|
+
|
|
19
|
+
serialize :context, JSON
|
|
20
|
+
|
|
21
|
+
# Check if decision has been recorded
|
|
22
|
+
def decision_recorded?
|
|
23
|
+
decision_result.present?
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Get the test this assignment belongs to
|
|
27
|
+
def test
|
|
28
|
+
ab_test_model
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Record decision result
|
|
32
|
+
def record_decision!(decision, confidence_score)
|
|
33
|
+
update!(
|
|
34
|
+
decision_result: decision,
|
|
35
|
+
confidence: confidence_score
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def set_defaults
|
|
42
|
+
self.timestamp ||= Time.current
|
|
43
|
+
self.context ||= {}
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# A/B Test model for decision_agent gem
|
|
2
|
+
# Stores A/B test configurations
|
|
3
|
+
class ABTestModel < ActiveRecord::Base
|
|
4
|
+
has_many :ab_test_assignment_models, dependent: :destroy
|
|
5
|
+
|
|
6
|
+
validates :name, presence: true
|
|
7
|
+
validates :champion_version_id, presence: true
|
|
8
|
+
validates :challenger_version_id, presence: true
|
|
9
|
+
validates :status, presence: true, inclusion: { in: %w[scheduled running completed cancelled] }
|
|
10
|
+
|
|
11
|
+
serialize :traffic_split, JSON
|
|
12
|
+
|
|
13
|
+
before_validation :set_defaults
|
|
14
|
+
|
|
15
|
+
# Scopes
|
|
16
|
+
scope :active, -> { where(status: "running") }
|
|
17
|
+
scope :scheduled, -> { where(status: "scheduled") }
|
|
18
|
+
scope :completed, -> { where(status: "completed") }
|
|
19
|
+
scope :running_or_scheduled, -> { where(status: %w[running scheduled]) }
|
|
20
|
+
|
|
21
|
+
# Check if test is currently running
|
|
22
|
+
def running?
|
|
23
|
+
status == "running" &&
|
|
24
|
+
(start_date.nil? || start_date <= Time.current) &&
|
|
25
|
+
(end_date.nil? || end_date > Time.current)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Get statistics for this test
|
|
29
|
+
def statistics
|
|
30
|
+
{
|
|
31
|
+
total_assignments: ab_test_assignment_models.count,
|
|
32
|
+
champion_count: ab_test_assignment_models.where(variant: "champion").count,
|
|
33
|
+
challenger_count: ab_test_assignment_models.where(variant: "challenger").count,
|
|
34
|
+
with_decisions: ab_test_assignment_models.where.not(decision_result: nil).count,
|
|
35
|
+
avg_confidence: ab_test_assignment_models.where.not(confidence: nil).average(:confidence)&.to_f
|
|
36
|
+
}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Get assignments by variant
|
|
40
|
+
def champion_assignments
|
|
41
|
+
ab_test_assignment_models.where(variant: "champion")
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def challenger_assignments
|
|
45
|
+
ab_test_assignment_models.where(variant: "challenger")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def set_defaults
|
|
51
|
+
self.traffic_split ||= { champion: 90, challenger: 10 }
|
|
52
|
+
self.status ||= "scheduled"
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
class CreateDecisionAgentABTestingTables < ActiveRecord::Migration[7.0]
|
|
2
|
+
def change
|
|
3
|
+
# A/B Tests table
|
|
4
|
+
create_table :ab_test_models do |t|
|
|
5
|
+
t.string :name, null: false
|
|
6
|
+
t.string :champion_version_id, null: false
|
|
7
|
+
t.string :challenger_version_id, null: false
|
|
8
|
+
t.text :traffic_split, null: false # JSON: { champion: 90, challenger: 10 }
|
|
9
|
+
t.datetime :start_date
|
|
10
|
+
t.datetime :end_date
|
|
11
|
+
t.string :status, null: false, default: "scheduled" # scheduled, running, completed, cancelled
|
|
12
|
+
t.timestamps
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
add_index :ab_test_models, :status
|
|
16
|
+
add_index :ab_test_models, :start_date
|
|
17
|
+
add_index :ab_test_models, %i[status start_date], name: "index_ab_tests_on_status_and_start_date"
|
|
18
|
+
|
|
19
|
+
# A/B Test Assignments table
|
|
20
|
+
create_table :ab_test_assignment_models do |t|
|
|
21
|
+
t.references :ab_test_model, null: false, foreign_key: true, index: true
|
|
22
|
+
t.string :user_id # Optional: for consistent assignment to same users
|
|
23
|
+
t.string :variant, null: false # "champion" or "challenger"
|
|
24
|
+
t.string :version_id, null: false
|
|
25
|
+
t.datetime :timestamp, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
26
|
+
|
|
27
|
+
# Decision results (populated after decision is made)
|
|
28
|
+
t.string :decision_result
|
|
29
|
+
t.float :confidence
|
|
30
|
+
t.text :context # JSON: additional context
|
|
31
|
+
|
|
32
|
+
t.timestamps
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
add_index :ab_test_assignment_models, :user_id
|
|
36
|
+
add_index :ab_test_assignment_models, :variant
|
|
37
|
+
add_index :ab_test_assignment_models, :timestamp
|
|
38
|
+
add_index :ab_test_assignment_models, %i[ab_test_model_id variant], name: "index_assignments_on_test_and_variant"
|
|
39
|
+
|
|
40
|
+
# Optional: Index for querying assignments with decisions
|
|
41
|
+
add_index :ab_test_assignment_models, :decision_result, where: "decision_result IS NOT NULL", name: "index_assignments_with_decisions"
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# rubocop:disable Metrics/BlockLength
|
|
2
|
+
namespace :decision_agent do
|
|
3
|
+
namespace :ab_testing do
|
|
4
|
+
desc "List all A/B tests"
|
|
5
|
+
task list: :environment do
|
|
6
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
7
|
+
|
|
8
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
9
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
tests = manager.list_tests
|
|
13
|
+
puts "\n📊 A/B Tests (Total: #{tests.size})\n"
|
|
14
|
+
puts "=" * 80
|
|
15
|
+
|
|
16
|
+
if tests.empty?
|
|
17
|
+
puts "No A/B tests found."
|
|
18
|
+
else
|
|
19
|
+
tests.each do |test|
|
|
20
|
+
puts "\nID: #{test.id}"
|
|
21
|
+
puts "Name: #{test.name}"
|
|
22
|
+
puts "Status: #{test.status}"
|
|
23
|
+
puts "Champion Version: #{test.champion_version_id}"
|
|
24
|
+
puts "Challenger Version: #{test.challenger_version_id}"
|
|
25
|
+
puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
|
|
26
|
+
puts "Start Date: #{test.start_date}"
|
|
27
|
+
puts "End Date: #{test.end_date || 'N/A'}"
|
|
28
|
+
puts "-" * 80
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
desc "Show A/B test results - Usage: rake decision_agent:ab_testing:results[test_id]"
|
|
34
|
+
task :results, [:test_id] => :environment do |_t, args|
|
|
35
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
36
|
+
|
|
37
|
+
test_id = args[:test_id]
|
|
38
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:results[123]" unless test_id
|
|
39
|
+
|
|
40
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
41
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
results = manager.get_results(test_id)
|
|
45
|
+
|
|
46
|
+
puts "\n📊 A/B Test Results"
|
|
47
|
+
puts "=" * 80
|
|
48
|
+
puts "\nTest: #{results[:test][:name]}"
|
|
49
|
+
puts "Status: #{results[:test][:status]}"
|
|
50
|
+
puts "Total Assignments: #{results[:total_assignments]}"
|
|
51
|
+
puts ""
|
|
52
|
+
|
|
53
|
+
puts "🏆 Champion (Version #{results[:test][:champion_version_id]})"
|
|
54
|
+
puts " Assignments: #{results[:champion][:total_assignments]}"
|
|
55
|
+
puts " Decisions Recorded: #{results[:champion][:decisions_recorded]}"
|
|
56
|
+
if results[:champion][:avg_confidence]
|
|
57
|
+
puts " Avg Confidence: #{results[:champion][:avg_confidence]}"
|
|
58
|
+
puts " Min/Max Confidence: #{results[:champion][:min_confidence]} / #{results[:champion][:max_confidence]}"
|
|
59
|
+
end
|
|
60
|
+
puts ""
|
|
61
|
+
|
|
62
|
+
puts "🆕 Challenger (Version #{results[:test][:challenger_version_id]})"
|
|
63
|
+
puts " Assignments: #{results[:challenger][:total_assignments]}"
|
|
64
|
+
puts " Decisions Recorded: #{results[:challenger][:decisions_recorded]}"
|
|
65
|
+
if results[:challenger][:avg_confidence]
|
|
66
|
+
puts " Avg Confidence: #{results[:challenger][:avg_confidence]}"
|
|
67
|
+
puts " Min/Max Confidence: #{results[:challenger][:min_confidence]} / #{results[:challenger][:max_confidence]}"
|
|
68
|
+
end
|
|
69
|
+
puts ""
|
|
70
|
+
|
|
71
|
+
if results[:comparison][:statistical_significance] == "insufficient_data"
|
|
72
|
+
puts "⚠️ Insufficient data for statistical comparison"
|
|
73
|
+
else
|
|
74
|
+
puts "📈 Statistical Comparison"
|
|
75
|
+
puts " Improvement: #{results[:comparison][:improvement_percentage]}%"
|
|
76
|
+
puts " Winner: #{results[:comparison][:winner]}"
|
|
77
|
+
puts " Statistical Significance: #{results[:comparison][:statistical_significance]}"
|
|
78
|
+
puts " Confidence Level: #{(results[:comparison][:confidence_level] * 100).round(0)}%"
|
|
79
|
+
puts ""
|
|
80
|
+
puts "💡 Recommendation: #{results[:comparison][:recommendation]}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
puts "=" * 80
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
desc "Start an A/B test - Usage: rake decision_agent:ab_testing:start[test_id]"
|
|
87
|
+
task :start, [:test_id] => :environment do |_t, args|
|
|
88
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
89
|
+
|
|
90
|
+
test_id = args[:test_id]
|
|
91
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:start[123]" unless test_id
|
|
92
|
+
|
|
93
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
94
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
manager.start_test(test_id)
|
|
98
|
+
puts "✅ A/B test #{test_id} started successfully!"
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
desc "Complete an A/B test - Usage: rake decision_agent:ab_testing:complete[test_id]"
|
|
102
|
+
task :complete, [:test_id] => :environment do |_t, args|
|
|
103
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
104
|
+
|
|
105
|
+
test_id = args[:test_id]
|
|
106
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:complete[123]" unless test_id
|
|
107
|
+
|
|
108
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
109
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
manager.complete_test(test_id)
|
|
113
|
+
puts "✅ A/B test #{test_id} completed successfully!"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
desc "Cancel an A/B test - Usage: rake decision_agent:ab_testing:cancel[test_id]"
|
|
117
|
+
task :cancel, [:test_id] => :environment do |_t, args|
|
|
118
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
119
|
+
|
|
120
|
+
test_id = args[:test_id]
|
|
121
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:cancel[123]" unless test_id
|
|
122
|
+
|
|
123
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
124
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
manager.cancel_test(test_id)
|
|
128
|
+
puts "✅ A/B test #{test_id} cancelled successfully!"
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
desc "Create a new A/B test - Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
|
|
132
|
+
task :create, %i[name champion_id challenger_id split] => :environment do |_t, args|
|
|
133
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
134
|
+
|
|
135
|
+
name = args[:name]
|
|
136
|
+
champion_id = args[:champion_id]
|
|
137
|
+
challenger_id = args[:challenger_id]
|
|
138
|
+
split = args[:split] || "90,10"
|
|
139
|
+
|
|
140
|
+
unless name && champion_id && challenger_id
|
|
141
|
+
raise "Missing arguments. Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
champion_pct, challenger_pct = split.split(",").map(&:to_i)
|
|
145
|
+
|
|
146
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
147
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
test = manager.create_test(
|
|
151
|
+
name: name,
|
|
152
|
+
champion_version_id: champion_id,
|
|
153
|
+
challenger_version_id: challenger_id,
|
|
154
|
+
traffic_split: { champion: champion_pct, challenger: challenger_pct }
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
puts "✅ A/B test created successfully!"
|
|
158
|
+
puts " ID: #{test.id}"
|
|
159
|
+
puts " Name: #{test.name}"
|
|
160
|
+
puts " Status: #{test.status}"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
desc "Show active A/B tests"
|
|
164
|
+
task active: :environment do
|
|
165
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
166
|
+
|
|
167
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
168
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
tests = manager.active_tests
|
|
172
|
+
puts "\n🔄 Active A/B Tests (Total: #{tests.size})\n"
|
|
173
|
+
puts "=" * 80
|
|
174
|
+
|
|
175
|
+
if tests.empty?
|
|
176
|
+
puts "No active A/B tests found."
|
|
177
|
+
else
|
|
178
|
+
tests.each do |test|
|
|
179
|
+
puts "\nID: #{test.id}"
|
|
180
|
+
puts "Name: #{test.name}"
|
|
181
|
+
puts "Champion vs Challenger: #{test.champion_version_id} vs #{test.challenger_version_id}"
|
|
182
|
+
puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
|
|
183
|
+
puts "-" * 80
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
# rubocop:enable Metrics/BlockLength
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :decision_agent do
|
|
4
|
+
namespace :monitoring do
|
|
5
|
+
desc "Cleanup old monitoring metrics (default: older than 30 days)"
|
|
6
|
+
task cleanup: :environment do
|
|
7
|
+
older_than = ENV.fetch("OLDER_THAN", 30 * 24 * 3600).to_i # Default: 30 days
|
|
8
|
+
|
|
9
|
+
puts "Cleaning up metrics older than #{older_than / 86_400} days..."
|
|
10
|
+
|
|
11
|
+
count = 0
|
|
12
|
+
count += DecisionLog.where("created_at < ?", Time.now - older_than).delete_all
|
|
13
|
+
count += EvaluationMetric.where("created_at < ?", Time.now - older_than).delete_all
|
|
14
|
+
count += PerformanceMetric.where("created_at < ?", Time.now - older_than).delete_all
|
|
15
|
+
count += ErrorMetric.where("created_at < ?", Time.now - older_than).delete_all
|
|
16
|
+
|
|
17
|
+
puts "Deleted #{count} old metric records"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
desc "Show monitoring statistics"
|
|
21
|
+
task stats: :environment do
|
|
22
|
+
time_range = ENV.fetch("TIME_RANGE", 3600).to_i # Default: 1 hour
|
|
23
|
+
|
|
24
|
+
puts "\n=== Decision Agent Monitoring Statistics ==="
|
|
25
|
+
puts "Time range: Last #{time_range / 3600.0} hours\n\n"
|
|
26
|
+
|
|
27
|
+
decisions = DecisionLog.recent(time_range)
|
|
28
|
+
evaluations = EvaluationMetric.recent(time_range)
|
|
29
|
+
performance = PerformanceMetric.recent(time_range)
|
|
30
|
+
errors = ErrorMetric.recent(time_range)
|
|
31
|
+
|
|
32
|
+
puts "Decisions:"
|
|
33
|
+
puts " Total: #{decisions.count}"
|
|
34
|
+
puts " Average confidence: #{decisions.average(:confidence)&.round(4) || 'N/A'}"
|
|
35
|
+
puts " Success rate: #{(DecisionLog.success_rate(time_range: time_range) * 100).round(2)}%"
|
|
36
|
+
puts " By decision: #{decisions.group(:decision).count}"
|
|
37
|
+
|
|
38
|
+
puts "\nEvaluations:"
|
|
39
|
+
puts " Total: #{evaluations.count}"
|
|
40
|
+
puts " By evaluator: #{evaluations.group(:evaluator_name).count}"
|
|
41
|
+
|
|
42
|
+
puts "\nPerformance:"
|
|
43
|
+
puts " Total operations: #{performance.count}"
|
|
44
|
+
puts " Average duration: #{performance.average_duration(time_range: time_range)&.round(2) || 'N/A'} ms"
|
|
45
|
+
puts " P95 latency: #{performance.p95(time_range: time_range).round(2)} ms"
|
|
46
|
+
puts " P99 latency: #{performance.p99(time_range: time_range).round(2)} ms"
|
|
47
|
+
puts " Success rate: #{(performance.success_rate(time_range: time_range) * 100).round(2)}%"
|
|
48
|
+
|
|
49
|
+
puts "\nErrors:"
|
|
50
|
+
puts " Total: #{errors.count}"
|
|
51
|
+
puts " By type: #{errors.group(:error_type).count}"
|
|
52
|
+
puts " By severity: #{errors.group(:severity).count}"
|
|
53
|
+
puts " Critical: #{errors.critical.count}"
|
|
54
|
+
|
|
55
|
+
puts "\n=== End of Statistics ===\n"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
desc "Archive old metrics to JSON file"
|
|
59
|
+
task :archive, [:output_file] => :environment do |_t, args|
|
|
60
|
+
output_file = args[:output_file] || "metrics_archive_#{Time.now.to_i}.json"
|
|
61
|
+
older_than = ENV.fetch("OLDER_THAN", 30 * 24 * 3600).to_i
|
|
62
|
+
cutoff_time = Time.now - older_than
|
|
63
|
+
|
|
64
|
+
puts "Archiving metrics older than #{older_than / 86_400} days to #{output_file}..."
|
|
65
|
+
|
|
66
|
+
archive_data = {
|
|
67
|
+
archived_at: Time.now.iso8601,
|
|
68
|
+
cutoff_time: cutoff_time.iso8601,
|
|
69
|
+
decisions: DecisionLog.where("created_at < ?", cutoff_time).map do |d|
|
|
70
|
+
{
|
|
71
|
+
decision: d.decision,
|
|
72
|
+
confidence: d.confidence,
|
|
73
|
+
context: d.parsed_context,
|
|
74
|
+
status: d.status,
|
|
75
|
+
created_at: d.created_at.iso8601
|
|
76
|
+
}
|
|
77
|
+
end,
|
|
78
|
+
evaluations: EvaluationMetric.where("created_at < ?", cutoff_time).map do |e|
|
|
79
|
+
{
|
|
80
|
+
evaluator_name: e.evaluator_name,
|
|
81
|
+
score: e.score,
|
|
82
|
+
success: e.success,
|
|
83
|
+
details: e.parsed_details,
|
|
84
|
+
created_at: e.created_at.iso8601
|
|
85
|
+
}
|
|
86
|
+
end,
|
|
87
|
+
performance: PerformanceMetric.where("created_at < ?", cutoff_time).map do |p|
|
|
88
|
+
{
|
|
89
|
+
operation: p.operation,
|
|
90
|
+
duration_ms: p.duration_ms,
|
|
91
|
+
status: p.status,
|
|
92
|
+
metadata: p.parsed_metadata,
|
|
93
|
+
created_at: p.created_at.iso8601
|
|
94
|
+
}
|
|
95
|
+
end,
|
|
96
|
+
errors: ErrorMetric.where("created_at < ?", cutoff_time).map do |e|
|
|
97
|
+
{
|
|
98
|
+
error_type: e.error_type,
|
|
99
|
+
message: e.message,
|
|
100
|
+
severity: e.severity,
|
|
101
|
+
context: e.parsed_context,
|
|
102
|
+
created_at: e.created_at.iso8601
|
|
103
|
+
}
|
|
104
|
+
end
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
File.write(output_file, JSON.pretty_generate(archive_data))
|
|
108
|
+
|
|
109
|
+
total = archive_data.values.sum { |v| v.is_a?(Array) ? v.size : 0 }
|
|
110
|
+
puts "Archived #{total} metrics to #{output_file}"
|
|
111
|
+
puts "Run 'rake decision_agent:monitoring:cleanup' to delete these records from the database"
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class DecisionLog < ApplicationRecord
|
|
4
|
+
has_many :evaluation_metrics, dependent: :destroy
|
|
5
|
+
|
|
6
|
+
validates :decision, presence: true
|
|
7
|
+
validates :confidence, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
|
|
8
|
+
validates :status, inclusion: { in: %w[success failure error] }, allow_nil: true
|
|
9
|
+
|
|
10
|
+
scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
|
|
11
|
+
scope :successful, -> { where(status: "success") }
|
|
12
|
+
scope :failed, -> { where(status: "failure") }
|
|
13
|
+
scope :with_errors, -> { where(status: "error") }
|
|
14
|
+
scope :by_decision, ->(decision) { where(decision: decision) }
|
|
15
|
+
scope :low_confidence, ->(threshold = 0.5) { where("confidence < ?", threshold) }
|
|
16
|
+
scope :high_confidence, ->(threshold = 0.8) { where("confidence >= ?", threshold) }
|
|
17
|
+
|
|
18
|
+
# Time series aggregation helpers
|
|
19
|
+
def self.count_by_time_bucket(bucket_size: 60, time_range: 3600)
|
|
20
|
+
recent(time_range)
|
|
21
|
+
.group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
|
|
22
|
+
.count
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.average_confidence_by_time(bucket_size: 60, time_range: 3600)
|
|
26
|
+
recent(time_range)
|
|
27
|
+
.where.not(confidence: nil)
|
|
28
|
+
.group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
|
|
29
|
+
.average(:confidence)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def self.success_rate(time_range: 3600)
|
|
33
|
+
total = recent(time_range).where.not(status: nil).count
|
|
34
|
+
return 0.0 if total.zero?
|
|
35
|
+
|
|
36
|
+
successful_count = recent(time_range).successful.count
|
|
37
|
+
successful_count.to_f / total
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Parse JSON context field
|
|
41
|
+
def parsed_context
|
|
42
|
+
return {} if context.nil?
|
|
43
|
+
|
|
44
|
+
JSON.parse(context, symbolize_names: true)
|
|
45
|
+
rescue JSON::ParserError
|
|
46
|
+
{}
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Parse JSON metadata field
|
|
50
|
+
def parsed_metadata
|
|
51
|
+
return {} if metadata.nil?
|
|
52
|
+
|
|
53
|
+
JSON.parse(metadata, symbolize_names: true)
|
|
54
|
+
rescue JSON::ParserError
|
|
55
|
+
{}
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class ErrorMetric < ApplicationRecord
|
|
4
|
+
validates :error_type, presence: true
|
|
5
|
+
validates :severity, inclusion: { in: %w[low medium high critical] }, allow_nil: true
|
|
6
|
+
|
|
7
|
+
scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
|
|
8
|
+
scope :by_type, ->(error_type) { where(error_type: error_type) }
|
|
9
|
+
scope :by_severity, ->(severity) { where(severity: severity) }
|
|
10
|
+
scope :critical, -> { where(severity: "critical") }
|
|
11
|
+
scope :high_severity, -> { where(severity: %w[high critical]) }
|
|
12
|
+
|
|
13
|
+
# Aggregation helpers
|
|
14
|
+
def self.count_by_type(time_range: 3600)
|
|
15
|
+
recent(time_range).group(:error_type).count
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.count_by_severity(time_range: 3600)
|
|
19
|
+
recent(time_range).group(:severity).count
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.error_rate(time_range: 3600, total_operations: nil)
|
|
23
|
+
error_count = recent(time_range).count
|
|
24
|
+
return 0.0 if total_operations.nil? || total_operations.zero?
|
|
25
|
+
|
|
26
|
+
error_count.to_f / total_operations
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Time series aggregation
|
|
30
|
+
def self.count_by_time_bucket(bucket_size: 60, time_range: 3600)
|
|
31
|
+
recent(time_range)
|
|
32
|
+
.group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
|
|
33
|
+
.count
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Parse JSON context field
|
|
37
|
+
def parsed_context
|
|
38
|
+
return {} if context.nil?
|
|
39
|
+
|
|
40
|
+
JSON.parse(context, symbolize_names: true)
|
|
41
|
+
rescue JSON::ParserError
|
|
42
|
+
{}
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Parse JSON stack_trace field
|
|
46
|
+
def parsed_stack_trace
|
|
47
|
+
return [] if stack_trace.nil?
|
|
48
|
+
|
|
49
|
+
JSON.parse(stack_trace)
|
|
50
|
+
rescue JSON::ParserError
|
|
51
|
+
[]
|
|
52
|
+
end
|
|
53
|
+
end
|