decision_agent 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +212 -35
- data/bin/decision_agent +3 -8
- data/lib/decision_agent/ab_testing/ab_test.rb +197 -0
- data/lib/decision_agent/ab_testing/ab_test_assignment.rb +76 -0
- data/lib/decision_agent/ab_testing/ab_test_manager.rb +317 -0
- data/lib/decision_agent/ab_testing/ab_testing_agent.rb +152 -0
- data/lib/decision_agent/ab_testing/storage/activerecord_adapter.rb +155 -0
- data/lib/decision_agent/ab_testing/storage/adapter.rb +67 -0
- data/lib/decision_agent/ab_testing/storage/memory_adapter.rb +116 -0
- data/lib/decision_agent/agent.rb +19 -26
- data/lib/decision_agent/audit/null_adapter.rb +1 -2
- data/lib/decision_agent/decision.rb +3 -1
- data/lib/decision_agent/dsl/condition_evaluator.rb +4 -3
- data/lib/decision_agent/dsl/rule_parser.rb +4 -6
- data/lib/decision_agent/dsl/schema_validator.rb +27 -31
- data/lib/decision_agent/errors.rb +11 -8
- data/lib/decision_agent/evaluation.rb +3 -1
- data/lib/decision_agent/evaluation_validator.rb +78 -0
- data/lib/decision_agent/evaluators/json_rule_evaluator.rb +26 -0
- data/lib/decision_agent/evaluators/static_evaluator.rb +2 -6
- data/lib/decision_agent/monitoring/alert_manager.rb +282 -0
- data/lib/decision_agent/monitoring/dashboard/public/dashboard.css +381 -0
- data/lib/decision_agent/monitoring/dashboard/public/dashboard.js +471 -0
- data/lib/decision_agent/monitoring/dashboard/public/index.html +161 -0
- data/lib/decision_agent/monitoring/dashboard_server.rb +340 -0
- data/lib/decision_agent/monitoring/metrics_collector.rb +423 -0
- data/lib/decision_agent/monitoring/monitored_agent.rb +71 -0
- data/lib/decision_agent/monitoring/prometheus_exporter.rb +247 -0
- data/lib/decision_agent/monitoring/storage/activerecord_adapter.rb +253 -0
- data/lib/decision_agent/monitoring/storage/base_adapter.rb +90 -0
- data/lib/decision_agent/monitoring/storage/memory_adapter.rb +222 -0
- data/lib/decision_agent/replay/replay.rb +12 -22
- data/lib/decision_agent/scoring/base.rb +1 -1
- data/lib/decision_agent/scoring/consensus.rb +5 -5
- data/lib/decision_agent/scoring/weighted_average.rb +1 -1
- data/lib/decision_agent/version.rb +1 -1
- data/lib/decision_agent/versioning/activerecord_adapter.rb +69 -33
- data/lib/decision_agent/versioning/adapter.rb +1 -3
- data/lib/decision_agent/versioning/file_storage_adapter.rb +143 -35
- data/lib/decision_agent/versioning/version_manager.rb +4 -12
- data/lib/decision_agent/web/public/index.html +1 -1
- data/lib/decision_agent/web/server.rb +19 -24
- data/lib/decision_agent.rb +14 -0
- data/lib/generators/decision_agent/install/install_generator.rb +42 -5
- data/lib/generators/decision_agent/install/templates/ab_test_assignment_model.rb +45 -0
- data/lib/generators/decision_agent/install/templates/ab_test_model.rb +54 -0
- data/lib/generators/decision_agent/install/templates/ab_testing_migration.rb +43 -0
- data/lib/generators/decision_agent/install/templates/ab_testing_tasks.rake +189 -0
- data/lib/generators/decision_agent/install/templates/decision_agent_tasks.rake +114 -0
- data/lib/generators/decision_agent/install/templates/decision_log.rb +57 -0
- data/lib/generators/decision_agent/install/templates/error_metric.rb +53 -0
- data/lib/generators/decision_agent/install/templates/evaluation_metric.rb +43 -0
- data/lib/generators/decision_agent/install/templates/migration.rb +17 -6
- data/lib/generators/decision_agent/install/templates/monitoring_migration.rb +109 -0
- data/lib/generators/decision_agent/install/templates/performance_metric.rb +76 -0
- data/lib/generators/decision_agent/install/templates/rule.rb +3 -3
- data/lib/generators/decision_agent/install/templates/rule_version.rb +13 -7
- data/spec/ab_testing/ab_test_manager_spec.rb +330 -0
- data/spec/ab_testing/ab_test_spec.rb +270 -0
- data/spec/activerecord_thread_safety_spec.rb +553 -0
- data/spec/agent_spec.rb +13 -13
- data/spec/api_contract_spec.rb +16 -16
- data/spec/audit_adapters_spec.rb +3 -3
- data/spec/comprehensive_edge_cases_spec.rb +86 -86
- data/spec/dsl_validation_spec.rb +83 -83
- data/spec/edge_cases_spec.rb +23 -23
- data/spec/examples/feedback_aware_evaluator_spec.rb +7 -7
- data/spec/examples.txt +612 -0
- data/spec/issue_verification_spec.rb +759 -0
- data/spec/json_rule_evaluator_spec.rb +15 -15
- data/spec/monitoring/alert_manager_spec.rb +378 -0
- data/spec/monitoring/metrics_collector_spec.rb +281 -0
- data/spec/monitoring/monitored_agent_spec.rb +222 -0
- data/spec/monitoring/prometheus_exporter_spec.rb +242 -0
- data/spec/monitoring/storage/activerecord_adapter_spec.rb +346 -0
- data/spec/monitoring/storage/memory_adapter_spec.rb +247 -0
- data/spec/replay_edge_cases_spec.rb +58 -58
- data/spec/replay_spec.rb +11 -11
- data/spec/rfc8785_canonicalization_spec.rb +215 -0
- data/spec/scoring_spec.rb +1 -1
- data/spec/spec_helper.rb +9 -0
- data/spec/thread_safety_spec.rb +482 -0
- data/spec/thread_safety_spec.rb.broken +878 -0
- data/spec/versioning_spec.rb +141 -37
- data/spec/web_ui_rack_spec.rb +135 -0
- metadata +93 -6
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
class CreateDecisionAgentABTestingTables < ActiveRecord::Migration[7.0]
|
|
2
|
+
def change
|
|
3
|
+
# A/B Tests table
|
|
4
|
+
create_table :ab_test_models do |t|
|
|
5
|
+
t.string :name, null: false
|
|
6
|
+
t.string :champion_version_id, null: false
|
|
7
|
+
t.string :challenger_version_id, null: false
|
|
8
|
+
t.text :traffic_split, null: false # JSON: { champion: 90, challenger: 10 }
|
|
9
|
+
t.datetime :start_date
|
|
10
|
+
t.datetime :end_date
|
|
11
|
+
t.string :status, null: false, default: "scheduled" # scheduled, running, completed, cancelled
|
|
12
|
+
t.timestamps
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
add_index :ab_test_models, :status
|
|
16
|
+
add_index :ab_test_models, :start_date
|
|
17
|
+
add_index :ab_test_models, %i[status start_date], name: "index_ab_tests_on_status_and_start_date"
|
|
18
|
+
|
|
19
|
+
# A/B Test Assignments table
|
|
20
|
+
create_table :ab_test_assignment_models do |t|
|
|
21
|
+
t.references :ab_test_model, null: false, foreign_key: true, index: true
|
|
22
|
+
t.string :user_id # Optional: for consistent assignment to same users
|
|
23
|
+
t.string :variant, null: false # "champion" or "challenger"
|
|
24
|
+
t.string :version_id, null: false
|
|
25
|
+
t.datetime :timestamp, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
26
|
+
|
|
27
|
+
# Decision results (populated after decision is made)
|
|
28
|
+
t.string :decision_result
|
|
29
|
+
t.float :confidence
|
|
30
|
+
t.text :context # JSON: additional context
|
|
31
|
+
|
|
32
|
+
t.timestamps
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
add_index :ab_test_assignment_models, :user_id
|
|
36
|
+
add_index :ab_test_assignment_models, :variant
|
|
37
|
+
add_index :ab_test_assignment_models, :timestamp
|
|
38
|
+
add_index :ab_test_assignment_models, %i[ab_test_model_id variant], name: "index_assignments_on_test_and_variant"
|
|
39
|
+
|
|
40
|
+
# Optional: Index for querying assignments with decisions
|
|
41
|
+
add_index :ab_test_assignment_models, :decision_result, where: "decision_result IS NOT NULL", name: "index_assignments_with_decisions"
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# rubocop:disable Metrics/BlockLength
|
|
2
|
+
namespace :decision_agent do
|
|
3
|
+
namespace :ab_testing do
|
|
4
|
+
desc "List all A/B tests"
|
|
5
|
+
task list: :environment do
|
|
6
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
7
|
+
|
|
8
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
9
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
tests = manager.list_tests
|
|
13
|
+
puts "\n📊 A/B Tests (Total: #{tests.size})\n"
|
|
14
|
+
puts "=" * 80
|
|
15
|
+
|
|
16
|
+
if tests.empty?
|
|
17
|
+
puts "No A/B tests found."
|
|
18
|
+
else
|
|
19
|
+
tests.each do |test|
|
|
20
|
+
puts "\nID: #{test.id}"
|
|
21
|
+
puts "Name: #{test.name}"
|
|
22
|
+
puts "Status: #{test.status}"
|
|
23
|
+
puts "Champion Version: #{test.champion_version_id}"
|
|
24
|
+
puts "Challenger Version: #{test.challenger_version_id}"
|
|
25
|
+
puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
|
|
26
|
+
puts "Start Date: #{test.start_date}"
|
|
27
|
+
puts "End Date: #{test.end_date || 'N/A'}"
|
|
28
|
+
puts "-" * 80
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
desc "Show A/B test results - Usage: rake decision_agent:ab_testing:results[test_id]"
|
|
34
|
+
task :results, [:test_id] => :environment do |_t, args|
|
|
35
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
36
|
+
|
|
37
|
+
test_id = args[:test_id]
|
|
38
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:results[123]" unless test_id
|
|
39
|
+
|
|
40
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
41
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
results = manager.get_results(test_id)
|
|
45
|
+
|
|
46
|
+
puts "\n📊 A/B Test Results"
|
|
47
|
+
puts "=" * 80
|
|
48
|
+
puts "\nTest: #{results[:test][:name]}"
|
|
49
|
+
puts "Status: #{results[:test][:status]}"
|
|
50
|
+
puts "Total Assignments: #{results[:total_assignments]}"
|
|
51
|
+
puts ""
|
|
52
|
+
|
|
53
|
+
puts "🏆 Champion (Version #{results[:test][:champion_version_id]})"
|
|
54
|
+
puts " Assignments: #{results[:champion][:total_assignments]}"
|
|
55
|
+
puts " Decisions Recorded: #{results[:champion][:decisions_recorded]}"
|
|
56
|
+
if results[:champion][:avg_confidence]
|
|
57
|
+
puts " Avg Confidence: #{results[:champion][:avg_confidence]}"
|
|
58
|
+
puts " Min/Max Confidence: #{results[:champion][:min_confidence]} / #{results[:champion][:max_confidence]}"
|
|
59
|
+
end
|
|
60
|
+
puts ""
|
|
61
|
+
|
|
62
|
+
puts "🆕 Challenger (Version #{results[:test][:challenger_version_id]})"
|
|
63
|
+
puts " Assignments: #{results[:challenger][:total_assignments]}"
|
|
64
|
+
puts " Decisions Recorded: #{results[:challenger][:decisions_recorded]}"
|
|
65
|
+
if results[:challenger][:avg_confidence]
|
|
66
|
+
puts " Avg Confidence: #{results[:challenger][:avg_confidence]}"
|
|
67
|
+
puts " Min/Max Confidence: #{results[:challenger][:min_confidence]} / #{results[:challenger][:max_confidence]}"
|
|
68
|
+
end
|
|
69
|
+
puts ""
|
|
70
|
+
|
|
71
|
+
if results[:comparison][:statistical_significance] == "insufficient_data"
|
|
72
|
+
puts "⚠️ Insufficient data for statistical comparison"
|
|
73
|
+
else
|
|
74
|
+
puts "📈 Statistical Comparison"
|
|
75
|
+
puts " Improvement: #{results[:comparison][:improvement_percentage]}%"
|
|
76
|
+
puts " Winner: #{results[:comparison][:winner]}"
|
|
77
|
+
puts " Statistical Significance: #{results[:comparison][:statistical_significance]}"
|
|
78
|
+
puts " Confidence Level: #{(results[:comparison][:confidence_level] * 100).round(0)}%"
|
|
79
|
+
puts ""
|
|
80
|
+
puts "💡 Recommendation: #{results[:comparison][:recommendation]}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
puts "=" * 80
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
desc "Start an A/B test - Usage: rake decision_agent:ab_testing:start[test_id]"
|
|
87
|
+
task :start, [:test_id] => :environment do |_t, args|
|
|
88
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
89
|
+
|
|
90
|
+
test_id = args[:test_id]
|
|
91
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:start[123]" unless test_id
|
|
92
|
+
|
|
93
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
94
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
manager.start_test(test_id)
|
|
98
|
+
puts "✅ A/B test #{test_id} started successfully!"
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
desc "Complete an A/B test - Usage: rake decision_agent:ab_testing:complete[test_id]"
|
|
102
|
+
task :complete, [:test_id] => :environment do |_t, args|
|
|
103
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
104
|
+
|
|
105
|
+
test_id = args[:test_id]
|
|
106
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:complete[123]" unless test_id
|
|
107
|
+
|
|
108
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
109
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
manager.complete_test(test_id)
|
|
113
|
+
puts "✅ A/B test #{test_id} completed successfully!"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
desc "Cancel an A/B test - Usage: rake decision_agent:ab_testing:cancel[test_id]"
|
|
117
|
+
task :cancel, [:test_id] => :environment do |_t, args|
|
|
118
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
119
|
+
|
|
120
|
+
test_id = args[:test_id]
|
|
121
|
+
raise "Test ID is required. Usage: rake decision_agent:ab_testing:cancel[123]" unless test_id
|
|
122
|
+
|
|
123
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
124
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
manager.cancel_test(test_id)
|
|
128
|
+
puts "✅ A/B test #{test_id} cancelled successfully!"
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
desc "Create a new A/B test - Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
|
|
132
|
+
task :create, %i[name champion_id challenger_id split] => :environment do |_t, args|
|
|
133
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
134
|
+
|
|
135
|
+
name = args[:name]
|
|
136
|
+
champion_id = args[:champion_id]
|
|
137
|
+
challenger_id = args[:challenger_id]
|
|
138
|
+
split = args[:split] || "90,10"
|
|
139
|
+
|
|
140
|
+
unless name && champion_id && challenger_id
|
|
141
|
+
raise "Missing arguments. Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
champion_pct, challenger_pct = split.split(",").map(&:to_i)
|
|
145
|
+
|
|
146
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
147
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
test = manager.create_test(
|
|
151
|
+
name: name,
|
|
152
|
+
champion_version_id: champion_id,
|
|
153
|
+
challenger_version_id: challenger_id,
|
|
154
|
+
traffic_split: { champion: champion_pct, challenger: challenger_pct }
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
puts "✅ A/B test created successfully!"
|
|
158
|
+
puts " ID: #{test.id}"
|
|
159
|
+
puts " Name: #{test.name}"
|
|
160
|
+
puts " Status: #{test.status}"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
desc "Show active A/B tests"
|
|
164
|
+
task active: :environment do
|
|
165
|
+
require "decision_agent/ab_testing/ab_test_manager"
|
|
166
|
+
|
|
167
|
+
manager = DecisionAgent::ABTesting::ABTestManager.new(
|
|
168
|
+
storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
tests = manager.active_tests
|
|
172
|
+
puts "\n🔄 Active A/B Tests (Total: #{tests.size})\n"
|
|
173
|
+
puts "=" * 80
|
|
174
|
+
|
|
175
|
+
if tests.empty?
|
|
176
|
+
puts "No active A/B tests found."
|
|
177
|
+
else
|
|
178
|
+
tests.each do |test|
|
|
179
|
+
puts "\nID: #{test.id}"
|
|
180
|
+
puts "Name: #{test.name}"
|
|
181
|
+
puts "Champion vs Challenger: #{test.champion_version_id} vs #{test.challenger_version_id}"
|
|
182
|
+
puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
|
|
183
|
+
puts "-" * 80
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
# rubocop:enable Metrics/BlockLength
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :decision_agent do
|
|
4
|
+
namespace :monitoring do
|
|
5
|
+
desc "Cleanup old monitoring metrics (default: older than 30 days)"
|
|
6
|
+
task cleanup: :environment do
|
|
7
|
+
older_than = ENV.fetch("OLDER_THAN", 30 * 24 * 3600).to_i # Default: 30 days
|
|
8
|
+
|
|
9
|
+
puts "Cleaning up metrics older than #{older_than / 86_400} days..."
|
|
10
|
+
|
|
11
|
+
count = 0
|
|
12
|
+
count += DecisionLog.where("created_at < ?", Time.now - older_than).delete_all
|
|
13
|
+
count += EvaluationMetric.where("created_at < ?", Time.now - older_than).delete_all
|
|
14
|
+
count += PerformanceMetric.where("created_at < ?", Time.now - older_than).delete_all
|
|
15
|
+
count += ErrorMetric.where("created_at < ?", Time.now - older_than).delete_all
|
|
16
|
+
|
|
17
|
+
puts "Deleted #{count} old metric records"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
desc "Show monitoring statistics"
|
|
21
|
+
task stats: :environment do
|
|
22
|
+
time_range = ENV.fetch("TIME_RANGE", 3600).to_i # Default: 1 hour
|
|
23
|
+
|
|
24
|
+
puts "\n=== Decision Agent Monitoring Statistics ==="
|
|
25
|
+
puts "Time range: Last #{time_range / 3600.0} hours\n\n"
|
|
26
|
+
|
|
27
|
+
decisions = DecisionLog.recent(time_range)
|
|
28
|
+
evaluations = EvaluationMetric.recent(time_range)
|
|
29
|
+
performance = PerformanceMetric.recent(time_range)
|
|
30
|
+
errors = ErrorMetric.recent(time_range)
|
|
31
|
+
|
|
32
|
+
puts "Decisions:"
|
|
33
|
+
puts " Total: #{decisions.count}"
|
|
34
|
+
puts " Average confidence: #{decisions.average(:confidence)&.round(4) || 'N/A'}"
|
|
35
|
+
puts " Success rate: #{(DecisionLog.success_rate(time_range: time_range) * 100).round(2)}%"
|
|
36
|
+
puts " By decision: #{decisions.group(:decision).count}"
|
|
37
|
+
|
|
38
|
+
puts "\nEvaluations:"
|
|
39
|
+
puts " Total: #{evaluations.count}"
|
|
40
|
+
puts " By evaluator: #{evaluations.group(:evaluator_name).count}"
|
|
41
|
+
|
|
42
|
+
puts "\nPerformance:"
|
|
43
|
+
puts " Total operations: #{performance.count}"
|
|
44
|
+
puts " Average duration: #{performance.average_duration(time_range: time_range)&.round(2) || 'N/A'} ms"
|
|
45
|
+
puts " P95 latency: #{performance.p95(time_range: time_range).round(2)} ms"
|
|
46
|
+
puts " P99 latency: #{performance.p99(time_range: time_range).round(2)} ms"
|
|
47
|
+
puts " Success rate: #{(performance.success_rate(time_range: time_range) * 100).round(2)}%"
|
|
48
|
+
|
|
49
|
+
puts "\nErrors:"
|
|
50
|
+
puts " Total: #{errors.count}"
|
|
51
|
+
puts " By type: #{errors.group(:error_type).count}"
|
|
52
|
+
puts " By severity: #{errors.group(:severity).count}"
|
|
53
|
+
puts " Critical: #{errors.critical.count}"
|
|
54
|
+
|
|
55
|
+
puts "\n=== End of Statistics ===\n"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
desc "Archive old metrics to JSON file"
|
|
59
|
+
task :archive, [:output_file] => :environment do |_t, args|
|
|
60
|
+
output_file = args[:output_file] || "metrics_archive_#{Time.now.to_i}.json"
|
|
61
|
+
older_than = ENV.fetch("OLDER_THAN", 30 * 24 * 3600).to_i
|
|
62
|
+
cutoff_time = Time.now - older_than
|
|
63
|
+
|
|
64
|
+
puts "Archiving metrics older than #{older_than / 86_400} days to #{output_file}..."
|
|
65
|
+
|
|
66
|
+
archive_data = {
|
|
67
|
+
archived_at: Time.now.iso8601,
|
|
68
|
+
cutoff_time: cutoff_time.iso8601,
|
|
69
|
+
decisions: DecisionLog.where("created_at < ?", cutoff_time).map do |d|
|
|
70
|
+
{
|
|
71
|
+
decision: d.decision,
|
|
72
|
+
confidence: d.confidence,
|
|
73
|
+
context: d.parsed_context,
|
|
74
|
+
status: d.status,
|
|
75
|
+
created_at: d.created_at.iso8601
|
|
76
|
+
}
|
|
77
|
+
end,
|
|
78
|
+
evaluations: EvaluationMetric.where("created_at < ?", cutoff_time).map do |e|
|
|
79
|
+
{
|
|
80
|
+
evaluator_name: e.evaluator_name,
|
|
81
|
+
score: e.score,
|
|
82
|
+
success: e.success,
|
|
83
|
+
details: e.parsed_details,
|
|
84
|
+
created_at: e.created_at.iso8601
|
|
85
|
+
}
|
|
86
|
+
end,
|
|
87
|
+
performance: PerformanceMetric.where("created_at < ?", cutoff_time).map do |p|
|
|
88
|
+
{
|
|
89
|
+
operation: p.operation,
|
|
90
|
+
duration_ms: p.duration_ms,
|
|
91
|
+
status: p.status,
|
|
92
|
+
metadata: p.parsed_metadata,
|
|
93
|
+
created_at: p.created_at.iso8601
|
|
94
|
+
}
|
|
95
|
+
end,
|
|
96
|
+
errors: ErrorMetric.where("created_at < ?", cutoff_time).map do |e|
|
|
97
|
+
{
|
|
98
|
+
error_type: e.error_type,
|
|
99
|
+
message: e.message,
|
|
100
|
+
severity: e.severity,
|
|
101
|
+
context: e.parsed_context,
|
|
102
|
+
created_at: e.created_at.iso8601
|
|
103
|
+
}
|
|
104
|
+
end
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
File.write(output_file, JSON.pretty_generate(archive_data))
|
|
108
|
+
|
|
109
|
+
total = archive_data.values.sum { |v| v.is_a?(Array) ? v.size : 0 }
|
|
110
|
+
puts "Archived #{total} metrics to #{output_file}"
|
|
111
|
+
puts "Run 'rake decision_agent:monitoring:cleanup' to delete these records from the database"
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class DecisionLog < ApplicationRecord
|
|
4
|
+
has_many :evaluation_metrics, dependent: :destroy
|
|
5
|
+
|
|
6
|
+
validates :decision, presence: true
|
|
7
|
+
validates :confidence, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
|
|
8
|
+
validates :status, inclusion: { in: %w[success failure error] }, allow_nil: true
|
|
9
|
+
|
|
10
|
+
scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
|
|
11
|
+
scope :successful, -> { where(status: "success") }
|
|
12
|
+
scope :failed, -> { where(status: "failure") }
|
|
13
|
+
scope :with_errors, -> { where(status: "error") }
|
|
14
|
+
scope :by_decision, ->(decision) { where(decision: decision) }
|
|
15
|
+
scope :low_confidence, ->(threshold = 0.5) { where("confidence < ?", threshold) }
|
|
16
|
+
scope :high_confidence, ->(threshold = 0.8) { where("confidence >= ?", threshold) }
|
|
17
|
+
|
|
18
|
+
# Time series aggregation helpers
|
|
19
|
+
def self.count_by_time_bucket(bucket_size: 60, time_range: 3600)
|
|
20
|
+
recent(time_range)
|
|
21
|
+
.group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
|
|
22
|
+
.count
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.average_confidence_by_time(bucket_size: 60, time_range: 3600)
|
|
26
|
+
recent(time_range)
|
|
27
|
+
.where.not(confidence: nil)
|
|
28
|
+
.group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
|
|
29
|
+
.average(:confidence)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def self.success_rate(time_range: 3600)
|
|
33
|
+
total = recent(time_range).where.not(status: nil).count
|
|
34
|
+
return 0.0 if total.zero?
|
|
35
|
+
|
|
36
|
+
successful_count = recent(time_range).successful.count
|
|
37
|
+
successful_count.to_f / total
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Parse JSON context field
|
|
41
|
+
def parsed_context
|
|
42
|
+
return {} if context.nil?
|
|
43
|
+
|
|
44
|
+
JSON.parse(context, symbolize_names: true)
|
|
45
|
+
rescue JSON::ParserError
|
|
46
|
+
{}
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Parse JSON metadata field
|
|
50
|
+
def parsed_metadata
|
|
51
|
+
return {} if metadata.nil?
|
|
52
|
+
|
|
53
|
+
JSON.parse(metadata, symbolize_names: true)
|
|
54
|
+
rescue JSON::ParserError
|
|
55
|
+
{}
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class ErrorMetric < ApplicationRecord
|
|
4
|
+
validates :error_type, presence: true
|
|
5
|
+
validates :severity, inclusion: { in: %w[low medium high critical] }, allow_nil: true
|
|
6
|
+
|
|
7
|
+
scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
|
|
8
|
+
scope :by_type, ->(error_type) { where(error_type: error_type) }
|
|
9
|
+
scope :by_severity, ->(severity) { where(severity: severity) }
|
|
10
|
+
scope :critical, -> { where(severity: "critical") }
|
|
11
|
+
scope :high_severity, -> { where(severity: %w[high critical]) }
|
|
12
|
+
|
|
13
|
+
# Aggregation helpers
|
|
14
|
+
def self.count_by_type(time_range: 3600)
|
|
15
|
+
recent(time_range).group(:error_type).count
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.count_by_severity(time_range: 3600)
|
|
19
|
+
recent(time_range).group(:severity).count
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.error_rate(time_range: 3600, total_operations: nil)
|
|
23
|
+
error_count = recent(time_range).count
|
|
24
|
+
return 0.0 if total_operations.nil? || total_operations.zero?
|
|
25
|
+
|
|
26
|
+
error_count.to_f / total_operations
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Time series aggregation
|
|
30
|
+
def self.count_by_time_bucket(bucket_size: 60, time_range: 3600)
|
|
31
|
+
recent(time_range)
|
|
32
|
+
.group("(EXTRACT(EPOCH FROM created_at)::bigint / #{bucket_size}) * #{bucket_size}")
|
|
33
|
+
.count
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Parse JSON context field
|
|
37
|
+
def parsed_context
|
|
38
|
+
return {} if context.nil?
|
|
39
|
+
|
|
40
|
+
JSON.parse(context, symbolize_names: true)
|
|
41
|
+
rescue JSON::ParserError
|
|
42
|
+
{}
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Parse JSON stack_trace field
|
|
46
|
+
def parsed_stack_trace
|
|
47
|
+
return [] if stack_trace.nil?
|
|
48
|
+
|
|
49
|
+
JSON.parse(stack_trace)
|
|
50
|
+
rescue JSON::ParserError
|
|
51
|
+
[]
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class EvaluationMetric < ApplicationRecord
|
|
4
|
+
belongs_to :decision_log, optional: true
|
|
5
|
+
|
|
6
|
+
validates :evaluator_name, presence: true
|
|
7
|
+
validates :score, numericality: true, allow_nil: true
|
|
8
|
+
|
|
9
|
+
scope :recent, ->(time_range = 3600) { where("created_at >= ?", Time.now - time_range) }
|
|
10
|
+
scope :by_evaluator, ->(evaluator) { where(evaluator_name: evaluator) }
|
|
11
|
+
scope :successful, -> { where(success: true) }
|
|
12
|
+
scope :failed, -> { where(success: false) }
|
|
13
|
+
|
|
14
|
+
# Aggregation helpers
|
|
15
|
+
def self.average_score_by_evaluator(time_range: 3600)
|
|
16
|
+
recent(time_range)
|
|
17
|
+
.where.not(score: nil)
|
|
18
|
+
.group(:evaluator_name)
|
|
19
|
+
.average(:score)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.success_rate_by_evaluator(time_range: 3600)
|
|
23
|
+
recent(time_range)
|
|
24
|
+
.where.not(success: nil)
|
|
25
|
+
.group(:evaluator_name)
|
|
26
|
+
.select("evaluator_name, AVG(CASE WHEN success THEN 1.0 ELSE 0.0 END) as success_rate")
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.count_by_evaluator(time_range: 3600)
|
|
30
|
+
recent(time_range)
|
|
31
|
+
.group(:evaluator_name)
|
|
32
|
+
.count
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Parse JSON details field
|
|
36
|
+
def parsed_details
|
|
37
|
+
return {} if details.nil?
|
|
38
|
+
|
|
39
|
+
JSON.parse(details, symbolize_names: true)
|
|
40
|
+
rescue JSON::ParserError
|
|
41
|
+
{}
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -5,7 +5,7 @@ class CreateDecisionAgentTables < ActiveRecord::Migration[7.0]
|
|
|
5
5
|
t.string :rule_id, null: false, index: { unique: true }
|
|
6
6
|
t.string :ruleset, null: false
|
|
7
7
|
t.text :description
|
|
8
|
-
t.string :status, default:
|
|
8
|
+
t.string :status, default: "active"
|
|
9
9
|
t.timestamps
|
|
10
10
|
end
|
|
11
11
|
|
|
@@ -13,14 +13,25 @@ class CreateDecisionAgentTables < ActiveRecord::Migration[7.0]
|
|
|
13
13
|
create_table :rule_versions do |t|
|
|
14
14
|
t.string :rule_id, null: false, index: true
|
|
15
15
|
t.integer :version_number, null: false
|
|
16
|
-
t.text :content, null: false
|
|
17
|
-
t.string :created_by, null: false, default:
|
|
16
|
+
t.text :content, null: false # JSON rule definition
|
|
17
|
+
t.string :created_by, null: false, default: "system"
|
|
18
18
|
t.text :changelog
|
|
19
|
-
t.string :status, null: false, default:
|
|
19
|
+
t.string :status, null: false, default: "draft" # draft, active, archived
|
|
20
20
|
t.timestamps
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
# ✅ CRITICAL: Unique constraint prevents duplicate version numbers per rule
|
|
24
|
+
# This protects against race conditions in concurrent version creation
|
|
25
|
+
add_index :rule_versions, %i[rule_id version_number], unique: true
|
|
26
|
+
|
|
27
|
+
# Index for efficient queries by rule_id and status
|
|
28
|
+
add_index :rule_versions, %i[rule_id status]
|
|
29
|
+
|
|
30
|
+
# Optional: Partial unique index for PostgreSQL to enforce one active version per rule
|
|
31
|
+
# Uncomment if using PostgreSQL:
|
|
32
|
+
# add_index :rule_versions, [:rule_id, :status],
|
|
33
|
+
# unique: true,
|
|
34
|
+
# where: "status = 'active'",
|
|
35
|
+
# name: 'index_rule_versions_one_active_per_rule'
|
|
25
36
|
end
|
|
26
37
|
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateDecisionAgentMonitoringTables < ActiveRecord::Migration[7.0]
|
|
4
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
5
|
+
def change
|
|
6
|
+
# Decision logs table
|
|
7
|
+
create_table :decision_logs do |t|
|
|
8
|
+
t.string :decision, null: false
|
|
9
|
+
t.float :confidence
|
|
10
|
+
t.integer :evaluations_count, default: 0
|
|
11
|
+
t.float :duration_ms
|
|
12
|
+
t.string :status # success, failure, error
|
|
13
|
+
t.text :context # JSON
|
|
14
|
+
t.text :metadata # JSON
|
|
15
|
+
|
|
16
|
+
t.timestamps
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
add_index :decision_logs, :decision
|
|
20
|
+
add_index :decision_logs, :status
|
|
21
|
+
add_index :decision_logs, :confidence
|
|
22
|
+
add_index :decision_logs, :created_at
|
|
23
|
+
add_index :decision_logs, %i[decision created_at]
|
|
24
|
+
add_index :decision_logs, %i[status created_at]
|
|
25
|
+
|
|
26
|
+
# Evaluation metrics table
|
|
27
|
+
create_table :evaluation_metrics do |t|
|
|
28
|
+
t.references :decision_log, foreign_key: true, index: true
|
|
29
|
+
t.string :evaluator_name, null: false
|
|
30
|
+
t.float :score
|
|
31
|
+
t.boolean :success
|
|
32
|
+
t.float :duration_ms
|
|
33
|
+
t.text :details # JSON
|
|
34
|
+
|
|
35
|
+
t.timestamps
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
add_index :evaluation_metrics, :evaluator_name
|
|
39
|
+
add_index :evaluation_metrics, :success
|
|
40
|
+
add_index :evaluation_metrics, :created_at
|
|
41
|
+
add_index :evaluation_metrics, %i[evaluator_name created_at]
|
|
42
|
+
add_index :evaluation_metrics, %i[evaluator_name success]
|
|
43
|
+
|
|
44
|
+
# Performance metrics table
|
|
45
|
+
create_table :performance_metrics do |t|
|
|
46
|
+
t.string :operation, null: false
|
|
47
|
+
t.float :duration_ms
|
|
48
|
+
t.string :status # success, failure, error
|
|
49
|
+
t.text :metadata # JSON
|
|
50
|
+
|
|
51
|
+
t.timestamps
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
add_index :performance_metrics, :operation
|
|
55
|
+
add_index :performance_metrics, :status
|
|
56
|
+
add_index :performance_metrics, :duration_ms
|
|
57
|
+
add_index :performance_metrics, :created_at
|
|
58
|
+
add_index :performance_metrics, %i[operation created_at]
|
|
59
|
+
add_index :performance_metrics, %i[status created_at]
|
|
60
|
+
|
|
61
|
+
# Error metrics table
|
|
62
|
+
create_table :error_metrics do |t|
|
|
63
|
+
t.string :error_type, null: false
|
|
64
|
+
t.text :message
|
|
65
|
+
t.text :stack_trace # JSON array
|
|
66
|
+
t.string :severity # low, medium, high, critical
|
|
67
|
+
t.text :context # JSON
|
|
68
|
+
|
|
69
|
+
t.timestamps
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
add_index :error_metrics, :error_type
|
|
73
|
+
add_index :error_metrics, :severity
|
|
74
|
+
add_index :error_metrics, :created_at
|
|
75
|
+
add_index :error_metrics, %i[error_type created_at]
|
|
76
|
+
add_index :error_metrics, %i[severity created_at]
|
|
77
|
+
|
|
78
|
+
# PostgreSQL-specific optimizations (optional)
|
|
79
|
+
return unless adapter_name == "PostgreSQL"
|
|
80
|
+
|
|
81
|
+
# Partial indexes for active records (recent data)
|
|
82
|
+
execute <<-SQL
|
|
83
|
+
CREATE INDEX index_decision_logs_on_recent
|
|
84
|
+
ON decision_logs (created_at DESC)
|
|
85
|
+
WHERE created_at >= NOW() - INTERVAL '7 days';
|
|
86
|
+
SQL
|
|
87
|
+
|
|
88
|
+
execute <<-SQL
|
|
89
|
+
CREATE INDEX index_performance_metrics_on_recent
|
|
90
|
+
ON performance_metrics (created_at DESC)
|
|
91
|
+
WHERE created_at >= NOW() - INTERVAL '7 days';
|
|
92
|
+
SQL
|
|
93
|
+
|
|
94
|
+
execute <<-SQL
|
|
95
|
+
CREATE INDEX index_error_metrics_on_recent_critical
|
|
96
|
+
ON error_metrics (created_at DESC)
|
|
97
|
+
WHERE severity IN ('high', 'critical') AND created_at >= NOW() - INTERVAL '7 days';
|
|
98
|
+
SQL
|
|
99
|
+
|
|
100
|
+
# Consider table partitioning for large-scale deployments
|
|
101
|
+
# Example: Partition by month for decision_logs
|
|
102
|
+
# This is commented out by default - enable if needed
|
|
103
|
+
# execute <<-SQL
|
|
104
|
+
# CREATE TABLE decision_logs_partitioned (LIKE decision_logs INCLUDING ALL)
|
|
105
|
+
# PARTITION BY RANGE (created_at);
|
|
106
|
+
# SQL
|
|
107
|
+
end
|
|
108
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
109
|
+
end
|