spec_scout 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.idea/.gitignore +10 -0
- data/.idea/Projects.iml +41 -0
- data/.idea/copilot.data.migration.ask2agent.xml +6 -0
- data/.idea/modules.xml +8 -0
- data/.idea/vcs.xml +6 -0
- data/.rspec_status +236 -0
- data/Gemfile +11 -0
- data/Gemfile.lock +72 -0
- data/LICENSE +21 -0
- data/README.md +433 -0
- data/Rakefile +12 -0
- data/examples/README.md +321 -0
- data/examples/best_practices.md +401 -0
- data/examples/configurations/basic_config.rb +24 -0
- data/examples/configurations/ci_config.rb +35 -0
- data/examples/configurations/conservative_config.rb +32 -0
- data/examples/configurations/development_config.rb +37 -0
- data/examples/configurations/performance_focused_config.rb +38 -0
- data/examples/output_formatter_demo.rb +67 -0
- data/examples/sample_outputs/console_output_high_confidence.txt +27 -0
- data/examples/sample_outputs/console_output_medium_confidence.txt +27 -0
- data/examples/sample_outputs/console_output_no_action.txt +27 -0
- data/examples/sample_outputs/console_output_risk_detected.txt +27 -0
- data/examples/sample_outputs/json_output_high_confidence.json +108 -0
- data/examples/sample_outputs/json_output_no_action.json +108 -0
- data/examples/workflows/basic_workflow.md +159 -0
- data/examples/workflows/ci_integration.md +372 -0
- data/exe/spec_scout +7 -0
- data/lib/spec_scout/agent_result.rb +44 -0
- data/lib/spec_scout/agents/database_agent.rb +113 -0
- data/lib/spec_scout/agents/factory_agent.rb +179 -0
- data/lib/spec_scout/agents/intent_agent.rb +223 -0
- data/lib/spec_scout/agents/risk_agent.rb +290 -0
- data/lib/spec_scout/base_agent.rb +72 -0
- data/lib/spec_scout/cli.rb +158 -0
- data/lib/spec_scout/configuration.rb +162 -0
- data/lib/spec_scout/consensus_engine.rb +535 -0
- data/lib/spec_scout/enforcement_handler.rb +182 -0
- data/lib/spec_scout/output_formatter.rb +307 -0
- data/lib/spec_scout/profile_data.rb +37 -0
- data/lib/spec_scout/profile_normalizer.rb +238 -0
- data/lib/spec_scout/recommendation.rb +62 -0
- data/lib/spec_scout/safety_validator.rb +127 -0
- data/lib/spec_scout/spec_scout.rb +519 -0
- data/lib/spec_scout/testprof_integration.rb +206 -0
- data/lib/spec_scout/version.rb +5 -0
- data/lib/spec_scout.rb +43 -0
- metadata +166 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SpecScout
|
|
4
|
+
# Abstract base class for all analysis agents
|
|
5
|
+
class BaseAgent
|
|
6
|
+
attr_reader :profile_data
|
|
7
|
+
|
|
8
|
+
def initialize(profile_data)
|
|
9
|
+
@profile_data = profile_data
|
|
10
|
+
validate_profile_data!
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Abstract method to be implemented by subclasses
|
|
14
|
+
# Returns an AgentResult with verdict, confidence, and reasoning
|
|
15
|
+
def evaluate
|
|
16
|
+
raise NotImplementedError, 'Subclasses must implement #evaluate'
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Agent name for identification
|
|
20
|
+
def agent_name
|
|
21
|
+
class_name = self.class.name || 'UnknownAgent'
|
|
22
|
+
class_name.split('::').last.downcase.gsub('agent', '').to_sym
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
protected
|
|
26
|
+
|
|
27
|
+
# Helper method to create AgentResult
|
|
28
|
+
def create_result(verdict:, confidence:, reasoning:, metadata: {})
|
|
29
|
+
AgentResult.new(
|
|
30
|
+
agent_name: agent_name,
|
|
31
|
+
verdict: verdict,
|
|
32
|
+
confidence: confidence,
|
|
33
|
+
reasoning: reasoning,
|
|
34
|
+
metadata: metadata
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Validate confidence level
|
|
39
|
+
def validate_confidence(confidence)
|
|
40
|
+
return if AgentResult::VALID_CONFIDENCE_LEVELS.include?(confidence)
|
|
41
|
+
|
|
42
|
+
raise ArgumentError, "Invalid confidence level: #{confidence}"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Check if database operations are present
|
|
46
|
+
def database_operations_present?
|
|
47
|
+
return false unless profile_data.db.is_a?(Hash)
|
|
48
|
+
|
|
49
|
+
total_queries = profile_data.db[:total_queries] || 0
|
|
50
|
+
inserts = profile_data.db[:inserts] || 0
|
|
51
|
+
|
|
52
|
+
total_queries.positive? || inserts.positive?
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Check if factories are present
|
|
56
|
+
def factories_present?
|
|
57
|
+
return false unless profile_data.factories.is_a?(Hash)
|
|
58
|
+
|
|
59
|
+
profile_data.factories.any?
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
def validate_profile_data!
|
|
65
|
+
raise ArgumentError, "Expected ProfileData, got #{profile_data.class}" unless profile_data.is_a?(ProfileData)
|
|
66
|
+
|
|
67
|
+
return if profile_data.valid?
|
|
68
|
+
|
|
69
|
+
raise ArgumentError, 'Invalid ProfileData structure'
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SpecScout
|
|
4
|
+
# Command Line Interface for SpecScout
|
|
5
|
+
class CLI
|
|
6
|
+
def self.run(args = ARGV)
|
|
7
|
+
if args.include?('--version') || args.include?('-v')
|
|
8
|
+
puts "SpecScout #{VERSION}"
|
|
9
|
+
exit(0)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
if args.include?('--help') || args.include?('-h')
|
|
13
|
+
print_help
|
|
14
|
+
exit(0)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Parse configuration from CLI args
|
|
18
|
+
config = parse_args(args)
|
|
19
|
+
|
|
20
|
+
# Create and run SpecScout
|
|
21
|
+
scout = SpecScout.new(config)
|
|
22
|
+
result = scout.analyze
|
|
23
|
+
|
|
24
|
+
# Handle output
|
|
25
|
+
handle_output(result, config)
|
|
26
|
+
|
|
27
|
+
# Exit with appropriate code for CI/enforcement mode
|
|
28
|
+
exit_code = determine_exit_code(result, config)
|
|
29
|
+
exit(exit_code) if exit_code > 0
|
|
30
|
+
|
|
31
|
+
result
|
|
32
|
+
rescue StandardError => e
|
|
33
|
+
handle_cli_error(e)
|
|
34
|
+
exit(1)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def self.parse_args(args)
|
|
38
|
+
config = ::SpecScout.configuration.dup
|
|
39
|
+
|
|
40
|
+
i = 0
|
|
41
|
+
while i < args.length
|
|
42
|
+
case args[i]
|
|
43
|
+
when '--disable'
|
|
44
|
+
config.enable = false
|
|
45
|
+
when '--no-testprof'
|
|
46
|
+
config.use_test_prof = false
|
|
47
|
+
when '--enforce'
|
|
48
|
+
config.enforcement_mode = true
|
|
49
|
+
when '--fail-on-high-confidence'
|
|
50
|
+
config.fail_on_high_confidence = true
|
|
51
|
+
when '--output', '-o'
|
|
52
|
+
i += 1
|
|
53
|
+
raise ArgumentError, '--output requires a format (console, json)' unless i < args.length
|
|
54
|
+
|
|
55
|
+
config.output_format = args[i]
|
|
56
|
+
|
|
57
|
+
when '--enable-agent'
|
|
58
|
+
i += 1
|
|
59
|
+
raise ArgumentError, '--enable-agent requires an agent name' unless i < args.length
|
|
60
|
+
|
|
61
|
+
config.enable_agent(args[i])
|
|
62
|
+
|
|
63
|
+
when '--disable-agent'
|
|
64
|
+
i += 1
|
|
65
|
+
raise ArgumentError, '--disable-agent requires an agent name' unless i < args.length
|
|
66
|
+
|
|
67
|
+
config.disable_agent(args[i])
|
|
68
|
+
|
|
69
|
+
when '--spec'
|
|
70
|
+
i += 1
|
|
71
|
+
raise ArgumentError, '--spec requires a spec file path' unless i < args.length
|
|
72
|
+
# Store spec location in config metadata for now
|
|
73
|
+
# This could be enhanced later if needed
|
|
74
|
+
|
|
75
|
+
when /^--/
|
|
76
|
+
raise ArgumentError, "Unknown option: #{args[i]}"
|
|
77
|
+
end
|
|
78
|
+
i += 1
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
config.validate!
|
|
82
|
+
config
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def self.handle_output(result, config)
|
|
86
|
+
return unless result
|
|
87
|
+
|
|
88
|
+
if result[:recommendation] && result[:profile_data]
|
|
89
|
+
formatter = OutputFormatter.new(result[:recommendation], result[:profile_data])
|
|
90
|
+
output = formatter.format_recommendation
|
|
91
|
+
puts output
|
|
92
|
+
elsif result[:disabled]
|
|
93
|
+
puts 'SpecScout is disabled' if config.console_output?
|
|
94
|
+
elsif result[:no_profile_data]
|
|
95
|
+
puts 'No profile data available - ensure TestProf is properly configured' if config.console_output?
|
|
96
|
+
elsif result[:no_agents]
|
|
97
|
+
puts 'No agents produced results' if config.console_output?
|
|
98
|
+
elsif result[:error]
|
|
99
|
+
puts "Analysis failed: #{result[:error].message}" if config.console_output?
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def self.determine_exit_code(result, config)
|
|
104
|
+
return 0 unless result
|
|
105
|
+
return 0 unless config.enforcement_mode?
|
|
106
|
+
return 0 unless result[:should_fail]
|
|
107
|
+
|
|
108
|
+
1 # Fail in enforcement mode with high confidence recommendations
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def self.handle_cli_error(error)
|
|
112
|
+
case error
|
|
113
|
+
when ArgumentError
|
|
114
|
+
puts "Error: #{error.message}"
|
|
115
|
+
puts 'Use --help for usage information'
|
|
116
|
+
else
|
|
117
|
+
puts "Unexpected error: #{error.message}"
|
|
118
|
+
puts error.backtrace.join("\n") if ENV['SPEC_SCOUT_DEBUG']
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def self.print_help
|
|
123
|
+
puts <<~HELP
|
|
124
|
+
SpecScout - Intelligent test optimization advisor
|
|
125
|
+
|
|
126
|
+
Usage: spec_scout [options] [spec_file]
|
|
127
|
+
|
|
128
|
+
Options:
|
|
129
|
+
--disable Disable SpecScout analysis
|
|
130
|
+
--no-testprof Disable TestProf integration
|
|
131
|
+
--enforce Enable enforcement mode (fail on high confidence)
|
|
132
|
+
--fail-on-high-confidence Fail on high confidence recommendations
|
|
133
|
+
--output FORMAT, -o FORMAT Output format (console, json)
|
|
134
|
+
--enable-agent AGENT Enable specific agent (database, factory, intent, risk)
|
|
135
|
+
--disable-agent AGENT Disable specific agent
|
|
136
|
+
--spec SPEC_FILE Analyze specific spec file
|
|
137
|
+
--version, -v Show version
|
|
138
|
+
--help, -h Show this help message
|
|
139
|
+
|
|
140
|
+
Agents:
|
|
141
|
+
database Analyze database usage patterns
|
|
142
|
+
factory Analyze FactoryBot strategy usage
|
|
143
|
+
intent Classify test intent and behavior
|
|
144
|
+
risk Identify potentially unsafe optimizations
|
|
145
|
+
|
|
146
|
+
Examples:
|
|
147
|
+
spec_scout # Run with default settings
|
|
148
|
+
spec_scout --enforce # Enable enforcement mode
|
|
149
|
+
spec_scout --output json # JSON output
|
|
150
|
+
spec_scout --disable-agent risk # Disable risk agent
|
|
151
|
+
spec_scout spec/models/user_spec.rb # Analyze specific spec file
|
|
152
|
+
|
|
153
|
+
Environment Variables:
|
|
154
|
+
SPEC_SCOUT_DEBUG=1 Enable debug output
|
|
155
|
+
HELP
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SpecScout
|
|
4
|
+
# Configuration class for Spec Scout settings
|
|
5
|
+
# Supports enabling/disabling agents selectively, enforcement modes, and backward compatibility
|
|
6
|
+
class Configuration
|
|
7
|
+
attr_accessor :enable, :use_test_prof, :fail_on_high_confidence,
|
|
8
|
+
:enabled_agents, :output_format, :enforcement_mode,
|
|
9
|
+
:auto_apply_enabled, :blocking_mode_enabled
|
|
10
|
+
|
|
11
|
+
VALID_OUTPUT_FORMATS = %i[console json].freeze
|
|
12
|
+
VALID_AGENTS = %i[database factory intent risk].freeze
|
|
13
|
+
|
|
14
|
+
def initialize
|
|
15
|
+
@enable = true
|
|
16
|
+
@use_test_prof = true
|
|
17
|
+
@fail_on_high_confidence = false
|
|
18
|
+
@enabled_agents = VALID_AGENTS.dup
|
|
19
|
+
@output_format = :console
|
|
20
|
+
@enforcement_mode = false
|
|
21
|
+
@auto_apply_enabled = false # Safety: never auto-apply by default
|
|
22
|
+
@blocking_mode_enabled = false # Safety: non-blocking by default
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def enabled?
|
|
26
|
+
@enable
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def test_prof_enabled?
|
|
30
|
+
@use_test_prof
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def enforcement_mode?
|
|
34
|
+
@enforcement_mode
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def auto_apply_enabled?
|
|
38
|
+
@auto_apply_enabled
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def blocking_mode_enabled?
|
|
42
|
+
@blocking_mode_enabled
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def agent_enabled?(agent_name)
|
|
46
|
+
@enabled_agents.include?(agent_name.to_sym)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def enable_agent(agent_name)
|
|
50
|
+
agent_sym = agent_name.to_sym
|
|
51
|
+
raise ArgumentError, "Unknown agent: #{agent_name}" unless VALID_AGENTS.include?(agent_sym)
|
|
52
|
+
|
|
53
|
+
@enabled_agents << agent_sym unless @enabled_agents.include?(agent_sym)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def disable_agent(agent_name)
|
|
57
|
+
@enabled_agents.delete(agent_name.to_sym)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def output_format=(format)
|
|
61
|
+
format_sym = format.to_sym
|
|
62
|
+
unless VALID_OUTPUT_FORMATS.include?(format_sym)
|
|
63
|
+
raise ArgumentError, "Invalid output format: #{format}. Valid formats: #{VALID_OUTPUT_FORMATS}"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
@output_format = format_sym
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def json_output?
|
|
70
|
+
@output_format == :json
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def console_output?
|
|
74
|
+
@output_format == :console
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def debug_mode?
|
|
78
|
+
ENV['SPEC_SCOUT_DEBUG'] == 'true'
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def validate!
|
|
82
|
+
unless @enabled_agents.all? { |agent| VALID_AGENTS.include?(agent) }
|
|
83
|
+
invalid_agents = @enabled_agents - VALID_AGENTS
|
|
84
|
+
raise ArgumentError, "Invalid agents: #{invalid_agents}. Valid agents: #{VALID_AGENTS}"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
unless VALID_OUTPUT_FORMATS.include?(@output_format)
|
|
88
|
+
raise ArgumentError, "Invalid output format: #{@output_format}. Valid formats: #{VALID_OUTPUT_FORMATS}"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
true
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def to_h
|
|
95
|
+
{
|
|
96
|
+
enable: @enable,
|
|
97
|
+
use_test_prof: @use_test_prof,
|
|
98
|
+
fail_on_high_confidence: @fail_on_high_confidence,
|
|
99
|
+
enabled_agents: @enabled_agents,
|
|
100
|
+
output_format: @output_format,
|
|
101
|
+
enforcement_mode: @enforcement_mode,
|
|
102
|
+
auto_apply_enabled: @auto_apply_enabled,
|
|
103
|
+
blocking_mode_enabled: @blocking_mode_enabled
|
|
104
|
+
}
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Create a duplicate configuration for CLI argument parsing
|
|
108
|
+
def dup
|
|
109
|
+
new_config = self.class.new
|
|
110
|
+
new_config.enable = @enable
|
|
111
|
+
new_config.use_test_prof = @use_test_prof
|
|
112
|
+
new_config.fail_on_high_confidence = @fail_on_high_confidence
|
|
113
|
+
new_config.enabled_agents = @enabled_agents.dup
|
|
114
|
+
new_config.output_format = @output_format
|
|
115
|
+
new_config.enforcement_mode = @enforcement_mode
|
|
116
|
+
new_config.auto_apply_enabled = @auto_apply_enabled
|
|
117
|
+
new_config.blocking_mode_enabled = @blocking_mode_enabled
|
|
118
|
+
new_config
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Merge configuration from hash (for backward compatibility)
|
|
122
|
+
def merge!(options = {})
|
|
123
|
+
options.each do |key, value|
|
|
124
|
+
case key.to_sym
|
|
125
|
+
when :enable
|
|
126
|
+
@enable = value
|
|
127
|
+
when :use_test_prof
|
|
128
|
+
@use_test_prof = value
|
|
129
|
+
when :fail_on_high_confidence
|
|
130
|
+
@fail_on_high_confidence = value
|
|
131
|
+
when :enabled_agents
|
|
132
|
+
@enabled_agents = Array(value).map(&:to_sym)
|
|
133
|
+
when :output_format
|
|
134
|
+
self.output_format = value
|
|
135
|
+
when :enforcement_mode
|
|
136
|
+
@enforcement_mode = value
|
|
137
|
+
when :auto_apply_enabled
|
|
138
|
+
@auto_apply_enabled = value
|
|
139
|
+
when :blocking_mode_enabled
|
|
140
|
+
@blocking_mode_enabled = value
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
validate!
|
|
144
|
+
self
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Check if enforcement should fail on high confidence
|
|
148
|
+
def should_fail_on_high_confidence?
|
|
149
|
+
@enforcement_mode && @fail_on_high_confidence
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Gracefully handle disabled TestProf integration
|
|
153
|
+
def graceful_testprof_disable
|
|
154
|
+
return self if @use_test_prof
|
|
155
|
+
|
|
156
|
+
# When TestProf is disabled, we can still run agents on mock data
|
|
157
|
+
# This maintains backward compatibility
|
|
158
|
+
warn 'TestProf integration disabled - running in analysis-only mode' if @enable
|
|
159
|
+
self
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|