mutation_tester 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +13 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +83 -0
- data/LICENSE.txt +19 -0
- data/Rakefile +57 -0
- data/docs/ci.md +158 -0
- data/docs/execution-runners.md +163 -0
- data/docs/json-schema.md +171 -0
- data/docs/mutation-types.md +207 -0
- data/examples/calculator.rb +35 -0
- data/examples/calculator_100_perc_spec.rb +121 -0
- data/examples/calculator_minitest.rb +53 -0
- data/examples/calculator_spec.rb +105 -0
- data/examples/github_actions/ai_mutation_gate.yml +75 -0
- data/examples/github_actions/mutation_test.yml +87 -0
- data/examples/hooks/pre-push +41 -0
- data/examples/run_example.rb +31 -0
- data/examples/run_example_minitest.rb +38 -0
- data/examples/run_example_parallel_minitest.rb +48 -0
- data/examples/run_example_parallel_rspec.rb +48 -0
- data/examples/run_example_rspec.rb +38 -0
- data/examples/spec_helper.rb +21 -0
- data/exe/mutation_test +345 -0
- data/lib/mutation_tester/batch_runner.rb +286 -0
- data/lib/mutation_tester/configuration.rb +105 -0
- data/lib/mutation_tester/core.rb +193 -0
- data/lib/mutation_tester/fork_runner/worker.rb +202 -0
- data/lib/mutation_tester/fork_runner.rb +382 -0
- data/lib/mutation_tester/framework_detector.rb +25 -0
- data/lib/mutation_tester/in_memory_loader.rb +25 -0
- data/lib/mutation_tester/mutation_runner.rb +644 -0
- data/lib/mutation_tester/mutator.rb +874 -0
- data/lib/mutation_tester/progress_display.rb +130 -0
- data/lib/mutation_tester/railtie.rb +7 -0
- data/lib/mutation_tester/rake_task.rb +18 -0
- data/lib/mutation_tester/reporters/base_reporter.rb +154 -0
- data/lib/mutation_tester/reporters/batch_json_reporter.rb +72 -0
- data/lib/mutation_tester/reporters/console_reporter.rb +130 -0
- data/lib/mutation_tester/reporters/html_reporter.rb +693 -0
- data/lib/mutation_tester/reporters/json_reporter.rb +67 -0
- data/lib/mutation_tester/test_command.rb +165 -0
- data/lib/mutation_tester/version.rb +3 -0
- data/lib/mutation_tester.rb +52 -0
- data/lib/tasks/mutation_tester.rake +80 -0
- data/readme.md +925 -0
- metadata +213 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
require 'time'
|
|
2
|
+
|
|
3
|
+
module MutationTester
|
|
4
|
+
module Reporters
|
|
5
|
+
class JsonReporter < BaseReporter
|
|
6
|
+
SCHEMA_VERSION = 1
|
|
7
|
+
|
|
8
|
+
def generate
|
|
9
|
+
output_file = File.join(@config.output_dir, 'mutation_report.json')
|
|
10
|
+
|
|
11
|
+
File.write(output_file, render)
|
|
12
|
+
puts Rainbow("✓ JSON report saved to: #{output_file}").green
|
|
13
|
+
output_file
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def render
|
|
17
|
+
@render ||= JSON.pretty_generate(report_data)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def report_data
|
|
21
|
+
{
|
|
22
|
+
schema_version: SCHEMA_VERSION,
|
|
23
|
+
interrupted: interrupted?,
|
|
24
|
+
metadata: {
|
|
25
|
+
version: MutationTester::VERSION,
|
|
26
|
+
generated_at: Time.now.iso8601,
|
|
27
|
+
source_file: @source_file,
|
|
28
|
+
spec_file: @spec_file
|
|
29
|
+
},
|
|
30
|
+
summary: {
|
|
31
|
+
total: total_count,
|
|
32
|
+
killed: effective_killed_count,
|
|
33
|
+
survived: survived_count,
|
|
34
|
+
mutation_score: mutation_score,
|
|
35
|
+
quality_rating: quality_rating,
|
|
36
|
+
categories: {
|
|
37
|
+
killed: killed_count,
|
|
38
|
+
survived: survived_count,
|
|
39
|
+
timeout: timeout_count,
|
|
40
|
+
stillborn: stillborn_count,
|
|
41
|
+
error: error_count
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
mutations: @results.map { |r| mutation_entry(r) }
|
|
45
|
+
}
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def mutation_entry(result)
|
|
51
|
+
status = status_of(result)
|
|
52
|
+
entry = result.merge(status: status)
|
|
53
|
+
entry.delete(:kill_phase)
|
|
54
|
+
entry[:diff] = diff_text(result) if DIFF_DETAIL_STATUSES.include?(status)
|
|
55
|
+
entry
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def diff_text(result)
|
|
59
|
+
diff_lines([result]).map { |kind, text, _| "#{diff_prefix(kind)}#{text}" }.join("\n")
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def diff_prefix(kind)
|
|
63
|
+
{ removed: '- ', added: '+ ', context: ' ' }.fetch(kind, '')
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
require 'tempfile'
|
|
2
|
+
|
|
3
|
+
module MutationTester
|
|
4
|
+
class TestCommand
|
|
5
|
+
Result = Struct.new(:passed, :timed_out, :output) do
|
|
6
|
+
alias_method :passed?, :passed
|
|
7
|
+
alias_method :timed_out?, :timed_out
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
POLL_INTERVAL = 0.05
|
|
11
|
+
|
|
12
|
+
attr_reader :spec_file, :framework, :use_bundle_exec, :example_filters
|
|
13
|
+
|
|
14
|
+
def initialize(spec_file, use_bundle_exec:, framework: nil, runner: :spawn, example_filters: [], worker_env_var: nil)
|
|
15
|
+
@spec_file = spec_file
|
|
16
|
+
@framework = framework || self.class.detect_framework(spec_file)
|
|
17
|
+
@use_bundle_exec = use_bundle_exec
|
|
18
|
+
@runner = runner
|
|
19
|
+
@example_filters = @framework == :rspec ? Array(example_filters) : []
|
|
20
|
+
@worker_env_var = worker_env_var
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def argv
|
|
24
|
+
parts = [runner, spec_file, *filter_args]
|
|
25
|
+
@use_bundle_exec ? ['bundle', 'exec', *parts] : parts
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def command
|
|
29
|
+
cmd = ([runner, spec_file] + example_filters.flat_map { |f| ['-e', "'#{f}'"] }).join(' ')
|
|
30
|
+
@use_bundle_exec ? "bundle exec #{cmd}" : cmd
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def shell_command(timeout: nil, quiet: false)
|
|
34
|
+
cmd = command
|
|
35
|
+
cmd = "timeout #{timeout}s #{cmd}" if timeout
|
|
36
|
+
cmd = "#{cmd} > /dev/null 2>&1" if quiet
|
|
37
|
+
cmd
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def run(timeout: nil, chdir: nil, capture: false)
|
|
41
|
+
if fork_execution?
|
|
42
|
+
fork_runner = ForkRunner.acquire(use_bundle_exec: @use_bundle_exec)
|
|
43
|
+
return fork_runner.execute(spec_file, timeout: timeout, chdir: chdir || Dir.pwd, capture: capture, args: filter_args) if fork_runner
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
return run_captured(timeout: timeout, chdir: chdir) if capture
|
|
47
|
+
|
|
48
|
+
spawn_options = { pgroup: true, %i[out err] => File::NULL }
|
|
49
|
+
spawn_options[:chdir] = chdir if chdir
|
|
50
|
+
|
|
51
|
+
pid = Process.spawn(*spawn_argv, spawn_options)
|
|
52
|
+
wait_with_deadline(pid, timeout)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def fork_execution?
|
|
56
|
+
return false if @framework == :minitest
|
|
57
|
+
return false if @runner == :spawn
|
|
58
|
+
|
|
59
|
+
ForkRunner.available?
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def self.detect_framework(spec_file)
|
|
63
|
+
FrameworkDetector.detect(spec_file)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.use_bundle_exec?(path)
|
|
67
|
+
dir = File.dirname(File.expand_path(path))
|
|
68
|
+
|
|
69
|
+
loop do
|
|
70
|
+
return true if File.exist?(File.join(dir, 'Gemfile'))
|
|
71
|
+
|
|
72
|
+
parent = File.dirname(dir)
|
|
73
|
+
break if parent == dir
|
|
74
|
+
|
|
75
|
+
dir = parent
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
false
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def spawn_argv
|
|
84
|
+
overrides = worker_env_overrides
|
|
85
|
+
overrides ? [overrides, *argv] : argv
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def worker_env_overrides
|
|
89
|
+
return nil unless @worker_env_var
|
|
90
|
+
|
|
91
|
+
index = defined?(Parallel) && Parallel.respond_to?(:worker_number) ? Parallel.worker_number : nil
|
|
92
|
+
{ @worker_env_var => Configuration.worker_env_value(index) }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def runner
|
|
96
|
+
@framework == :minitest ? 'ruby' : 'rspec'
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def filter_args
|
|
100
|
+
example_filters.flat_map { |filter| ['-e', filter] }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def run_captured(timeout:, chdir:)
|
|
104
|
+
log = Tempfile.new(['mutation_tester_baseline', '.log'])
|
|
105
|
+
spawn_options = { pgroup: true, %i[out err] => log.path }
|
|
106
|
+
spawn_options[:chdir] = chdir if chdir
|
|
107
|
+
|
|
108
|
+
pid = Process.spawn(*spawn_argv, spawn_options)
|
|
109
|
+
result = wait_with_deadline(pid, timeout)
|
|
110
|
+
result.output = File.read(log.path)
|
|
111
|
+
result
|
|
112
|
+
ensure
|
|
113
|
+
log&.close
|
|
114
|
+
log&.unlink
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def wait_with_deadline(pid, timeout)
|
|
118
|
+
unless timeout
|
|
119
|
+
_pid, status = Process.waitpid2(pid)
|
|
120
|
+
return Result.new(status.success?, false)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
deadline = monotonic_time + timeout
|
|
124
|
+
loop do
|
|
125
|
+
_pid, status = Process.waitpid2(pid, Process::WNOHANG)
|
|
126
|
+
return Result.new(status.success?, false) if status
|
|
127
|
+
|
|
128
|
+
break if monotonic_time >= deadline
|
|
129
|
+
|
|
130
|
+
sleep(POLL_INTERVAL)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
_pid, status = Process.waitpid2(pid, Process::WNOHANG)
|
|
134
|
+
return Result.new(status.success?, false) if status
|
|
135
|
+
|
|
136
|
+
terminate_process_group(pid)
|
|
137
|
+
Result.new(false, true)
|
|
138
|
+
ensure
|
|
139
|
+
terminate_process_group(pid) if pid && process_running?(pid)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def terminate_process_group(pid)
|
|
143
|
+
Process.kill('KILL', -pid)
|
|
144
|
+
rescue Errno::ESRCH
|
|
145
|
+
ensure
|
|
146
|
+
reap(pid)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def reap(pid)
|
|
150
|
+
Process.waitpid(pid)
|
|
151
|
+
rescue Errno::ECHILD, Errno::ESRCH
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def process_running?(pid)
|
|
155
|
+
Process.kill(0, pid)
|
|
156
|
+
true
|
|
157
|
+
rescue Errno::ESRCH
|
|
158
|
+
false
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def monotonic_time
|
|
162
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
require 'parser/current'
|
|
2
|
+
require 'unparser'
|
|
3
|
+
require 'parallel'
|
|
4
|
+
require 'rainbow'
|
|
5
|
+
require 'fileutils'
|
|
6
|
+
require 'erb'
|
|
7
|
+
require 'json'
|
|
8
|
+
|
|
9
|
+
require_relative 'mutation_tester/version'
|
|
10
|
+
require_relative 'mutation_tester/configuration'
|
|
11
|
+
require_relative 'mutation_tester/framework_detector'
|
|
12
|
+
require_relative 'mutation_tester/in_memory_loader'
|
|
13
|
+
require_relative 'mutation_tester/fork_runner'
|
|
14
|
+
require_relative 'mutation_tester/test_command'
|
|
15
|
+
require_relative 'mutation_tester/core'
|
|
16
|
+
require_relative 'mutation_tester/batch_runner'
|
|
17
|
+
require_relative 'mutation_tester/mutator'
|
|
18
|
+
require_relative 'mutation_tester/mutation_runner'
|
|
19
|
+
require_relative 'mutation_tester/reporters/base_reporter'
|
|
20
|
+
require_relative 'mutation_tester/reporters/console_reporter'
|
|
21
|
+
require_relative 'mutation_tester/reporters/html_reporter'
|
|
22
|
+
require_relative 'mutation_tester/reporters/json_reporter'
|
|
23
|
+
require_relative 'mutation_tester/reporters/batch_json_reporter'
|
|
24
|
+
require_relative 'mutation_tester/progress_display'
|
|
25
|
+
|
|
26
|
+
require_relative 'mutation_tester/railtie' if defined?(Rails::Railtie)
|
|
27
|
+
|
|
28
|
+
module MutationTester
|
|
29
|
+
class Error < StandardError; end
|
|
30
|
+
|
|
31
|
+
class << self
|
|
32
|
+
attr_writer :configuration
|
|
33
|
+
|
|
34
|
+
def configuration
|
|
35
|
+
@configuration ||= Configuration.new
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def configure
|
|
39
|
+
yield(configuration)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def reset_configuration!
|
|
43
|
+
@configuration = Configuration.new
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def run(source_file, spec_file, options = {})
|
|
47
|
+
config = configuration.merge(options)
|
|
48
|
+
core = Core.new(source_file, spec_file, config)
|
|
49
|
+
core.run
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
task :environment unless Rake::Task.task_defined?(:environment)
|
|
2
|
+
|
|
3
|
+
namespace :mutation do
|
|
4
|
+
desc 'Run mutation tests on a specific file'
|
|
5
|
+
task :test, %i[source_file spec_file] => :environment do |_t, args|
|
|
6
|
+
require 'mutation_tester'
|
|
7
|
+
|
|
8
|
+
unless args[:source_file] && args[:spec_file]
|
|
9
|
+
puts Rainbow('Usage: rake mutation:test[app/models/user.rb,spec/models/user_spec.rb]').yellow
|
|
10
|
+
abort
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
source_file = args[:source_file]
|
|
14
|
+
spec_file = args[:spec_file]
|
|
15
|
+
|
|
16
|
+
unless File.exist?(source_file)
|
|
17
|
+
puts Rainbow("❌ Source file not found: #{source_file}").red
|
|
18
|
+
abort
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
unless File.exist?(spec_file)
|
|
22
|
+
puts Rainbow("❌ Spec file not found: #{spec_file}").red
|
|
23
|
+
abort
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
abort unless MutationTester.run(source_file, spec_file)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
desc 'Run mutation tests on all models'
|
|
30
|
+
task test_models: :environment do
|
|
31
|
+
require 'mutation_tester'
|
|
32
|
+
|
|
33
|
+
all_passed = true
|
|
34
|
+
|
|
35
|
+
Dir.glob('app/models/**/*.rb').each do |source_file|
|
|
36
|
+
spec_file = source_file.gsub('app/', 'spec/').gsub('.rb', '_spec.rb')
|
|
37
|
+
|
|
38
|
+
next unless File.exist?(spec_file)
|
|
39
|
+
|
|
40
|
+
puts "\n" + Rainbow('=' * 80).bright
|
|
41
|
+
puts Rainbow("Testing: #{source_file}").cyan
|
|
42
|
+
puts Rainbow('=' * 80).bright
|
|
43
|
+
|
|
44
|
+
all_passed = false unless MutationTester.run(source_file, spec_file)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
unless all_passed
|
|
48
|
+
puts Rainbow("\n❌ One or more files did not meet the mutation score threshold").red
|
|
49
|
+
abort
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
desc 'Run mutation tests (usage: rake "mutation_test[SOURCE_FILE,TEST_FILE]")'
|
|
55
|
+
task :mutation_test, [:source_file, :spec_file] do |_t, args|
|
|
56
|
+
require 'mutation_tester'
|
|
57
|
+
|
|
58
|
+
unless args[:source_file] && args[:spec_file]
|
|
59
|
+
puts Rainbow('Usage: rake "mutation_test[SOURCE_FILE,TEST_FILE]"').yellow
|
|
60
|
+
puts Rainbow('Example: rake "mutation_test[app/models/user.rb,spec/models/user_spec.rb]"').cyan
|
|
61
|
+
abort
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
source_file = args[:source_file]
|
|
65
|
+
spec_file = args[:spec_file]
|
|
66
|
+
|
|
67
|
+
unless File.exist?(source_file)
|
|
68
|
+
puts Rainbow("❌ Source file not found: #{source_file}").red
|
|
69
|
+
puts Rainbow('💡 Tip: Make sure the path is correct and the file exists').yellow
|
|
70
|
+
abort
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
unless File.exist?(spec_file)
|
|
74
|
+
puts Rainbow("❌ Test file not found: #{spec_file}").red
|
|
75
|
+
puts Rainbow('💡 Tip: Make sure the path is correct and the file exists').yellow
|
|
76
|
+
abort
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
abort unless MutationTester.run(source_file, spec_file)
|
|
80
|
+
end
|