mutaterb 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -5
- data/lib/mutaterb/cli.rb +16 -4
- data/lib/mutaterb/config.rb +22 -7
- data/lib/mutaterb/flag_parser.rb +1 -0
- data/lib/mutaterb/mutator.rb +13 -8
- data/lib/mutaterb/project_detector.rb +33 -7
- data/lib/mutaterb/reporter.rb +1 -0
- data/lib/mutaterb/test_adapters/minitest_adapter.rb +63 -0
- data/lib/mutaterb/test_adapters/rspec_adapter.rb +34 -0
- data/lib/mutaterb/test_runner.rb +39 -49
- data/lib/mutaterb/test_suite.rb +6 -4
- data/lib/mutaterb/version.rb +1 -1
- data/lib/mutaterb.rb +2 -0
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 71841b126edbdca7f12d77aaca2622dde361a8011a49fbd2973cde656737c134
|
|
4
|
+
data.tar.gz: 2c45a913486f515d16ab0217d331b35e98f84f620a43b9a6496ad8c94681c940
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1b716efda6a38584d538c4b76d82113be716dea2f466efd293afcef20d9b40aaf628a80ffd5aef8d3c8f80848314e3a6269bcb2877cf42d8c6c9b5bcb75dabe6
|
|
7
|
+
data.tar.gz: 24655446fcfc728007c10e89d7b8be5df414aaf99baf3d441f154add892c1dfcb59251570ca9747b132d606c167acb3ecb9e3ea71bcc230c8df6105dd912440c
|
data/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# MutateRB
|
|
2
2
|
|
|
3
3
|
A mutation testing tool for Ruby and Ruby on Rails projects: it modifies code covered
|
|
4
|
-
by your
|
|
5
|
-
|
|
4
|
+
by your test suite and checks whether it catches the change. If a mutated test does not
|
|
5
|
+
fail, that test is identified as weak. Supports both RSpec and Minitest.
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
@@ -25,9 +25,11 @@ mutaterb
|
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
Without flags, it automatically detects whether the project is pure Ruby or Rails,
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
which test framework it uses (RSpec if there's a `spec/` folder, Minitest if there's a
|
|
29
|
+
`test/` folder — RSpec wins if both are present), applies mutations to covered code,
|
|
30
|
+
and displays a summary: how many mutations were "killed" (caught by a test) and how
|
|
31
|
+
many "survived" (no test caught them — weak tests), with file, line, and related
|
|
32
|
+
test(s) for each survivor.
|
|
31
33
|
|
|
32
34
|
### Main flags
|
|
33
35
|
|
|
@@ -37,6 +39,7 @@ caught them — weak tests), with file, line, and related test(s) for each survi
|
|
|
37
39
|
| `--include PATHS` | Restrict analysis to these paths (comma-separated) |
|
|
38
40
|
| `--exclude PATHS` | Exclude these paths (comma-separated) |
|
|
39
41
|
| `--strictness LEVEL` | `low`, `default`, or `high` |
|
|
42
|
+
| `--framework FRAMEWORK` | `auto`, `rspec`, or `minitest` — forces the test framework instead of auto-detecting it |
|
|
40
43
|
| `--mutation-types TYPES` | Mutation types to apply, comma-separated |
|
|
41
44
|
| `--exit-zero` | Do not fail (exit 0) even if "survived" mutations exist |
|
|
42
45
|
| `--json-output PATH` | Export the results summary to a JSON file |
|
data/lib/mutaterb/cli.rb
CHANGED
|
@@ -39,10 +39,11 @@ module MutateRB
|
|
|
39
39
|
Signal.trap("INT") { interrupted = true }
|
|
40
40
|
|
|
41
41
|
detection = ProjectDetector.new(config).detect
|
|
42
|
-
if detection.
|
|
42
|
+
if detection.test_files.empty?
|
|
43
43
|
warn "mutaterb: no tests found in #{config.target_dir}"
|
|
44
44
|
return EXIT_OPERATIONAL_ERROR
|
|
45
45
|
end
|
|
46
|
+
warn_ambiguous_frameworks(detection)
|
|
46
47
|
|
|
47
48
|
test_suite = build_test_suite(config, detection)
|
|
48
49
|
run = MutationRun.new(config: config, test_suite: test_suite)
|
|
@@ -57,10 +58,21 @@ module MutateRB
|
|
|
57
58
|
run.exit_code
|
|
58
59
|
end
|
|
59
60
|
|
|
61
|
+
# FR-002/SC-003: when both frameworks are present and none was forced
|
|
62
|
+
# explicitly, the choice must be visible, never silent.
|
|
63
|
+
def warn_ambiguous_frameworks(detection)
|
|
64
|
+
return unless detection.ambiguous_frameworks
|
|
65
|
+
|
|
66
|
+
warn "mutaterb: detected both RSpec and Minitest — using #{detection.test_framework} " \
|
|
67
|
+
"(force one explicitly with --framework)"
|
|
68
|
+
end
|
|
69
|
+
|
|
60
70
|
def build_test_suite(config, detection)
|
|
61
|
-
test_runner = TestRunner.new(config: config, project_type: detection.project_type
|
|
62
|
-
|
|
63
|
-
|
|
71
|
+
test_runner = TestRunner.new(config: config, project_type: detection.project_type,
|
|
72
|
+
framework: detection.test_framework)
|
|
73
|
+
baseline_examples = test_runner.run_baseline(detection.test_files)
|
|
74
|
+
TestSuite.from_baseline(project_type: detection.project_type, framework: detection.test_framework,
|
|
75
|
+
baseline_examples: baseline_examples)
|
|
64
76
|
end
|
|
65
77
|
|
|
66
78
|
def run_mutations(run, test_suite, config)
|
data/lib/mutaterb/config.rb
CHANGED
|
@@ -8,14 +8,15 @@ module MutateRB
|
|
|
8
8
|
class Config
|
|
9
9
|
ALL_MUTATION_TYPES = %i[conditional_boundary boolean_literal nil_literal arithmetic_comparison].freeze
|
|
10
10
|
VALID_STRICTNESS = %i[low default high].freeze
|
|
11
|
+
VALID_TEST_FRAMEWORKS = %i[auto rspec minitest].freeze
|
|
11
12
|
DEFAULT_FILE_NAME = ".mutaterb.yml"
|
|
12
13
|
|
|
13
14
|
attr_accessor :target_dir, :include_paths, :exclude_paths, :strictness,
|
|
14
|
-
:mutation_types, :exit_on_survivors, :json_output_path
|
|
15
|
+
:mutation_types, :exit_on_survivors, :json_output_path, :test_framework
|
|
15
16
|
|
|
16
17
|
def initialize(target_dir: ".", include_paths: [], exclude_paths: [],
|
|
17
18
|
strictness: :default, mutation_types: ALL_MUTATION_TYPES.dup,
|
|
18
|
-
exit_on_survivors: true, json_output_path: nil)
|
|
19
|
+
exit_on_survivors: true, json_output_path: nil, test_framework: :auto)
|
|
19
20
|
@target_dir = target_dir
|
|
20
21
|
@include_paths = include_paths
|
|
21
22
|
@exclude_paths = exclude_paths
|
|
@@ -23,6 +24,7 @@ module MutateRB
|
|
|
23
24
|
@mutation_types = mutation_types
|
|
24
25
|
@exit_on_survivors = exit_on_survivors
|
|
25
26
|
@json_output_path = json_output_path
|
|
27
|
+
@test_framework = test_framework
|
|
26
28
|
validate!
|
|
27
29
|
end
|
|
28
30
|
|
|
@@ -46,7 +48,7 @@ module MutateRB
|
|
|
46
48
|
|
|
47
49
|
def self.attributes_from_yaml(raw)
|
|
48
50
|
known_keys = %w[target_dir include_paths exclude_paths strictness mutation_types
|
|
49
|
-
exit_on_survivors json_output_path]
|
|
51
|
+
exit_on_survivors json_output_path test_framework]
|
|
50
52
|
raw.each_key do |key|
|
|
51
53
|
warn "mutaterb: ignoring unknown config key #{key.inspect}" unless known_keys.include?(key)
|
|
52
54
|
end
|
|
@@ -64,7 +66,8 @@ module MutateRB
|
|
|
64
66
|
ALL_MUTATION_TYPES.dup
|
|
65
67
|
end,
|
|
66
68
|
exit_on_survivors: raw.fetch("exit_on_survivors", true),
|
|
67
|
-
json_output_path: raw["json_output_path"]
|
|
69
|
+
json_output_path: raw["json_output_path"],
|
|
70
|
+
test_framework: raw.key?("test_framework") ? symbolize(raw["test_framework"], "test_framework") : :auto
|
|
68
71
|
}
|
|
69
72
|
end
|
|
70
73
|
private_class_method :attributes_from_yaml
|
|
@@ -94,15 +97,27 @@ module MutateRB
|
|
|
94
97
|
raise ConfigError, "target_dir #{target_dir.inspect} does not exist" unless Dir.exist?(target_dir)
|
|
95
98
|
raise ConfigError, "include_paths must be an Array" unless include_paths.is_a?(Array)
|
|
96
99
|
raise ConfigError, "exclude_paths must be an Array" unless exclude_paths.is_a?(Array)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
+
|
|
101
|
+
validate_enum!(:strictness, strictness, VALID_STRICTNESS)
|
|
102
|
+
validate_enum!(:test_framework, test_framework, VALID_TEST_FRAMEWORKS)
|
|
100
103
|
raise ConfigError, "mutation_types must be an Array" unless mutation_types.is_a?(Array)
|
|
101
104
|
|
|
102
105
|
unknown = mutation_types - ALL_MUTATION_TYPES
|
|
103
106
|
raise ConfigError, "unknown mutation_types: #{unknown.join(', ')}" unless unknown.empty?
|
|
104
107
|
raise ConfigError, "exit_on_survivors must be true or false" unless [true, false].include?(exit_on_survivors)
|
|
105
108
|
|
|
109
|
+
validate_json_output_path!
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
private
|
|
113
|
+
|
|
114
|
+
def validate_enum!(field, value, allowed)
|
|
115
|
+
return if allowed.include?(value)
|
|
116
|
+
|
|
117
|
+
raise ConfigError, "#{field} must be one of #{allowed.join(', ')}, got #{value.inspect}"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def validate_json_output_path!
|
|
106
121
|
return unless json_output_path
|
|
107
122
|
|
|
108
123
|
parent = File.dirname(File.expand_path(json_output_path))
|
data/lib/mutaterb/flag_parser.rb
CHANGED
|
@@ -35,6 +35,7 @@ module MutateRB
|
|
|
35
35
|
flags[:exclude_paths] = v.split(",")
|
|
36
36
|
end
|
|
37
37
|
opts.on("--strictness LEVEL", "low|default|high") { |v| flags[:strictness] = v.to_sym }
|
|
38
|
+
opts.on("--framework FRAMEWORK", "auto|rspec|minitest") { |v| flags[:test_framework] = v.to_sym }
|
|
38
39
|
opts.on("--mutation-types TYPES", "Mutation types to apply, comma-separated") do |v|
|
|
39
40
|
flags[:mutation_types] = v.split(",").map(&:to_sym)
|
|
40
41
|
end
|
data/lib/mutaterb/mutator.rb
CHANGED
|
@@ -15,7 +15,8 @@ module MutateRB
|
|
|
15
15
|
def initialize(config:, test_suite:, test_runner: nil)
|
|
16
16
|
@config = config
|
|
17
17
|
@test_suite = test_suite
|
|
18
|
-
@test_runner = test_runner || TestRunner.new(config: config, project_type: test_suite.project_type
|
|
18
|
+
@test_runner = test_runner || TestRunner.new(config: config, project_type: test_suite.project_type,
|
|
19
|
+
framework: test_suite.framework)
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
# Yields each finished Mutant, one at a time.
|
|
@@ -31,7 +32,7 @@ module MutateRB
|
|
|
31
32
|
|
|
32
33
|
def source_files
|
|
33
34
|
Dir.glob(File.join(config.target_dir, "**", "*.rb"))
|
|
34
|
-
.reject { |f| f.include?("/spec/") || f.
|
|
35
|
+
.reject { |f| f.include?("/spec/") || f.include?("/test/") }
|
|
35
36
|
.select { |f| in_scope?(f) }
|
|
36
37
|
end
|
|
37
38
|
|
|
@@ -127,22 +128,26 @@ module MutateRB
|
|
|
127
128
|
end
|
|
128
129
|
end
|
|
129
130
|
|
|
130
|
-
# Convention-based coverage mapping: lib/foo/bar.rb -> spec/foo/bar_spec.rb
|
|
131
|
+
# Convention-based coverage mapping: lib/foo/bar.rb -> spec/foo/bar_spec.rb
|
|
132
|
+
# (or test/foo/bar_test.rb for Minitest, feature 004 research.md #5).
|
|
131
133
|
# ponytail: no real coverage tracking yet; falls back to the whole suite
|
|
132
|
-
# when no matching
|
|
134
|
+
# when no matching test file exists. Upgrade path: integrate SimpleCov
|
|
133
135
|
# coverage data if this heuristic proves too coarse for real projects.
|
|
134
136
|
def related_tests_for(source_file)
|
|
135
|
-
mapped =
|
|
137
|
+
mapped = mapped_test_file(source_file)
|
|
136
138
|
matches = test_suite.test_cases.select { |t| t.file_path == mapped }
|
|
137
139
|
matches = test_suite.test_cases.dup if matches.empty?
|
|
138
140
|
matches.reject(&:baseline_broken?)
|
|
139
141
|
end
|
|
140
142
|
|
|
141
|
-
def
|
|
143
|
+
def mapped_test_file(source_file)
|
|
142
144
|
relative = source_file.sub(%r{\A#{Regexp.escape(config.target_dir)}/?}, "")
|
|
143
145
|
relative = relative.sub(%r{\A(lib|app)/}, "")
|
|
144
|
-
|
|
145
|
-
|
|
146
|
+
if test_suite.framework == :minitest
|
|
147
|
+
File.join(config.target_dir, "test", relative.sub(/\.rb\z/, "_test.rb"))
|
|
148
|
+
else
|
|
149
|
+
File.join(config.target_dir, "spec", relative.sub(/\.rb\z/, "_spec.rb"))
|
|
150
|
+
end
|
|
146
151
|
end
|
|
147
152
|
end
|
|
148
153
|
end
|
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module MutateRB
|
|
4
|
-
# Detects whether the target project is Ruby or Rails
|
|
5
|
-
# files (FR-001,
|
|
4
|
+
# Detects whether the target project is Ruby or Rails, which test framework
|
|
5
|
+
# it uses (RSpec and/or Minitest), and locates its test files (FR-001,
|
|
6
|
+
# FR-002, FR-003).
|
|
6
7
|
class ProjectDetector
|
|
7
|
-
Detection = Struct.new(:project_type, :
|
|
8
|
+
Detection = Struct.new(:project_type, :test_framework, :test_files, :ambiguous_frameworks)
|
|
8
9
|
|
|
9
10
|
def initialize(config)
|
|
10
11
|
@config = config
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
def detect
|
|
14
|
-
|
|
15
|
+
framework, ambiguous = resolve_test_framework
|
|
16
|
+
Detection.new(project_type, framework, discover_test_files(framework), ambiguous)
|
|
15
17
|
end
|
|
16
18
|
|
|
17
19
|
private
|
|
@@ -24,9 +26,33 @@ module MutateRB
|
|
|
24
26
|
File.exist?(rails_marker) || File.exist?(rails_bin) ? :rails : :ruby
|
|
25
27
|
end
|
|
26
28
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
# research.md #1: an explicit config.test_framework always wins; otherwise
|
|
30
|
+
# detect by which test files actually exist, RSpec winning when both do.
|
|
31
|
+
def resolve_test_framework
|
|
32
|
+
return [config.test_framework, false] unless config.test_framework == :auto
|
|
33
|
+
|
|
34
|
+
has_rspec = test_files?(:rspec)
|
|
35
|
+
has_minitest = test_files?(:minitest)
|
|
36
|
+
return [:rspec, true] if has_rspec && has_minitest
|
|
37
|
+
return [:minitest, false] if has_minitest && !has_rspec
|
|
38
|
+
|
|
39
|
+
[:rspec, false]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def test_files?(framework)
|
|
43
|
+
!Dir.glob(test_file_pattern(framework)).empty?
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def discover_test_files(framework)
|
|
47
|
+
apply_scope(Dir.glob(test_file_pattern(framework)))
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def test_file_pattern(framework)
|
|
51
|
+
if framework == :minitest
|
|
52
|
+
File.join(config.target_dir, "test", "**", "*_test.rb")
|
|
53
|
+
else
|
|
54
|
+
File.join(config.target_dir, "spec", "**", "*_spec.rb")
|
|
55
|
+
end
|
|
30
56
|
end
|
|
31
57
|
|
|
32
58
|
def apply_scope(files)
|
data/lib/mutaterb/reporter.rb
CHANGED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "shellwords"
|
|
4
|
+
|
|
5
|
+
module MutateRB
|
|
6
|
+
module TestAdapters
|
|
7
|
+
# Builds a Minitest invocation and parses its native `--verbose` output —
|
|
8
|
+
# no added dependency in the target project (FR-009, research.md #2/#4).
|
|
9
|
+
#
|
|
10
|
+
# Minitest's own runner only knows how to run one file's worth of tests
|
|
11
|
+
# per process, and its verbose line format doesn't name the source file,
|
|
12
|
+
# so this adapter spawns one process per file (joined with `&&` into a
|
|
13
|
+
# single shell command, so TestRunner still only tracks one pid) and
|
|
14
|
+
# emits a marker line between them to attribute each parsed example back
|
|
15
|
+
# to the file it came from.
|
|
16
|
+
class MinitestAdapter
|
|
17
|
+
LINE_PATTERN = /^(\S+)\s*=\s*([\d.]+)\s*s\s*=\s*([.FES])/
|
|
18
|
+
FILE_MARKER = "@@MUTATERB_FILE@@"
|
|
19
|
+
|
|
20
|
+
def self.command_for(files, project_type:)
|
|
21
|
+
files.map { |file| "echo #{FILE_MARKER}#{Shellwords.escape(file)} && #{run_one(file, project_type)}" }
|
|
22
|
+
.join(" && ")
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.run_one(file, project_type)
|
|
26
|
+
runner = project_type == :rails ? "bin/rails test" : "bundle exec ruby -Itest -Ilib"
|
|
27
|
+
"#{runner} #{Shellwords.escape(file)} -v"
|
|
28
|
+
end
|
|
29
|
+
private_class_method :run_one
|
|
30
|
+
|
|
31
|
+
def self.parse(raw, files: [])
|
|
32
|
+
current_file = files.first
|
|
33
|
+
examples = []
|
|
34
|
+
raw.each_line do |line|
|
|
35
|
+
if (marker = line[/\A#{Regexp.escape(FILE_MARKER)}(.+)/, 1])
|
|
36
|
+
current_file = marker.strip
|
|
37
|
+
next
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
append_example(examples, line, current_file)
|
|
41
|
+
end
|
|
42
|
+
{ status: :completed, examples: examples }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.append_example(examples, line, file)
|
|
46
|
+
match = line.match(LINE_PATTERN)
|
|
47
|
+
return unless match
|
|
48
|
+
|
|
49
|
+
result_char = match[3]
|
|
50
|
+
return if result_char == "S" # skipped: excluded, not passed/failed (research.md #2)
|
|
51
|
+
|
|
52
|
+
examples << {
|
|
53
|
+
id: match[1],
|
|
54
|
+
description: match[1],
|
|
55
|
+
file_path: file,
|
|
56
|
+
status: result_char == "." ? :passed : :failed,
|
|
57
|
+
duration: match[2].to_f
|
|
58
|
+
}
|
|
59
|
+
end
|
|
60
|
+
private_class_method :append_example
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "shellwords"
|
|
5
|
+
|
|
6
|
+
module MutateRB
|
|
7
|
+
module TestAdapters
|
|
8
|
+
# Builds the `bundle exec rspec` command and parses its `--format json`
|
|
9
|
+
# output (extracted from TestRunner, research.md #3 of feature 004 —
|
|
10
|
+
# MutateRB never bundles its own RSpec, it shells out to the target
|
|
11
|
+
# project's own).
|
|
12
|
+
class RspecAdapter
|
|
13
|
+
def self.command_for(files, project_type:) # rubocop:disable Lint/UnusedMethodArgument
|
|
14
|
+
"bundle exec rspec --format json #{Shellwords.join(files)}"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.parse(raw, files: []) # rubocop:disable Lint/UnusedMethodArgument
|
|
18
|
+
data = JSON.parse(raw)
|
|
19
|
+
examples = data.fetch("examples", []).map do |example|
|
|
20
|
+
{
|
|
21
|
+
id: example["id"],
|
|
22
|
+
description: example["full_description"],
|
|
23
|
+
file_path: example["file_path"],
|
|
24
|
+
status: example["status"] == "passed" ? :passed : :failed,
|
|
25
|
+
duration: example["run_time"].to_f
|
|
26
|
+
}
|
|
27
|
+
end
|
|
28
|
+
{ status: :completed, examples: examples }
|
|
29
|
+
rescue JSON::ParserError
|
|
30
|
+
{ status: :error, examples: [] }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
data/lib/mutaterb/test_runner.rb
CHANGED
|
@@ -1,28 +1,34 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "json"
|
|
4
3
|
require "timeout"
|
|
5
4
|
require "bundler"
|
|
6
5
|
|
|
7
6
|
module MutateRB
|
|
8
|
-
# Runs the target project's own
|
|
9
|
-
# MutateRB never bundles its own RSpec, it shells
|
|
10
|
-
#
|
|
11
|
-
#
|
|
7
|
+
# Runs the target project's own test suite as a subprocess (research.md #2
|
|
8
|
+
# of feature 001): MutateRB never bundles its own RSpec/Minitest, it shells
|
|
9
|
+
# out to the target project's own. The actual command and output parsing
|
|
10
|
+
# are delegated to a framework-specific adapter (research.md #3 of feature
|
|
11
|
+
# 004); this class owns spawning, timeouts, and killing hung processes.
|
|
12
12
|
class TestRunner
|
|
13
13
|
MIN_TIMEOUT_SECONDS = 5
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
ADAPTERS = {
|
|
16
|
+
rspec: TestAdapters::RspecAdapter,
|
|
17
|
+
minitest: TestAdapters::MinitestAdapter
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
def initialize(config:, project_type:, framework: :rspec)
|
|
16
21
|
@config = config
|
|
17
22
|
@project_type = project_type
|
|
23
|
+
@adapter = ADAPTERS.fetch(framework)
|
|
18
24
|
end
|
|
19
25
|
|
|
20
|
-
# Runs the given
|
|
21
|
-
#
|
|
26
|
+
# Runs the given test files once, unmutated, and returns one Hash per
|
|
27
|
+
# test example: { id:, description:, file_path:, status:, duration: }.
|
|
22
28
|
# Used to build TestCase instances (FR-012's baseline_status) and as the
|
|
23
29
|
# basis for the per-mutant timeout (FR-014).
|
|
24
|
-
def run_baseline(
|
|
25
|
-
|
|
30
|
+
def run_baseline(files)
|
|
31
|
+
spawn_test_command(files, timeout_seconds: nil).fetch(:examples, [])
|
|
26
32
|
end
|
|
27
33
|
|
|
28
34
|
# Runs the given test cases against a mutation. Timeout = 2x the slowest
|
|
@@ -35,82 +41,66 @@ module MutateRB
|
|
|
35
41
|
slowest_baseline = test_cases.filter_map(&:baseline_duration_seconds).max || 0
|
|
36
42
|
timeout_seconds = [slowest_baseline * 2, MIN_TIMEOUT_SECONDS].max
|
|
37
43
|
files = test_cases.map(&:file_path).uniq
|
|
38
|
-
|
|
44
|
+
spawn_test_command(files, timeout_seconds: timeout_seconds)
|
|
39
45
|
end
|
|
40
46
|
|
|
41
47
|
private
|
|
42
48
|
|
|
43
|
-
attr_reader :config, :project_type
|
|
49
|
+
attr_reader :config, :project_type, :adapter
|
|
44
50
|
|
|
45
51
|
# Runs inside Bundler.with_unbundled_env (research.md #2): when `mutaterb`
|
|
46
52
|
# itself is invoked via `bundle exec`, BUNDLE_GEMFILE/RUBYOPT point at
|
|
47
|
-
# MutateRB's own Gemfile and would otherwise leak into this child process
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
|
|
53
|
+
# MutateRB's own Gemfile and would otherwise leak into this child process.
|
|
54
|
+
# Spawned in its own process group (pgroup: true) so that when the
|
|
55
|
+
# adapter's command is a shell pipeline (Minitest's multi-file case joins
|
|
56
|
+
# several invocations with `&&`), killing the group also reaches the
|
|
57
|
+
# actual test process, not just the intermediate shell.
|
|
58
|
+
def spawn_test_command(files, timeout_seconds:)
|
|
52
59
|
env = project_type == :rails ? { "RAILS_ENV" => "test" } : {}
|
|
60
|
+
command = adapter.command_for(files, project_type: project_type)
|
|
53
61
|
stdout_read, stdout_write = IO.pipe
|
|
54
62
|
pid = Bundler.with_unbundled_env do
|
|
55
|
-
Process.spawn(env,
|
|
56
|
-
|
|
63
|
+
Process.spawn(env, command, out: stdout_write, err: File::NULL, pgroup: true,
|
|
64
|
+
chdir: File.expand_path(config.target_dir))
|
|
57
65
|
end
|
|
58
66
|
stdout_write.close
|
|
59
67
|
|
|
60
|
-
output = wait_with_timeout(pid, stdout_read, timeout_seconds)
|
|
68
|
+
output = wait_with_timeout(pid, stdout_read, timeout_seconds, files)
|
|
61
69
|
stdout_read.close unless stdout_read.closed?
|
|
62
70
|
output
|
|
63
71
|
end
|
|
64
72
|
|
|
65
|
-
def wait_with_timeout(pid, stdout_read, timeout_seconds)
|
|
73
|
+
def wait_with_timeout(pid, stdout_read, timeout_seconds, files)
|
|
66
74
|
if timeout_seconds
|
|
67
|
-
Timeout.timeout(timeout_seconds) { read_and_wait(pid, stdout_read) }
|
|
75
|
+
Timeout.timeout(timeout_seconds) { read_and_wait(pid, stdout_read, files) }
|
|
68
76
|
else
|
|
69
|
-
read_and_wait(pid, stdout_read)
|
|
77
|
+
read_and_wait(pid, stdout_read, files)
|
|
70
78
|
end
|
|
71
79
|
rescue Timeout::Error
|
|
72
80
|
kill(pid)
|
|
73
81
|
{ status: :timeout, examples: [] }
|
|
74
82
|
end
|
|
75
83
|
|
|
76
|
-
def read_and_wait(pid, stdout_read)
|
|
84
|
+
def read_and_wait(pid, stdout_read, files)
|
|
77
85
|
raw = stdout_read.read
|
|
78
86
|
Process.wait(pid)
|
|
79
|
-
|
|
87
|
+
adapter.parse(raw, files: files)
|
|
80
88
|
end
|
|
81
89
|
|
|
82
|
-
# Kills a hung child process: TERM first, KILL if it ignores TERM
|
|
83
|
-
# (research.md #3 — Timeout alone never touches the
|
|
90
|
+
# Kills a hung child process group: TERM first, KILL if it ignores TERM
|
|
91
|
+
# for 1s (research.md #3 of feature 001 — Timeout alone never touches the
|
|
92
|
+
# child process). Signals the whole process group (negative pid) so a
|
|
93
|
+
# shell-wrapped command's real child is reached too.
|
|
84
94
|
def kill(pid)
|
|
85
|
-
Process.kill("TERM", pid)
|
|
95
|
+
Process.kill("TERM", -pid)
|
|
86
96
|
begin
|
|
87
97
|
Timeout.timeout(1) { Process.wait(pid) }
|
|
88
98
|
rescue Timeout::Error
|
|
89
|
-
Process.kill("KILL", pid)
|
|
99
|
+
Process.kill("KILL", -pid)
|
|
90
100
|
Process.wait(pid)
|
|
91
101
|
end
|
|
92
102
|
rescue Errno::ESRCH, Errno::ECHILD
|
|
93
103
|
nil
|
|
94
104
|
end
|
|
95
|
-
|
|
96
|
-
def rspec_command(files)
|
|
97
|
-
["bundle", "exec", "rspec", "--format", "json", *files]
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
def parse_output(raw)
|
|
101
|
-
data = JSON.parse(raw)
|
|
102
|
-
examples = data.fetch("examples", []).map do |example|
|
|
103
|
-
{
|
|
104
|
-
id: example["id"],
|
|
105
|
-
description: example["full_description"],
|
|
106
|
-
file_path: example["file_path"],
|
|
107
|
-
status: example["status"] == "passed" ? :passed : :failed,
|
|
108
|
-
duration: example["run_time"].to_f
|
|
109
|
-
}
|
|
110
|
-
end
|
|
111
|
-
{ status: :completed, examples: examples }
|
|
112
|
-
rescue JSON::ParserError
|
|
113
|
-
{ status: :error, examples: [] }
|
|
114
|
-
end
|
|
115
105
|
end
|
|
116
106
|
end
|
data/lib/mutaterb/test_suite.rb
CHANGED
|
@@ -4,23 +4,25 @@ module MutateRB
|
|
|
4
4
|
# The collection of tests detected for the target project (data-model.md).
|
|
5
5
|
class TestSuite
|
|
6
6
|
VALID_PROJECT_TYPES = %i[ruby rails].freeze
|
|
7
|
+
VALID_FRAMEWORKS = %i[rspec minitest].freeze
|
|
7
8
|
|
|
8
9
|
attr_reader :framework, :project_type, :test_cases
|
|
9
10
|
|
|
10
|
-
def initialize(project_type:, test_cases: [])
|
|
11
|
+
def initialize(project_type:, framework: :rspec, test_cases: [])
|
|
11
12
|
unless VALID_PROJECT_TYPES.include?(project_type)
|
|
12
13
|
raise ArgumentError,
|
|
13
14
|
"invalid project_type #{project_type.inspect}"
|
|
14
15
|
end
|
|
16
|
+
raise ArgumentError, "invalid framework #{framework.inspect}" unless VALID_FRAMEWORKS.include?(framework)
|
|
15
17
|
|
|
16
|
-
@framework =
|
|
18
|
+
@framework = framework
|
|
17
19
|
@project_type = project_type
|
|
18
20
|
@test_cases = Array(test_cases)
|
|
19
21
|
end
|
|
20
22
|
|
|
21
23
|
# Builds a TestSuite from the raw example hashes returned by
|
|
22
24
|
# TestRunner#run_baseline (FR-012's baseline_status comes from here).
|
|
23
|
-
def self.from_baseline(project_type:, baseline_examples:)
|
|
25
|
+
def self.from_baseline(project_type:, framework:, baseline_examples:)
|
|
24
26
|
test_cases = baseline_examples.map do |example|
|
|
25
27
|
TestCase.new(
|
|
26
28
|
id: example.fetch(:id),
|
|
@@ -30,7 +32,7 @@ module MutateRB
|
|
|
30
32
|
baseline_duration_seconds: example.fetch(:duration)
|
|
31
33
|
)
|
|
32
34
|
end
|
|
33
|
-
new(project_type: project_type, test_cases: test_cases)
|
|
35
|
+
new(project_type: project_type, framework: framework, test_cases: test_cases)
|
|
34
36
|
end
|
|
35
37
|
|
|
36
38
|
def find(id)
|
data/lib/mutaterb/version.rb
CHANGED
data/lib/mutaterb.rb
CHANGED
|
@@ -9,6 +9,8 @@ require_relative "mutaterb/config"
|
|
|
9
9
|
require_relative "mutaterb/mutation_run"
|
|
10
10
|
require_relative "mutaterb/flag_parser"
|
|
11
11
|
require_relative "mutaterb/project_detector"
|
|
12
|
+
require_relative "mutaterb/test_adapters/rspec_adapter"
|
|
13
|
+
require_relative "mutaterb/test_adapters/minitest_adapter"
|
|
12
14
|
require_relative "mutaterb/test_runner"
|
|
13
15
|
require_relative "mutaterb/mutation_operators/base_operator"
|
|
14
16
|
require_relative "mutaterb/mutation_operators/conditional_boundary_operator"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mutaterb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- MarceloM47
|
|
@@ -78,6 +78,8 @@ files:
|
|
|
78
78
|
- lib/mutaterb/mutator.rb
|
|
79
79
|
- lib/mutaterb/project_detector.rb
|
|
80
80
|
- lib/mutaterb/reporter.rb
|
|
81
|
+
- lib/mutaterb/test_adapters/minitest_adapter.rb
|
|
82
|
+
- lib/mutaterb/test_adapters/rspec_adapter.rb
|
|
81
83
|
- lib/mutaterb/test_case.rb
|
|
82
84
|
- lib/mutaterb/test_runner.rb
|
|
83
85
|
- lib/mutaterb/test_suite.rb
|