evilution 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +15 -0
- data/.claude/prompts/architect.md +14 -1
- data/.claude/skills/create-issue/SKILL.md +55 -0
- data/.rubocop_todo.yml +7 -0
- data/CHANGELOG.md +38 -0
- data/README.md +57 -3
- data/lib/evilution/ast/constant_names.rb +34 -0
- data/lib/evilution/cache.rb +2 -0
- data/lib/evilution/child_output.rb +24 -0
- data/lib/evilution/cli/commands/run.rb +9 -0
- data/lib/evilution/cli/commands/version.rb +2 -0
- data/lib/evilution/cli/parser/options_builder.rb +16 -2
- data/lib/evilution/compare/invalid_input.rb +12 -0
- data/lib/evilution/compare.rb +1 -10
- data/lib/evilution/config/builders/spec_resolver.rb +15 -0
- data/lib/evilution/config/builders/spec_selector.rb +16 -0
- data/lib/evilution/config/builders.rb +4 -0
- data/lib/evilution/config/env_loader.rb +12 -0
- data/lib/evilution/config/file_loader.rb +22 -0
- data/lib/evilution/config/sources.rb +14 -0
- data/lib/evilution/config/validators/base.rb +37 -0
- data/lib/evilution/config/validators/example_targeting_cache.rb +37 -0
- data/lib/evilution/config/validators/example_targeting_fallback.rb +22 -0
- data/lib/evilution/config/validators/fail_fast.rb +11 -0
- data/lib/evilution/config/validators/hooks.rb +12 -0
- data/lib/evilution/config/validators/ignore_patterns.rb +16 -0
- data/lib/evilution/config/validators/integration.rb +11 -0
- data/lib/evilution/config/validators/isolation.rb +19 -0
- data/lib/evilution/config/validators/jobs.rb +9 -0
- data/lib/evilution/config/validators/preload.rb +13 -0
- data/lib/evilution/config/validators/spec_mappings.rb +56 -0
- data/lib/evilution/config/validators/spec_pattern.rb +12 -0
- data/lib/evilution/config/validators.rb +4 -0
- data/lib/evilution/config.rb +78 -268
- data/lib/evilution/feedback/detector.rb +15 -0
- data/lib/evilution/feedback/messages.rb +42 -0
- data/lib/evilution/feedback.rb +5 -0
- data/lib/evilution/integration/base.rb +4 -155
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +49 -0
- data/lib/evilution/integration/loading/constant_pinner.rb +24 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +52 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +54 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +15 -0
- data/lib/evilution/integration/loading/syntax_validator.rb +19 -0
- data/lib/evilution/integration/loading.rb +6 -0
- data/lib/evilution/integration/rspec/baseline_runner.rb +16 -0
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +17 -0
- data/lib/evilution/integration/rspec/example_filter_applier.rb +21 -0
- data/lib/evilution/integration/rspec/framework_loader.rb +28 -0
- data/lib/evilution/integration/rspec/result_builder.rb +40 -0
- data/lib/evilution/integration/rspec/state_guard/example_groups_constants.rb +28 -0
- data/lib/evilution/integration/rspec/state_guard/internals.rb +19 -0
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +35 -0
- data/lib/evilution/integration/rspec/state_guard/reporter_arrays.rb +32 -0
- data/lib/evilution/integration/rspec/state_guard/world_example_groups.rb +20 -0
- data/lib/evilution/integration/rspec/state_guard/world_filtered_examples.rb +20 -0
- data/lib/evilution/integration/rspec/state_guard/world_sources_by_path.rb +20 -0
- data/lib/evilution/integration/rspec/state_guard.rb +40 -0
- data/lib/evilution/integration/rspec/test_file_resolver.rb +30 -0
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +18 -0
- data/lib/evilution/integration/rspec.rb +61 -232
- data/lib/evilution/isolation/fork.rb +7 -2
- data/lib/evilution/load_path/subpath_resolver.rb +25 -0
- data/lib/evilution/load_path.rb +4 -0
- data/lib/evilution/mcp/info_tool/actions/base.rb +22 -0
- data/lib/evilution/mcp/info_tool/actions/environment.rb +42 -0
- data/lib/evilution/mcp/info_tool/actions/feedback.rb +16 -0
- data/lib/evilution/mcp/info_tool/actions/statuses.rb +10 -0
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +47 -0
- data/lib/evilution/mcp/info_tool/actions/tests.rb +60 -0
- data/lib/evilution/mcp/info_tool/actions.rb +16 -0
- data/lib/evilution/mcp/info_tool/config_factory.rb +24 -0
- data/lib/evilution/mcp/info_tool/error_mapper.rb +15 -0
- data/lib/evilution/mcp/info_tool/request_parser.rb +34 -0
- data/lib/evilution/mcp/info_tool/response_formatter.rb +24 -0
- data/lib/evilution/mcp/info_tool/status_glossary.rb +75 -0
- data/lib/evilution/mcp/info_tool.rb +43 -261
- data/lib/evilution/mcp/mutate_tool/error_payload.rb +8 -1
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +13 -1
- data/lib/evilution/mcp/mutate_tool.rb +5 -2
- data/lib/evilution/mutator/operator/block_removal.rb +1 -1
- data/lib/evilution/mutator/operator/method_body_replacement.rb +18 -2
- data/lib/evilution/parallel/work_queue/channel/frame.rb +21 -0
- data/lib/evilution/parallel/work_queue/channel.rb +23 -0
- data/lib/evilution/parallel/work_queue/collection_state.rb +14 -0
- data/lib/evilution/parallel/work_queue/dispatcher.rb +133 -0
- data/lib/evilution/parallel/work_queue/validators/optional_positive_int.rb +11 -0
- data/lib/evilution/parallel/work_queue/validators/optional_positive_number.rb +11 -0
- data/lib/evilution/parallel/work_queue/validators/positive_int.rb +11 -0
- data/lib/evilution/parallel/work_queue/validators.rb +6 -0
- data/lib/evilution/parallel/work_queue/worker/loop.rb +45 -0
- data/lib/evilution/parallel/work_queue/worker.rb +114 -0
- data/lib/evilution/parallel/work_queue/worker_stat.rb +17 -0
- data/lib/evilution/parallel/work_queue.rb +42 -327
- data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +18 -0
- data/lib/evilution/reporter/cli/item_formatters/disabled.rb +9 -0
- data/lib/evilution/reporter/cli/item_formatters/error.rb +14 -0
- data/lib/evilution/reporter/cli/item_formatters/result_location.rb +10 -0
- data/lib/evilution/reporter/cli/item_formatters.rb +6 -0
- data/lib/evilution/reporter/cli/line_formatters/duration.rb +9 -0
- data/lib/evilution/reporter/cli/line_formatters/efficiency.rb +18 -0
- data/lib/evilution/reporter/cli/line_formatters/feedback_footer.rb +13 -0
- data/lib/evilution/reporter/cli/line_formatters/header.rb +10 -0
- data/lib/evilution/reporter/cli/line_formatters/mutations.rb +16 -0
- data/lib/evilution/reporter/cli/line_formatters/peak_memory.rb +12 -0
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +20 -0
- data/lib/evilution/reporter/cli/line_formatters/score.rb +14 -0
- data/lib/evilution/reporter/cli/line_formatters/truncation_notice.rb +11 -0
- data/lib/evilution/reporter/cli/line_formatters.rb +6 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +26 -0
- data/lib/evilution/reporter/cli/pct.rb +9 -0
- data/lib/evilution/reporter/cli/section.rb +13 -0
- data/lib/evilution/reporter/cli/section_renderer.rb +15 -0
- data/lib/evilution/reporter/cli/trailer.rb +22 -0
- data/lib/evilution/reporter/cli.rb +79 -162
- data/lib/evilution/runner/isolation_resolver.rb +20 -2
- data/lib/evilution/runner/mutation_executor/mutation_runner.rb +32 -0
- data/lib/evilution/runner/mutation_executor/neutralization_pipeline.rb +16 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +46 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +75 -0
- data/lib/evilution/runner/mutation_executor/result_cache.rb +69 -0
- data/lib/evilution/runner/mutation_executor/result_notifier.rb +48 -0
- data/lib/evilution/runner/mutation_executor/result_packer.rb +39 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +80 -0
- data/lib/evilution/runner/mutation_executor/strategy/sequential.rb +34 -0
- data/lib/evilution/runner/mutation_executor.rb +58 -289
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +21 -0
- data/lib/evilution/version.rb +1 -1
- metadata +125 -5
- data/lib/evilution/mcp/session_diff_tool.rb +0 -63
- data/lib/evilution/mcp/session_list_tool.rb +0 -50
- data/lib/evilution/mcp/session_show_tool.rb +0 -57
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../cli"
|
|
4
|
+
require_relative "line_formatters/truncation_notice"
|
|
5
|
+
require_relative "line_formatters/result_line"
|
|
6
|
+
require_relative "line_formatters/feedback_footer"
|
|
7
|
+
|
|
8
|
+
class Evilution::Reporter::CLI::Trailer
|
|
9
|
+
DEFAULT_LINES = [
|
|
10
|
+
Evilution::Reporter::CLI::LineFormatters::TruncationNotice.new,
|
|
11
|
+
Evilution::Reporter::CLI::LineFormatters::ResultLine.new,
|
|
12
|
+
Evilution::Reporter::CLI::LineFormatters::FeedbackFooter.new
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
15
|
+
def initialize(lines: DEFAULT_LINES)
|
|
16
|
+
@lines = lines
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def call(summary)
|
|
20
|
+
@lines.filter_map { |line| line.format(summary) }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -5,172 +5,89 @@ require_relative "../reporter"
|
|
|
5
5
|
class Evilution::Reporter::CLI
|
|
6
6
|
SEPARATOR = "=" * 44
|
|
7
7
|
|
|
8
|
-
def
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
8
|
+
def initialize(
|
|
9
|
+
header: LineFormatters::Header.new,
|
|
10
|
+
metrics_block: MetricsBlock.new,
|
|
11
|
+
section_renderer: SectionRenderer.new,
|
|
12
|
+
sections: DEFAULT_SECTIONS,
|
|
13
|
+
trailer: Trailer.new
|
|
14
|
+
)
|
|
15
|
+
@header = header
|
|
16
|
+
@metrics_block = metrics_block
|
|
17
|
+
@section_renderer = section_renderer
|
|
18
|
+
@sections = sections
|
|
19
|
+
@trailer = trailer
|
|
16
20
|
end
|
|
17
21
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
lines << header
|
|
22
|
+
def call(summary)
|
|
23
|
+
lines = []
|
|
24
|
+
lines << @header.format(summary)
|
|
22
25
|
lines << SEPARATOR
|
|
23
26
|
lines << ""
|
|
24
|
-
lines
|
|
25
|
-
|
|
26
|
-
lines << duration_line(summary)
|
|
27
|
-
lines << efficiency_line(summary) if summary.duration.positive?
|
|
28
|
-
peak = summary.peak_memory_mb
|
|
29
|
-
lines << peak_memory_line(peak) if peak
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
def append_sections(lines, summary)
|
|
33
|
-
append_survived(lines, summary)
|
|
34
|
-
append_neutral(lines, summary)
|
|
35
|
-
append_equivalent(lines, summary)
|
|
36
|
-
append_unresolved(lines, summary)
|
|
37
|
-
append_unparseable(lines, summary)
|
|
38
|
-
append_errors(lines, summary)
|
|
39
|
-
append_disabled(lines, summary)
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def append_survived(lines, summary)
|
|
43
|
-
gaps = summary.coverage_gaps
|
|
44
|
-
return unless gaps.any?
|
|
45
|
-
|
|
46
|
-
lines << ""
|
|
47
|
-
lines << "Survived mutations (#{gaps.length} coverage gap#{"s" unless gaps.length == 1}):"
|
|
48
|
-
gaps.each { |gap| lines << format_coverage_gap(gap) }
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def append_neutral(lines, summary)
|
|
52
|
-
return unless summary.neutral_results.any?
|
|
53
|
-
|
|
54
|
-
lines << ""
|
|
55
|
-
lines << "Neutral mutations (test already failing):"
|
|
56
|
-
summary.neutral_results.each { |result| lines << format_neutral(result) }
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def append_equivalent(lines, summary)
|
|
60
|
-
return unless summary.equivalent_results.any?
|
|
61
|
-
|
|
62
|
-
lines << ""
|
|
63
|
-
lines << "Equivalent mutations (provably identical behavior):"
|
|
64
|
-
summary.equivalent_results.each { |result| lines << format_neutral(result) }
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
def append_unresolved(lines, summary)
|
|
68
|
-
return unless summary.unresolved_results.any?
|
|
69
|
-
|
|
70
|
-
lines << ""
|
|
71
|
-
lines << "Unresolved mutations (no test file resolved):"
|
|
72
|
-
summary.unresolved_results.each { |result| lines << format_neutral(result) }
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
def append_unparseable(lines, summary)
|
|
76
|
-
return unless summary.unparseable_results.any?
|
|
77
|
-
|
|
78
|
-
lines << ""
|
|
79
|
-
lines << "Unparseable mutations (mutated source did not parse):"
|
|
80
|
-
summary.unparseable_results.each { |result| lines << format_neutral(result) }
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def append_errors(lines, summary)
|
|
84
|
-
errored = summary.results.select(&:error?)
|
|
85
|
-
return if errored.empty?
|
|
86
|
-
|
|
27
|
+
lines.concat(@metrics_block.call(summary))
|
|
28
|
+
@sections.each { |section| lines.concat(@section_renderer.call(section, summary)) }
|
|
87
29
|
lines << ""
|
|
88
|
-
lines
|
|
89
|
-
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
def format_error(result)
|
|
93
|
-
mutation = result.mutation
|
|
94
|
-
header = " #{mutation.operator_name}: #{mutation.file_path}:#{mutation.line}"
|
|
95
|
-
return header unless result.error_message
|
|
96
|
-
|
|
97
|
-
indented = result.error_message.lines.map { |line| " #{line.chomp}" }.join("\n")
|
|
98
|
-
"#{header}\n#{indented}"
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
def append_disabled(lines, summary)
|
|
102
|
-
return unless summary.disabled_mutations.any?
|
|
103
|
-
|
|
104
|
-
lines << ""
|
|
105
|
-
lines << "Disabled mutations (skipped by # evilution:disable):"
|
|
106
|
-
summary.disabled_mutations.each { |mutation| lines << format_disabled(mutation) }
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
def header
|
|
110
|
-
"Evilution v#{Evilution::VERSION} — Mutation Testing Results"
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
def mutations_line(summary)
|
|
114
|
-
parts = "Mutations: #{summary.total} total, #{summary.killed} killed, " \
|
|
115
|
-
"#{summary.survived} survived, #{summary.timed_out} timed out"
|
|
116
|
-
parts += ", #{summary.neutral} neutral" if summary.neutral.positive?
|
|
117
|
-
parts += ", #{summary.equivalent} equivalent" if summary.equivalent.positive?
|
|
118
|
-
parts += ", #{summary.unresolved} unresolved" if summary.unresolved.positive?
|
|
119
|
-
parts += ", #{summary.unparseable} unparseable" if summary.unparseable.positive?
|
|
120
|
-
parts += ", #{summary.skipped} skipped" if summary.skipped.positive?
|
|
121
|
-
parts
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
def score_line(summary)
|
|
125
|
-
score_pct = format_pct(summary.score)
|
|
126
|
-
"Score: #{score_pct} (#{summary.killed}/#{summary.score_denominator})"
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
def duration_line(summary)
|
|
130
|
-
"Duration: #{format("%.2f", summary.duration)}s"
|
|
131
|
-
end
|
|
132
|
-
|
|
133
|
-
def efficiency_line(summary)
|
|
134
|
-
pct = format("%.2f%%", summary.efficiency * 100)
|
|
135
|
-
rate = format("%.2f", summary.mutations_per_second)
|
|
136
|
-
"Efficiency: #{pct} killtime, #{rate} mutations/s"
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
def format_coverage_gap(gap)
|
|
140
|
-
location = "#{gap.file_path}:#{gap.line}"
|
|
141
|
-
header = if gap.single?
|
|
142
|
-
" #{gap.primary_operator}: #{location} (#{gap.subject_name})"
|
|
143
|
-
else
|
|
144
|
-
operators = gap.operator_names.join(", ")
|
|
145
|
-
" #{location} (#{gap.subject_name}) [#{gap.count} mutations: #{operators}]"
|
|
146
|
-
end
|
|
147
|
-
body = gap.mutation_results.first.mutation.unified_diff || gap.primary_diff
|
|
148
|
-
indented = body.split("\n").map { |l| " #{l}" }.join("\n")
|
|
149
|
-
"#{header}\n#{indented}"
|
|
150
|
-
end
|
|
151
|
-
|
|
152
|
-
def format_neutral(result)
|
|
153
|
-
mutation = result.mutation
|
|
154
|
-
" #{mutation.operator_name}: #{mutation.file_path}:#{mutation.line}"
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
def format_disabled(mutation)
|
|
158
|
-
" #{mutation.operator_name}: #{mutation.file_path}:#{mutation.line}"
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
def result_line(summary)
|
|
162
|
-
min_score = 0.8
|
|
163
|
-
pass_fail = summary.success?(min_score: min_score) ? "PASS" : "FAIL"
|
|
164
|
-
score_pct = format_pct(summary.score)
|
|
165
|
-
threshold_pct = format_pct(min_score)
|
|
166
|
-
"Result: #{pass_fail} (score #{score_pct} #{pass_fail == "PASS" ? ">=" : "<"} #{threshold_pct})"
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
def peak_memory_line(peak_mb)
|
|
170
|
-
format("Peak memory: %<mb>.1f MB", mb: peak_mb)
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
def format_pct(value)
|
|
174
|
-
format("%.2f%%", value * 100)
|
|
30
|
+
lines.concat(@trailer.call(summary))
|
|
31
|
+
lines.join("\n")
|
|
175
32
|
end
|
|
176
33
|
end
|
|
34
|
+
|
|
35
|
+
require_relative "cli/pct"
|
|
36
|
+
require_relative "cli/section"
|
|
37
|
+
require_relative "cli/section_renderer"
|
|
38
|
+
require_relative "cli/line_formatters/header"
|
|
39
|
+
require_relative "cli/line_formatters/mutations"
|
|
40
|
+
require_relative "cli/line_formatters/score"
|
|
41
|
+
require_relative "cli/line_formatters/duration"
|
|
42
|
+
require_relative "cli/line_formatters/efficiency"
|
|
43
|
+
require_relative "cli/line_formatters/peak_memory"
|
|
44
|
+
require_relative "cli/line_formatters/truncation_notice"
|
|
45
|
+
require_relative "cli/line_formatters/result_line"
|
|
46
|
+
require_relative "cli/line_formatters/feedback_footer"
|
|
47
|
+
require_relative "cli/item_formatters/coverage_gap"
|
|
48
|
+
require_relative "cli/item_formatters/result_location"
|
|
49
|
+
require_relative "cli/item_formatters/error"
|
|
50
|
+
require_relative "cli/item_formatters/disabled"
|
|
51
|
+
require_relative "cli/metrics_block"
|
|
52
|
+
require_relative "cli/trailer"
|
|
53
|
+
|
|
54
|
+
Evilution::Reporter::CLI.const_set(
|
|
55
|
+
:DEFAULT_SECTIONS,
|
|
56
|
+
[
|
|
57
|
+
Evilution::Reporter::CLI::Section.new(
|
|
58
|
+
title: ->(gaps) { "Survived mutations (#{gaps.length} coverage gap#{"s" unless gaps.length == 1}):" },
|
|
59
|
+
fetcher: lambda(&:coverage_gaps),
|
|
60
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::CoverageGap.new
|
|
61
|
+
),
|
|
62
|
+
Evilution::Reporter::CLI::Section.new(
|
|
63
|
+
title: "Neutral mutations (test already failing):",
|
|
64
|
+
fetcher: lambda(&:neutral_results),
|
|
65
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::ResultLocation.new
|
|
66
|
+
),
|
|
67
|
+
Evilution::Reporter::CLI::Section.new(
|
|
68
|
+
title: "Equivalent mutations (provably identical behavior):",
|
|
69
|
+
fetcher: lambda(&:equivalent_results),
|
|
70
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::ResultLocation.new
|
|
71
|
+
),
|
|
72
|
+
Evilution::Reporter::CLI::Section.new(
|
|
73
|
+
title: "Unresolved mutations (no test file resolved):",
|
|
74
|
+
fetcher: lambda(&:unresolved_results),
|
|
75
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::ResultLocation.new
|
|
76
|
+
),
|
|
77
|
+
Evilution::Reporter::CLI::Section.new(
|
|
78
|
+
title: "Unparseable mutations (mutated source did not parse):",
|
|
79
|
+
fetcher: lambda(&:unparseable_results),
|
|
80
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::ResultLocation.new
|
|
81
|
+
),
|
|
82
|
+
Evilution::Reporter::CLI::Section.new(
|
|
83
|
+
title: "Errored mutations:",
|
|
84
|
+
fetcher: ->(s) { s.results.select(&:error?) },
|
|
85
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::Error.new
|
|
86
|
+
),
|
|
87
|
+
Evilution::Reporter::CLI::Section.new(
|
|
88
|
+
title: "Disabled mutations (skipped by # evilution:disable):",
|
|
89
|
+
fetcher: lambda(&:disabled_mutations),
|
|
90
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::Disabled.new
|
|
91
|
+
)
|
|
92
|
+
].freeze
|
|
93
|
+
)
|
|
@@ -9,6 +9,7 @@ class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disabl
|
|
|
9
9
|
class Evilution::Runner::IsolationResolver
|
|
10
10
|
PRELOAD_CANDIDATES = [
|
|
11
11
|
File.join("spec", "rails_helper.rb"),
|
|
12
|
+
File.join("spec", "spec_helper.rb"),
|
|
12
13
|
File.join("test", "test_helper.rb")
|
|
13
14
|
].freeze
|
|
14
15
|
|
|
@@ -30,13 +31,15 @@ class Evilution::Runner::IsolationResolver
|
|
|
30
31
|
|
|
31
32
|
def perform_preload
|
|
32
33
|
return if config.preload == false
|
|
33
|
-
return unless
|
|
34
|
+
return unless should_preload?
|
|
34
35
|
|
|
35
36
|
path = resolve_preload_path
|
|
36
37
|
return unless path
|
|
37
38
|
|
|
38
39
|
prepare_load_path_for_preload
|
|
39
40
|
require File.expand_path(path)
|
|
41
|
+
rescue Evilution::ConfigError
|
|
42
|
+
raise
|
|
40
43
|
rescue ScriptError, StandardError => e
|
|
41
44
|
raise Evilution::ConfigError.new(
|
|
42
45
|
"failed to preload #{path.inspect}: #{e.class}: #{e.message}",
|
|
@@ -48,6 +51,17 @@ class Evilution::Runner::IsolationResolver
|
|
|
48
51
|
|
|
49
52
|
attr_reader :config, :hooks
|
|
50
53
|
|
|
54
|
+
# Under :fork, allow preloading — caller resolves whether a path exists (an
|
|
55
|
+
# explicit --preload / preload: value, or an auto-detected rails_helper) and
|
|
56
|
+
# bails early when none does. Under :in_process, only allow preloading when
|
|
57
|
+
# the user explicitly asked via --preload or preload: in YAML — don't
|
|
58
|
+
# auto-load spec/rails_helper.rb for a user who opted out of fork.
|
|
59
|
+
def should_preload?
|
|
60
|
+
return true if resolve_isolation == :fork
|
|
61
|
+
|
|
62
|
+
config.preload.is_a?(String)
|
|
63
|
+
end
|
|
64
|
+
|
|
51
65
|
def target_files
|
|
52
66
|
@target_files ||= @target_files_callback.call
|
|
53
67
|
end
|
|
@@ -112,7 +126,11 @@ class Evilution::Runner::IsolationResolver
|
|
|
112
126
|
abs = File.join(root, rel)
|
|
113
127
|
return abs if File.file?(abs)
|
|
114
128
|
end
|
|
115
|
-
|
|
129
|
+
|
|
130
|
+
raise Evilution::ConfigError,
|
|
131
|
+
"Preload file not found. Tried: [#{PRELOAD_CANDIDATES.join(", ")}]. " \
|
|
132
|
+
"Pass --preload <file> or set preload: in .evilution.yml. " \
|
|
133
|
+
"Use --no-preload (or preload: false) to disable preloading entirely."
|
|
116
134
|
end
|
|
117
135
|
|
|
118
136
|
# When the user explicitly requests InProcess on a Rails project, warn once
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../result/mutation_result"
|
|
4
|
+
|
|
5
|
+
class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disable Lint/EmptyClass
|
|
6
|
+
class Evilution::Runner::MutationExecutor; end unless defined?(Evilution::Runner::MutationExecutor) # rubocop:disable Lint/EmptyClass
|
|
7
|
+
|
|
8
|
+
class Evilution::Runner::MutationExecutor::MutationRunner
|
|
9
|
+
def initialize(config:, cache:, isolator:)
|
|
10
|
+
@config = config
|
|
11
|
+
@cache = cache
|
|
12
|
+
@isolator = isolator
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(mutation, integration:)
|
|
16
|
+
return unparseable_result(mutation) if mutation.unparseable?
|
|
17
|
+
|
|
18
|
+
cached = @cache.fetch(mutation)
|
|
19
|
+
return cached if cached
|
|
20
|
+
|
|
21
|
+
test_command = ->(m) { integration.call(m) }
|
|
22
|
+
result = @isolator.call(mutation: mutation, test_command: test_command, timeout: @config.timeout)
|
|
23
|
+
@cache.store(mutation, result)
|
|
24
|
+
result
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def unparseable_result(mutation)
|
|
30
|
+
Evilution::Result::MutationResult.new(mutation: mutation, status: :unparseable)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disable Lint/EmptyClass
|
|
4
|
+
class Evilution::Runner::MutationExecutor; end unless defined?(Evilution::Runner::MutationExecutor) # rubocop:disable Lint/EmptyClass
|
|
5
|
+
|
|
6
|
+
class Evilution::Runner::MutationExecutor::NeutralizationPipeline
|
|
7
|
+
def initialize(neutralizers)
|
|
8
|
+
@neutralizers = neutralizers
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def call(result, **ctx)
|
|
12
|
+
@neutralizers.reduce(result) do |acc, nz|
|
|
13
|
+
ctx.empty? ? nz.call(acc) : nz.call(acc, **ctx)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../../result/mutation_result"
|
|
4
|
+
|
|
5
|
+
class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disable Lint/EmptyClass
|
|
6
|
+
class Evilution::Runner::MutationExecutor; end unless defined?(Evilution::Runner::MutationExecutor) # rubocop:disable Lint/EmptyClass
|
|
7
|
+
module Evilution::Runner::MutationExecutor::Neutralizer; end unless defined?(Evilution::Runner::MutationExecutor::Neutralizer)
|
|
8
|
+
|
|
9
|
+
class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
|
|
10
|
+
def initialize(config:, spec_resolver:, fallback_dir:)
|
|
11
|
+
@config = config
|
|
12
|
+
@spec_resolver = spec_resolver
|
|
13
|
+
@fallback_dir = fallback_dir
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call(result, baseline_result:)
|
|
17
|
+
return result unless result.survived? && baseline_result && baseline_result.failed?
|
|
18
|
+
|
|
19
|
+
if @config.spec_files.any?
|
|
20
|
+
should_neutralize = true
|
|
21
|
+
else
|
|
22
|
+
spec_file = @spec_resolver.call(result.mutation.file_path) || @fallback_dir
|
|
23
|
+
should_neutralize = baseline_result.failed_spec_files.include?(spec_file)
|
|
24
|
+
end
|
|
25
|
+
return result unless should_neutralize
|
|
26
|
+
|
|
27
|
+
neutralize(result)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def neutralize(result)
|
|
33
|
+
Evilution::Result::MutationResult.new(
|
|
34
|
+
mutation: result.mutation,
|
|
35
|
+
status: :neutral,
|
|
36
|
+
duration: result.duration,
|
|
37
|
+
test_command: result.test_command,
|
|
38
|
+
child_rss_kb: result.child_rss_kb,
|
|
39
|
+
memory_delta_kb: result.memory_delta_kb,
|
|
40
|
+
parent_rss_kb: result.parent_rss_kb,
|
|
41
|
+
error_message: result.error_message,
|
|
42
|
+
error_class: result.error_class,
|
|
43
|
+
error_backtrace: result.error_backtrace
|
|
44
|
+
)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../../result/mutation_result"
|
|
4
|
+
|
|
5
|
+
class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disable Lint/EmptyClass
|
|
6
|
+
class Evilution::Runner::MutationExecutor; end unless defined?(Evilution::Runner::MutationExecutor) # rubocop:disable Lint/EmptyClass
|
|
7
|
+
module Evilution::Runner::MutationExecutor::Neutralizer; end unless defined?(Evilution::Runner::MutationExecutor::Neutralizer)
|
|
8
|
+
|
|
9
|
+
# Reclassify results as :neutral when the failure was caused by test
|
|
10
|
+
# infrastructure rather than by the mutation. Two independent paths:
|
|
11
|
+
#
|
|
12
|
+
# 1) :error from a missing require / spec_helper / rails_helper / spec/support
|
|
13
|
+
# initialization — detected by error_class ∈ INFRA_ERROR_CLASSES and
|
|
14
|
+
# first backtrace frame matching INFRA_BACKTRACE_PATHS. Origin-only match
|
|
15
|
+
# (not `any?`): Ruby backtraces typically carry spec_helper frames below
|
|
16
|
+
# mutation-caused errors, so matching any frame would misclassify real
|
|
17
|
+
# mutation NameError/LoadError as :neutral.
|
|
18
|
+
#
|
|
19
|
+
# 2) :killed from a CrashDetector test_crashed whose sole crash class is in
|
|
20
|
+
# INFRA_CRASH_CLASSES (ActiveRecord::StatementTimeout, Timeout::Error,
|
|
21
|
+
# etc.). These surface under parallel workers sharing a DB file or on a
|
|
22
|
+
# slow CI; fork.rb initially reports them as :killed, and without this
|
|
23
|
+
# demotion the kill count inflates with infra noise. No backtrace check:
|
|
24
|
+
# the single-class signal from CrashDetector already rules out mixed
|
|
25
|
+
# mutation-caused failures. See EV-toid / GH #814.
|
|
26
|
+
class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
|
|
27
|
+
INFRA_ERROR_CLASSES = %w[LoadError NameError].freeze
|
|
28
|
+
INFRA_BACKTRACE_PATHS = %r{(?:^|/)(?:spec_helper\.rb|rails_helper\.rb|spec/support/)}
|
|
29
|
+
INFRA_CRASH_CLASSES = %w[
|
|
30
|
+
Timeout::Error
|
|
31
|
+
ActiveRecord::StatementTimeout
|
|
32
|
+
ActiveRecord::Deadlocked
|
|
33
|
+
ActiveRecord::ConnectionTimeoutError
|
|
34
|
+
ActiveRecord::LockWaitTimeout
|
|
35
|
+
SQLite3::BusyException
|
|
36
|
+
].freeze
|
|
37
|
+
private_constant :INFRA_ERROR_CLASSES, :INFRA_BACKTRACE_PATHS, :INFRA_CRASH_CLASSES
|
|
38
|
+
|
|
39
|
+
def call(result, **_ctx)
|
|
40
|
+
return neutralize(result) if infra_crash?(result)
|
|
41
|
+
return result unless result.error?
|
|
42
|
+
return result unless INFRA_ERROR_CLASSES.include?(result.error_class)
|
|
43
|
+
return result unless infra_origin?(result.error_backtrace)
|
|
44
|
+
|
|
45
|
+
neutralize(result)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def infra_crash?(result)
|
|
51
|
+
result.killed? && INFRA_CRASH_CLASSES.include?(result.error_class)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def infra_origin?(backtrace)
|
|
55
|
+
frames = Array(backtrace)
|
|
56
|
+
return false if frames.empty?
|
|
57
|
+
|
|
58
|
+
frames.first =~ INFRA_BACKTRACE_PATHS ? true : false
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def neutralize(result)
|
|
62
|
+
Evilution::Result::MutationResult.new(
|
|
63
|
+
mutation: result.mutation,
|
|
64
|
+
status: :neutral,
|
|
65
|
+
duration: result.duration,
|
|
66
|
+
test_command: result.test_command,
|
|
67
|
+
child_rss_kb: result.child_rss_kb,
|
|
68
|
+
memory_delta_kb: result.memory_delta_kb,
|
|
69
|
+
parent_rss_kb: result.parent_rss_kb,
|
|
70
|
+
error_message: result.error_message,
|
|
71
|
+
error_class: result.error_class,
|
|
72
|
+
error_backtrace: result.error_backtrace
|
|
73
|
+
)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../result/mutation_result"
|
|
4
|
+
|
|
5
|
+
class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disable Lint/EmptyClass
|
|
6
|
+
class Evilution::Runner::MutationExecutor; end unless defined?(Evilution::Runner::MutationExecutor) # rubocop:disable Lint/EmptyClass
|
|
7
|
+
|
|
8
|
+
class Evilution::Runner::MutationExecutor::ResultCache
|
|
9
|
+
CACHEABLE_STATUSES = %i[killed timeout].freeze
|
|
10
|
+
private_constant :CACHEABLE_STATUSES
|
|
11
|
+
|
|
12
|
+
def initialize(backend)
|
|
13
|
+
@backend = backend
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def fetch(mutation)
|
|
17
|
+
return nil unless @backend
|
|
18
|
+
|
|
19
|
+
data = @backend.fetch(mutation)
|
|
20
|
+
return nil unless data
|
|
21
|
+
return nil unless CACHEABLE_STATUSES.include?(data[:status])
|
|
22
|
+
|
|
23
|
+
Evilution::Result::MutationResult.new(
|
|
24
|
+
mutation: mutation,
|
|
25
|
+
status: data[:status],
|
|
26
|
+
duration: data[:duration],
|
|
27
|
+
killing_test: data[:killing_test],
|
|
28
|
+
test_command: data[:test_command]
|
|
29
|
+
)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def store(mutation, result)
|
|
33
|
+
return unless @backend
|
|
34
|
+
return unless result.killed? || result.timeout?
|
|
35
|
+
|
|
36
|
+
@backend.store(mutation,
|
|
37
|
+
status: result.status,
|
|
38
|
+
duration: result.duration,
|
|
39
|
+
killing_test: result.killing_test,
|
|
40
|
+
test_command: result.test_command)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def partition(batch, packer:)
|
|
44
|
+
uncached_indices = []
|
|
45
|
+
cached_results = {}
|
|
46
|
+
|
|
47
|
+
batch.each_with_index do |mutation, i|
|
|
48
|
+
if mutation.unparseable?
|
|
49
|
+
cached_results[i] = packer.compact(unparseable_result(mutation))
|
|
50
|
+
next
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
cached = fetch(mutation)
|
|
54
|
+
if cached
|
|
55
|
+
cached_results[i] = packer.compact(cached)
|
|
56
|
+
else
|
|
57
|
+
uncached_indices << i
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
[uncached_indices, cached_results]
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def unparseable_result(mutation)
|
|
67
|
+
Evilution::Result::MutationResult.new(mutation: mutation, status: :unparseable)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../reporter/progress_bar"
|
|
4
|
+
|
|
5
|
+
class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disable Lint/EmptyClass
|
|
6
|
+
class Evilution::Runner::MutationExecutor; end unless defined?(Evilution::Runner::MutationExecutor) # rubocop:disable Lint/EmptyClass
|
|
7
|
+
|
|
8
|
+
class Evilution::Runner::MutationExecutor::ResultNotifier
|
|
9
|
+
def initialize(config, diagnostics:, on_result:)
|
|
10
|
+
@config = config
|
|
11
|
+
@diagnostics = diagnostics
|
|
12
|
+
@on_result = on_result
|
|
13
|
+
@survived_count = 0
|
|
14
|
+
@progress_bar = nil
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
attr_reader :survived_count
|
|
18
|
+
|
|
19
|
+
def start(total)
|
|
20
|
+
@survived_count = 0
|
|
21
|
+
@progress_bar = build_progress_bar(total)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def notify(result, index)
|
|
25
|
+
@on_result.call(result) if @on_result
|
|
26
|
+
@progress_bar.tick(status: result.status) if @progress_bar
|
|
27
|
+
@diagnostics.log_progress(index, result.status)
|
|
28
|
+
@diagnostics.log_mutation_diagnostics(result)
|
|
29
|
+
@survived_count += 1 if result.survived?
|
|
30
|
+
truncate? ? :truncate : :continue
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def finish
|
|
34
|
+
@progress_bar.finish if @progress_bar
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def truncate?
|
|
40
|
+
@config.fail_fast? && @survived_count >= @config.fail_fast
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def build_progress_bar(total)
|
|
44
|
+
return nil if !@config.progress? || @config.quiet || @config.verbose || !@config.text? || !$stderr.tty?
|
|
45
|
+
|
|
46
|
+
Evilution::Reporter::ProgressBar.new(total: total, output: $stderr)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../result/mutation_result"
|
|
4
|
+
|
|
5
|
+
class Evilution::Runner; end unless defined?(Evilution::Runner) # rubocop:disable Lint/EmptyClass
|
|
6
|
+
class Evilution::Runner::MutationExecutor; end unless defined?(Evilution::Runner::MutationExecutor) # rubocop:disable Lint/EmptyClass
|
|
7
|
+
|
|
8
|
+
class Evilution::Runner::MutationExecutor::ResultPacker
|
|
9
|
+
def compact(result)
|
|
10
|
+
{
|
|
11
|
+
status: result.status,
|
|
12
|
+
duration: result.duration,
|
|
13
|
+
killing_test: result.killing_test,
|
|
14
|
+
test_command: result.test_command,
|
|
15
|
+
child_rss_kb: result.child_rss_kb,
|
|
16
|
+
memory_delta_kb: result.memory_delta_kb,
|
|
17
|
+
parent_rss_kb: result.parent_rss_kb,
|
|
18
|
+
error_message: result.error_message,
|
|
19
|
+
error_class: result.error_class,
|
|
20
|
+
error_backtrace: result.error_backtrace
|
|
21
|
+
}
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def rebuild(mutation, data)
|
|
25
|
+
Evilution::Result::MutationResult.new(
|
|
26
|
+
mutation: mutation,
|
|
27
|
+
status: data[:status],
|
|
28
|
+
duration: data[:duration],
|
|
29
|
+
killing_test: data[:killing_test],
|
|
30
|
+
test_command: data[:test_command],
|
|
31
|
+
child_rss_kb: data[:child_rss_kb],
|
|
32
|
+
memory_delta_kb: data[:memory_delta_kb],
|
|
33
|
+
parent_rss_kb: data[:parent_rss_kb],
|
|
34
|
+
error_message: data[:error_message],
|
|
35
|
+
error_class: data[:error_class],
|
|
36
|
+
error_backtrace: data[:error_backtrace]
|
|
37
|
+
)
|
|
38
|
+
end
|
|
39
|
+
end
|