evilution 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +62 -0
- data/.rubocop_todo.yml +5 -0
- data/CHANGELOG.md +77 -0
- data/README.md +80 -24
- data/docs/architecture.md +89 -9
- data/docs/ast_pattern_syntax.md +50 -5
- data/docs/isolation.md +39 -2
- data/docs/migration-from-mutant.md +1 -1
- data/lib/evilution/ast/aasm_declaration.rb +133 -0
- data/lib/evilution/ast/callback_declaration.rb +91 -0
- data/lib/evilution/ast/included_block.rb +27 -0
- data/lib/evilution/ast/literal_callable.rb +21 -0
- data/lib/evilution/ast/parser.rb +119 -16
- data/lib/evilution/ast/pattern/method_name.rb +63 -0
- data/lib/evilution/ast/pattern/parser.rb +14 -15
- data/lib/evilution/ast/regexp_pattern.rb +104 -0
- data/lib/evilution/ast/scope_declaration.rb +45 -0
- data/lib/evilution/ast/uncovered_code.rb +88 -0
- data/lib/evilution/ast/value_object_definition.rb +25 -0
- data/lib/evilution/baseline/failure_formatter.rb +27 -0
- data/lib/evilution/baseline/report.rb +66 -0
- data/lib/evilution/baseline/spec_failure.rb +38 -0
- data/lib/evilution/baseline.rb +67 -30
- data/lib/evilution/cli/parser/file_args.rb +2 -1
- data/lib/evilution/cli/parser/options_builder.rb +1 -1
- data/lib/evilution/cli.rb +3 -2
- data/lib/evilution/config/validators/spec_mappings.rb +2 -1
- data/lib/evilution/config.rb +3 -2
- data/lib/evilution/equivalent/detector.rb +3 -1
- data/lib/evilution/equivalent/heuristic/guarded_index_fetch/condition.rb +66 -0
- data/lib/evilution/equivalent/heuristic/guarded_index_fetch/disturbance.rb +54 -0
- data/lib/evilution/equivalent/heuristic/guarded_index_fetch/early_exit.rb +73 -0
- data/lib/evilution/equivalent/heuristic/guarded_index_fetch/guard.rb +36 -0
- data/lib/evilution/equivalent/heuristic/guarded_index_fetch/index_read.rb +82 -0
- data/lib/evilution/equivalent/heuristic/guarded_index_fetch/node_path.rb +43 -0
- data/lib/evilution/equivalent/heuristic/guarded_index_fetch.rb +65 -0
- data/lib/evilution/example_filter.rb +45 -0
- data/lib/evilution/hooks/registry.rb +2 -1
- data/lib/evilution/integration/base.rb +2 -0
- data/lib/evilution/integration/known_failures.rb +26 -0
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +132 -13
- data/lib/evilution/integration/loading/callback_redeclaration.rb +186 -0
- data/lib/evilution/integration/loading/concern_redeclaration.rb +61 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -12
- data/lib/evilution/integration/loading/mutation_applier.rb +16 -0
- data/lib/evilution/integration/loading/test_class_cache.rb +46 -0
- data/lib/evilution/integration/minitest/test_ids.rb +21 -0
- data/lib/evilution/integration/minitest.rb +45 -6
- data/lib/evilution/integration/rspec/baseline_runner.rb +47 -3
- data/lib/evilution/integration/rspec/example_ids.rb +35 -0
- data/lib/evilution/integration/rspec/result_builder.rb +6 -1
- data/lib/evilution/integration/rspec/state_guard/anonymous_example_group_examples.rb +36 -0
- data/lib/evilution/integration/rspec/state_guard.rb +4 -1
- data/lib/evilution/integration/rspec.rb +22 -4
- data/lib/evilution/integration/test_unit/result_builder.rb +6 -0
- data/lib/evilution/integration/test_unit/test_ids.rb +18 -0
- data/lib/evilution/integration/test_unit.rb +37 -9
- data/lib/evilution/isolation/fork.rb +2 -1
- data/lib/evilution/isolation/in_process.rb +21 -4
- data/lib/evilution/mcp/info_tool/status_glossary.rb +2 -2
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +2 -1
- data/lib/evilution/memory/leak_check.rb +27 -3
- data/lib/evilution/mutation.rb +7 -2
- data/lib/evilution/mutator/base.rb +49 -5
- data/lib/evilution/mutator/operator/alias_removal.rb +126 -0
- data/lib/evilution/mutator/operator/argument_order_permutation.rb +101 -0
- data/lib/evilution/mutator/operator/argument_removal.rb +9 -1
- data/lib/evilution/mutator/operator/comparison_operand_swap.rb +54 -0
- data/lib/evilution/mutator/operator/data_struct_member.rb +82 -0
- data/lib/evilution/mutator/operator/exception_swallow.rb +84 -0
- data/lib/evilution/mutator/operator/format_specifier_swap.rb +100 -0
- data/lib/evilution/mutator/operator/forwarded_argument_drop.rb +142 -0
- data/lib/evilution/mutator/operator/integer_division_to_fdiv.rb +62 -0
- data/lib/evilution/mutator/operator/keyword_argument.rb +25 -2
- data/lib/evilution/mutator/operator/keyword_value_swap.rb +75 -0
- data/lib/evilution/mutator/operator/no_matching_pattern_else.rb +47 -0
- data/lib/evilution/mutator/operator/numbered_parameter_swap.rb +78 -0
- data/lib/evilution/mutator/operator/off_by_one_boundary.rb +68 -0
- data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +14 -3
- data/lib/evilution/mutator/operator/pattern_matching_array.rb +12 -1
- data/lib/evilution/mutator/operator/pattern_wildcard_widening.rb +117 -0
- data/lib/evilution/mutator/operator/pin_operator_removal.rb +48 -0
- data/lib/evilution/mutator/operator/regex_simplification.rb +53 -132
- data/lib/evilution/mutator/operator/regexp_alternation_branch_deletion.rb +54 -0
- data/lib/evilution/mutator/operator/regexp_anchor_promotion.rb +43 -0
- data/lib/evilution/mutator/operator/regexp_capture_to_passive.rb +61 -0
- data/lib/evilution/mutator/operator/regexp_character_type_complement.rb +54 -0
- data/lib/evilution/mutator/operator/regexp_named_group_rename.rb +106 -0
- data/lib/evilution/mutator/operator/regexp_option_removal.rb +51 -0
- data/lib/evilution/mutator/operator/regexp_quantifier_minimum_swap.rb +45 -0
- data/lib/evilution/mutator/operator/rescue_else_concatenation.rb +62 -0
- data/lib/evilution/mutator/operator/rescue_handler_concatenation.rb +69 -0
- data/lib/evilution/mutator/operator/rescue_handler_promotion.rb +65 -0
- data/lib/evilution/mutator/operator/return_keyword_removal.rb +79 -0
- data/lib/evilution/mutator/operator/rightward_assignment.rb +46 -0
- data/lib/evilution/mutator/operator/send_mutation.rb +2 -0
- data/lib/evilution/mutator/operator/splat_operator.rb +38 -13
- data/lib/evilution/mutator/operator/statement_reorder.rb +153 -0
- data/lib/evilution/mutator/registry.rb +31 -2
- data/lib/evilution/mutator/rescue_handlers.rb +81 -0
- data/lib/evilution/process_supervisor.rb +3 -2
- data/lib/evilution/reporter/cli/line_formatters/baseline_neutralized_notice.rb +55 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +2 -0
- data/lib/evilution/reporter/json/baseline.rb +15 -0
- data/lib/evilution/reporter/json.rb +7 -2
- data/lib/evilution/result/baseline_neutralization.rb +10 -0
- data/lib/evilution/result/mutation_result.rb +9 -1
- data/lib/evilution/result/summary.rb +42 -2
- data/lib/evilution/runner/baseline_runner.rb +9 -4
- data/lib/evilution/runner/canary.rb +2 -57
- data/lib/evilution/runner/canary_failure_message.rb +101 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +22 -8
- data/lib/evilution/runner/mutation_executor/result_cache.rb +3 -0
- data/lib/evilution/runner/mutation_executor/result_packer.rb +4 -2
- data/lib/evilution/runner/mutation_executor.rb +1 -1
- data/lib/evilution/runner/mutation_planner.rb +11 -2
- data/lib/evilution/runner/report_publisher.rb +3 -2
- data/lib/evilution/runner/subject_pipeline.rb +46 -1
- data/lib/evilution/runner.rb +4 -1
- data/lib/evilution/subject.rb +8 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +28 -0
- data/script/memory_check +62 -16
- metadata +83 -4
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../runner"
|
|
4
|
+
|
|
5
|
+
# Explains why the proof-of-life canary failed, from the result its synthetic
|
|
6
|
+
# mutation came back with. Kept apart from Canary, which only runs the probe:
|
|
7
|
+
# the wording changes for different reasons than the mechanics do.
|
|
8
|
+
class Evilution::Runner::CanaryFailureMessage
|
|
9
|
+
# Raised by RubyGems when it cannot load a gem at a version that satisfies
|
|
10
|
+
# every requirement already in play.
|
|
11
|
+
GEM_ACTIVATION_ERRORS = %w[
|
|
12
|
+
Gem::ConflictError Gem::LoadError Gem::MissingSpecError Gem::MissingSpecVersionError
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
15
|
+
def initialize(result)
|
|
16
|
+
@result = result
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# When the child reported an error, that error IS the diagnosis -- naming it
|
|
20
|
+
# beats guessing. The speculative list stays only for the cases that carry no
|
|
21
|
+
# error at all (:killed, :timeout), where guesses are the only help there is.
|
|
22
|
+
# Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
|
|
23
|
+
# this message used to drop, and none of the four guesses was the cause.
|
|
24
|
+
# EV-65nf / GH #1586.
|
|
25
|
+
def to_s
|
|
26
|
+
return gem_activation_message(@result) if gem_activation_failure?(@result)
|
|
27
|
+
|
|
28
|
+
"#{failure_preamble(@result.status)} #{diagnosis(@result)} " \
|
|
29
|
+
"Re-run with --no-canary to bypass this check."
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
# The usual cause is evilution running outside `bundle exec`: RubyGems then
|
|
35
|
+
# activates the newest installed copy of a shared dependency, which the test
|
|
36
|
+
# framework may not accept. Every mutation would fail the same way, so
|
|
37
|
+
# skipping the canary would not help.
|
|
38
|
+
def gem_activation_message(result)
|
|
39
|
+
"evilution proof-of-life canary failed: the test framework could not be loaded " \
|
|
40
|
+
"because of a gem activation conflict. This is an environment problem, not a " \
|
|
41
|
+
"mutation-pipeline defect, and every mutation would fail the same way. Run " \
|
|
42
|
+
"evilution under `bundle exec` so the project's Gemfile.lock decides gem versions " \
|
|
43
|
+
"(for the MCP server: command `bundle`, args [\"exec\", \"evilution\", \"mcp\"]). " \
|
|
44
|
+
"The child reported: #{reported_error(result)}"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# The class may arrive in its own field or only as the message prefix,
|
|
48
|
+
# depending on which path packed the error.
|
|
49
|
+
def gem_activation_failure?(result)
|
|
50
|
+
message = result.error_message.to_s
|
|
51
|
+
|
|
52
|
+
GEM_ACTIVATION_ERRORS.any? do |klass|
|
|
53
|
+
result.error_class == klass || message.start_with?("#{klass}:")
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def failure_preamble(status)
|
|
58
|
+
"evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
|
|
59
|
+
"mutation was scored #{status.inspect} instead of :survived. The mutation " \
|
|
60
|
+
"pipeline is misreporting — every score this run would produce is unreliable."
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def diagnosis(result)
|
|
64
|
+
reported = reported_error(result)
|
|
65
|
+
return speculative_causes if reported.nil?
|
|
66
|
+
|
|
67
|
+
"The child reported: #{reported}"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def reported_error(result)
|
|
71
|
+
message = result.error_message
|
|
72
|
+
return nil if message.nil? || message.empty?
|
|
73
|
+
|
|
74
|
+
described = with_class_prefix(message, result.error_class)
|
|
75
|
+
frame = first_frame(result)
|
|
76
|
+
frame.nil? ? described : "#{described} (at #{frame})"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# MutationApplier already prefixes the class onto the message it packs
|
|
80
|
+
# ("#{e.class}: #{e.message}"), while other paths pack the bare message and
|
|
81
|
+
# leave the class in its own field. Prefixing unconditionally would print
|
|
82
|
+
# "NameError: NameError: ..." for the former.
|
|
83
|
+
def with_class_prefix(message, klass)
|
|
84
|
+
return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
|
|
85
|
+
|
|
86
|
+
"#{klass}: #{message}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def first_frame(result)
|
|
90
|
+
backtrace = result.error_backtrace
|
|
91
|
+
return nil if backtrace.nil? || backtrace.empty?
|
|
92
|
+
|
|
93
|
+
backtrace.first
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def speculative_causes
|
|
97
|
+
"Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
|
|
98
|
+
"RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
|
|
99
|
+
"an isolation-mode defect."
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -4,6 +4,17 @@ require_relative "../neutralizer"
|
|
|
4
4
|
require_relative "../../../result/mutation_result"
|
|
5
5
|
require_relative "../../../result/neutral_reason"
|
|
6
6
|
|
|
7
|
+
# Reclassifies a kill as :neutral when the tests failed on nothing but
|
|
8
|
+
# examples that were already failing before any mutation ran.
|
|
9
|
+
#
|
|
10
|
+
# Such an example fails whatever the mutation is, so every mutation it runs
|
|
11
|
+
# against would count as killed and a red spec file would score full marks.
|
|
12
|
+
# The failure says nothing about the mutation: it has no verdict.
|
|
13
|
+
#
|
|
14
|
+
# A survivor is left alone, whatever the baseline did. Its tests passed, the
|
|
15
|
+
# examples that were red in the baseline included, so the gap it reports is
|
|
16
|
+
# real -- and neutralizing it, as used to happen for every survivor a red spec
|
|
17
|
+
# file covered, hid exactly the mutations a reader needs to see.
|
|
7
18
|
class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
|
|
8
19
|
def initialize(config:, spec_resolver:, fallback_dir:)
|
|
9
20
|
@config = config
|
|
@@ -12,19 +23,22 @@ class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
|
|
|
12
23
|
end
|
|
13
24
|
|
|
14
25
|
def call(result, baseline_result:)
|
|
15
|
-
return result unless result.
|
|
26
|
+
return result unless result.killed? && result.known_failures_only?
|
|
16
27
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
failed = baseline_result.failed_spec_files
|
|
20
|
-
spec_file = covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
|
|
21
|
-
return result unless spec_file
|
|
22
|
-
|
|
23
|
-
neutralize(result, spec_file)
|
|
28
|
+
neutralize(result, red_spec(result, baseline_result))
|
|
24
29
|
end
|
|
25
30
|
|
|
26
31
|
private
|
|
27
32
|
|
|
33
|
+
# The red spec file to name: one covering the mutated source. A run given
|
|
34
|
+
# its spec files explicitly has no single one to point at.
|
|
35
|
+
def red_spec(result, baseline_result)
|
|
36
|
+
return nil if @config.spec_files.any? || baseline_result.nil?
|
|
37
|
+
|
|
38
|
+
failed = baseline_result.failed_spec_files
|
|
39
|
+
covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
|
|
40
|
+
end
|
|
41
|
+
|
|
28
42
|
# The spec files the baseline ran for this source: every file the resolver
|
|
29
43
|
# returns (a spec_selector may map one source to several), or the fallback
|
|
30
44
|
# directory when nothing resolves -- but only if the run falls back to the
|
|
@@ -34,6 +34,9 @@ class Evilution::Runner::MutationExecutor::ResultCache
|
|
|
34
34
|
return unless @backend
|
|
35
35
|
return unless result.killed? || result.timeout?
|
|
36
36
|
return if infra_crash?(result)
|
|
37
|
+
# Nor is one where only already-failing examples failed: the baseline
|
|
38
|
+
# that says so belongs to this run, and the cache keeps no trace of it.
|
|
39
|
+
return if result.known_failures_only?
|
|
37
40
|
|
|
38
41
|
@backend.store(mutation,
|
|
39
42
|
status: result.status,
|
|
@@ -15,7 +15,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
|
|
|
15
15
|
parent_rss_kb: result.parent_rss_kb,
|
|
16
16
|
error_message: result.error_message,
|
|
17
17
|
error_class: result.error_class,
|
|
18
|
-
error_backtrace: result.error_backtrace
|
|
18
|
+
error_backtrace: result.error_backtrace,
|
|
19
|
+
known_failures_only: result.known_failures_only?
|
|
19
20
|
}
|
|
20
21
|
end
|
|
21
22
|
|
|
@@ -35,7 +36,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
|
|
|
35
36
|
message: data[:error_message],
|
|
36
37
|
klass: data[:error_class],
|
|
37
38
|
backtrace: data[:error_backtrace]
|
|
38
|
-
)
|
|
39
|
+
),
|
|
40
|
+
known_failures_only: data[:known_failures_only]
|
|
39
41
|
)
|
|
40
42
|
end
|
|
41
43
|
end
|
|
@@ -30,7 +30,7 @@ class Evilution::Runner::MutationExecutor
|
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
def call(mutations, baseline_result = nil)
|
|
33
|
-
integration = @baseline_runner.build_integration
|
|
33
|
+
integration = @baseline_runner.build_integration(baseline_result)
|
|
34
34
|
spec_resolver = baseline_failed?(baseline_result) ? @baseline_runner.neutralization_resolver : nil
|
|
35
35
|
notifier = build_notifier
|
|
36
36
|
pipeline = build_pipeline(spec_resolver)
|
|
@@ -7,7 +7,7 @@ require_relative "../ast/pattern/filter"
|
|
|
7
7
|
require_relative "../equivalent/detector"
|
|
8
8
|
|
|
9
9
|
class Evilution::Runner::MutationPlanner
|
|
10
|
-
Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, keyword_init: true)
|
|
10
|
+
Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, :mutated_lines, keyword_init: true)
|
|
11
11
|
|
|
12
12
|
GenerationResult = Data.define(:mutations, :skipped)
|
|
13
13
|
DisabledFilterResult = Data.define(:enabled, :disabled)
|
|
@@ -33,13 +33,22 @@ class Evilution::Runner::MutationPlanner
|
|
|
33
33
|
sig_filter = filter_sig_blocks(disabled_filter.enabled)
|
|
34
34
|
equivalent_filter = filter_equivalent(sig_filter.enabled)
|
|
35
35
|
|
|
36
|
-
build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
|
|
36
|
+
plan = build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
|
|
37
|
+
plan.mutated_lines = mutated_lines(deduped)
|
|
38
|
+
plan
|
|
37
39
|
end
|
|
38
40
|
|
|
39
41
|
private
|
|
40
42
|
|
|
41
43
|
attr_reader :config, :registry
|
|
42
44
|
|
|
45
|
+
# The lines mutations were generated for, per file. Taken before any filter:
|
|
46
|
+
# a line whose mutations were all disabled or judged equivalent was still
|
|
47
|
+
# reached by an operator.
|
|
48
|
+
def mutated_lines(mutations)
|
|
49
|
+
mutations.group_by(&:file_path).transform_values { |group| group.map(&:line).uniq.sort }
|
|
50
|
+
end
|
|
51
|
+
|
|
43
52
|
def compute_disabled_mutations(disabled_filter)
|
|
44
53
|
return [] unless config.show_disabled?
|
|
45
54
|
|
|
@@ -5,6 +5,7 @@ require_relative "../reporter/json"
|
|
|
5
5
|
require_relative "../reporter/cli"
|
|
6
6
|
require_relative "../reporter/html"
|
|
7
7
|
require_relative "../session/store"
|
|
8
|
+
require_relative "../diagnostic"
|
|
8
9
|
|
|
9
10
|
class Evilution::Runner::ReportPublisher
|
|
10
11
|
def initialize(config)
|
|
@@ -46,7 +47,7 @@ class Evilution::Runner::ReportPublisher
|
|
|
46
47
|
def publish_html(output)
|
|
47
48
|
path = "evilution-report.html"
|
|
48
49
|
File.write(path, output)
|
|
49
|
-
warn
|
|
50
|
+
Evilution::Diagnostic.warn("HTML report written to #{path}")
|
|
50
51
|
end
|
|
51
52
|
|
|
52
53
|
def save_session(summary)
|
|
@@ -54,7 +55,7 @@ class Evilution::Runner::ReportPublisher
|
|
|
54
55
|
|
|
55
56
|
Evilution::Session::Store.new.save(summary)
|
|
56
57
|
rescue StandardError => e
|
|
57
|
-
warn
|
|
58
|
+
Evilution::Diagnostic.warn("[evilution] failed to save session: #{e.message}") unless config.quiet
|
|
58
59
|
end
|
|
59
60
|
|
|
60
61
|
private
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../runner"
|
|
4
4
|
require_relative "../ast/inheritance_scanner"
|
|
5
|
+
require_relative "../ast/uncovered_code"
|
|
6
|
+
require_relative "../diagnostic"
|
|
5
7
|
require_relative "../git/changed_files"
|
|
6
8
|
|
|
7
9
|
class Evilution::Runner::SubjectPipeline
|
|
@@ -12,6 +14,7 @@ class Evilution::Runner::SubjectPipeline
|
|
|
12
14
|
|
|
13
15
|
def call
|
|
14
16
|
subjects = parse_subjects
|
|
17
|
+
@subjects_by_file = subjects.group_by(&:file_path)
|
|
15
18
|
subjects = filter_by_descendants(subjects) if descendants_target?
|
|
16
19
|
subjects = filter_by_target(subjects) if method_target?
|
|
17
20
|
subjects = filter_by_line_ranges(subjects) if config.line_ranges?
|
|
@@ -22,6 +25,28 @@ class Evilution::Runner::SubjectPipeline
|
|
|
22
25
|
@target_files ||= resolve_target_files
|
|
23
26
|
end
|
|
24
27
|
|
|
28
|
+
# Warns about the targeted lines that hold code outside every subject and
|
|
29
|
+
# that no mutation reached either. mutated_lines: the lines mutations were
|
|
30
|
+
# generated for, per file -- known only once #call's subjects are planned.
|
|
31
|
+
#
|
|
32
|
+
# A line range is checked as given; a whole file only when it has no
|
|
33
|
+
# subjects at all -- every class with an `include` or a constant would
|
|
34
|
+
# otherwise warn.
|
|
35
|
+
def report_uncovered_code(mutated_lines = {})
|
|
36
|
+
by_file = @subjects_by_file || {}
|
|
37
|
+
@subjects_by_file = nil
|
|
38
|
+
@uncovered_code = target_files.filter_map do |file|
|
|
39
|
+
uncovered_entry(file, by_file.fetch(file, []), mutated_lines.fetch(file, []))
|
|
40
|
+
end
|
|
41
|
+
@uncovered_code.each { |entry| warn_uncovered(entry) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Those lines, as `{ file:, lines: ["2-4", "9"] }` per file, once
|
|
45
|
+
# #report_uncovered_code has run.
|
|
46
|
+
def uncovered_code
|
|
47
|
+
@uncovered_code || []
|
|
48
|
+
end
|
|
49
|
+
|
|
25
50
|
private
|
|
26
51
|
|
|
27
52
|
attr_reader :config, :parser
|
|
@@ -30,6 +55,26 @@ class Evilution::Runner::SubjectPipeline
|
|
|
30
55
|
target_files.flat_map { |file| parser.call(file) }
|
|
31
56
|
end
|
|
32
57
|
|
|
58
|
+
def uncovered_entry(file, file_subjects, mutated_lines)
|
|
59
|
+
range = config.line_ranges[file]
|
|
60
|
+
return unless range || file_subjects.empty?
|
|
61
|
+
|
|
62
|
+
lines = Evilution::AST::UncoveredCode.call(file, file_subjects, lines: range, mutated_lines: mutated_lines)
|
|
63
|
+
{ file: file, lines: lines.map { |run| line_label(run) } } unless lines.empty?
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Spelled the way a range is passed on the command line: `2-4`, or `9`.
|
|
67
|
+
def line_label(run)
|
|
68
|
+
run.size == 1 ? run.first.to_s : "#{run.first}-#{run.last}"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def warn_uncovered(entry)
|
|
72
|
+
Evilution::Diagnostic.warn(
|
|
73
|
+
"[evilution] #{entry[:file]}:#{entry[:lines].join(", ")} holds code outside every subject " \
|
|
74
|
+
"(class-body code such as DSL calls and constants is not mutated); no mutations target those lines."
|
|
75
|
+
)
|
|
76
|
+
end
|
|
77
|
+
|
|
33
78
|
def source_glob_target?
|
|
34
79
|
config.target&.start_with?("source:")
|
|
35
80
|
end
|
|
@@ -124,7 +169,7 @@ class Evilution::Runner::SubjectPipeline
|
|
|
124
169
|
end
|
|
125
170
|
|
|
126
171
|
def class_matcher(target)
|
|
127
|
-
->(s) { s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
|
|
172
|
+
->(s) { s.name == target || s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
|
|
128
173
|
end
|
|
129
174
|
|
|
130
175
|
def filter_by_line_ranges(subjects)
|
data/lib/evilution/runner.rb
CHANGED
|
@@ -38,6 +38,7 @@ class Evilution::Runner
|
|
|
38
38
|
baseline_result = run_baseline(subjects)
|
|
39
39
|
|
|
40
40
|
plan = mutation_planner.call(subjects)
|
|
41
|
+
subject_pipeline.report_uncovered_code(plan.mutated_lines)
|
|
41
42
|
release_subject_nodes(subjects)
|
|
42
43
|
clear_operator_caches
|
|
43
44
|
execution = run_mutations(plan.enabled, baseline_result)
|
|
@@ -52,7 +53,9 @@ class Evilution::Runner
|
|
|
52
53
|
disabled_mutations: plan.disabled_mutations,
|
|
53
54
|
unresolved_target_files: target_spec_audit.call(audited_files),
|
|
54
55
|
target_file_count: audited_files.length,
|
|
55
|
-
infra_retried: execution.infra_retried
|
|
56
|
+
infra_retried: execution.infra_retried,
|
|
57
|
+
uncovered_code: subject_pipeline.uncovered_code,
|
|
58
|
+
baseline_failures: baseline_result ? baseline_result.failures : [])
|
|
56
59
|
output_report(summary)
|
|
57
60
|
save_session(summary)
|
|
58
61
|
|
data/lib/evilution/subject.rb
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# What mutations are generated for and reported against. A :method subject is
|
|
4
|
+
# a method definition; a :constant subject is a definition assigned to a
|
|
5
|
+
# constant outside any method, such as `Point = Data.define(:x, :y)`. A :scope
|
|
6
|
+
# or :aasm subject is the body of a callable handed to a class-body
|
|
7
|
+
# declaration.
|
|
3
8
|
class Evilution::Subject
|
|
4
|
-
attr_reader :name, :file_path, :line_number, :source, :node
|
|
9
|
+
attr_reader :name, :file_path, :line_number, :source, :node, :kind
|
|
5
10
|
|
|
6
|
-
def initialize(name:, file_path:, line_number:, source:, node:)
|
|
11
|
+
def initialize(name:, file_path:, line_number:, source:, node:, kind: :method)
|
|
7
12
|
@name = name
|
|
8
13
|
@file_path = file_path
|
|
9
14
|
@line_number = line_number
|
|
10
15
|
@source = source
|
|
11
16
|
@node = node
|
|
17
|
+
@kind = kind
|
|
12
18
|
end
|
|
13
19
|
|
|
14
20
|
def release_node!
|
data/lib/evilution/version.rb
CHANGED
data/lib/evilution.rb
CHANGED
|
@@ -14,6 +14,7 @@ require_relative "evilution/ast/inheritance_scanner"
|
|
|
14
14
|
require_relative "evilution/ast/pattern"
|
|
15
15
|
require_relative "evilution/ast/sorbet_sig_detector"
|
|
16
16
|
require_relative "evilution/ast/local_reads"
|
|
17
|
+
require_relative "evilution/ast/regexp_pattern"
|
|
17
18
|
require_relative "evilution/ast/pattern/matcher"
|
|
18
19
|
require_relative "evilution/ast/pattern/parser"
|
|
19
20
|
require_relative "evilution/hooks"
|
|
@@ -53,6 +54,33 @@ require_relative "evilution/mutator/operator/receiver_constructor_swap"
|
|
|
53
54
|
require_relative "evilution/mutator/operator/block_body_to_nil"
|
|
54
55
|
require_relative "evilution/mutator/operator/block_body_to_raise"
|
|
55
56
|
require_relative "evilution/mutator/operator/block_body_promotion"
|
|
57
|
+
require_relative "evilution/mutator/operator/data_struct_member"
|
|
58
|
+
require_relative "evilution/mutator/operator/pin_operator_removal"
|
|
59
|
+
require_relative "evilution/mutator/operator/rightward_assignment"
|
|
60
|
+
require_relative "evilution/mutator/operator/numbered_parameter_swap"
|
|
61
|
+
require_relative "evilution/mutator/operator/forwarded_argument_drop"
|
|
62
|
+
require_relative "evilution/mutator/operator/pattern_wildcard_widening"
|
|
63
|
+
require_relative "evilution/mutator/operator/no_matching_pattern_else"
|
|
64
|
+
require_relative "evilution/mutator/operator/argument_order_permutation"
|
|
65
|
+
require_relative "evilution/mutator/operator/keyword_value_swap"
|
|
66
|
+
require_relative "evilution/mutator/operator/comparison_operand_swap"
|
|
67
|
+
require_relative "evilution/mutator/operator/integer_division_to_fdiv"
|
|
68
|
+
require_relative "evilution/mutator/operator/off_by_one_boundary"
|
|
69
|
+
require_relative "evilution/mutator/operator/format_specifier_swap"
|
|
70
|
+
require_relative "evilution/mutator/operator/exception_swallow"
|
|
71
|
+
require_relative "evilution/mutator/operator/statement_reorder"
|
|
72
|
+
require_relative "evilution/mutator/operator/alias_removal"
|
|
73
|
+
require_relative "evilution/mutator/operator/regexp_character_type_complement"
|
|
74
|
+
require_relative "evilution/mutator/operator/regexp_anchor_promotion"
|
|
75
|
+
require_relative "evilution/mutator/operator/regexp_alternation_branch_deletion"
|
|
76
|
+
require_relative "evilution/mutator/operator/regexp_quantifier_minimum_swap"
|
|
77
|
+
require_relative "evilution/mutator/operator/regexp_capture_to_passive"
|
|
78
|
+
require_relative "evilution/mutator/operator/regexp_named_group_rename"
|
|
79
|
+
require_relative "evilution/mutator/operator/regexp_option_removal"
|
|
80
|
+
require_relative "evilution/mutator/operator/rescue_handler_promotion"
|
|
81
|
+
require_relative "evilution/mutator/operator/rescue_handler_concatenation"
|
|
82
|
+
require_relative "evilution/mutator/operator/rescue_else_concatenation"
|
|
83
|
+
require_relative "evilution/mutator/operator/return_keyword_removal"
|
|
56
84
|
require_relative "evilution/mutator/operator/arithmetic_replacement"
|
|
57
85
|
require_relative "evilution/mutator/operator/string_literal"
|
|
58
86
|
require_relative "evilution/mutator/operator/array_literal"
|
data/script/memory_check
CHANGED
|
@@ -11,6 +11,13 @@ COMPLEX_FIXTURE = File.expand_path("../spec/support/fixtures/memory_check/target
|
|
|
11
11
|
COMPLEX_FIXTURE_SPEC = File.expand_path("../spec/support/fixtures/memory_check/target_examples.rb", __dir__)
|
|
12
12
|
ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_ITERATIONS", 50))
|
|
13
13
|
MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_MAX_GROWTH_KB", 10_240))
|
|
14
|
+
# Every check that picks mutations at random draws from its own Random seeded
|
|
15
|
+
# with this, so a run exercises the same mutations every time and a failure can
|
|
16
|
+
# be replayed by passing the printed seed back.
|
|
17
|
+
SEED = Integer(ENV.fetch("MEMORY_CHECK_SEED", 42))
|
|
18
|
+
COMPLEX_POOL_SIZE = 10
|
|
19
|
+
COMPLEX_ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_ITERATIONS", 100))
|
|
20
|
+
COMPLEX_MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_MAX_GROWTH_KB", 10_240))
|
|
14
21
|
|
|
15
22
|
def setup_workload
|
|
16
23
|
parser = Evilution::AST::Parser.new
|
|
@@ -21,25 +28,40 @@ def setup_workload
|
|
|
21
28
|
mutations
|
|
22
29
|
end
|
|
23
30
|
|
|
24
|
-
def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB,
|
|
25
|
-
|
|
31
|
+
def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB, picked: nil,
|
|
32
|
+
warmup_iterations: Evilution::Memory::LeakCheck::WARMUP_ITERATIONS, &)
|
|
33
|
+
check = Evilution::Memory::LeakCheck.new(iterations: iterations, max_growth_kb: max_growth_kb,
|
|
34
|
+
warmup_iterations: warmup_iterations)
|
|
26
35
|
result = check.run(&)
|
|
27
|
-
report(name, result)
|
|
36
|
+
report(name, result, picked)
|
|
28
37
|
result[:passed]
|
|
29
38
|
end
|
|
30
39
|
|
|
31
|
-
def report(name, result)
|
|
40
|
+
def report(name, result, picked)
|
|
32
41
|
status = result[:passed] ? "PASS" : "FAIL"
|
|
33
42
|
metrics = format_metrics(result)
|
|
34
43
|
|
|
35
44
|
puts "[#{status}] #{name}"
|
|
36
|
-
puts "
|
|
45
|
+
puts " Sustained growth: #{metrics[:sustained]} (max: #{metrics[:max]})"
|
|
46
|
+
puts " Growth: #{metrics[:growth]}"
|
|
37
47
|
puts " Samples (MB): #{metrics[:samples]}"
|
|
48
|
+
report_picked(picked) unless result[:passed]
|
|
38
49
|
puts
|
|
39
50
|
end
|
|
40
51
|
|
|
52
|
+
# What a failed run exercised, so it can be replayed: the seed, and how often
|
|
53
|
+
# each operator came up among the mutations the check picked.
|
|
54
|
+
def report_picked(picked)
|
|
55
|
+
puts " Seed: #{SEED} (MEMORY_CHECK_SEED)"
|
|
56
|
+
return if picked.nil? || picked.empty?
|
|
57
|
+
|
|
58
|
+
tally = picked.map(&:operator_name).tally.sort_by { |operator, count| [-count, operator.to_s] }
|
|
59
|
+
puts " Operators (#{picked.size} picks): #{tally.map { |operator, count| "#{operator} x#{count}" }.join(", ")}"
|
|
60
|
+
end
|
|
61
|
+
|
|
41
62
|
def format_metrics(result)
|
|
42
63
|
{
|
|
64
|
+
sustained: result[:sustained_growth_mb] ? format("%.1f MB", result[:sustained_growth_mb]) : "N/A",
|
|
43
65
|
growth: result[:growth_mb] ? format("%.1f MB", result[:growth_mb]) : "N/A",
|
|
44
66
|
max: format("%.1f MB", result[:max_growth_kb] / 1024.0),
|
|
45
67
|
samples: result[:samples].map { |s| s ? format("%.1f", s / 1024.0) : "N/A" }.join(" -> ")
|
|
@@ -53,15 +75,21 @@ stub_test_command = ->(_m) { { passed: false } }
|
|
|
53
75
|
all_passed = true
|
|
54
76
|
|
|
55
77
|
# 1. InProcess isolation
|
|
56
|
-
|
|
57
|
-
|
|
78
|
+
random = Random.new(SEED)
|
|
79
|
+
picked = []
|
|
80
|
+
all_passed &= run_check("InProcess isolation", picked: picked) do
|
|
81
|
+
mutation = mutations.sample(random: random)
|
|
82
|
+
picked << mutation
|
|
58
83
|
isolator = Evilution::Isolation::InProcess.new
|
|
59
84
|
isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
|
|
60
85
|
end
|
|
61
86
|
|
|
62
87
|
# 2. Fork isolation
|
|
63
|
-
|
|
64
|
-
|
|
88
|
+
random = Random.new(SEED)
|
|
89
|
+
picked = []
|
|
90
|
+
all_passed &= run_check("Fork isolation", picked: picked) do
|
|
91
|
+
mutation = mutations.sample(random: random)
|
|
92
|
+
picked << mutation
|
|
65
93
|
isolator = Evilution::Isolation::Fork.new
|
|
66
94
|
isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
|
|
67
95
|
end
|
|
@@ -113,13 +141,31 @@ complex_subjects = complex_parser.call(COMPLEX_FIXTURE)
|
|
|
113
141
|
complex_mutations = complex_subjects.flat_map { |s| complex_registry.mutations_for(s) }
|
|
114
142
|
|
|
115
143
|
integration = Evilution::Integration::RSpec.new(test_files: [COMPLEX_FIXTURE_SPEC])
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
#
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
144
|
+
complex_pool = complex_mutations.sample(COMPLEX_POOL_SIZE, random: Random.new(SEED))
|
|
145
|
+
complex_turn = complex_pool.cycle
|
|
146
|
+
|
|
147
|
+
# This check used to grow 20-41 MB over 20 mutations, which was put down to
|
|
148
|
+
# per-mutation require() and redefinition warnings and given a 40 MB budget. It
|
|
149
|
+
# was a leak: every run left its examples reachable from
|
|
150
|
+
# RSpec::Core::AnonymousExampleGroup, about 1 MB per mutation (fixed by
|
|
151
|
+
# StateGuard::AnonymousExampleGroupExamples).
|
|
152
|
+
#
|
|
153
|
+
# The first run of a mutation still grows the process, as the heap reaches the
|
|
154
|
+
# size that mutation needs, and the growth depends on which mutation it is. So
|
|
155
|
+
# the check cycles through a fixed pool: warm-up runs each mutation twice, and
|
|
156
|
+
# every measured run repeats work the warm-up already did. Growth from there
|
|
157
|
+
# on is retained per run.
|
|
158
|
+
#
|
|
159
|
+
# The budget is 10 MB over 100 runs, about 100 KB per mutation: a leak of 200 KB
|
|
160
|
+
# per mutation grows about 18 MB once its largest step is taken out, and fails.
|
|
161
|
+
# Measured on Linux (aarch64) over seeds 1, 7, 42 and 99, sustained growth was
|
|
162
|
+
# -0.9 to 0.6 MB on Ruby 3.4.10 and 0.7 to 2.8 MB on 4.0.7. Ruby 4.0.7 creeps
|
|
163
|
+
# by about 30 KB per mutation in RSS with no Ruby objects retained, which is
|
|
164
|
+
# below what this check can tell apart from allocator noise.
|
|
165
|
+
all_passed &= run_check("RSpec integration per-mutation (Config)",
|
|
166
|
+
iterations: COMPLEX_ITERATIONS, max_growth_kb: COMPLEX_MAX_GROWTH_KB,
|
|
167
|
+
warmup_iterations: COMPLEX_POOL_SIZE * 2, picked: complex_pool) do
|
|
168
|
+
integration.call(complex_turn.next)
|
|
123
169
|
end
|
|
124
170
|
|
|
125
171
|
puts all_passed ? "All memory checks passed." : "Some memory checks failed!"
|