evilution 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +58 -0
- data/.rubocop_todo.yml +9 -0
- data/CHANGELOG.md +86 -0
- data/README.md +202 -14
- data/docs/architecture.md +54 -5
- data/docs/isolation.md +3 -4
- data/exe/evil +8 -1
- data/exe/evilution +8 -1
- data/lib/evilution/ast/local_reads.rb +43 -0
- data/lib/evilution/ast/parser.rb +1 -0
- data/lib/evilution/ast/pattern/filter.rb +1 -0
- data/lib/evilution/ast/pattern/parser.rb +1 -0
- data/lib/evilution/ast/pattern.rb +2 -0
- data/lib/evilution/ast/sorbet_sig_detector.rb +1 -0
- data/lib/evilution/baseline.rb +37 -15
- data/lib/evilution/child_output.rb +1 -1
- data/lib/evilution/cli/command.rb +1 -0
- data/lib/evilution/cli/commands/tests_list.rb +4 -3
- data/lib/evilution/cli/commands.rb +2 -0
- data/lib/evilution/cli/dispatcher.rb +2 -0
- data/lib/evilution/cli/exit_guard.rb +61 -0
- data/lib/evilution/cli/parsed_args.rb +2 -0
- data/lib/evilution/cli/parser/command_extractor.rb +2 -0
- data/lib/evilution/cli/parser/file_args.rb +2 -0
- data/lib/evilution/cli/parser/options_builder.rb +4 -0
- data/lib/evilution/cli/parser/stdin_reader.rb +1 -0
- data/lib/evilution/cli/parser.rb +1 -0
- data/lib/evilution/cli/printers/tests_list.rb +5 -4
- data/lib/evilution/cli/printers.rb +2 -0
- data/lib/evilution/cli/result.rb +2 -0
- data/lib/evilution/cli.rb +6 -0
- data/lib/evilution/config/builders.rb +2 -0
- data/lib/evilution/config/env_loader.rb +2 -0
- data/lib/evilution/config/file_loader.rb +1 -0
- data/lib/evilution/config/sources.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_cache.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_fallback.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_strategy.rb +1 -0
- data/lib/evilution/config/validators/fail_fast.rb +1 -0
- data/lib/evilution/config/validators/hooks.rb +1 -0
- data/lib/evilution/config/validators/ignore_patterns.rb +1 -0
- data/lib/evilution/config/validators/integration.rb +1 -0
- data/lib/evilution/config/validators/isolation.rb +1 -0
- data/lib/evilution/config/validators/jobs.rb +1 -0
- data/lib/evilution/config/validators/preload.rb +1 -0
- data/lib/evilution/config/validators/profile.rb +1 -0
- data/lib/evilution/config/validators/spec_mappings.rb +1 -0
- data/lib/evilution/config/validators/spec_pattern.rb +1 -0
- data/lib/evilution/config/validators/warmup.rb +17 -0
- data/lib/evilution/config/validators.rb +2 -0
- data/lib/evilution/config.rb +7 -5
- data/lib/evilution/coverage.rb +1 -1
- data/lib/evilution/coverage_example_filter.rb +1 -1
- data/lib/evilution/diagnostic.rb +22 -0
- data/lib/evilution/example_filter.rb +1 -1
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +10 -2
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
- data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
- data/lib/evilution/integration/loading/test_load_path.rb +21 -13
- data/lib/evilution/integration/minitest.rb +2 -1
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_state.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +5 -1
- data/lib/evilution/integration/rspec/state_guard/example_groups_constants.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/internals.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/reporter_arrays.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_example_groups.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_filtered_examples.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_sources_by_path.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard.rb +6 -0
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
- data/lib/evilution/integration/rspec.rb +48 -2
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
- data/lib/evilution/isolation/fork.rb +29 -8
- data/lib/evilution/mcp/info_tool/actions/environment.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/feedback.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/statuses.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/tests.rb +1 -0
- data/lib/evilution/mutation.rb +1 -1
- data/lib/evilution/mutator/operator/argument_list_removal.rb +52 -0
- data/lib/evilution/mutator/operator/argument_propagation.rb +54 -0
- data/lib/evilution/mutator/operator/array_coercion_to_literal.rb +34 -0
- data/lib/evilution/mutator/operator/attribute_write_to_read.rb +34 -0
- data/lib/evilution/mutator/operator/bang_method.rb +11 -3
- data/lib/evilution/mutator/operator/binary_operand_promotion.rb +63 -0
- data/lib/evilution/mutator/operator/block_body_promotion.rb +45 -0
- data/lib/evilution/mutator/operator/block_body_to_nil.rb +52 -0
- data/lib/evilution/mutator/operator/block_body_to_raise.rb +34 -0
- data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
- data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
- data/lib/evilution/mutator/operator/call_to_nil.rb +34 -0
- data/lib/evilution/mutator/operator/coercion_emptying.rb +60 -0
- data/lib/evilution/mutator/operator/collection_replacement.rb +19 -9
- data/lib/evilution/mutator/operator/comparison_replacement.rb +37 -12
- data/lib/evilution/mutator/operator/const_get_to_constant_path.rb +50 -0
- data/lib/evilution/mutator/operator/dig_to_fetch_chain.rb +45 -0
- data/lib/evilution/mutator/operator/double_negation_removal.rb +22 -0
- data/lib/evilution/mutator/operator/dynamic_dispatch_resolution.rb +79 -0
- data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
- data/lib/evilution/mutator/operator/inequality_to_negated_identity.rb +43 -0
- data/lib/evilution/mutator/operator/keyword_argument_removal.rb +47 -0
- data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
- data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
- data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
- data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
- data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
- data/lib/evilution/mutator/operator/proc_to_lambda.rb +43 -0
- data/lib/evilution/mutator/operator/receiver_constructor_swap.rb +51 -0
- data/lib/evilution/mutator/operator/reduce_to_sum.rb +58 -0
- data/lib/evilution/mutator/operator/regexp_anchor_to_predicate.rb +136 -0
- data/lib/evilution/mutator/operator/safe_navigation_removal.rb +67 -0
- data/lib/evilution/mutator/operator/send_mutation.rb +42 -6
- data/lib/evilution/mutator/operator/symbol_to_proc_replacement.rb +59 -0
- data/lib/evilution/mutator/operator/to_i_to_integer.rb +38 -0
- data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
- data/lib/evilution/mutator/primitives.rb +25 -0
- data/lib/evilution/mutator/registry.rb +32 -1
- data/lib/evilution/parallel/pool.rb +1 -0
- data/lib/evilution/parallel_db_warning.rb +1 -1
- data/lib/evilution/process_supervisor.rb +20 -8
- data/lib/evilution/rails_warmup.rb +61 -0
- data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
- data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
- data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
- data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +11 -7
- data/lib/evilution/reporter/cli.rb +20 -4
- data/lib/evilution/reporter/html/report.rb +1 -0
- data/lib/evilution/reporter/json/subjects.rb +29 -0
- data/lib/evilution/reporter/json.rb +23 -1
- data/lib/evilution/result/coverage_gap_grouper.rb +1 -0
- data/lib/evilution/result/mutation_result.rb +3 -2
- data/lib/evilution/result/neutral_reason.rb +35 -0
- data/lib/evilution/result/subject_score.rb +28 -0
- data/lib/evilution/result/subject_scorer.rb +38 -0
- data/lib/evilution/result/summary.rb +44 -2
- data/lib/evilution/runner/baseline_runner.rb +17 -7
- data/lib/evilution/runner/canary.rb +52 -5
- data/lib/evilution/runner/isolation_resolver.rb +23 -2
- data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +23 -10
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
- data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
- data/lib/evilution/runner/mutation_executor.rb +32 -4
- data/lib/evilution/runner/report_publisher.rb +31 -9
- data/lib/evilution/runner/target_spec_audit.rb +43 -0
- data/lib/evilution/runner.rb +10 -1
- data/lib/evilution/source_ast_cache.rb +1 -1
- data/lib/evilution/spec_ast_cache.rb +1 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +35 -0
- metadata +50 -2
|
@@ -13,7 +13,9 @@ require_relative "../loading"
|
|
|
13
13
|
# Strategy: walk Prism tree, find CallNodes that sit directly under a class
|
|
14
14
|
# or module body (not inside a def). Calls on a small allowlist of patterns
|
|
15
15
|
# known to be idempotent (`include`, `attr_*`, visibility modifiers, etc.)
|
|
16
|
-
# are preserved; everything else
|
|
16
|
+
# are preserved; everything else has its source range replaced with `()`.
|
|
17
|
+
# The replacement is shorter than the call it replaces, so edits are applied
|
|
18
|
+
# from the end of the source backwards to keep earlier offsets valid.
|
|
17
19
|
class Evilution::Integration::Loading::BodyCallNeutralizer
|
|
18
20
|
IDEMPOTENT_CALLS = %i[
|
|
19
21
|
include extend prepend using
|
|
@@ -25,6 +27,12 @@ class Evilution::Integration::Loading::BodyCallNeutralizer
|
|
|
25
27
|
require require_relative autoload
|
|
26
28
|
].to_set.freeze
|
|
27
29
|
|
|
30
|
+
# `()` rather than `nil`: under -w Ruby flags a bare `nil` statement as
|
|
31
|
+
# "possibly useless use of nil in void context", which a project raising on
|
|
32
|
+
# warnings turns into an error for every mutation of the file. `()` evaluates
|
|
33
|
+
# to nil without the warning on every supported Ruby (`(nil)` still warns on 3.3).
|
|
34
|
+
REPLACEMENT = "()"
|
|
35
|
+
|
|
28
36
|
class << self
|
|
29
37
|
attr_writer :preloaded_features
|
|
30
38
|
|
|
@@ -80,7 +88,7 @@ class Evilution::Integration::Loading::BodyCallNeutralizer
|
|
|
80
88
|
def apply_edits(source, edits)
|
|
81
89
|
bytes = source.b
|
|
82
90
|
edits.sort_by!(&:first).reverse_each do |start_offset, end_offset|
|
|
83
|
-
bytes[start_offset, end_offset - start_offset] =
|
|
91
|
+
bytes[start_offset, end_offset - start_offset] = REPLACEMENT
|
|
84
92
|
end
|
|
85
93
|
bytes.force_encoding(source.encoding)
|
|
86
94
|
end
|
|
@@ -11,6 +11,22 @@ require_relative "../../load_path/subpath_resolver"
|
|
|
11
11
|
class Evilution::Integration::Loading::ConcernStateCleaner
|
|
12
12
|
IVARS = %i[@_included_block @_prepended_block].freeze
|
|
13
13
|
|
|
14
|
+
# ObjectSpace hands us every Module in the VM, including ones that answer no
|
|
15
|
+
# message honestly. ActiveSupport::Deprecation::DeprecatedConstantProxy
|
|
16
|
+
# subclasses Module and undefines all of its instance methods, so a plain
|
|
17
|
+
# `mod.singleton_class` lands in its method_missing and emits the
|
|
18
|
+
# deprecation; where the deprecator's behaviour is :raise (grape's
|
|
19
|
+
# spec_helper sets that), the sweep blows up on a module it merely walked
|
|
20
|
+
# past. Calling through UnboundMethods bypasses the object's own method
|
|
21
|
+
# table, so a proxy is never given the chance to intercept -- and this holds
|
|
22
|
+
# for any method_missing-based wrapper, not just ActiveSupport's.
|
|
23
|
+
# EV-v2rc / GH #1581.
|
|
24
|
+
SINGLETON_CLASS = Object.instance_method(:singleton_class)
|
|
25
|
+
IVAR_DEFINED = Object.instance_method(:instance_variable_defined?)
|
|
26
|
+
IVAR_GET = Object.instance_method(:instance_variable_get)
|
|
27
|
+
REMOVE_IVAR = Object.instance_method(:remove_instance_variable)
|
|
28
|
+
private_constant :SINGLETON_CLASS, :IVAR_DEFINED, :IVAR_GET, :REMOVE_IVAR
|
|
29
|
+
|
|
14
30
|
def initialize(subpath_resolver: Evilution::LoadPath::SubpathResolver.new)
|
|
15
31
|
@subpath_resolver = subpath_resolver
|
|
16
32
|
end
|
|
@@ -22,7 +38,7 @@ class Evilution::Integration::Loading::ConcernStateCleaner
|
|
|
22
38
|
subpath = @subpath_resolver.call(file_path)
|
|
23
39
|
|
|
24
40
|
ObjectSpace.each_object(Module) do |mod|
|
|
25
|
-
next unless mod.
|
|
41
|
+
next unless SINGLETON_CLASS.bind_call(mod).ancestors.include?(ActiveSupport::Concern)
|
|
26
42
|
|
|
27
43
|
clear_concern_ivars(mod, absolute, subpath)
|
|
28
44
|
end
|
|
@@ -32,14 +48,14 @@ class Evilution::Integration::Loading::ConcernStateCleaner
|
|
|
32
48
|
|
|
33
49
|
def clear_concern_ivars(mod, absolute, subpath)
|
|
34
50
|
IVARS.each do |ivar|
|
|
35
|
-
next unless
|
|
51
|
+
next unless IVAR_DEFINED.bind_call(mod, ivar)
|
|
36
52
|
|
|
37
|
-
block =
|
|
53
|
+
block = IVAR_GET.bind_call(mod, ivar)
|
|
38
54
|
block_file = block.source_location&.first
|
|
39
55
|
next unless block_file
|
|
40
56
|
|
|
41
57
|
expanded = File.expand_path(block_file)
|
|
42
|
-
|
|
58
|
+
REMOVE_IVAR.bind_call(mod, ivar) if source_matches?(expanded, absolute, subpath)
|
|
43
59
|
end
|
|
44
60
|
end
|
|
45
61
|
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../loading"
|
|
4
|
+
|
|
5
|
+
# Suppresses exactly the Ruby warnings that re-evaluating a file is guaranteed
|
|
6
|
+
# to produce, and nothing else.
|
|
7
|
+
#
|
|
8
|
+
# Applying a mutation re-evaluates the file, which redefines its methods and
|
|
9
|
+
# reassigns its constants. Ruby reports that in verbose mode, and a project
|
|
10
|
+
# that also promotes warnings to exceptions -- dry-schema's `.rspec` carries
|
|
11
|
+
# `--warnings` while its spec_helper installs `Warning.process { |w| raise w }`
|
|
12
|
+
# -- fails every mutation on a message describing evilution's mechanism rather
|
|
13
|
+
# than the code under test.
|
|
14
|
+
#
|
|
15
|
+
# The blunt fix, `$VERBOSE = nil` around the eval, costs too much: the
|
|
16
|
+
# evaluated file then sees a different $VERBOSE than a normal load would (code
|
|
17
|
+
# branching on it takes the other path), and genuine warnings introduced BY a
|
|
18
|
+
# mutation are swallowed along with ours -- losing a kill that a raising
|
|
19
|
+
# handler would otherwise hand us. So the filter matches the specific
|
|
20
|
+
# mechanism messages instead and delegates everything else to whatever handler
|
|
21
|
+
# the project installed.
|
|
22
|
+
#
|
|
23
|
+
# EV-df7u / GH #1588.
|
|
24
|
+
class Evilution::Integration::Loading::ReevalWarningFilter
|
|
25
|
+
MECHANISM_WARNINGS = Regexp.union(
|
|
26
|
+
/method redefined; discarding old /,
|
|
27
|
+
/already initialized constant /,
|
|
28
|
+
/previous definition of .+ was here/
|
|
29
|
+
).freeze
|
|
30
|
+
|
|
31
|
+
ACTIVE_KEY = :__evilution_reeval_warning_filter_active
|
|
32
|
+
|
|
33
|
+
# Prepended to Warning's singleton, so it sits AHEAD of any handler the
|
|
34
|
+
# project extended in (the `warning` gem extends Warning::Processor), and
|
|
35
|
+
# `super` still reaches that handler for everything it does not drop.
|
|
36
|
+
module Filter
|
|
37
|
+
def warn(message, *args, **kwargs)
|
|
38
|
+
filter = Evilution::Integration::Loading::ReevalWarningFilter
|
|
39
|
+
return if filter.active? && filter.mechanism_warning?(message)
|
|
40
|
+
|
|
41
|
+
super
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
class << self
|
|
46
|
+
def suppress
|
|
47
|
+
install
|
|
48
|
+
previous = Thread.current[ACTIVE_KEY]
|
|
49
|
+
Thread.current[ACTIVE_KEY] = true
|
|
50
|
+
yield
|
|
51
|
+
ensure
|
|
52
|
+
Thread.current[ACTIVE_KEY] = previous
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def active?
|
|
56
|
+
Thread.current[ACTIVE_KEY] ? true : false
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def mechanism_warning?(message)
|
|
60
|
+
MECHANISM_WARNINGS.match?(message)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Idempotent: prepending twice would run the filter twice per warning.
|
|
64
|
+
# `target` exists so a test can install into a throwaway module -- once
|
|
65
|
+
# Warning's singleton has the filter there is no way to take it back off.
|
|
66
|
+
def install(target = Warning.singleton_class)
|
|
67
|
+
return if target.ancestors.include?(Filter)
|
|
68
|
+
|
|
69
|
+
target.prepend(Filter)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../loading"
|
|
4
|
+
require_relative "reeval_warning_filter"
|
|
4
5
|
|
|
5
6
|
# Evaluate source with __FILE__ set to the absolute original path so that
|
|
6
7
|
# `require_relative` and `__dir__` resolve against the real source tree, where
|
|
@@ -17,6 +18,8 @@ class Evilution::Integration::Loading::SourceEvaluator
|
|
|
17
18
|
# GH #1278), anchor the eval __FILE__ against PROJECT_ROOT so siblings
|
|
18
19
|
# `require_relative` can find each other from the real source tree.
|
|
19
20
|
absolute = File.expand_path(file_path, Evilution.project_base_dir)
|
|
20
|
-
|
|
21
|
+
Evilution::Integration::Loading::ReevalWarningFilter.suppress do
|
|
22
|
+
eval(source, TOPLEVEL_BINDING, absolute, 1)
|
|
23
|
+
end
|
|
21
24
|
end
|
|
22
25
|
end
|
|
@@ -22,29 +22,37 @@ module Evilution::Integration::Loading::TestLoadPath
|
|
|
22
22
|
# Prepend every relevant test directory to $LOAD_PATH (idempotently).
|
|
23
23
|
# Iterate in reverse so the first entry from #dirs_for ends up frontmost,
|
|
24
24
|
# preserving its order (mirrors how `ruby -Ia -Ib` lands a before b).
|
|
25
|
-
def add!(files, base: Evilution.project_base_dir)
|
|
26
|
-
dirs_for(files, base).reverse_each do |dir|
|
|
25
|
+
def add!(files, base: Evilution.project_base_dir, outside_root_dirs: true)
|
|
26
|
+
dirs_for(files, base, outside_root_dirs: outside_root_dirs).reverse_each do |dir|
|
|
27
27
|
$LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
|
|
28
28
|
end
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
# The directories to put on $LOAD_PATH for the given
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
|
|
31
|
+
# The directories to put on $LOAD_PATH for the given files: the
|
|
32
|
+
# conventional test/ and spec/ roots under base, each file's own directory,
|
|
33
|
+
# and the topmost test/spec ancestor of each file (covers nested layouts
|
|
34
|
+
# like test/unit, spec/lib, spec/unit). Existing directories only, and only
|
|
35
|
+
# those inside the project base -- never a broad outside-project dir (e.g. a
|
|
36
|
+
# /tmp test file), which would over-widen $LOAD_PATH for the whole process
|
|
37
|
+
# (the baseline runs in the long-lived parent).
|
|
38
|
+
#
|
|
39
|
+
# A resolved test file outside every test root (a tests/ layout, a test
|
|
40
|
+
# reached through --spec or spec_mappings) still contributes its own
|
|
41
|
+
# directory, mirroring `ruby -Itests`. A preload does not: with
|
|
42
|
+
# `outside_root_dirs: false`, a `--preload config/evilution_preload.rb`
|
|
43
|
+
# would otherwise put config/ ahead of the gems, and `require "puma"` during
|
|
44
|
+
# Bundler.require would load the app's config/puma.rb (GH #1597).
|
|
45
|
+
def dirs_for(files, base, outside_root_dirs: true)
|
|
39
46
|
base = File.expand_path(base)
|
|
40
47
|
dirs = conventional_roots(base)
|
|
41
48
|
Array(files).each do |file|
|
|
42
49
|
file_dir = File.dirname(File.expand_path(file, base))
|
|
43
|
-
dirs << file_dir
|
|
44
50
|
root = root_ancestor(file_dir, base)
|
|
45
|
-
|
|
51
|
+
next unless root || outside_root_dirs
|
|
52
|
+
|
|
53
|
+
dirs << file_dir << root
|
|
46
54
|
end
|
|
47
|
-
dirs.uniq.select { |dir| File.directory?(dir) && within?(dir, base) }
|
|
55
|
+
dirs.compact.uniq.select { |dir| File.directory?(dir) && within?(dir, base) }
|
|
48
56
|
end
|
|
49
57
|
|
|
50
58
|
def within?(dir, base)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
require_relative "base"
|
|
5
|
+
require_relative "../diagnostic"
|
|
5
6
|
require_relative "minitest_crash_detector"
|
|
6
7
|
require_relative "loading/test_load_path"
|
|
7
8
|
require_relative "../spec_resolver"
|
|
@@ -276,7 +277,7 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
|
|
|
276
277
|
return if @warned_files.include?(file_path)
|
|
277
278
|
|
|
278
279
|
@warned_files << file_path
|
|
279
|
-
warn
|
|
280
|
+
Evilution::Diagnostic.warn(unresolved_test_message(file_path))
|
|
280
281
|
end
|
|
281
282
|
|
|
282
283
|
# Name both recovery paths when skipping :unresolved; when already falling
|
|
@@ -10,8 +10,16 @@ class Evilution::Integration::RSpec::CrashDetectorLifecycle
|
|
|
10
10
|
@detector.reset
|
|
11
11
|
else
|
|
12
12
|
@detector = Evilution::Integration::CrashDetector.new(StringIO.new)
|
|
13
|
-
::RSpec.configuration.add_formatter(@detector)
|
|
14
13
|
end
|
|
15
14
|
@detector
|
|
16
15
|
end
|
|
16
|
+
|
|
17
|
+
# The run rebuilds RSpec's formatter loader so its formatters are bound to the
|
|
18
|
+
# run's own streams (EV-m6xc / GH #1627), which drops any formatter registered
|
|
19
|
+
# before it — the detector included. It is registered per run instead.
|
|
20
|
+
def register
|
|
21
|
+
detector = current
|
|
22
|
+
::RSpec.configuration.add_formatter(detector)
|
|
23
|
+
detector
|
|
24
|
+
end
|
|
17
25
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../state_guard"
|
|
3
4
|
require_relative "../../rspec"
|
|
4
5
|
require_relative "internals"
|
|
5
6
|
|
|
@@ -18,7 +19,10 @@ require_relative "internals"
|
|
|
18
19
|
# writing the ivars directly: configuration#output_stream= is guarded (it warns
|
|
19
20
|
# and no-ops once a reporter exists), so the public setter cannot put it back.
|
|
20
21
|
class Evilution::Integration::RSpec::StateGuard::ConfigurationStreams
|
|
21
|
-
|
|
22
|
+
# - @reporter and @formatter_loader -- both hold the stream they were built
|
|
23
|
+
# with, so the run drops them to rebuild against its own
|
|
24
|
+
# (EV-m6xc / GH #1627); the host's are restored here.
|
|
25
|
+
IVARS = %i[@color_mode @output_stream @error_stream @reporter @formatter_loader].freeze
|
|
22
26
|
|
|
23
27
|
def snapshot
|
|
24
28
|
config = ::RSpec.configuration
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../state_guard"
|
|
3
4
|
require_relative "../../rspec"
|
|
4
5
|
|
|
5
|
-
class Evilution::Integration::RSpec::StateGuard; end unless defined?(Evilution::Integration::RSpec::StateGuard) # rubocop:disable Lint/EmptyClass
|
|
6
|
-
|
|
7
6
|
class Evilution::Integration::RSpec::StateGuard::ExampleGroupsConstants
|
|
8
7
|
def snapshot
|
|
9
8
|
return nil unless defined?(::RSpec::ExampleGroups)
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../state_guard"
|
|
3
4
|
require_relative "../../rspec"
|
|
4
5
|
|
|
5
|
-
class Evilution::Integration::RSpec::StateGuard; end unless defined?(Evilution::Integration::RSpec::StateGuard) # rubocop:disable Lint/EmptyClass
|
|
6
|
-
|
|
7
6
|
module Evilution::Integration::RSpec::StateGuard::Internals
|
|
8
7
|
module_function
|
|
9
8
|
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../state_guard"
|
|
3
4
|
require_relative "../../rspec"
|
|
4
5
|
|
|
5
|
-
class Evilution::Integration::RSpec::StateGuard; end unless defined?(Evilution::Integration::RSpec::StateGuard) # rubocop:disable Lint/EmptyClass
|
|
6
|
-
|
|
7
6
|
class Evilution::Integration::RSpec::StateGuard::ObjectSpaceExampleGroups
|
|
8
7
|
def snapshot
|
|
9
8
|
groups = Set.new
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../rspec"
|
|
4
|
+
|
|
5
|
+
# Declared before the children are required: each child requires this file
|
|
6
|
+
# and nests under StateGuard, while DEFAULT_STRATEGIES below needs the
|
|
7
|
+
# children loaded, so the namespace must exist first.
|
|
8
|
+
class Evilution::Integration::RSpec::StateGuard; end # rubocop:disable Lint/EmptyClass
|
|
9
|
+
|
|
4
10
|
require_relative "state_guard/object_space_example_groups"
|
|
5
11
|
require_relative "state_guard/world_example_groups"
|
|
6
12
|
require_relative "state_guard/world_sources_by_path"
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../rspec"
|
|
4
|
+
require_relative "../../diagnostic"
|
|
4
5
|
|
|
5
6
|
class Evilution::Integration::RSpec::UnresolvedSpecWarner
|
|
6
7
|
def initialize
|
|
@@ -11,7 +12,7 @@ class Evilution::Integration::RSpec::UnresolvedSpecWarner
|
|
|
11
12
|
return if @warned.include?(file_path)
|
|
12
13
|
|
|
13
14
|
@warned << file_path
|
|
14
|
-
warn
|
|
15
|
+
Evilution::Diagnostic.warn(message(file_path, fallback_to_full_suite))
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
private
|
|
@@ -70,6 +70,26 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
70
70
|
targets = @example_filter_applier.call(mutation, files)
|
|
71
71
|
return @result_builder.unresolved_example(mutation) if targets.nil?
|
|
72
72
|
|
|
73
|
+
result = run_targets(targets)
|
|
74
|
+
return result unless confirm_survivor?(result, targets, files)
|
|
75
|
+
|
|
76
|
+
run_targets(files)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Targeting runs a subset of the examples in a spec file, chosen by name. A
|
|
80
|
+
# mutation that subset does not catch looks like a survivor whether or not the
|
|
81
|
+
# rest of the file would have caught it, and a survivor nobody can reproduce
|
|
82
|
+
# is worse than a missed kill: it sends the reader to write a test that is
|
|
83
|
+
# already there (EV-f8h3 / GH #1624).
|
|
84
|
+
#
|
|
85
|
+
# So a survivor is re-run against the whole resolved file, and that run is the
|
|
86
|
+
# one reported. Only survivors pay for it, and only where the subset was
|
|
87
|
+
# actually narrower than the file.
|
|
88
|
+
def confirm_survivor?(result, targets, files)
|
|
89
|
+
result[:passed] && targets != files
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def run_targets(targets)
|
|
73
93
|
args = ["--format", "progress", "--no-color", "--order", "defined", *resolve_targets(targets)]
|
|
74
94
|
command = "rspec #{args.join(" ")}"
|
|
75
95
|
|
|
@@ -115,10 +135,13 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
115
135
|
end
|
|
116
136
|
|
|
117
137
|
def execute_run(args, command)
|
|
118
|
-
detector = @crash_detector_lifecycle.current
|
|
119
138
|
snapshot = @state_guard.snapshot
|
|
120
139
|
begin
|
|
121
|
-
|
|
140
|
+
out = StringIO.new
|
|
141
|
+
err = StringIO.new
|
|
142
|
+
claim_rspec_streams(out, err)
|
|
143
|
+
detector = @crash_detector_lifecycle.register
|
|
144
|
+
status = ::RSpec::Core::Runner.run(args, err, out)
|
|
122
145
|
@result_builder.from_run(status, command, detector, examples_loaded:)
|
|
123
146
|
rescue StandardError => e
|
|
124
147
|
{ passed: false, error: e.message, test_command: command }
|
|
@@ -127,6 +150,29 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
127
150
|
end
|
|
128
151
|
end
|
|
129
152
|
|
|
153
|
+
# RSpec redirects its own output to the streams it is handed only when the
|
|
154
|
+
# stream its configuration already holds is the current `$stdout`. In-process
|
|
155
|
+
# isolation swaps `$stdout` for a null IO around every mutation, and a
|
|
156
|
+
# configuration built before that swap — which is what `--preload` produces,
|
|
157
|
+
# since the project's spec_helper calls RSpec.configure — still holds the real
|
|
158
|
+
# stdout. The two no longer match, RSpec keeps its own stream, and the run's
|
|
159
|
+
# output lands on the real stdout ahead of the report (EV-m6xc / GH #1627).
|
|
160
|
+
#
|
|
161
|
+
# So the streams are claimed outright rather than offered. The ivars are
|
|
162
|
+
# written directly because `output_stream=` is guarded once a reporter exists.
|
|
163
|
+
# The reporter and its formatters are dropped along with them: both hold the
|
|
164
|
+
# stream they were built with, so leaving them in place would keep writing
|
|
165
|
+
# wherever the host pointed. StateGuard snapshots all four ivars and puts the
|
|
166
|
+
# host's back afterwards.
|
|
167
|
+
def claim_rspec_streams(out, err)
|
|
168
|
+
config = ::RSpec.configuration
|
|
169
|
+
config.instance_variable_set(:@output_stream, out)
|
|
170
|
+
config.instance_variable_set(:@error_stream, err)
|
|
171
|
+
%i[@reporter @formatter_loader].each do |ivar|
|
|
172
|
+
config.remove_instance_variable(ivar) if config.instance_variable_defined?(ivar)
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
130
176
|
# Count of examples loaded into RSpec.world by this run. Used to distinguish
|
|
131
177
|
# "test failed → mutation killed" (positive count, nonzero status) from
|
|
132
178
|
# "spec file failed to load / RSpec returned nonzero with nothing observed"
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../test_unit"
|
|
4
|
+
require_relative "../../diagnostic"
|
|
4
5
|
|
|
5
6
|
# Resolves the list of test files to load for a given mutation. Encapsulates
|
|
6
7
|
# the explicit-override path, spec-selector lookup, fallback glob, and the
|
|
@@ -41,7 +42,7 @@ class Evilution::Integration::TestUnit::TestFileResolver
|
|
|
41
42
|
return if @warned_files.include?(file_path)
|
|
42
43
|
|
|
43
44
|
@warned_files << file_path
|
|
44
|
-
warn
|
|
45
|
+
Evilution::Diagnostic.warn(unresolved_message(file_path))
|
|
45
46
|
end
|
|
46
47
|
|
|
47
48
|
# Name both recovery paths when skipping :unresolved; when already falling
|
|
@@ -142,9 +142,9 @@ class Evilution::Isolation::Fork
|
|
|
142
142
|
# Child exited. Drain any final payload that arrived between
|
|
143
143
|
# wait_readable timeout and the reap (race) before declaring empty.
|
|
144
144
|
final = read_payload(read_io, Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.1)
|
|
145
|
-
return decode_payload(final) if final
|
|
145
|
+
return decode_payload(final, handle) if final
|
|
146
146
|
|
|
147
|
-
return empty_result
|
|
147
|
+
return empty_result(handle)
|
|
148
148
|
end
|
|
149
149
|
end
|
|
150
150
|
|
|
@@ -161,11 +161,13 @@ class Evilution::Isolation::Fork
|
|
|
161
161
|
|
|
162
162
|
if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
163
163
|
@supervisor.terminate(handle, grace: GRACE_PERIOD)
|
|
164
|
-
|
|
164
|
+
# The status now records OUR signal, so it cannot be reported as the
|
|
165
|
+
# child's own cause of death.
|
|
166
|
+
return decode_payload(payload, handle, terminated: true)
|
|
165
167
|
end
|
|
166
168
|
sleep 0.05
|
|
167
169
|
end
|
|
168
|
-
decode_payload(payload)
|
|
170
|
+
decode_payload(payload, handle)
|
|
169
171
|
end
|
|
170
172
|
|
|
171
173
|
def read_payload(read_io, deadline)
|
|
@@ -201,14 +203,33 @@ class Evilution::Isolation::Fork
|
|
|
201
203
|
buf
|
|
202
204
|
end
|
|
203
205
|
|
|
204
|
-
def decode_payload(data)
|
|
205
|
-
return empty_result if data.nil? || data.empty?
|
|
206
|
+
def decode_payload(data, handle = nil, terminated: false)
|
|
207
|
+
return empty_result(handle, terminated: terminated) if data.nil? || data.empty?
|
|
206
208
|
|
|
207
209
|
{ timeout: false }.merge(Marshal.load(data))
|
|
208
210
|
end
|
|
209
211
|
|
|
210
|
-
def empty_result
|
|
211
|
-
{ timeout: false, passed: false, error:
|
|
212
|
+
def empty_result(handle, terminated: false)
|
|
213
|
+
{ timeout: false, passed: false, error: child_death_reason(handle, terminated: terminated) }
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# A child that wrote no payload has usually left its stdout/stderr capture
|
|
217
|
+
# empty too, so its termination status is the only evidence of what happened.
|
|
218
|
+
# Reporting it separates a segfault from an OOM kill from a clean early exit,
|
|
219
|
+
# all of which reach this path identically.
|
|
220
|
+
def child_death_reason(handle, terminated: false)
|
|
221
|
+
return "child was terminated after the reap deadline without writing a result" if terminated
|
|
222
|
+
|
|
223
|
+
status = handle.nil? ? nil : handle.status
|
|
224
|
+
return "empty result from child" unless status.is_a?(Process::Status)
|
|
225
|
+
return "child exited #{status.exitstatus} without writing a result" unless status.signaled?
|
|
226
|
+
|
|
227
|
+
"child died on #{signal_name(status.termsig)} without writing a result"
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def signal_name(termsig)
|
|
231
|
+
name = Signal.signame(termsig)
|
|
232
|
+
name.nil? ? "signal #{termsig}" : "SIG#{name}"
|
|
212
233
|
end
|
|
213
234
|
|
|
214
235
|
def timeout_result(handle)
|
data/lib/evilution/mutation.rb
CHANGED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../operator"
|
|
4
|
+
|
|
5
|
+
# Drop a call's whole argument list: `combine(a, b)` becomes `combine`,
|
|
6
|
+
# `normalize(value)` becomes `normalize`. Every argument kind goes —
|
|
7
|
+
# positional, splat, keywords and `...` — while a block-pass stays attached
|
|
8
|
+
# (`each_slice(2, &block)` becomes `each_slice(&block)`) and so does a
|
|
9
|
+
# literal block.
|
|
10
|
+
#
|
|
11
|
+
# A kill by ArgumentError proves arity is asserted; a survivor means the
|
|
12
|
+
# arguments never mattered. argument_removal drops one argument at a time and
|
|
13
|
+
# only from calls with two or more.
|
|
14
|
+
#
|
|
15
|
+
# Operator methods (`a + b`, `a[i]`) are left to binary operand promotion and
|
|
16
|
+
# attribute writes to attribute_write_to_read.
|
|
17
|
+
class Evilution::Mutator::Operator::ArgumentListRemoval < Evilution::Mutator::Base
|
|
18
|
+
IDENTIFIER = /\A[[:alpha:]_]/
|
|
19
|
+
private_constant :IDENTIFIER
|
|
20
|
+
|
|
21
|
+
def visit_call_node(node)
|
|
22
|
+
remove_argument_list(node) if removable?(node)
|
|
23
|
+
super
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def removable?(node)
|
|
29
|
+
node.arguments && node.message_loc && node.name.match?(IDENTIFIER) && !node.attribute_write?
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def remove_argument_list(node)
|
|
33
|
+
block_pass = node.block if node.block.is_a?(Prism::BlockArgumentNode)
|
|
34
|
+
start = node.message_loc.end_offset
|
|
35
|
+
|
|
36
|
+
add_mutation(
|
|
37
|
+
offset: start,
|
|
38
|
+
length: argument_list_end(node, block_pass) - start,
|
|
39
|
+
replacement: block_pass ? "(#{block_pass.slice})" : "",
|
|
40
|
+
node: node
|
|
41
|
+
)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Without parentheses the list ends at its last element: the block-pass
|
|
45
|
+
# when there is one (it always comes last), else the last argument. The
|
|
46
|
+
# call node itself would reach past a literal `do ... end` block.
|
|
47
|
+
def argument_list_end(node, block_pass)
|
|
48
|
+
return node.closing_loc.end_offset if node.closing_loc
|
|
49
|
+
|
|
50
|
+
(block_pass || node.arguments).end_offset
|
|
51
|
+
end
|
|
52
|
+
end
|