evilution 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +41 -0
- data/CHANGELOG.md +65 -0
- data/README.md +149 -13
- data/docs/architecture.md +20 -5
- data/docs/isolation.md +3 -4
- data/exe/evil +8 -1
- data/exe/evilution +8 -1
- data/lib/evilution/ast/local_reads.rb +43 -0
- data/lib/evilution/cli/exit_guard.rb +59 -0
- data/lib/evilution/cli/parser/options_builder.rb +1 -0
- data/lib/evilution/cli.rb +1 -0
- data/lib/evilution/config.rb +3 -2
- data/lib/evilution/diagnostic.rb +22 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
- data/lib/evilution/integration/loading/redefinition_recovery.rb +1 -1
- data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
- data/lib/evilution/integration/minitest.rb +2 -1
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +4 -1
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
- data/lib/evilution/integration/rspec.rb +48 -2
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
- data/lib/evilution/isolation/fork.rb +29 -8
- data/lib/evilution/mcp/complete_result_server.rb +41 -0
- data/lib/evilution/mcp/server.rb +2 -1
- data/lib/evilution/mutator/base.rb +14 -2
- data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
- data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
- data/lib/evilution/mutator/operator/boolean_expression_to_nil.rb +22 -0
- data/lib/evilution/mutator/operator/boolean_operand_promotion.rb +31 -0
- data/lib/evilution/mutator/operator/case_in.rb +64 -0
- data/lib/evilution/mutator/operator/case_when.rb +72 -1
- data/lib/evilution/mutator/operator/conditional_branch.rb +20 -7
- data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
- data/lib/evilution/mutator/operator/if_branch_swap.rb +48 -0
- data/lib/evilution/mutator/operator/loop_body_to_raise.rb +68 -0
- data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
- data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
- data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
- data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
- data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
- data/lib/evilution/mutator/operator/pattern_predicate.rb +28 -0
- data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
- data/lib/evilution/mutator/primitives.rb +52 -0
- data/lib/evilution/mutator/registry.rb +14 -0
- data/lib/evilution/process_supervisor.rb +20 -8
- data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
- data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
- data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
- data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +11 -7
- data/lib/evilution/reporter/cli.rb +20 -4
- data/lib/evilution/reporter/json/subjects.rb +29 -0
- data/lib/evilution/reporter/json.rb +23 -1
- data/lib/evilution/result/mutation_result.rb +3 -2
- data/lib/evilution/result/neutral_reason.rb +35 -0
- data/lib/evilution/result/subject_score.rb +28 -0
- data/lib/evilution/result/subject_scorer.rb +37 -0
- data/lib/evilution/result/summary.rb +44 -2
- data/lib/evilution/runner/canary.rb +52 -5
- data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +9 -3
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
- data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
- data/lib/evilution/runner/mutation_executor.rb +32 -4
- data/lib/evilution/runner/report_publisher.rb +31 -9
- data/lib/evilution/runner/target_spec_audit.rb +43 -0
- data/lib/evilution/runner.rb +10 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +18 -0
- data/scripts/compare_targeting +4 -2
- metadata +33 -2
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "prism"
|
|
4
|
+
|
|
5
|
+
require_relative "../ast"
|
|
6
|
+
|
|
7
|
+
# Answers whether a scope body ever reads a given local name — the question an
|
|
8
|
+
# operator asks before changing a parameter, since a parameter nothing reads
|
|
9
|
+
# makes most such mutations behaviour-preserving.
|
|
10
|
+
#
|
|
11
|
+
# Scope-aware in the ways that matter here. A block shares the scope it is
|
|
12
|
+
# written in, so a read inside one counts — unless the block binds the same name
|
|
13
|
+
# itself, as a parameter or a block-local, in which case the read is of the
|
|
14
|
+
# block's own variable. A nested def opens a scope of its own, where a local of
|
|
15
|
+
# the same name is unrelated. A write is not a read.
|
|
16
|
+
#
|
|
17
|
+
# Shadowing is settled by Prism's own answer rather than by inspecting each
|
|
18
|
+
# block's parameter list: a LocalVariableReadNode carries the number of scopes
|
|
19
|
+
# between the read and the variable's declaration, so a read refers to the local
|
|
20
|
+
# in question exactly when that depth matches the number of block scopes
|
|
21
|
+
# descended to reach it.
|
|
22
|
+
class Evilution::AST::LocalReads
|
|
23
|
+
def call(node, name, depth = 0)
|
|
24
|
+
return false if node.nil?
|
|
25
|
+
return true if reads?(node, name, depth)
|
|
26
|
+
|
|
27
|
+
node.compact_child_nodes.any? do |child|
|
|
28
|
+
next false if child.is_a?(Prism::DefNode)
|
|
29
|
+
|
|
30
|
+
call(child, name, depth + (scope?(child) ? 1 : 0))
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def reads?(node, name, depth)
|
|
37
|
+
node.is_a?(Prism::LocalVariableReadNode) && node.name.to_s == name && node.depth == depth
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def scope?(node)
|
|
41
|
+
node.is_a?(Prism::BlockNode) || node.is_a?(Prism::LambdaNode)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Gives evilution the final word on the process exit status.
|
|
4
|
+
#
|
|
5
|
+
# `--preload` loads the project's own spec helper into the parent process, and
|
|
6
|
+
# whatever that helper installs comes along with it. SimpleCov's at_exit hook
|
|
7
|
+
# calls `exit` with its own status when coverage is below the configured
|
|
8
|
+
# minimum, which replaces the status evilution computed and breaks a CI step
|
|
9
|
+
# reading it (EV-g8ya / GH #1608).
|
|
10
|
+
#
|
|
11
|
+
# at_exit hooks run last-registered-first, so a hook installed before the
|
|
12
|
+
# preload is the last one standing. It exits with `exit!`, which is immediate
|
|
13
|
+
# and cannot be overridden by anything left in the queue — by then every other
|
|
14
|
+
# hook, evilution's own temp-directory cleanup included, has already run.
|
|
15
|
+
#
|
|
16
|
+
# With no status recorded, evilution did not finish normally: an exception is on
|
|
17
|
+
# its way out, and Ruby's own handling decides the status instead.
|
|
18
|
+
#
|
|
19
|
+
# The hook belongs to the process that installed it. Evilution forks workers,
|
|
20
|
+
# and a fork inherits its parent's at_exit hooks; a worker running this one
|
|
21
|
+
# would exit with the parent's status and skip its own ending. The guard
|
|
22
|
+
# therefore remembers its process and stands down anywhere else.
|
|
23
|
+
class Evilution::CLI::ExitGuard
|
|
24
|
+
attr_accessor :status
|
|
25
|
+
|
|
26
|
+
def initialize(register: nil, exiter: nil, pid_source: nil, streams: nil)
|
|
27
|
+
@register = register || ->(&hook) { at_exit(&hook) }
|
|
28
|
+
@exiter = exiter || ->(code) { exit!(code) }
|
|
29
|
+
@streams = streams || -> { [$stdout, $stderr] }
|
|
30
|
+
@pid_source = pid_source || -> { Process.pid }
|
|
31
|
+
@status = nil
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def install
|
|
35
|
+
@pid = @pid_source.call
|
|
36
|
+
@register.call { fire }
|
|
37
|
+
self
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def fire
|
|
43
|
+
return if @status.nil?
|
|
44
|
+
return unless @pid_source.call == @pid
|
|
45
|
+
|
|
46
|
+
flush_streams
|
|
47
|
+
@exiter.call(@status)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# `exit!` leaves buffered output where it stands, so anything written by the
|
|
51
|
+
# report, or by a hook that ran before this one, has to be pushed out first.
|
|
52
|
+
def flush_streams
|
|
53
|
+
@streams.call.each do |stream|
|
|
54
|
+
stream.flush
|
|
55
|
+
rescue IOError, Errno::EBADF
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -55,6 +55,7 @@ class Evilution::CLI::Parser::OptionsBuilder
|
|
|
55
55
|
|
|
56
56
|
def add_spec_filter_options(opts)
|
|
57
57
|
opts.on("--min-score FLOAT", Float, "Minimum mutation score to pass") { |s| @options[:min_score] = s }
|
|
58
|
+
opts.on("--output FILE", "Write the report to FILE instead of stdout") { |f| @options[:output] = f }
|
|
58
59
|
opts.on("--spec FILES", Array, "Spec files to run (comma-separated)") { |f| @options[:spec_files] = f }
|
|
59
60
|
opts.on("--spec-dir DIR", "Include all specs in DIR") { |d| expand_spec_dir(d) }
|
|
60
61
|
opts.on("--spec-pattern GLOB",
|
data/lib/evilution/cli.rb
CHANGED
data/lib/evilution/config.rb
CHANGED
|
@@ -17,7 +17,7 @@ class Evilution::Config
|
|
|
17
17
|
save_session: false, line_ranges: {}, spec_files: [], ignore_patterns: [],
|
|
18
18
|
show_disabled: false, baseline_session: nil, skip_heredoc_literals: false,
|
|
19
19
|
related_specs_heuristic: false, fallback_to_full_suite: false, preload: nil,
|
|
20
|
-
spec_mappings: {}, spec_pattern: nil, example_targeting: true,
|
|
20
|
+
output: nil, spec_mappings: {}, spec_pattern: nil, example_targeting: true,
|
|
21
21
|
example_targeting_fallback: :full_file, example_targeting_strategy: :lexical,
|
|
22
22
|
example_targeting_cache: { max_files: 50, max_blocks: 10_000 },
|
|
23
23
|
quiet_children: false, quiet_children_dir: "tmp/evilution_children",
|
|
@@ -25,7 +25,7 @@ class Evilution::Config
|
|
|
25
25
|
}.freeze
|
|
26
26
|
|
|
27
27
|
attr_reader :target_files, :schema_version, :timeout, :format,
|
|
28
|
-
:target, :min_score, :integration, :verbose, :quiet,
|
|
28
|
+
:target, :min_score, :integration, :verbose, :quiet, :output,
|
|
29
29
|
:jobs, :fail_fast, :baseline, :isolation, :incremental, :suggest_tests,
|
|
30
30
|
:progress, :save_session, :line_ranges, :spec_files, :hooks,
|
|
31
31
|
:ignore_patterns, :show_disabled, :baseline_session,
|
|
@@ -243,6 +243,7 @@ class Evilution::Config
|
|
|
243
243
|
format: :to_sym.to_proc,
|
|
244
244
|
target: nil,
|
|
245
245
|
min_score: :to_f.to_proc,
|
|
246
|
+
output: nil,
|
|
246
247
|
verbose: nil,
|
|
247
248
|
quiet: nil,
|
|
248
249
|
baseline: nil,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../evilution"
|
|
4
|
+
|
|
5
|
+
# Evilution's own advisory messages -- "no matching spec found", and the like.
|
|
6
|
+
#
|
|
7
|
+
# These deliberately bypass Kernel#warn. That routes through Warning.warn,
|
|
8
|
+
# which a project is free to replace: the `warning` gem with a raising handler
|
|
9
|
+
# turns every Ruby warning into an exception, and both dry-schema and
|
|
10
|
+
# dry-monads configure exactly that in their spec_helper. Under such a project
|
|
11
|
+
# our advisory message would be promoted to a fatal error, so a mutation we
|
|
12
|
+
# meant to classify :unresolved would be reported :error instead -- 239 of
|
|
13
|
+
# dry-monads' 240 mutations, in the run that found this.
|
|
14
|
+
#
|
|
15
|
+
# A message about how evilution is behaving must never be able to fail a
|
|
16
|
+
# user's mutation. Writing straight to the stream keeps it a diagnostic.
|
|
17
|
+
# EV-df7u / GH #1588.
|
|
18
|
+
module Evilution::Diagnostic
|
|
19
|
+
def self.warn(message, stream: $stderr)
|
|
20
|
+
stream.puts(message)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -11,6 +11,22 @@ require_relative "../../load_path/subpath_resolver"
|
|
|
11
11
|
class Evilution::Integration::Loading::ConcernStateCleaner
|
|
12
12
|
IVARS = %i[@_included_block @_prepended_block].freeze
|
|
13
13
|
|
|
14
|
+
# ObjectSpace hands us every Module in the VM, including ones that answer no
|
|
15
|
+
# message honestly. ActiveSupport::Deprecation::DeprecatedConstantProxy
|
|
16
|
+
# subclasses Module and undefines all of its instance methods, so a plain
|
|
17
|
+
# `mod.singleton_class` lands in its method_missing and emits the
|
|
18
|
+
# deprecation; where the deprecator's behaviour is :raise (grape's
|
|
19
|
+
# spec_helper sets that), the sweep blows up on a module it merely walked
|
|
20
|
+
# past. Calling through UnboundMethods bypasses the object's own method
|
|
21
|
+
# table, so a proxy is never given the chance to intercept -- and this holds
|
|
22
|
+
# for any method_missing-based wrapper, not just ActiveSupport's.
|
|
23
|
+
# EV-v2rc / GH #1581.
|
|
24
|
+
SINGLETON_CLASS = Object.instance_method(:singleton_class)
|
|
25
|
+
IVAR_DEFINED = Object.instance_method(:instance_variable_defined?)
|
|
26
|
+
IVAR_GET = Object.instance_method(:instance_variable_get)
|
|
27
|
+
REMOVE_IVAR = Object.instance_method(:remove_instance_variable)
|
|
28
|
+
private_constant :SINGLETON_CLASS, :IVAR_DEFINED, :IVAR_GET, :REMOVE_IVAR
|
|
29
|
+
|
|
14
30
|
def initialize(subpath_resolver: Evilution::LoadPath::SubpathResolver.new)
|
|
15
31
|
@subpath_resolver = subpath_resolver
|
|
16
32
|
end
|
|
@@ -22,7 +38,7 @@ class Evilution::Integration::Loading::ConcernStateCleaner
|
|
|
22
38
|
subpath = @subpath_resolver.call(file_path)
|
|
23
39
|
|
|
24
40
|
ObjectSpace.each_object(Module) do |mod|
|
|
25
|
-
next unless mod.
|
|
41
|
+
next unless SINGLETON_CLASS.bind_call(mod).ancestors.include?(ActiveSupport::Concern)
|
|
26
42
|
|
|
27
43
|
clear_concern_ivars(mod, absolute, subpath)
|
|
28
44
|
end
|
|
@@ -32,14 +48,14 @@ class Evilution::Integration::Loading::ConcernStateCleaner
|
|
|
32
48
|
|
|
33
49
|
def clear_concern_ivars(mod, absolute, subpath)
|
|
34
50
|
IVARS.each do |ivar|
|
|
35
|
-
next unless
|
|
51
|
+
next unless IVAR_DEFINED.bind_call(mod, ivar)
|
|
36
52
|
|
|
37
|
-
block =
|
|
53
|
+
block = IVAR_GET.bind_call(mod, ivar)
|
|
38
54
|
block_file = block.source_location&.first
|
|
39
55
|
next unless block_file
|
|
40
56
|
|
|
41
57
|
expanded = File.expand_path(block_file)
|
|
42
|
-
|
|
58
|
+
REMOVE_IVAR.bind_call(mod, ivar) if source_matches?(expanded, absolute, subpath)
|
|
43
59
|
end
|
|
44
60
|
end
|
|
45
61
|
|
|
@@ -65,7 +65,7 @@ class Evilution::Integration::Loading::RedefinitionRecovery
|
|
|
65
65
|
msg = error.message
|
|
66
66
|
# `msg` is a String and patterns are substrings, so `include?` is a
|
|
67
67
|
# substring test — `intersect?` would compare array elements, not text.
|
|
68
|
-
IDEMPOTENCY_PATTERNS.any? {
|
|
68
|
+
IDEMPOTENCY_PATTERNS.any? { msg.include?(_1) }
|
|
69
69
|
end
|
|
70
70
|
|
|
71
71
|
def superclass_mismatch?(error)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../loading"
|
|
4
|
+
|
|
5
|
+
# Suppresses exactly the Ruby warnings that re-evaluating a file is guaranteed
|
|
6
|
+
# to produce, and nothing else.
|
|
7
|
+
#
|
|
8
|
+
# Applying a mutation re-evaluates the file, which redefines its methods and
|
|
9
|
+
# reassigns its constants. Ruby reports that in verbose mode, and a project
|
|
10
|
+
# that also promotes warnings to exceptions -- dry-schema's `.rspec` carries
|
|
11
|
+
# `--warnings` while its spec_helper installs `Warning.process { |w| raise w }`
|
|
12
|
+
# -- fails every mutation on a message describing evilution's mechanism rather
|
|
13
|
+
# than the code under test.
|
|
14
|
+
#
|
|
15
|
+
# The blunt fix, `$VERBOSE = nil` around the eval, costs too much: the
|
|
16
|
+
# evaluated file then sees a different $VERBOSE than a normal load would (code
|
|
17
|
+
# branching on it takes the other path), and genuine warnings introduced BY a
|
|
18
|
+
# mutation are swallowed along with ours -- losing a kill that a raising
|
|
19
|
+
# handler would otherwise hand us. So the filter matches the specific
|
|
20
|
+
# mechanism messages instead and delegates everything else to whatever handler
|
|
21
|
+
# the project installed.
|
|
22
|
+
#
|
|
23
|
+
# EV-df7u / GH #1588.
|
|
24
|
+
class Evilution::Integration::Loading::ReevalWarningFilter
|
|
25
|
+
MECHANISM_WARNINGS = Regexp.union(
|
|
26
|
+
/method redefined; discarding old /,
|
|
27
|
+
/already initialized constant /,
|
|
28
|
+
/previous definition of .+ was here/
|
|
29
|
+
).freeze
|
|
30
|
+
|
|
31
|
+
ACTIVE_KEY = :__evilution_reeval_warning_filter_active
|
|
32
|
+
|
|
33
|
+
# Prepended to Warning's singleton, so it sits AHEAD of any handler the
|
|
34
|
+
# project extended in (the `warning` gem extends Warning::Processor), and
|
|
35
|
+
# `super` still reaches that handler for everything it does not drop.
|
|
36
|
+
module Filter
|
|
37
|
+
def warn(message, *args, **kwargs)
|
|
38
|
+
filter = Evilution::Integration::Loading::ReevalWarningFilter
|
|
39
|
+
return if filter.active? && filter.mechanism_warning?(message)
|
|
40
|
+
|
|
41
|
+
super
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
class << self
|
|
46
|
+
def suppress
|
|
47
|
+
install
|
|
48
|
+
previous = Thread.current[ACTIVE_KEY]
|
|
49
|
+
Thread.current[ACTIVE_KEY] = true
|
|
50
|
+
yield
|
|
51
|
+
ensure
|
|
52
|
+
Thread.current[ACTIVE_KEY] = previous
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def active?
|
|
56
|
+
Thread.current[ACTIVE_KEY] ? true : false
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def mechanism_warning?(message)
|
|
60
|
+
MECHANISM_WARNINGS.match?(message)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Idempotent: prepending twice would run the filter twice per warning.
|
|
64
|
+
# `target` exists so a test can install into a throwaway module -- once
|
|
65
|
+
# Warning's singleton has the filter there is no way to take it back off.
|
|
66
|
+
def install(target = Warning.singleton_class)
|
|
67
|
+
return if target.ancestors.include?(Filter)
|
|
68
|
+
|
|
69
|
+
target.prepend(Filter)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../loading"
|
|
4
|
+
require_relative "reeval_warning_filter"
|
|
4
5
|
|
|
5
6
|
# Evaluate source with __FILE__ set to the absolute original path so that
|
|
6
7
|
# `require_relative` and `__dir__` resolve against the real source tree, where
|
|
@@ -17,6 +18,8 @@ class Evilution::Integration::Loading::SourceEvaluator
|
|
|
17
18
|
# GH #1278), anchor the eval __FILE__ against PROJECT_ROOT so siblings
|
|
18
19
|
# `require_relative` can find each other from the real source tree.
|
|
19
20
|
absolute = File.expand_path(file_path, Evilution.project_base_dir)
|
|
20
|
-
|
|
21
|
+
Evilution::Integration::Loading::ReevalWarningFilter.suppress do
|
|
22
|
+
eval(source, TOPLEVEL_BINDING, absolute, 1)
|
|
23
|
+
end
|
|
21
24
|
end
|
|
22
25
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
require_relative "base"
|
|
5
|
+
require_relative "../diagnostic"
|
|
5
6
|
require_relative "minitest_crash_detector"
|
|
6
7
|
require_relative "loading/test_load_path"
|
|
7
8
|
require_relative "../spec_resolver"
|
|
@@ -276,7 +277,7 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
|
|
|
276
277
|
return if @warned_files.include?(file_path)
|
|
277
278
|
|
|
278
279
|
@warned_files << file_path
|
|
279
|
-
warn
|
|
280
|
+
Evilution::Diagnostic.warn(unresolved_test_message(file_path))
|
|
280
281
|
end
|
|
281
282
|
|
|
282
283
|
# Name both recovery paths when skipping :unresolved; when already falling
|
|
@@ -10,8 +10,16 @@ class Evilution::Integration::RSpec::CrashDetectorLifecycle
|
|
|
10
10
|
@detector.reset
|
|
11
11
|
else
|
|
12
12
|
@detector = Evilution::Integration::CrashDetector.new(StringIO.new)
|
|
13
|
-
::RSpec.configuration.add_formatter(@detector)
|
|
14
13
|
end
|
|
15
14
|
@detector
|
|
16
15
|
end
|
|
16
|
+
|
|
17
|
+
# The run rebuilds RSpec's formatter loader so its formatters are bound to the
|
|
18
|
+
# run's own streams (EV-m6xc / GH #1627), which drops any formatter registered
|
|
19
|
+
# before it — the detector included. It is registered per run instead.
|
|
20
|
+
def register
|
|
21
|
+
detector = current
|
|
22
|
+
::RSpec.configuration.add_formatter(detector)
|
|
23
|
+
detector
|
|
24
|
+
end
|
|
17
25
|
end
|
|
@@ -18,7 +18,10 @@ require_relative "internals"
|
|
|
18
18
|
# writing the ivars directly: configuration#output_stream= is guarded (it warns
|
|
19
19
|
# and no-ops once a reporter exists), so the public setter cannot put it back.
|
|
20
20
|
class Evilution::Integration::RSpec::StateGuard::ConfigurationStreams
|
|
21
|
-
|
|
21
|
+
# - @reporter and @formatter_loader -- both hold the stream they were built
|
|
22
|
+
# with, so the run drops them to rebuild against its own
|
|
23
|
+
# (EV-m6xc / GH #1627); the host's are restored here.
|
|
24
|
+
IVARS = %i[@color_mode @output_stream @error_stream @reporter @formatter_loader].freeze
|
|
22
25
|
|
|
23
26
|
def snapshot
|
|
24
27
|
config = ::RSpec.configuration
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../rspec"
|
|
4
|
+
require_relative "../../diagnostic"
|
|
4
5
|
|
|
5
6
|
class Evilution::Integration::RSpec::UnresolvedSpecWarner
|
|
6
7
|
def initialize
|
|
@@ -11,7 +12,7 @@ class Evilution::Integration::RSpec::UnresolvedSpecWarner
|
|
|
11
12
|
return if @warned.include?(file_path)
|
|
12
13
|
|
|
13
14
|
@warned << file_path
|
|
14
|
-
warn
|
|
15
|
+
Evilution::Diagnostic.warn(message(file_path, fallback_to_full_suite))
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
private
|
|
@@ -70,6 +70,26 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
70
70
|
targets = @example_filter_applier.call(mutation, files)
|
|
71
71
|
return @result_builder.unresolved_example(mutation) if targets.nil?
|
|
72
72
|
|
|
73
|
+
result = run_targets(targets)
|
|
74
|
+
return result unless confirm_survivor?(result, targets, files)
|
|
75
|
+
|
|
76
|
+
run_targets(files)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Targeting runs a subset of the examples in a spec file, chosen by name. A
|
|
80
|
+
# mutation that subset does not catch looks like a survivor whether or not the
|
|
81
|
+
# rest of the file would have caught it, and a survivor nobody can reproduce
|
|
82
|
+
# is worse than a missed kill: it sends the reader to write a test that is
|
|
83
|
+
# already there (EV-f8h3 / GH #1624).
|
|
84
|
+
#
|
|
85
|
+
# So a survivor is re-run against the whole resolved file, and that run is the
|
|
86
|
+
# one reported. Only survivors pay for it, and only where the subset was
|
|
87
|
+
# actually narrower than the file.
|
|
88
|
+
def confirm_survivor?(result, targets, files)
|
|
89
|
+
result[:passed] && targets != files
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def run_targets(targets)
|
|
73
93
|
args = ["--format", "progress", "--no-color", "--order", "defined", *resolve_targets(targets)]
|
|
74
94
|
command = "rspec #{args.join(" ")}"
|
|
75
95
|
|
|
@@ -115,10 +135,13 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
115
135
|
end
|
|
116
136
|
|
|
117
137
|
def execute_run(args, command)
|
|
118
|
-
detector = @crash_detector_lifecycle.current
|
|
119
138
|
snapshot = @state_guard.snapshot
|
|
120
139
|
begin
|
|
121
|
-
|
|
140
|
+
out = StringIO.new
|
|
141
|
+
err = StringIO.new
|
|
142
|
+
claim_rspec_streams(out, err)
|
|
143
|
+
detector = @crash_detector_lifecycle.register
|
|
144
|
+
status = ::RSpec::Core::Runner.run(args, err, out)
|
|
122
145
|
@result_builder.from_run(status, command, detector, examples_loaded:)
|
|
123
146
|
rescue StandardError => e
|
|
124
147
|
{ passed: false, error: e.message, test_command: command }
|
|
@@ -127,6 +150,29 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
127
150
|
end
|
|
128
151
|
end
|
|
129
152
|
|
|
153
|
+
# RSpec redirects its own output to the streams it is handed only when the
|
|
154
|
+
# stream its configuration already holds is the current `$stdout`. In-process
|
|
155
|
+
# isolation swaps `$stdout` for a null IO around every mutation, and a
|
|
156
|
+
# configuration built before that swap — which is what `--preload` produces,
|
|
157
|
+
# since the project's spec_helper calls RSpec.configure — still holds the real
|
|
158
|
+
# stdout. The two no longer match, RSpec keeps its own stream, and the run's
|
|
159
|
+
# output lands on the real stdout ahead of the report (EV-m6xc / GH #1627).
|
|
160
|
+
#
|
|
161
|
+
# So the streams are claimed outright rather than offered. The ivars are
|
|
162
|
+
# written directly because `output_stream=` is guarded once a reporter exists.
|
|
163
|
+
# The reporter and its formatters are dropped along with them: both hold the
|
|
164
|
+
# stream they were built with, so leaving them in place would keep writing
|
|
165
|
+
# wherever the host pointed. StateGuard snapshots all four ivars and puts the
|
|
166
|
+
# host's back afterwards.
|
|
167
|
+
def claim_rspec_streams(out, err)
|
|
168
|
+
config = ::RSpec.configuration
|
|
169
|
+
config.instance_variable_set(:@output_stream, out)
|
|
170
|
+
config.instance_variable_set(:@error_stream, err)
|
|
171
|
+
%i[@reporter @formatter_loader].each do |ivar|
|
|
172
|
+
config.remove_instance_variable(ivar) if config.instance_variable_defined?(ivar)
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
130
176
|
# Count of examples loaded into RSpec.world by this run. Used to distinguish
|
|
131
177
|
# "test failed → mutation killed" (positive count, nonzero status) from
|
|
132
178
|
# "spec file failed to load / RSpec returned nonzero with nothing observed"
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../test_unit"
|
|
4
|
+
require_relative "../../diagnostic"
|
|
4
5
|
|
|
5
6
|
# Resolves the list of test files to load for a given mutation. Encapsulates
|
|
6
7
|
# the explicit-override path, spec-selector lookup, fallback glob, and the
|
|
@@ -41,7 +42,7 @@ class Evilution::Integration::TestUnit::TestFileResolver
|
|
|
41
42
|
return if @warned_files.include?(file_path)
|
|
42
43
|
|
|
43
44
|
@warned_files << file_path
|
|
44
|
-
warn
|
|
45
|
+
Evilution::Diagnostic.warn(unresolved_message(file_path))
|
|
45
46
|
end
|
|
46
47
|
|
|
47
48
|
# Name both recovery paths when skipping :unresolved; when already falling
|
|
@@ -142,9 +142,9 @@ class Evilution::Isolation::Fork
|
|
|
142
142
|
# Child exited. Drain any final payload that arrived between
|
|
143
143
|
# wait_readable timeout and the reap (race) before declaring empty.
|
|
144
144
|
final = read_payload(read_io, Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.1)
|
|
145
|
-
return decode_payload(final) if final
|
|
145
|
+
return decode_payload(final, handle) if final
|
|
146
146
|
|
|
147
|
-
return empty_result
|
|
147
|
+
return empty_result(handle)
|
|
148
148
|
end
|
|
149
149
|
end
|
|
150
150
|
|
|
@@ -161,11 +161,13 @@ class Evilution::Isolation::Fork
|
|
|
161
161
|
|
|
162
162
|
if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
163
163
|
@supervisor.terminate(handle, grace: GRACE_PERIOD)
|
|
164
|
-
|
|
164
|
+
# The status now records OUR signal, so it cannot be reported as the
|
|
165
|
+
# child's own cause of death.
|
|
166
|
+
return decode_payload(payload, handle, terminated: true)
|
|
165
167
|
end
|
|
166
168
|
sleep 0.05
|
|
167
169
|
end
|
|
168
|
-
decode_payload(payload)
|
|
170
|
+
decode_payload(payload, handle)
|
|
169
171
|
end
|
|
170
172
|
|
|
171
173
|
def read_payload(read_io, deadline)
|
|
@@ -201,14 +203,33 @@ class Evilution::Isolation::Fork
|
|
|
201
203
|
buf
|
|
202
204
|
end
|
|
203
205
|
|
|
204
|
-
def decode_payload(data)
|
|
205
|
-
return empty_result if data.nil? || data.empty?
|
|
206
|
+
def decode_payload(data, handle = nil, terminated: false)
|
|
207
|
+
return empty_result(handle, terminated: terminated) if data.nil? || data.empty?
|
|
206
208
|
|
|
207
209
|
{ timeout: false }.merge(Marshal.load(data))
|
|
208
210
|
end
|
|
209
211
|
|
|
210
|
-
def empty_result
|
|
211
|
-
{ timeout: false, passed: false, error:
|
|
212
|
+
def empty_result(handle, terminated: false)
|
|
213
|
+
{ timeout: false, passed: false, error: child_death_reason(handle, terminated: terminated) }
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# A child that wrote no payload has usually left its stdout/stderr capture
|
|
217
|
+
# empty too, so its termination status is the only evidence of what happened.
|
|
218
|
+
# Reporting it separates a segfault from an OOM kill from a clean early exit,
|
|
219
|
+
# all of which reach this path identically.
|
|
220
|
+
def child_death_reason(handle, terminated: false)
|
|
221
|
+
return "child was terminated after the reap deadline without writing a result" if terminated
|
|
222
|
+
|
|
223
|
+
status = handle.nil? ? nil : handle.status
|
|
224
|
+
return "empty result from child" unless status.is_a?(Process::Status)
|
|
225
|
+
return "child exited #{status.exitstatus} without writing a result" unless status.signaled?
|
|
226
|
+
|
|
227
|
+
"child died on #{signal_name(status.termsig)} without writing a result"
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def signal_name(termsig)
|
|
231
|
+
name = Signal.signame(termsig)
|
|
232
|
+
name.nil? ? "signal #{termsig}" : "SIG#{name}"
|
|
212
233
|
end
|
|
213
234
|
|
|
214
235
|
def timeout_result(handle)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "mcp"
|
|
4
|
+
|
|
5
|
+
require_relative "../mcp"
|
|
6
|
+
|
|
7
|
+
# MCP protocol revision 2026-07-28 (SEP-2322) makes `resultType` mandatory on list results: a
|
|
8
|
+
# client may only treat a missing field as "complete" for servers that negotiated an earlier
|
|
9
|
+
# revision. The mcp gem echoes back any client-offered revision it supports — 2026-07-28
|
|
10
|
+
# included — yet its list handlers still omit the field, so strict clients reject the whole
|
|
11
|
+
# list and the server appears to expose no tools at all.
|
|
12
|
+
#
|
|
13
|
+
# Every list result this server produces is final (there is no `input_required` round trip in
|
|
14
|
+
# the evilution tool surface), so the field is always `"complete"`. The value is written as a
|
|
15
|
+
# literal rather than `MCP::ResultType::COMPLETE` because the gem dependency allows versions
|
|
16
|
+
# that predate that constant. Older revisions ignore the extra key.
|
|
17
|
+
class Evilution::MCP::CompleteResultServer < MCP::Server
|
|
18
|
+
RESULT_TYPE_COMPLETE = "complete"
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def list_tools(request)
|
|
23
|
+
mark_complete(super)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def list_prompts(request)
|
|
27
|
+
mark_complete(super)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def list_resources(request)
|
|
31
|
+
mark_complete(super)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def list_resource_templates(request)
|
|
35
|
+
mark_complete(super)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def mark_complete(result)
|
|
39
|
+
result.merge(resultType: RESULT_TYPE_COMPLETE)
|
|
40
|
+
end
|
|
41
|
+
end
|
data/lib/evilution/mcp/server.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "mcp"
|
|
4
4
|
require_relative "../version"
|
|
5
|
+
require_relative "complete_result_server"
|
|
5
6
|
require_relative "mutate_tool"
|
|
6
7
|
require_relative "session_tool"
|
|
7
8
|
require_relative "info_tool"
|
|
@@ -10,7 +11,7 @@ require_relative "../mcp"
|
|
|
10
11
|
|
|
11
12
|
class Evilution::MCP::Server
|
|
12
13
|
def self.build
|
|
13
|
-
::MCP::
|
|
14
|
+
Evilution::MCP::CompleteResultServer.new(
|
|
14
15
|
name: "evilution",
|
|
15
16
|
version: Evilution::VERSION,
|
|
16
17
|
tools: [Evilution::MCP::MutateTool, Evilution::MCP::SessionTool, Evilution::MCP::InfoTool]
|
|
@@ -3,10 +3,13 @@
|
|
|
3
3
|
require "prism"
|
|
4
4
|
|
|
5
5
|
require_relative "../mutator"
|
|
6
|
+
require_relative "primitives"
|
|
6
7
|
require_relative "../integration/loading/body_call_neutralizer"
|
|
7
8
|
require_relative "../ast/heredoc_span"
|
|
8
9
|
|
|
9
10
|
class Evilution::Mutator::Base < Prism::Visitor
|
|
11
|
+
include Evilution::Mutator::Primitives
|
|
12
|
+
|
|
10
13
|
AffectedSlices = Data.define(:original, :mutated)
|
|
11
14
|
private_constant :AffectedSlices
|
|
12
15
|
|
|
@@ -41,7 +44,12 @@ class Evilution::Mutator::Base < Prism::Visitor
|
|
|
41
44
|
|
|
42
45
|
private
|
|
43
46
|
|
|
44
|
-
|
|
47
|
+
# `skip_unparseable` makes the mutation vanish rather than be recorded when
|
|
48
|
+
# the surgery does not parse. Off by default: the point operators rely on
|
|
49
|
+
# unparseable bytes reaching the reporter as their own bucket. The shared
|
|
50
|
+
# primitives opt in, since a nil-ification or promotion that does not parse
|
|
51
|
+
# carries no signal.
|
|
52
|
+
def add_mutation(offset:, length:, replacement:, node:, skip_unparseable: false)
|
|
45
53
|
return if @filter && @filter.skip?(node)
|
|
46
54
|
|
|
47
55
|
# When the byte range opens a heredoc but stops at its inline anchor,
|
|
@@ -67,6 +75,8 @@ class Evilution::Mutator::Base < Prism::Visitor
|
|
|
67
75
|
surgery = Evilution::AST::SourceSurgeon.apply(
|
|
68
76
|
@file_source, offset: offset, length: length, replacement: replacement
|
|
69
77
|
)
|
|
78
|
+
return if skip_unparseable && surgery.unparseable?
|
|
79
|
+
|
|
70
80
|
slices = slice_affected_lines(
|
|
71
81
|
mutated_source: surgery.source,
|
|
72
82
|
offset: offset,
|
|
@@ -74,7 +84,9 @@ class Evilution::Mutator::Base < Prism::Visitor
|
|
|
74
84
|
replacement_bytesize: replacement.bytesize
|
|
75
85
|
)
|
|
76
86
|
|
|
77
|
-
|
|
87
|
+
mutation = build_mutation_record(node, surgery, slices)
|
|
88
|
+
@mutations << mutation
|
|
89
|
+
mutation
|
|
78
90
|
end
|
|
79
91
|
|
|
80
92
|
def build_mutation_record(node, surgery, slices)
|