evilution 0.30.4 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +22 -0
- data/.rubocop_todo.yml +6 -0
- data/CHANGELOG.md +22 -0
- data/README.md +9 -7
- data/docs/integrations.md +126 -0
- data/docs/isolation.md +28 -0
- data/lib/evilution/cli/parser/options_builder.rb +6 -1
- data/lib/evilution/config/validators/integration.rb +5 -1
- data/lib/evilution/config.rb +14 -4
- data/lib/evilution/integration/loading/mutation_applier.rb +16 -8
- data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
- data/lib/evilution/integration/minitest.rb +1 -1
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -1
- data/lib/evilution/integration/rspec/framework_loader.rb +5 -1
- data/lib/evilution/integration/rspec.rb +38 -1
- data/lib/evilution/integration/test_unit/dispatcher.rb +26 -0
- data/lib/evilution/integration/test_unit/framework_loader.rb +33 -0
- data/lib/evilution/integration/test_unit/result_builder.rb +53 -0
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +26 -0
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +48 -0
- data/lib/evilution/integration/test_unit.rb +124 -0
- data/lib/evilution/integration/test_unit_crash_detector.rb +61 -0
- data/lib/evilution/isolation/fork.rb +26 -1
- data/lib/evilution/isolation/in_process.rb +20 -3
- data/lib/evilution/mcp/info_tool.rb +2 -2
- data/lib/evilution/mcp/mutate_tool.rb +3 -2
- data/lib/evilution/runner/baseline_runner.rb +3 -1
- data/lib/evilution/runner/canary.rb +130 -0
- data/lib/evilution/runner.rb +24 -1
- data/lib/evilution/spec_ast_cache.rb +20 -3
- data/lib/evilution/spec_resolver.rb +16 -2
- data/lib/evilution/spec_selector.rb +14 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +39 -0
- data/script/run_self_baseline +2 -2
- metadata +11 -2
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Loads the test-unit gem and disables its at_exit auto-run handler.
|
|
6
|
+
# Mirrors Evilution::Integration::RSpec::FrameworkLoader's role: framework
|
|
7
|
+
# setup is one responsibility separated from dispatch + result building so
|
|
8
|
+
# integrations can compose it independently in tests.
|
|
9
|
+
class Evilution::Integration::TestUnit::FrameworkLoader
|
|
10
|
+
def loaded?
|
|
11
|
+
@loaded == true
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def call
|
|
15
|
+
return if @loaded
|
|
16
|
+
|
|
17
|
+
require "test-unit"
|
|
18
|
+
self.class.stub_autorun!
|
|
19
|
+
@loaded = true
|
|
20
|
+
rescue LoadError => e
|
|
21
|
+
raise Evilution::Error, "test-unit is required but not available: #{e.message}"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# User code that `require "test-unit"` installs an at_exit hook that calls
|
|
25
|
+
# Test::Unit::AutoRunner.run when need_auto_run? is true. At evilution exit
|
|
26
|
+
# ARGV still holds evilution flags and the runner prints a misleading banner.
|
|
27
|
+
# Flipping need_auto_run = false prevents the handler from firing.
|
|
28
|
+
def self.stub_autorun!
|
|
29
|
+
return unless defined?(::Test::Unit::AutoRunner)
|
|
30
|
+
|
|
31
|
+
::Test::Unit::AutoRunner.need_auto_run = false
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Shapes the result Hash that flows back to Evilution::Result::MutationResult
|
|
6
|
+
# / classify_status. Three orthogonal flavours — pass/fail/crash, no tests
|
|
7
|
+
# executed, and unresolved spec — each have their own change axis (e.g. the
|
|
8
|
+
# no-tests-ran error string evolved separately as the test-unit framework
|
|
9
|
+
# diagnostic improved). Putting them behind a single object documents the
|
|
10
|
+
# contract and lets the integration class drop them.
|
|
11
|
+
module Evilution::Integration::TestUnit::ResultBuilder
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def call(passed:, command:, detector:)
|
|
15
|
+
if passed
|
|
16
|
+
{ passed: true, test_command: command }
|
|
17
|
+
elsif detector.only_crashes?
|
|
18
|
+
crash(command, detector)
|
|
19
|
+
else
|
|
20
|
+
{ passed: false, test_command: command }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def no_tests_ran(command)
|
|
25
|
+
{
|
|
26
|
+
passed: false,
|
|
27
|
+
error: "no Test::Unit tests executed (0 test methods ran) — the resolved " \
|
|
28
|
+
"spec registered no Test::Unit suite. Check --integration/--spec.",
|
|
29
|
+
error_class: "Evilution::Error",
|
|
30
|
+
test_command: command
|
|
31
|
+
}
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def unresolved(mutation_file_path)
|
|
35
|
+
{
|
|
36
|
+
passed: false,
|
|
37
|
+
unresolved: true,
|
|
38
|
+
error: "no matching test resolved for #{mutation_file_path}",
|
|
39
|
+
test_command: "ruby -Itest (skipped: no test resolved for #{mutation_file_path})"
|
|
40
|
+
}
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def crash(command, detector)
|
|
44
|
+
classes = detector.unique_crash_classes
|
|
45
|
+
{
|
|
46
|
+
passed: false,
|
|
47
|
+
test_crashed: true,
|
|
48
|
+
error: "test crashes: #{detector.crash_summary}",
|
|
49
|
+
error_class: (classes.first if classes.length == 1),
|
|
50
|
+
test_command: command
|
|
51
|
+
}
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Tracks Test::Unit::TestCase descendants in the host process. Test::Unit has
|
|
6
|
+
# no public registry-clear method analogous to Minitest::Runnable.runnables;
|
|
7
|
+
# the integration scopes each dispatch to classes that appeared during *this*
|
|
8
|
+
# round by diffing the descendant set before and after #load. Keeping that
|
|
9
|
+
# responsibility in its own object makes it cheap to stub in tests and lets
|
|
10
|
+
# the integration's main class read as orchestration.
|
|
11
|
+
module Evilution::Integration::TestUnit::SubjectClassRegistry
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def descendants
|
|
15
|
+
return [] unless defined?(::Test::Unit::TestCase)
|
|
16
|
+
|
|
17
|
+
ObjectSpace.each_object(Class).select { |c| c < ::Test::Unit::TestCase }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Yields, captures the descendant set before/after, and returns the diff.
|
|
21
|
+
def newly_loaded
|
|
22
|
+
before = descendants
|
|
23
|
+
yield
|
|
24
|
+
descendants - before
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Resolves the list of test files to load for a given mutation. Encapsulates
|
|
6
|
+
# the explicit-override path, spec-selector lookup, fallback glob, and the
|
|
7
|
+
# warn-once behaviour for unresolved sources. The integration class would
|
|
8
|
+
# otherwise carry both per-instance test resolution state (@test_files,
|
|
9
|
+
# @spec_selector, @warned_files, @fallback_to_full_suite) and dispatch
|
|
10
|
+
# orchestration in the same object; splitting them gives the resolver its
|
|
11
|
+
# own change axis (e.g. adding new resolution heuristics) independent of
|
|
12
|
+
# the runner.
|
|
13
|
+
class Evilution::Integration::TestUnit::TestFileResolver
|
|
14
|
+
def initialize(test_files:, spec_selector:, fallback_to_full_suite:)
|
|
15
|
+
@test_files = test_files
|
|
16
|
+
@spec_selector = spec_selector
|
|
17
|
+
@fallback_to_full_suite = fallback_to_full_suite
|
|
18
|
+
@warned_files = Set.new
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Returns the resolved file list, or nil if the source could not be
|
|
22
|
+
# resolved and fallback is disabled.
|
|
23
|
+
def call(mutation_file_path)
|
|
24
|
+
return @test_files if @test_files
|
|
25
|
+
|
|
26
|
+
resolved = Array(@spec_selector.call(mutation_file_path))
|
|
27
|
+
return resolved unless resolved.empty?
|
|
28
|
+
|
|
29
|
+
warn_unresolved(mutation_file_path)
|
|
30
|
+
@fallback_to_full_suite ? glob_test_files : nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def glob_test_files
|
|
36
|
+
files = Dir.glob("test/**/*_test.rb")
|
|
37
|
+
files.empty? ? ["test"] : files
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def warn_unresolved(file_path)
|
|
41
|
+
return if @warned_files.include?(file_path)
|
|
42
|
+
|
|
43
|
+
@warned_files << file_path
|
|
44
|
+
action = @fallback_to_full_suite ? "running full suite" : "marking mutation unresolved"
|
|
45
|
+
warn "[evilution] No matching test found for #{file_path}, #{action}. " \
|
|
46
|
+
"Use --spec to specify the test file."
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
require_relative "test_unit_crash_detector"
|
|
5
|
+
require_relative "../spec_resolver"
|
|
6
|
+
require_relative "../spec_selector"
|
|
7
|
+
|
|
8
|
+
require_relative "../integration"
|
|
9
|
+
|
|
10
|
+
# Test::Unit integration. Decomposed under lib/evilution/integration/test_unit/
|
|
11
|
+
# mirroring the RSpec integration's layout. This class is the orchestrator:
|
|
12
|
+
# it wires the framework loader, dispatcher, subject-class registry,
|
|
13
|
+
# test-file resolver, and result builder. The class is registered under
|
|
14
|
+
# Evilution::Runner::INTEGRATIONS[:test_unit] and reachable via the
|
|
15
|
+
# `--integration test-unit` CLI flag.
|
|
16
|
+
class Evilution::Integration::TestUnit < Evilution::Integration::Base
|
|
17
|
+
def self.baseline_runner
|
|
18
|
+
->(test_file) { run_baseline_test_file(test_file) }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# SpecResolver tuned for the dominant Test::Unit layout: tests live under
|
|
22
|
+
# test/, named with the _test.rb suffix (the same convention Minitest uses).
|
|
23
|
+
# Rails plugins on the test-unit gem (e.g. kaminari-core) follow this layout.
|
|
24
|
+
# The test/test_<name>.rb prefix-style convention is rare enough in practice
|
|
25
|
+
# that we defer support to a follow-up if a project surfaces needing it.
|
|
26
|
+
def self.spec_resolver
|
|
27
|
+
Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.baseline_options
|
|
31
|
+
{
|
|
32
|
+
runner: baseline_runner,
|
|
33
|
+
spec_resolver: spec_resolver,
|
|
34
|
+
fallback_dir: "test"
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def self.run_baseline_test_file(test_file)
|
|
39
|
+
require_relative "test_unit/framework_loader"
|
|
40
|
+
require_relative "test_unit/subject_class_registry"
|
|
41
|
+
require_relative "test_unit/dispatcher"
|
|
42
|
+
FrameworkLoader.new.call
|
|
43
|
+
new_classes = SubjectClassRegistry.newly_loaded do
|
|
44
|
+
baseline_test_files(test_file).each { |f| load(File.expand_path(f)) }
|
|
45
|
+
end
|
|
46
|
+
Dispatcher.call(new_classes, name: "evilution baseline").passed?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def self.baseline_test_files(test_file)
|
|
50
|
+
File.directory?(test_file) ? Dir.glob(File.join(test_file, "**/*_test.rb")) : [test_file]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil)
|
|
54
|
+
require_relative "test_unit/framework_loader"
|
|
55
|
+
require_relative "test_unit/subject_class_registry"
|
|
56
|
+
require_relative "test_unit/dispatcher"
|
|
57
|
+
require_relative "test_unit/test_file_resolver"
|
|
58
|
+
require_relative "test_unit/result_builder"
|
|
59
|
+
@framework_loader = FrameworkLoader.new
|
|
60
|
+
@file_resolver = TestFileResolver.new(
|
|
61
|
+
test_files: test_files,
|
|
62
|
+
spec_selector: spec_selector || Evilution::SpecSelector.new(spec_resolver: self.class.spec_resolver),
|
|
63
|
+
fallback_to_full_suite: fallback_to_full_suite
|
|
64
|
+
)
|
|
65
|
+
@crash_detector = nil
|
|
66
|
+
super(hooks: hooks)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def ensure_framework_loaded
|
|
72
|
+
return if @framework_loader.loaded?
|
|
73
|
+
|
|
74
|
+
fire_hook(:setup_integration_pre, integration: :test_unit)
|
|
75
|
+
@framework_loader.call
|
|
76
|
+
fire_hook(:setup_integration_post, integration: :test_unit)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def run_tests(mutation)
|
|
80
|
+
ensure_framework_loaded
|
|
81
|
+
reset_state
|
|
82
|
+
files = @file_resolver.call(mutation.file_path)
|
|
83
|
+
return ResultBuilder.unresolved(mutation.file_path) if files.nil?
|
|
84
|
+
|
|
85
|
+
command = "ruby -Itest #{files.join(" ")}"
|
|
86
|
+
execute_test_unit(files, command)
|
|
87
|
+
rescue StandardError => e
|
|
88
|
+
{ passed: false, error: e.message, test_command: command }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def execute_test_unit(files, command)
|
|
92
|
+
new_classes = SubjectClassRegistry.newly_loaded do
|
|
93
|
+
files.each { |f| load(File.expand_path(f, Evilution.project_base_dir)) }
|
|
94
|
+
end
|
|
95
|
+
return ResultBuilder.no_tests_ran(command) if Dispatcher.test_method_count(new_classes).zero?
|
|
96
|
+
|
|
97
|
+
detector = reset_crash_detector
|
|
98
|
+
result = Dispatcher.call(new_classes, name: "evilution-mutation")
|
|
99
|
+
result.faults.each { |fault| detector.record(fault) }
|
|
100
|
+
ResultBuilder.call(passed: result.passed?, command: command, detector: detector)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Test::Unit has no public registry-clear analogous to
|
|
104
|
+
# Minitest::Runnable.runnables.clear. SubjectClassRegistry's newly_loaded
|
|
105
|
+
# block scopes each dispatch to classes loaded in *this* round, so stale
|
|
106
|
+
# classes from prior mutations sit dormant on ObjectSpace without polluting
|
|
107
|
+
# the run. #reset_state stays as a contract no-op for parity with Minitest.
|
|
108
|
+
def reset_state
|
|
109
|
+
# no-op — see comment above
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def build_args(_mutation)
|
|
113
|
+
[]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def reset_crash_detector
|
|
117
|
+
if @crash_detector
|
|
118
|
+
@crash_detector.reset
|
|
119
|
+
else
|
|
120
|
+
@crash_detector = Evilution::Integration::TestUnitCrashDetector.new
|
|
121
|
+
end
|
|
122
|
+
@crash_detector
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../integration"
|
|
4
|
+
|
|
5
|
+
# Test::Unit analog of Evilution::Integration::MinitestCrashDetector. Tracks
|
|
6
|
+
# whether a Test::Unit test run produced only crashes (exceptions captured as
|
|
7
|
+
# Test::Unit::Error) vs assertion failures (Test::Unit::Failure). When only
|
|
8
|
+
# crashes occur, the mutation result can carry crash details while remaining
|
|
9
|
+
# classified as :killed — see classify_status / Result::MutationResult.
|
|
10
|
+
#
|
|
11
|
+
# Hook the detector into a Test::Unit::TestResult via .attach(result), or
|
|
12
|
+
# call #record(fault) directly when iterating a finished result's #faults.
|
|
13
|
+
class Evilution::Integration::TestUnitCrashDetector
|
|
14
|
+
def initialize
|
|
15
|
+
reset
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def reset
|
|
19
|
+
@assertion_failures = 0
|
|
20
|
+
@crashes = []
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def attach(test_result)
|
|
24
|
+
require "test/unit/testresult"
|
|
25
|
+
test_result.add_listener(Test::Unit::TestResult::FAULT) { |fault| record(fault) }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def record(fault)
|
|
29
|
+
if fault.is_a?(Test::Unit::Error)
|
|
30
|
+
@crashes << fault.exception
|
|
31
|
+
elsif fault.is_a?(Test::Unit::Failure)
|
|
32
|
+
@assertion_failures += 1
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def passed?
|
|
37
|
+
@assertion_failures.zero? && @crashes.empty?
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def assertion_failure?
|
|
41
|
+
@assertion_failures.positive?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def crashed?
|
|
45
|
+
@crashes.any?
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def only_crashes?
|
|
49
|
+
@crashes.any? && @assertion_failures.zero?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def unique_crash_classes
|
|
53
|
+
@crashes.map { |e| e.class.name }.uniq
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def crash_summary
|
|
57
|
+
return nil if @crashes.empty?
|
|
58
|
+
|
|
59
|
+
"#{unique_crash_classes.join(", ")} (#{@crashes.length} crash#{"es" unless @crashes.length == 1})"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -11,6 +11,7 @@ require_relative "../isolation"
|
|
|
11
11
|
|
|
12
12
|
class Evilution::Isolation::Fork
|
|
13
13
|
GRACE_PERIOD = 2
|
|
14
|
+
REAP_DEADLINE = 1.0
|
|
14
15
|
|
|
15
16
|
def initialize(hooks: nil)
|
|
16
17
|
@hooks = hooks
|
|
@@ -48,6 +49,15 @@ class Evilution::Isolation::Fork
|
|
|
48
49
|
def fork_child(read_io, write_io, sandbox_dir, mutation, test_command)
|
|
49
50
|
::Process.fork do
|
|
50
51
|
ENV["TMPDIR"] = sandbox_dir
|
|
52
|
+
# Path-relativizing mutations (e.g. File.join(dir, name) -> name) would
|
|
53
|
+
# otherwise write into the parent's CWD (typically the repo root) and
|
|
54
|
+
# leak past the run. chdir here keeps such writes inside sandbox_dir,
|
|
55
|
+
# which the ensure block of #call removes. The in_isolated_worker! flag
|
|
56
|
+
# signals the rest of evilution (SpecResolver/SpecSelector/SpecAstCache/
|
|
57
|
+
# MutationApplier/SourceEvaluator/Integration) to anchor project-relative
|
|
58
|
+
# paths to Evilution::PROJECT_ROOT instead of the sandbox CWD.
|
|
59
|
+
Dir.chdir(sandbox_dir)
|
|
60
|
+
Evilution.in_isolated_worker!
|
|
51
61
|
read_io.close
|
|
52
62
|
suppress_child_output
|
|
53
63
|
@hooks.fire(:worker_process_start, mutation:) if @hooks
|
|
@@ -121,8 +131,23 @@ class Evilution::Isolation::Fork
|
|
|
121
131
|
end
|
|
122
132
|
end
|
|
123
133
|
|
|
134
|
+
# Process.wait without a deadline can hang the parent indefinitely. The
|
|
135
|
+
# per-mutation child may already have written a payload (or a grandchild may
|
|
136
|
+
# have written garbage that looks like one) while the child itself is stuck
|
|
137
|
+
# in execute_in_child waiting on a subject grandchild the mutation broke.
|
|
138
|
+
# wait_for_result has already returned by this point, so the per-mutation
|
|
139
|
+
# timeout cannot fire. Bound the wait and fall back to the TERM/KILL ladder.
|
|
124
140
|
def reap_and_decode(pid, payload)
|
|
125
|
-
|
|
141
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + REAP_DEADLINE
|
|
142
|
+
loop do
|
|
143
|
+
break if ::Process.waitpid(pid, ::Process::WNOHANG)
|
|
144
|
+
|
|
145
|
+
if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
146
|
+
terminate_child(pid)
|
|
147
|
+
break
|
|
148
|
+
end
|
|
149
|
+
sleep 0.05
|
|
150
|
+
end
|
|
126
151
|
decode_payload(payload)
|
|
127
152
|
end
|
|
128
153
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "fileutils"
|
|
3
4
|
require "timeout"
|
|
5
|
+
require "tmpdir"
|
|
4
6
|
require_relative "../memory"
|
|
5
7
|
require_relative "../result/mutation_result"
|
|
6
8
|
|
|
@@ -17,19 +19,34 @@ class Evilution::Isolation::InProcess
|
|
|
17
19
|
def call(mutation:, test_command:, timeout:)
|
|
18
20
|
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
19
21
|
rss_before = Evilution::Memory.rss_kb
|
|
20
|
-
|
|
22
|
+
sandbox_dir = Dir.mktmpdir("evilution-run")
|
|
23
|
+
result = execute_with_timeout(mutation, test_command, timeout, sandbox_dir)
|
|
21
24
|
rss_after = Evilution::Memory.rss_kb
|
|
22
25
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
|
|
23
26
|
delta = compute_memory_delta(rss_before, rss_after, result)
|
|
24
27
|
|
|
25
28
|
build_mutation_result(mutation, result, duration, rss_before, rss_after, delta)
|
|
29
|
+
ensure
|
|
30
|
+
FileUtils.rm_rf(sandbox_dir) if sandbox_dir
|
|
26
31
|
end
|
|
27
32
|
|
|
28
33
|
private
|
|
29
34
|
|
|
30
|
-
|
|
35
|
+
# The Dir.chdir block is inside the Timeout.timeout block so that a
|
|
36
|
+
# Timeout::Error raised mid-call still unwinds through Dir.chdir's ensure
|
|
37
|
+
# and restores the parent CWD before the rescue clause runs. The sandbox
|
|
38
|
+
# contains any relative-path writes from path-relativizing mutations
|
|
39
|
+
# (EV-wqxu / GH #1278). Evilution.with_isolated_worker signals the rest of
|
|
40
|
+
# evilution (SpecResolver/SpecSelector/SpecAstCache/MutationApplier/
|
|
41
|
+
# SourceEvaluator/Integration) to anchor project-relative paths to
|
|
42
|
+
# PROJECT_ROOT for the duration of the call.
|
|
43
|
+
def execute_with_timeout(mutation, test_command, timeout, sandbox_dir)
|
|
31
44
|
result = Timeout.timeout(timeout) do
|
|
32
|
-
|
|
45
|
+
Evilution.with_isolated_worker do
|
|
46
|
+
Dir.chdir(sandbox_dir) do
|
|
47
|
+
suppress_output { test_command.call(mutation) }
|
|
48
|
+
end
|
|
49
|
+
end
|
|
33
50
|
end
|
|
34
51
|
{ timeout: false }.merge(result)
|
|
35
52
|
rescue Timeout::Error
|
|
@@ -58,8 +58,8 @@ class Evilution::MCP::InfoTool < MCP::Tool
|
|
|
58
58
|
},
|
|
59
59
|
integration: {
|
|
60
60
|
type: "string",
|
|
61
|
-
description: "[subjects, tests] Test integration (rspec, minitest) — 'tests' selects " \
|
|
62
|
-
"the matching spec resolver (spec/*_spec.rb for rspec, test/*_test.rb for minitest)"
|
|
61
|
+
description: "[subjects, tests] Test integration (rspec, minitest, test-unit) — 'tests' selects " \
|
|
62
|
+
"the matching spec resolver (spec/*_spec.rb for rspec, test/*_test.rb for minitest/test-unit)"
|
|
63
63
|
},
|
|
64
64
|
skip_config: {
|
|
65
65
|
type: "boolean",
|
|
@@ -72,8 +72,9 @@ class Evilution::MCP::MutateTool < MCP::Tool
|
|
|
72
72
|
},
|
|
73
73
|
integration: {
|
|
74
74
|
type: "string",
|
|
75
|
-
enum: %w[rspec minitest],
|
|
76
|
-
description: "Test integration to use (default: rspec)"
|
|
75
|
+
enum: %w[rspec minitest test-unit],
|
|
76
|
+
description: "Test integration to use (default: rspec). " \
|
|
77
|
+
"Use test-unit for projects whose suites run on the test-unit gem."
|
|
77
78
|
},
|
|
78
79
|
isolation: {
|
|
79
80
|
type: "string",
|
|
@@ -5,6 +5,7 @@ require_relative "../baseline"
|
|
|
5
5
|
require_relative "../spec_resolver"
|
|
6
6
|
require_relative "../integration/rspec"
|
|
7
7
|
require_relative "../integration/minitest"
|
|
8
|
+
require_relative "../integration/test_unit"
|
|
8
9
|
require_relative "../example_filter"
|
|
9
10
|
require_relative "../spec_ast_cache"
|
|
10
11
|
require_relative "../source_ast_cache"
|
|
@@ -12,7 +13,8 @@ require_relative "../source_ast_cache"
|
|
|
12
13
|
unless defined?(Evilution::Runner::INTEGRATIONS)
|
|
13
14
|
Evilution::Runner::INTEGRATIONS = {
|
|
14
15
|
rspec: Evilution::Integration::RSpec,
|
|
15
|
-
minitest: Evilution::Integration::Minitest
|
|
16
|
+
minitest: Evilution::Integration::Minitest,
|
|
17
|
+
test_unit: Evilution::Integration::TestUnit
|
|
16
18
|
}.freeze
|
|
17
19
|
end
|
|
18
20
|
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
require_relative "../runner"
|
|
7
|
+
require_relative "../mutation"
|
|
8
|
+
require_relative "../subject"
|
|
9
|
+
|
|
10
|
+
# Runs one guaranteed-unobservable synthetic mutation through the configured
|
|
11
|
+
# integration + isolation at session start. The synthetic spec never references
|
|
12
|
+
# the synthetic class, so mutating the class cannot change any test outcome — a
|
|
13
|
+
# healthy pipeline must score the mutation :survived. Any other status means the
|
|
14
|
+
# mutation infrastructure is misreporting, so the run aborts before producing
|
|
15
|
+
# numbers that would all be unreliable. Mirrors the configured --isolation so
|
|
16
|
+
# isolation-specific defects are caught too.
|
|
17
|
+
class Evilution::Runner::Canary
|
|
18
|
+
class Failed < Evilution::Error; end
|
|
19
|
+
|
|
20
|
+
def initialize(config:, isolator:, integration_class:, hooks: nil)
|
|
21
|
+
@config = config
|
|
22
|
+
@isolator = isolator
|
|
23
|
+
@integration_class = integration_class
|
|
24
|
+
@hooks = hooks
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def call
|
|
28
|
+
dir = Dir.mktmpdir("evilution-canary")
|
|
29
|
+
class_path = write_target_class(dir)
|
|
30
|
+
spec_path = write_spec(dir)
|
|
31
|
+
|
|
32
|
+
result = @isolator.call(
|
|
33
|
+
mutation: build_mutation(class_path),
|
|
34
|
+
test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
|
|
35
|
+
timeout: @config.timeout
|
|
36
|
+
)
|
|
37
|
+
raise Failed, failure_message(result.status) unless result.status == :survived
|
|
38
|
+
|
|
39
|
+
nil
|
|
40
|
+
ensure
|
|
41
|
+
FileUtils.remove_entry(dir) if dir
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
# pid + random hex keeps the synthetic class/spec names unique across
|
|
47
|
+
# concurrent sessions and across repeated re-eval into the same VM.
|
|
48
|
+
def suffix
|
|
49
|
+
@suffix ||= "#{Process.pid}_#{SecureRandom.hex(4)}"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def class_name
|
|
53
|
+
@class_name ||= "EvilutionCanary_#{suffix}"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def original_source
|
|
57
|
+
<<~RUBY
|
|
58
|
+
class #{class_name}
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def __evilution_canary_probe
|
|
62
|
+
:original
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
RUBY
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def mutated_source
|
|
69
|
+
original_source.sub(":original", "nil")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def write_target_class(dir)
|
|
73
|
+
path = File.join(dir, "#{class_name.downcase}.rb")
|
|
74
|
+
File.write(path, original_source)
|
|
75
|
+
path
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def write_spec(dir)
|
|
79
|
+
minitest = @config.integration == :minitest
|
|
80
|
+
name = minitest ? "canary_#{suffix}_test.rb" : "canary_#{suffix}_spec.rb"
|
|
81
|
+
path = File.join(dir, name)
|
|
82
|
+
File.write(path, minitest ? minitest_spec_source : rspec_spec_source)
|
|
83
|
+
path
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def rspec_spec_source
|
|
87
|
+
<<~RUBY
|
|
88
|
+
RSpec.describe("evilution proof-of-life canary") do
|
|
89
|
+
it "pipeline is alive" do
|
|
90
|
+
expect(true).to be(true)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
RUBY
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def minitest_spec_source
|
|
97
|
+
<<~RUBY
|
|
98
|
+
class EvilutionCanaryTest_#{suffix} < Minitest::Test
|
|
99
|
+
def test_pipeline_is_alive
|
|
100
|
+
assert true
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
RUBY
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def build_mutation(class_path)
|
|
107
|
+
Evilution::Mutation.new(
|
|
108
|
+
subject: Evilution::Subject.new(
|
|
109
|
+
name: "#{class_name}#__evilution_canary_probe",
|
|
110
|
+
file_path: class_path, line_number: 1, source: original_source, node: nil
|
|
111
|
+
),
|
|
112
|
+
operator_name: :canary_probe,
|
|
113
|
+
sources: Evilution::Mutation::Sources.new(original: original_source, mutated: mutated_source),
|
|
114
|
+
location: Evilution::Mutation::Location.new(file_path: class_path, line: 5, column: 4)
|
|
115
|
+
)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def build_integration(spec_path)
|
|
119
|
+
@integration_class.new(test_files: [spec_path], hooks: @hooks)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def failure_message(status)
|
|
123
|
+
"evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
|
|
124
|
+
"mutation was scored #{status.inspect} instead of :survived. The mutation " \
|
|
125
|
+
"pipeline is misreporting — every score this run would produce is unreliable. " \
|
|
126
|
+
"Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
|
|
127
|
+
"RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
|
|
128
|
+
"an isolation-mode defect. Re-run with --no-canary to bypass this check."
|
|
129
|
+
end
|
|
130
|
+
end
|
data/lib/evilution/runner.rb
CHANGED
|
@@ -4,6 +4,11 @@ require "fileutils"
|
|
|
4
4
|
require_relative "../evilution"
|
|
5
5
|
|
|
6
6
|
class Evilution::Runner
|
|
7
|
+
# Autoloaded: canary.rb subclasses Evilution::Error at class-body eval
|
|
8
|
+
# time, which is not yet defined while evilution.rb is mid-load. Deferring
|
|
9
|
+
# the load until first reference lets evilution.rb finish defining Error.
|
|
10
|
+
autoload :Canary, File.expand_path("runner/canary", __dir__)
|
|
11
|
+
|
|
7
12
|
attr_reader :config
|
|
8
13
|
|
|
9
14
|
def initialize(config: Evilution::Config.new, on_result: nil, hooks: nil)
|
|
@@ -27,6 +32,8 @@ class Evilution::Runner
|
|
|
27
32
|
perform_preload
|
|
28
33
|
log_memory("after preload") if rails_root_detected?
|
|
29
34
|
|
|
35
|
+
run_canary
|
|
36
|
+
|
|
30
37
|
baseline_result = run_baseline(subjects)
|
|
31
38
|
|
|
32
39
|
plan = mutation_planner.call(subjects)
|
|
@@ -122,6 +129,17 @@ class Evilution::Runner
|
|
|
122
129
|
baseline_runner.call(subjects)
|
|
123
130
|
end
|
|
124
131
|
|
|
132
|
+
def run_canary
|
|
133
|
+
return unless config.canary?
|
|
134
|
+
|
|
135
|
+
Evilution::Runner::Canary.new(
|
|
136
|
+
config: config,
|
|
137
|
+
isolator: isolator,
|
|
138
|
+
integration_class: baseline_runner.integration_class,
|
|
139
|
+
hooks: @hooks
|
|
140
|
+
).call
|
|
141
|
+
end
|
|
142
|
+
|
|
125
143
|
def run_mutations(mutations, baseline_result = nil)
|
|
126
144
|
mutation_executor.call(mutations, baseline_result)
|
|
127
145
|
end
|
|
@@ -136,7 +154,12 @@ class Evilution::Runner
|
|
|
136
154
|
return
|
|
137
155
|
end
|
|
138
156
|
|
|
139
|
-
|
|
157
|
+
# Resolve to absolute now — fork children chdir into a per-mutation
|
|
158
|
+
# sandbox (EV-wqxu / GH #1278) before suppress_child_output runs, so a
|
|
159
|
+
# relative log_dir would reopen $stdout/$stderr at <sandbox>/<log_dir>
|
|
160
|
+
# which does not exist and the child dies with Errno::ENOENT before
|
|
161
|
+
# marshaling any result back.
|
|
162
|
+
dir = File.expand_path(config.quiet_children_dir)
|
|
140
163
|
begin
|
|
141
164
|
FileUtils.rm_rf(dir)
|
|
142
165
|
FileUtils.mkdir_p(dir)
|