evilution 0.30.4 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +22 -0
  3. data/.rubocop_todo.yml +6 -0
  4. data/CHANGELOG.md +22 -0
  5. data/README.md +9 -7
  6. data/docs/integrations.md +126 -0
  7. data/docs/isolation.md +28 -0
  8. data/lib/evilution/cli/parser/options_builder.rb +6 -1
  9. data/lib/evilution/config/validators/integration.rb +5 -1
  10. data/lib/evilution/config.rb +14 -4
  11. data/lib/evilution/integration/loading/mutation_applier.rb +16 -8
  12. data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
  13. data/lib/evilution/integration/minitest.rb +1 -1
  14. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -1
  15. data/lib/evilution/integration/rspec/framework_loader.rb +5 -1
  16. data/lib/evilution/integration/rspec.rb +38 -1
  17. data/lib/evilution/integration/test_unit/dispatcher.rb +26 -0
  18. data/lib/evilution/integration/test_unit/framework_loader.rb +33 -0
  19. data/lib/evilution/integration/test_unit/result_builder.rb +53 -0
  20. data/lib/evilution/integration/test_unit/subject_class_registry.rb +26 -0
  21. data/lib/evilution/integration/test_unit/test_file_resolver.rb +48 -0
  22. data/lib/evilution/integration/test_unit.rb +124 -0
  23. data/lib/evilution/integration/test_unit_crash_detector.rb +61 -0
  24. data/lib/evilution/isolation/fork.rb +26 -1
  25. data/lib/evilution/isolation/in_process.rb +20 -3
  26. data/lib/evilution/mcp/info_tool.rb +2 -2
  27. data/lib/evilution/mcp/mutate_tool.rb +3 -2
  28. data/lib/evilution/runner/baseline_runner.rb +3 -1
  29. data/lib/evilution/runner/canary.rb +130 -0
  30. data/lib/evilution/runner.rb +24 -1
  31. data/lib/evilution/spec_ast_cache.rb +20 -3
  32. data/lib/evilution/spec_resolver.rb +16 -2
  33. data/lib/evilution/spec_selector.rb +14 -2
  34. data/lib/evilution/version.rb +1 -1
  35. data/lib/evilution.rb +39 -0
  36. data/script/run_self_baseline +2 -2
  37. metadata +11 -2
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../test_unit"
4
+
5
+ # Loads the test-unit gem and disables its at_exit auto-run handler.
6
+ # Mirrors Evilution::Integration::RSpec::FrameworkLoader's role: framework
7
+ # setup is one responsibility separated from dispatch + result building so
8
+ # integrations can compose it independently in tests.
9
+ class Evilution::Integration::TestUnit::FrameworkLoader
10
+ def loaded?
11
+ @loaded == true
12
+ end
13
+
14
+ def call
15
+ return if @loaded
16
+
17
+ require "test-unit"
18
+ self.class.stub_autorun!
19
+ @loaded = true
20
+ rescue LoadError => e
21
+ raise Evilution::Error, "test-unit is required but not available: #{e.message}"
22
+ end
23
+
24
+ # User code that `require "test-unit"` installs an at_exit hook that calls
25
+ # Test::Unit::AutoRunner.run when need_auto_run? is true. At evilution exit
26
+ # ARGV still holds evilution flags and the runner prints a misleading banner.
27
+ # Flipping need_auto_run = false prevents the handler from firing.
28
+ def self.stub_autorun!
29
+ return unless defined?(::Test::Unit::AutoRunner)
30
+
31
+ ::Test::Unit::AutoRunner.need_auto_run = false
32
+ end
33
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../test_unit"
4
+
5
+ # Shapes the result Hash that flows back to Evilution::Result::MutationResult
6
+ # / classify_status. Three orthogonal flavours — pass/fail/crash, no tests
7
+ # executed, and unresolved spec — each have their own change axis (e.g. the
8
+ # no-tests-ran error string evolved separately as the test-unit framework
9
+ # diagnostic improved). Putting them behind a single object documents the
10
+ # contract and lets the integration class drop them.
11
+ module Evilution::Integration::TestUnit::ResultBuilder
12
+ module_function
13
+
14
+ def call(passed:, command:, detector:)
15
+ if passed
16
+ { passed: true, test_command: command }
17
+ elsif detector.only_crashes?
18
+ crash(command, detector)
19
+ else
20
+ { passed: false, test_command: command }
21
+ end
22
+ end
23
+
24
+ def no_tests_ran(command)
25
+ {
26
+ passed: false,
27
+ error: "no Test::Unit tests executed (0 test methods ran) — the resolved " \
28
+ "spec registered no Test::Unit suite. Check --integration/--spec.",
29
+ error_class: "Evilution::Error",
30
+ test_command: command
31
+ }
32
+ end
33
+
34
+ def unresolved(mutation_file_path)
35
+ {
36
+ passed: false,
37
+ unresolved: true,
38
+ error: "no matching test resolved for #{mutation_file_path}",
39
+ test_command: "ruby -Itest (skipped: no test resolved for #{mutation_file_path})"
40
+ }
41
+ end
42
+
43
+ def crash(command, detector)
44
+ classes = detector.unique_crash_classes
45
+ {
46
+ passed: false,
47
+ test_crashed: true,
48
+ error: "test crashes: #{detector.crash_summary}",
49
+ error_class: (classes.first if classes.length == 1),
50
+ test_command: command
51
+ }
52
+ end
53
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../test_unit"
4
+
5
+ # Tracks Test::Unit::TestCase descendants in the host process. Test::Unit has
6
+ # no public registry-clear method analogous to Minitest::Runnable.runnables;
7
+ # the integration scopes each dispatch to classes that appeared during *this*
8
+ # round by diffing the descendant set before and after #load. Keeping that
9
+ # responsibility in its own object makes it cheap to stub in tests and lets
10
+ # the integration's main class read as orchestration.
11
+ module Evilution::Integration::TestUnit::SubjectClassRegistry
12
+ module_function
13
+
14
+ def descendants
15
+ return [] unless defined?(::Test::Unit::TestCase)
16
+
17
+ ObjectSpace.each_object(Class).select { |c| c < ::Test::Unit::TestCase }
18
+ end
19
+
20
+ # Yields, captures the descendant set before/after, and returns the diff.
21
+ def newly_loaded
22
+ before = descendants
23
+ yield
24
+ descendants - before
25
+ end
26
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../test_unit"
4
+
5
+ # Resolves the list of test files to load for a given mutation. Encapsulates
6
+ # the explicit-override path, spec-selector lookup, fallback glob, and the
7
+ # warn-once behaviour for unresolved sources. The integration class would
8
+ # otherwise carry both per-instance test resolution state (@test_files,
9
+ # @spec_selector, @warned_files, @fallback_to_full_suite) and dispatch
10
+ # orchestration in the same object; splitting them gives the resolver its
11
+ # own change axis (e.g. adding new resolution heuristics) independent of
12
+ # the runner.
13
+ class Evilution::Integration::TestUnit::TestFileResolver
14
+ def initialize(test_files:, spec_selector:, fallback_to_full_suite:)
15
+ @test_files = test_files
16
+ @spec_selector = spec_selector
17
+ @fallback_to_full_suite = fallback_to_full_suite
18
+ @warned_files = Set.new
19
+ end
20
+
21
+ # Returns the resolved file list, or nil if the source could not be
22
+ # resolved and fallback is disabled.
23
+ def call(mutation_file_path)
24
+ return @test_files if @test_files
25
+
26
+ resolved = Array(@spec_selector.call(mutation_file_path))
27
+ return resolved unless resolved.empty?
28
+
29
+ warn_unresolved(mutation_file_path)
30
+ @fallback_to_full_suite ? glob_test_files : nil
31
+ end
32
+
33
+ private
34
+
35
+ def glob_test_files
36
+ files = Dir.glob("test/**/*_test.rb")
37
+ files.empty? ? ["test"] : files
38
+ end
39
+
40
+ def warn_unresolved(file_path)
41
+ return if @warned_files.include?(file_path)
42
+
43
+ @warned_files << file_path
44
+ action = @fallback_to_full_suite ? "running full suite" : "marking mutation unresolved"
45
+ warn "[evilution] No matching test found for #{file_path}, #{action}. " \
46
+ "Use --spec to specify the test file."
47
+ end
48
+ end
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+ require_relative "test_unit_crash_detector"
5
+ require_relative "../spec_resolver"
6
+ require_relative "../spec_selector"
7
+
8
+ require_relative "../integration"
9
+
10
+ # Test::Unit integration. Decomposed under lib/evilution/integration/test_unit/
11
+ # mirroring the RSpec integration's layout. This class is the orchestrator:
12
+ # it wires the framework loader, dispatcher, subject-class registry,
13
+ # test-file resolver, and result builder. The class is registered under
14
+ # Evilution::Runner::INTEGRATIONS[:test_unit] and reachable via the
15
+ # `--integration test-unit` CLI flag.
16
+ class Evilution::Integration::TestUnit < Evilution::Integration::Base
17
+ def self.baseline_runner
18
+ ->(test_file) { run_baseline_test_file(test_file) }
19
+ end
20
+
21
+ # SpecResolver tuned for the dominant Test::Unit layout: tests live under
22
+ # test/, named with the _test.rb suffix (the same convention Minitest uses).
23
+ # Rails plugins on the test-unit gem (e.g. kaminari-core) follow this layout.
24
+ # The test/test_<name>.rb prefix-style convention is rare enough in practice
25
+ # that we defer support to a follow-up if a project surfaces needing it.
26
+ def self.spec_resolver
27
+ Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration")
28
+ end
29
+
30
+ def self.baseline_options
31
+ {
32
+ runner: baseline_runner,
33
+ spec_resolver: spec_resolver,
34
+ fallback_dir: "test"
35
+ }
36
+ end
37
+
38
+ def self.run_baseline_test_file(test_file)
39
+ require_relative "test_unit/framework_loader"
40
+ require_relative "test_unit/subject_class_registry"
41
+ require_relative "test_unit/dispatcher"
42
+ FrameworkLoader.new.call
43
+ new_classes = SubjectClassRegistry.newly_loaded do
44
+ baseline_test_files(test_file).each { |f| load(File.expand_path(f)) }
45
+ end
46
+ Dispatcher.call(new_classes, name: "evilution baseline").passed?
47
+ end
48
+
49
+ def self.baseline_test_files(test_file)
50
+ File.directory?(test_file) ? Dir.glob(File.join(test_file, "**/*_test.rb")) : [test_file]
51
+ end
52
+
53
+ def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil)
54
+ require_relative "test_unit/framework_loader"
55
+ require_relative "test_unit/subject_class_registry"
56
+ require_relative "test_unit/dispatcher"
57
+ require_relative "test_unit/test_file_resolver"
58
+ require_relative "test_unit/result_builder"
59
+ @framework_loader = FrameworkLoader.new
60
+ @file_resolver = TestFileResolver.new(
61
+ test_files: test_files,
62
+ spec_selector: spec_selector || Evilution::SpecSelector.new(spec_resolver: self.class.spec_resolver),
63
+ fallback_to_full_suite: fallback_to_full_suite
64
+ )
65
+ @crash_detector = nil
66
+ super(hooks: hooks)
67
+ end
68
+
69
+ private
70
+
71
+ def ensure_framework_loaded
72
+ return if @framework_loader.loaded?
73
+
74
+ fire_hook(:setup_integration_pre, integration: :test_unit)
75
+ @framework_loader.call
76
+ fire_hook(:setup_integration_post, integration: :test_unit)
77
+ end
78
+
79
+ def run_tests(mutation)
80
+ ensure_framework_loaded
81
+ reset_state
82
+ files = @file_resolver.call(mutation.file_path)
83
+ return ResultBuilder.unresolved(mutation.file_path) if files.nil?
84
+
85
+ command = "ruby -Itest #{files.join(" ")}"
86
+ execute_test_unit(files, command)
87
+ rescue StandardError => e
88
+ { passed: false, error: e.message, test_command: command }
89
+ end
90
+
91
+ def execute_test_unit(files, command)
92
+ new_classes = SubjectClassRegistry.newly_loaded do
93
+ files.each { |f| load(File.expand_path(f, Evilution.project_base_dir)) }
94
+ end
95
+ return ResultBuilder.no_tests_ran(command) if Dispatcher.test_method_count(new_classes).zero?
96
+
97
+ detector = reset_crash_detector
98
+ result = Dispatcher.call(new_classes, name: "evilution-mutation")
99
+ result.faults.each { |fault| detector.record(fault) }
100
+ ResultBuilder.call(passed: result.passed?, command: command, detector: detector)
101
+ end
102
+
103
+ # Test::Unit has no public registry-clear analogous to
104
+ # Minitest::Runnable.runnables.clear. SubjectClassRegistry's newly_loaded
105
+ # block scopes each dispatch to classes loaded in *this* round, so stale
106
+ # classes from prior mutations sit dormant on ObjectSpace without polluting
107
+ # the run. #reset_state stays as a contract no-op for parity with Minitest.
108
+ def reset_state
109
+ # no-op — see comment above
110
+ end
111
+
112
+ def build_args(_mutation)
113
+ []
114
+ end
115
+
116
+ def reset_crash_detector
117
+ if @crash_detector
118
+ @crash_detector.reset
119
+ else
120
+ @crash_detector = Evilution::Integration::TestUnitCrashDetector.new
121
+ end
122
+ @crash_detector
123
+ end
124
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../integration"
4
+
5
+ # Test::Unit analog of Evilution::Integration::MinitestCrashDetector. Tracks
6
+ # whether a Test::Unit test run produced only crashes (exceptions captured as
7
+ # Test::Unit::Error) vs assertion failures (Test::Unit::Failure). When only
8
+ # crashes occur, the mutation result can carry crash details while remaining
9
+ # classified as :killed — see classify_status / Result::MutationResult.
10
+ #
11
+ # Hook the detector into a Test::Unit::TestResult via .attach(result), or
12
+ # call #record(fault) directly when iterating a finished result's #faults.
13
+ class Evilution::Integration::TestUnitCrashDetector
14
+ def initialize
15
+ reset
16
+ end
17
+
18
+ def reset
19
+ @assertion_failures = 0
20
+ @crashes = []
21
+ end
22
+
23
+ def attach(test_result)
24
+ require "test/unit/testresult"
25
+ test_result.add_listener(Test::Unit::TestResult::FAULT) { |fault| record(fault) }
26
+ end
27
+
28
+ def record(fault)
29
+ if fault.is_a?(Test::Unit::Error)
30
+ @crashes << fault.exception
31
+ elsif fault.is_a?(Test::Unit::Failure)
32
+ @assertion_failures += 1
33
+ end
34
+ end
35
+
36
+ def passed?
37
+ @assertion_failures.zero? && @crashes.empty?
38
+ end
39
+
40
+ def assertion_failure?
41
+ @assertion_failures.positive?
42
+ end
43
+
44
+ def crashed?
45
+ @crashes.any?
46
+ end
47
+
48
+ def only_crashes?
49
+ @crashes.any? && @assertion_failures.zero?
50
+ end
51
+
52
+ def unique_crash_classes
53
+ @crashes.map { |e| e.class.name }.uniq
54
+ end
55
+
56
+ def crash_summary
57
+ return nil if @crashes.empty?
58
+
59
+ "#{unique_crash_classes.join(", ")} (#{@crashes.length} crash#{"es" unless @crashes.length == 1})"
60
+ end
61
+ end
@@ -11,6 +11,7 @@ require_relative "../isolation"
11
11
 
12
12
  class Evilution::Isolation::Fork
13
13
  GRACE_PERIOD = 2
14
+ REAP_DEADLINE = 1.0
14
15
 
15
16
  def initialize(hooks: nil)
16
17
  @hooks = hooks
@@ -48,6 +49,15 @@ class Evilution::Isolation::Fork
48
49
  def fork_child(read_io, write_io, sandbox_dir, mutation, test_command)
49
50
  ::Process.fork do
50
51
  ENV["TMPDIR"] = sandbox_dir
52
+ # Path-relativizing mutations (e.g. File.join(dir, name) -> name) would
53
+ # otherwise write into the parent's CWD (typically the repo root) and
54
+ # leak past the run. chdir here keeps such writes inside sandbox_dir,
55
+ # which the ensure block of #call removes. The in_isolated_worker! flag
56
+ # signals the rest of evilution (SpecResolver/SpecSelector/SpecAstCache/
57
+ # MutationApplier/SourceEvaluator/Integration) to anchor project-relative
58
+ # paths to Evilution::PROJECT_ROOT instead of the sandbox CWD.
59
+ Dir.chdir(sandbox_dir)
60
+ Evilution.in_isolated_worker!
51
61
  read_io.close
52
62
  suppress_child_output
53
63
  @hooks.fire(:worker_process_start, mutation:) if @hooks
@@ -121,8 +131,23 @@ class Evilution::Isolation::Fork
121
131
  end
122
132
  end
123
133
 
134
+ # Process.wait without a deadline can hang the parent indefinitely. The
135
+ # per-mutation child may already have written a payload (or a grandchild may
136
+ # have written garbage that looks like one) while the child itself is stuck
137
+ # in execute_in_child waiting on a subject grandchild the mutation broke.
138
+ # wait_for_result has already returned by this point, so the per-mutation
139
+ # timeout cannot fire. Bound the wait and fall back to the TERM/KILL ladder.
124
140
  def reap_and_decode(pid, payload)
125
- ::Process.wait(pid)
141
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + REAP_DEADLINE
142
+ loop do
143
+ break if ::Process.waitpid(pid, ::Process::WNOHANG)
144
+
145
+ if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
146
+ terminate_child(pid)
147
+ break
148
+ end
149
+ sleep 0.05
150
+ end
126
151
  decode_payload(payload)
127
152
  end
128
153
 
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "fileutils"
3
4
  require "timeout"
5
+ require "tmpdir"
4
6
  require_relative "../memory"
5
7
  require_relative "../result/mutation_result"
6
8
 
@@ -17,19 +19,34 @@ class Evilution::Isolation::InProcess
17
19
  def call(mutation:, test_command:, timeout:)
18
20
  start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
19
21
  rss_before = Evilution::Memory.rss_kb
20
- result = execute_with_timeout(mutation, test_command, timeout)
22
+ sandbox_dir = Dir.mktmpdir("evilution-run")
23
+ result = execute_with_timeout(mutation, test_command, timeout, sandbox_dir)
21
24
  rss_after = Evilution::Memory.rss_kb
22
25
  duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
23
26
  delta = compute_memory_delta(rss_before, rss_after, result)
24
27
 
25
28
  build_mutation_result(mutation, result, duration, rss_before, rss_after, delta)
29
+ ensure
30
+ FileUtils.rm_rf(sandbox_dir) if sandbox_dir
26
31
  end
27
32
 
28
33
  private
29
34
 
30
- def execute_with_timeout(mutation, test_command, timeout)
35
+ # The Dir.chdir block is inside the Timeout.timeout block so that a
36
+ # Timeout::Error raised mid-call still unwinds through Dir.chdir's ensure
37
+ # and restores the parent CWD before the rescue clause runs. The sandbox
38
+ # contains any relative-path writes from path-relativizing mutations
39
+ # (EV-wqxu / GH #1278). Evilution.with_isolated_worker signals the rest of
40
+ # evilution (SpecResolver/SpecSelector/SpecAstCache/MutationApplier/
41
+ # SourceEvaluator/Integration) to anchor project-relative paths to
42
+ # PROJECT_ROOT for the duration of the call.
43
+ def execute_with_timeout(mutation, test_command, timeout, sandbox_dir)
31
44
  result = Timeout.timeout(timeout) do
32
- suppress_output { test_command.call(mutation) }
45
+ Evilution.with_isolated_worker do
46
+ Dir.chdir(sandbox_dir) do
47
+ suppress_output { test_command.call(mutation) }
48
+ end
49
+ end
33
50
  end
34
51
  { timeout: false }.merge(result)
35
52
  rescue Timeout::Error
@@ -58,8 +58,8 @@ class Evilution::MCP::InfoTool < MCP::Tool
58
58
  },
59
59
  integration: {
60
60
  type: "string",
61
- description: "[subjects, tests] Test integration (rspec, minitest) — 'tests' selects " \
62
- "the matching spec resolver (spec/*_spec.rb for rspec, test/*_test.rb for minitest)"
61
+ description: "[subjects, tests] Test integration (rspec, minitest, test-unit) — 'tests' selects " \
62
+ "the matching spec resolver (spec/*_spec.rb for rspec, test/*_test.rb for minitest/test-unit)"
63
63
  },
64
64
  skip_config: {
65
65
  type: "boolean",
@@ -72,8 +72,9 @@ class Evilution::MCP::MutateTool < MCP::Tool
72
72
  },
73
73
  integration: {
74
74
  type: "string",
75
- enum: %w[rspec minitest],
76
- description: "Test integration to use (default: rspec)"
75
+ enum: %w[rspec minitest test-unit],
76
+ description: "Test integration to use (default: rspec). " \
77
+ "Use test-unit for projects whose suites run on the test-unit gem."
77
78
  },
78
79
  isolation: {
79
80
  type: "string",
@@ -5,6 +5,7 @@ require_relative "../baseline"
5
5
  require_relative "../spec_resolver"
6
6
  require_relative "../integration/rspec"
7
7
  require_relative "../integration/minitest"
8
+ require_relative "../integration/test_unit"
8
9
  require_relative "../example_filter"
9
10
  require_relative "../spec_ast_cache"
10
11
  require_relative "../source_ast_cache"
@@ -12,7 +13,8 @@ require_relative "../source_ast_cache"
12
13
  unless defined?(Evilution::Runner::INTEGRATIONS)
13
14
  Evilution::Runner::INTEGRATIONS = {
14
15
  rspec: Evilution::Integration::RSpec,
15
- minitest: Evilution::Integration::Minitest
16
+ minitest: Evilution::Integration::Minitest,
17
+ test_unit: Evilution::Integration::TestUnit
16
18
  }.freeze
17
19
  end
18
20
 
@@ -0,0 +1,130 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tmpdir"
4
+ require "fileutils"
5
+ require "securerandom"
6
+ require_relative "../runner"
7
+ require_relative "../mutation"
8
+ require_relative "../subject"
9
+
10
+ # Runs one guaranteed-unobservable synthetic mutation through the configured
11
+ # integration + isolation at session start. The synthetic spec never references
12
+ # the synthetic class, so mutating the class cannot change any test outcome — a
13
+ # healthy pipeline must score the mutation :survived. Any other status means the
14
+ # mutation infrastructure is misreporting, so the run aborts before producing
15
+ # numbers that would all be unreliable. Mirrors the configured --isolation so
16
+ # isolation-specific defects are caught too.
17
+ class Evilution::Runner::Canary
18
+ class Failed < Evilution::Error; end
19
+
20
+ def initialize(config:, isolator:, integration_class:, hooks: nil)
21
+ @config = config
22
+ @isolator = isolator
23
+ @integration_class = integration_class
24
+ @hooks = hooks
25
+ end
26
+
27
+ def call
28
+ dir = Dir.mktmpdir("evilution-canary")
29
+ class_path = write_target_class(dir)
30
+ spec_path = write_spec(dir)
31
+
32
+ result = @isolator.call(
33
+ mutation: build_mutation(class_path),
34
+ test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
35
+ timeout: @config.timeout
36
+ )
37
+ raise Failed, failure_message(result.status) unless result.status == :survived
38
+
39
+ nil
40
+ ensure
41
+ FileUtils.remove_entry(dir) if dir
42
+ end
43
+
44
+ private
45
+
46
+ # pid + random hex keeps the synthetic class/spec names unique across
47
+ # concurrent sessions and across repeated re-eval into the same VM.
48
+ def suffix
49
+ @suffix ||= "#{Process.pid}_#{SecureRandom.hex(4)}"
50
+ end
51
+
52
+ def class_name
53
+ @class_name ||= "EvilutionCanary_#{suffix}"
54
+ end
55
+
56
+ def original_source
57
+ <<~RUBY
58
+ class #{class_name}
59
+ private
60
+
61
+ def __evilution_canary_probe
62
+ :original
63
+ end
64
+ end
65
+ RUBY
66
+ end
67
+
68
+ def mutated_source
69
+ original_source.sub(":original", "nil")
70
+ end
71
+
72
+ def write_target_class(dir)
73
+ path = File.join(dir, "#{class_name.downcase}.rb")
74
+ File.write(path, original_source)
75
+ path
76
+ end
77
+
78
+ def write_spec(dir)
79
+ minitest = @config.integration == :minitest
80
+ name = minitest ? "canary_#{suffix}_test.rb" : "canary_#{suffix}_spec.rb"
81
+ path = File.join(dir, name)
82
+ File.write(path, minitest ? minitest_spec_source : rspec_spec_source)
83
+ path
84
+ end
85
+
86
+ def rspec_spec_source
87
+ <<~RUBY
88
+ RSpec.describe("evilution proof-of-life canary") do
89
+ it "pipeline is alive" do
90
+ expect(true).to be(true)
91
+ end
92
+ end
93
+ RUBY
94
+ end
95
+
96
+ def minitest_spec_source
97
+ <<~RUBY
98
+ class EvilutionCanaryTest_#{suffix} < Minitest::Test
99
+ def test_pipeline_is_alive
100
+ assert true
101
+ end
102
+ end
103
+ RUBY
104
+ end
105
+
106
+ def build_mutation(class_path)
107
+ Evilution::Mutation.new(
108
+ subject: Evilution::Subject.new(
109
+ name: "#{class_name}#__evilution_canary_probe",
110
+ file_path: class_path, line_number: 1, source: original_source, node: nil
111
+ ),
112
+ operator_name: :canary_probe,
113
+ sources: Evilution::Mutation::Sources.new(original: original_source, mutated: mutated_source),
114
+ location: Evilution::Mutation::Location.new(file_path: class_path, line: 5, column: 4)
115
+ )
116
+ end
117
+
118
+ def build_integration(spec_path)
119
+ @integration_class.new(test_files: [spec_path], hooks: @hooks)
120
+ end
121
+
122
+ def failure_message(status)
123
+ "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
124
+ "mutation was scored #{status.inspect} instead of :survived. The mutation " \
125
+ "pipeline is misreporting — every score this run would produce is unreliable. " \
126
+ "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
127
+ "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
128
+ "an isolation-mode defect. Re-run with --no-canary to bypass this check."
129
+ end
130
+ end
@@ -4,6 +4,11 @@ require "fileutils"
4
4
  require_relative "../evilution"
5
5
 
6
6
  class Evilution::Runner
7
+ # Autoloaded: canary.rb subclasses Evilution::Error at class-body eval
8
+ # time, which is not yet defined while evilution.rb is mid-load. Deferring
9
+ # the load until first reference lets evilution.rb finish defining Error.
10
+ autoload :Canary, File.expand_path("runner/canary", __dir__)
11
+
7
12
  attr_reader :config
8
13
 
9
14
  def initialize(config: Evilution::Config.new, on_result: nil, hooks: nil)
@@ -27,6 +32,8 @@ class Evilution::Runner
27
32
  perform_preload
28
33
  log_memory("after preload") if rails_root_detected?
29
34
 
35
+ run_canary
36
+
30
37
  baseline_result = run_baseline(subjects)
31
38
 
32
39
  plan = mutation_planner.call(subjects)
@@ -122,6 +129,17 @@ class Evilution::Runner
122
129
  baseline_runner.call(subjects)
123
130
  end
124
131
 
132
+ def run_canary
133
+ return unless config.canary?
134
+
135
+ Evilution::Runner::Canary.new(
136
+ config: config,
137
+ isolator: isolator,
138
+ integration_class: baseline_runner.integration_class,
139
+ hooks: @hooks
140
+ ).call
141
+ end
142
+
125
143
  def run_mutations(mutations, baseline_result = nil)
126
144
  mutation_executor.call(mutations, baseline_result)
127
145
  end
@@ -136,7 +154,12 @@ class Evilution::Runner
136
154
  return
137
155
  end
138
156
 
139
- dir = config.quiet_children_dir
157
+ # Resolve to absolute now — fork children chdir into a per-mutation
158
+ # sandbox (EV-wqxu / GH #1278) before suppress_child_output runs, so a
159
+ # relative log_dir would reopen $stdout/$stderr at <sandbox>/<log_dir>
160
+ # which does not exist and the child dies with Errno::ENOENT before
161
+ # marshaling any result back.
162
+ dir = File.expand_path(config.quiet_children_dir)
140
163
  begin
141
164
  FileUtils.rm_rf(dir)
142
165
  FileUtils.mkdir_p(dir)