henitai 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +222 -1
- data/README.md +100 -13
- data/assets/schema/henitai.schema.json +20 -0
- data/lib/henitai/arid_node_filter.rb +4 -19
- data/lib/henitai/bootstrap_error_message.rb +20 -0
- data/lib/henitai/canonical_report_merger.rb +23 -2
- data/lib/henitai/cli/run_command.rb +25 -7
- data/lib/henitai/cli/run_options.rb +1 -1
- data/lib/henitai/cli.rb +4 -1
- data/lib/henitai/configuration.rb +4 -1
- data/lib/henitai/configuration_validator/rules.rb +8 -0
- data/lib/henitai/configuration_validator/scalars.rb +9 -0
- data/lib/henitai/configuration_validator.rb +2 -1
- data/lib/henitai/constant_path.rb +48 -0
- data/lib/henitai/control_run.rb +73 -0
- data/lib/henitai/coverage_bootstrapper.rb +36 -10
- data/lib/henitai/coverage_formatter.rb +26 -3
- data/lib/henitai/coverage_report_reader.rb +21 -0
- data/lib/henitai/equivalence_detector/operand_predicates.rb +16 -0
- data/lib/henitai/equivalence_detector.rb +46 -38
- data/lib/henitai/example_summary.rb +24 -0
- data/lib/henitai/execution_engine/fork_server_scope.rb +57 -0
- data/lib/henitai/execution_engine.rb +75 -60
- data/lib/henitai/fidelity_check.rb +173 -0
- data/lib/henitai/flaky_detection_baseline.rb +61 -0
- data/lib/henitai/flaky_mutant_retrier.rb +43 -0
- data/lib/henitai/flaky_retry_decision.rb +23 -0
- data/lib/henitai/flaky_test_registry.rb +63 -0
- data/lib/henitai/fork_server/client.rb +231 -0
- data/lib/henitai/fork_server/group_killer.rb +48 -0
- data/lib/henitai/fork_server/group_terminator.rb +47 -0
- data/lib/henitai/fork_server/launcher.rb +38 -0
- data/lib/henitai/fork_server/mutant_runner.rb +141 -0
- data/lib/henitai/fork_server/protocol.rb +73 -0
- data/lib/henitai/fork_server/reaper.rb +24 -0
- data/lib/henitai/fork_server/runtime.rb +109 -0
- data/lib/henitai/fork_server/server.rb +108 -0
- data/lib/henitai/fork_server/server_process.rb +73 -0
- data/lib/henitai/fork_server/status.rb +52 -0
- data/lib/henitai/fork_server.rb +43 -0
- data/lib/henitai/generated_artifacts.rb +3 -0
- data/lib/henitai/git_diff_analyzer.rb +28 -11
- data/lib/henitai/incremental_filter.rb +20 -10
- data/lib/henitai/integration/base.rb +72 -5
- data/lib/henitai/integration/capped_log_sink.rb +34 -0
- data/lib/henitai/integration/child_bootstrap.rb +7 -0
- data/lib/henitai/integration/child_report_store.rb +36 -0
- data/lib/henitai/integration/minitest.rb +7 -5
- data/lib/henitai/integration/minitest_autorun.rb +20 -0
- data/lib/henitai/integration/minitest_outcome_reporter.rb +28 -0
- data/lib/henitai/integration/minitest_preload.rb +30 -0
- data/lib/henitai/integration/minitest_runnable_filter.rb +61 -0
- data/lib/henitai/integration/minitest_test_runner.rb +32 -15
- data/lib/henitai/integration/mutant_run_support.rb +73 -13
- data/lib/henitai/integration/rspec_child_runner.rb +44 -9
- data/lib/henitai/integration/rspec_preload.rb +35 -0
- data/lib/henitai/integration/rspec_process_runner.rb +4 -1
- data/lib/henitai/integration/scenario_log_support.rb +31 -18
- data/lib/henitai/integration.rb +16 -3
- data/lib/henitai/minitest_coverage_reporter.rb +22 -6
- data/lib/henitai/minitest_result_file.rb +62 -0
- data/lib/henitai/mutant/activation.rb +32 -0
- data/lib/henitai/mutant/activator.rb +75 -55
- data/lib/henitai/mutant/def_injection.rb +136 -0
- data/lib/henitai/mutant/lexical_scope_check.rb +120 -0
- data/lib/henitai/mutant/lexical_target.rb +105 -0
- data/lib/henitai/mutant/replacement.rb +52 -0
- data/lib/henitai/mutant.rb +27 -8
- data/lib/henitai/mutant_generator.rb +62 -18
- data/lib/henitai/mutant_history_store/verdict_cache.rb +12 -11
- data/lib/henitai/mutant_history_store.rb +14 -4
- data/lib/henitai/mutant_identity.rb +5 -1
- data/lib/henitai/mutant_verdict.rb +71 -0
- data/lib/henitai/operators/array_declaration.rb +4 -0
- data/lib/henitai/operators/assignment_expression.rb +2 -0
- data/lib/henitai/operators/conditional_expression.rb +21 -1
- data/lib/henitai/operators/hash_literal.rb +7 -4
- data/lib/henitai/operators/method_expression.rb +14 -3
- data/lib/henitai/operators/regex_metacharacters.rb +89 -0
- data/lib/henitai/operators/regex_mutator.rb +23 -28
- data/lib/henitai/operators/return_value.rb +1 -1
- data/lib/henitai/operators.rb +1 -0
- data/lib/henitai/per_test_coverage.rb +11 -7
- data/lib/henitai/per_test_coverage_collector.rb +30 -31
- data/lib/henitai/per_test_coverage_delta.rb +60 -0
- data/lib/henitai/per_test_coverage_selector.rb +26 -13
- data/lib/henitai/process_wakeup.rb +8 -3
- data/lib/henitai/process_worker_runner.rb +19 -5
- data/lib/henitai/reporter.rb +15 -8
- data/lib/henitai/result.rb +32 -16
- data/lib/henitai/reuse_context.rb +50 -0
- data/lib/henitai/rspec_coverage_formatter.rb +4 -0
- data/lib/henitai/rspec_outcomes.rb +72 -0
- data/lib/henitai/runner.rb +58 -19
- data/lib/henitai/runner_dependencies.rb +22 -1
- data/lib/henitai/sampling_strategy.rb +36 -6
- data/lib/henitai/scenario_execution_result.rb +28 -5
- data/lib/henitai/signature_line.rb +21 -0
- data/lib/henitai/slot_scheduler/draining.rb +12 -2
- data/lib/henitai/slot_scheduler/process_control.rb +33 -0
- data/lib/henitai/slot_scheduler/retry_policy.rb +8 -6
- data/lib/henitai/slot_scheduler.rb +47 -8
- data/lib/henitai/source_file_selection.rb +1 -1
- data/lib/henitai/source_parser.rb +29 -9
- data/lib/henitai/static_filter.rb +45 -14
- data/lib/henitai/stillborn_filter.rb +70 -2
- data/lib/henitai/subject.rb +8 -3
- data/lib/henitai/subject_resolver.rb +65 -31
- data/lib/henitai/suite_boot_clock.rb +48 -0
- data/lib/henitai/survivor_activation_cache.rb +18 -11
- data/lib/henitai/survivor_rerun_strategy.rb +47 -43
- data/lib/henitai/test_file_outcomes.rb +79 -0
- data/lib/henitai/timeout_calibrator.rb +31 -9
- data/lib/henitai/timeout_ceiling.rb +37 -0
- data/lib/henitai/verdict_fingerprint.rb +29 -30
- data/lib/henitai/version.rb +1 -1
- data/lib/henitai.rb +17 -1
- data/sig/configuration_validator.rbs +1 -0
- data/sig/henitai.rbs +225 -45
- metadata +46 -3
- data/lib/henitai/survivor_test_filter.rb +0 -72
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
module Integration
|
|
5
|
+
# Writes captured child output to a log file up to a byte cap, then
|
|
6
|
+
# discards the overflow and leaves a single truncation marker behind.
|
|
7
|
+
# Not interrupt-safe: the drain thread must be stopped cooperatively,
|
|
8
|
+
# never killed mid-write.
|
|
9
|
+
class CappedLogSink
|
|
10
|
+
def initialize(file, cap)
|
|
11
|
+
@file = file
|
|
12
|
+
@cap = cap
|
|
13
|
+
@written = 0
|
|
14
|
+
@truncated = false
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def write(chunk)
|
|
18
|
+
return unless @written < @cap
|
|
19
|
+
|
|
20
|
+
slice = chunk.byteslice(0, @cap - @written)
|
|
21
|
+
@file.write(slice)
|
|
22
|
+
@written += slice.bytesize
|
|
23
|
+
write_truncation_marker if @written >= @cap && !@truncated
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def write_truncation_marker
|
|
29
|
+
@file.write("\n[henitai] output truncated at #{@cap} bytes\n")
|
|
30
|
+
@truncated = true
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -9,6 +9,12 @@ module Henitai
|
|
|
9
9
|
# without forking, and so both the RSpec and Minitest paths -- which share
|
|
10
10
|
# MutantRunSupport#spawn_mutant -- get identical treatment.
|
|
11
11
|
module ChildBootstrap
|
|
12
|
+
# Signals the parent traps for its own shutdown (INT/TERM/HUP) and for
|
|
13
|
+
# its event-loop wakeup (CHLD). A child that kept those handlers would
|
|
14
|
+
# ignore the SIGTERM sent while draining it and could write into the
|
|
15
|
+
# parent's wakeup pipe.
|
|
16
|
+
INHERITED_TRAPS = %w[INT TERM HUP CHLD].freeze
|
|
17
|
+
|
|
12
18
|
# @param parent_pid [Integer] captured in the parent *before* Process.fork.
|
|
13
19
|
# Reading Process.ppid here instead would race the very death the
|
|
14
20
|
# watchdog is looking for: a parent that dies between fork and this
|
|
@@ -18,6 +24,7 @@ module Henitai
|
|
|
18
24
|
# First, so that even a crash later in this method releases the
|
|
19
25
|
# reports-directory lock rather than pinning it with an inherited fd.
|
|
20
26
|
InheritedFdRegistry.close_all!
|
|
27
|
+
INHERITED_TRAPS.each { |signal| Signal.trap(signal, "DEFAULT") }
|
|
21
28
|
Process.setpgid(0, 0)
|
|
22
29
|
OrphanWatchdog.start(parent_pid:)
|
|
23
30
|
nil
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Henitai
|
|
7
|
+
module Integration
|
|
8
|
+
# Carries a mutant child's report to the parent through a small JSON file
|
|
9
|
+
# next to the child's logs. The parent clears the file before each fork so
|
|
10
|
+
# a retry can never read the previous attempt's report.
|
|
11
|
+
class ChildReportStore
|
|
12
|
+
def write(path, data)
|
|
13
|
+
return unless path
|
|
14
|
+
|
|
15
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
16
|
+
temp_path = "#{path}.#{Process.pid}.tmp"
|
|
17
|
+
File.write(temp_path, JSON.generate(data))
|
|
18
|
+
File.rename(temp_path, path)
|
|
19
|
+
nil
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def read(path)
|
|
23
|
+
return unless path && File.exist?(path)
|
|
24
|
+
|
|
25
|
+
JSON.parse(File.read(path))
|
|
26
|
+
rescue JSON::ParserError
|
|
27
|
+
nil
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def clear(path)
|
|
31
|
+
FileUtils.rm_f(path) if path
|
|
32
|
+
nil
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -5,6 +5,7 @@ require "fileutils"
|
|
|
5
5
|
# tail, by which point Base is already defined, so the require is skipped.
|
|
6
6
|
require_relative "../integration" unless defined?(Henitai::Integration::Base)
|
|
7
7
|
require_relative "minitest_load_path"
|
|
8
|
+
require_relative "minitest_preload"
|
|
8
9
|
require_relative "minitest_suite_command"
|
|
9
10
|
require_relative "minitest_test_runner"
|
|
10
11
|
require_relative "rails_environment_preloader"
|
|
@@ -18,6 +19,7 @@ module Henitai
|
|
|
18
19
|
# but implements its own Minitest-specific test invocation and suite command.
|
|
19
20
|
# Per-test coverage collection is not yet wired into this path.
|
|
20
21
|
class Minitest < Base
|
|
22
|
+
include MinitestPreload
|
|
21
23
|
include MutantRunSupport
|
|
22
24
|
include RspecTestSelection
|
|
23
25
|
|
|
@@ -60,8 +62,8 @@ module Henitai
|
|
|
60
62
|
|
|
61
63
|
private
|
|
62
64
|
|
|
63
|
-
def run_tests(test_files)
|
|
64
|
-
MinitestTestRunner.new.call(test_files)
|
|
65
|
+
def run_tests(test_files, outcomes, &)
|
|
66
|
+
MinitestTestRunner.new(preloaded: @minitest_preloaded == true).call(test_files, outcomes, &)
|
|
65
67
|
end
|
|
66
68
|
|
|
67
69
|
def subprocess_env
|
|
@@ -81,7 +83,7 @@ module Henitai
|
|
|
81
83
|
File.open(log_paths[:stdout_path], "w") do |stdout_file|
|
|
82
84
|
File.open(log_paths[:stderr_path], "w") do |stderr_file|
|
|
83
85
|
Process.spawn(
|
|
84
|
-
|
|
86
|
+
suite_process_env,
|
|
85
87
|
*suite_command(test_files),
|
|
86
88
|
out: stdout_file,
|
|
87
89
|
err: stderr_file,
|
|
@@ -101,8 +103,8 @@ module Henitai
|
|
|
101
103
|
end
|
|
102
104
|
|
|
103
105
|
def spec_files
|
|
104
|
-
(Dir.glob("test/**/*_test.rb") + Dir.glob("test/**/*_spec.rb"))
|
|
105
|
-
|
|
106
|
+
@spec_files ||= (Dir.glob("test/**/*_test.rb") + Dir.glob("test/**/*_spec.rb"))
|
|
107
|
+
.reject { |f| f.start_with?("test/system/") }
|
|
106
108
|
end
|
|
107
109
|
end
|
|
108
110
|
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
module Integration
|
|
5
|
+
# Turns Minitest.autorun into a no-op, so that requiring test files never
|
|
6
|
+
# registers the at_exit hook that would run the suite a second time.
|
|
7
|
+
module MinitestAutorun
|
|
8
|
+
SUPPRESSOR = Module.new { def autorun = nil }
|
|
9
|
+
|
|
10
|
+
def self.suppress!
|
|
11
|
+
require "minitest"
|
|
12
|
+
singleton_class = ::Minitest.singleton_class
|
|
13
|
+
singleton_class.prepend(SUPPRESSOR) unless singleton_class.ancestors.include?(SUPPRESSOR)
|
|
14
|
+
nil
|
|
15
|
+
rescue LoadError, NameError
|
|
16
|
+
nil
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "minitest"
|
|
4
|
+
require "henitai/test_file_outcomes"
|
|
5
|
+
require "henitai/minitest_result_file"
|
|
6
|
+
|
|
7
|
+
module Henitai
|
|
8
|
+
module Integration
|
|
9
|
+
# Records the outcome of each finished Minitest result per test file.
|
|
10
|
+
class MinitestOutcomeReporter < ::Minitest::Reporter
|
|
11
|
+
def initialize(outcomes, report_path: nil, io: $stdout)
|
|
12
|
+
super(io, {})
|
|
13
|
+
@outcomes = outcomes
|
|
14
|
+
@report_path = report_path
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def record(result)
|
|
18
|
+
super
|
|
19
|
+
MinitestResultFile.record(@outcomes, result)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def report
|
|
23
|
+
super
|
|
24
|
+
@outcomes.write(@report_path) if @report_path
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "minitest_autorun"
|
|
4
|
+
require_relative "minitest_load_path"
|
|
5
|
+
require_relative "rails_environment_preloader"
|
|
6
|
+
|
|
7
|
+
module Henitai
|
|
8
|
+
module Integration
|
|
9
|
+
# Fork-server support for the Minitest integration (ADR-15): the server
|
|
10
|
+
# loads the environment and the union of the run's test files once, without
|
|
11
|
+
# running anything; each mutant process then runs only its own files.
|
|
12
|
+
module MinitestPreload
|
|
13
|
+
def fork_server_supported?
|
|
14
|
+
true
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def preload_environment(test_files)
|
|
18
|
+
MinitestAutorun.suppress!
|
|
19
|
+
CoverageRuntimeSuppressors.suppress_simplecov!
|
|
20
|
+
CoverageRuntimeSuppressors.suppress_coverage!
|
|
21
|
+
subprocess_env.each { |key, value| ENV[key] = value }
|
|
22
|
+
MinitestLoadPath.ensure!
|
|
23
|
+
RailsEnvironmentPreloader.new.call
|
|
24
|
+
test_files.each { |file| require File.expand_path(file) }
|
|
25
|
+
@minitest_preloaded = true
|
|
26
|
+
nil
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "henitai/minitest_result_file"
|
|
4
|
+
|
|
5
|
+
module Henitai
|
|
6
|
+
module Integration
|
|
7
|
+
# Narrows Minitest's registry of test classes to those defined in the given
|
|
8
|
+
# test files, for a process that inherited the union of a run's test files
|
|
9
|
+
# from the fork server (ADR-15). A class belongs to a file when one of its
|
|
10
|
+
# test methods is defined there.
|
|
11
|
+
class MinitestRunnableFilter
|
|
12
|
+
def initialize(test_files)
|
|
13
|
+
@wanted = test_files.to_set { |file| canonical(file) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def apply!
|
|
17
|
+
with_seed { ::Minitest::Runnable.runnables.select! { |suite| defined_in_wanted_file?(suite) } }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
# runnable_methods shuffles with Minitest.seed, which is unset before a run.
|
|
23
|
+
def with_seed
|
|
24
|
+
original = ::Minitest.seed
|
|
25
|
+
::Minitest.seed ||= 0
|
|
26
|
+
yield
|
|
27
|
+
ensure
|
|
28
|
+
::Minitest.seed = original
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# A suite belongs to a file when its class is defined there, or when one
|
|
32
|
+
# of its test methods is. The class file covers tests inherited from a
|
|
33
|
+
# shared module, which report the module's file; the method check covers
|
|
34
|
+
# a class reopened in another file.
|
|
35
|
+
def defined_in_wanted_file?(suite)
|
|
36
|
+
wanted_class_file?(suite) || method_in_wanted_file?(suite)
|
|
37
|
+
rescue NotImplementedError, NameError
|
|
38
|
+
false
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def wanted_class_file?(suite)
|
|
42
|
+
file = suite.name && MinitestResultFile.class_file_of(suite.name)
|
|
43
|
+
!file.nil? && @wanted.include?(canonical(file))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def method_in_wanted_file?(suite)
|
|
47
|
+
suite.runnable_methods.any? do |name|
|
|
48
|
+
location = suite.instance_method(name).source_location
|
|
49
|
+
location && @wanted.include?(canonical(location.first))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Symlinked temp or checkout directories must not make one file two paths.
|
|
54
|
+
def canonical(path)
|
|
55
|
+
File.realpath(path)
|
|
56
|
+
rescue SystemCallError
|
|
57
|
+
File.expand_path(path)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -1,16 +1,32 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
require "henitai/integration/minitest_autorun"
|
|
4
|
+
require "henitai/integration/minitest_outcome_reporter"
|
|
5
|
+
require "henitai/integration/minitest_runnable_filter"
|
|
6
|
+
require "henitai/test_file_outcomes"
|
|
4
7
|
|
|
5
8
|
module Henitai
|
|
6
9
|
module Integration
|
|
7
10
|
# Runs Minitest test files in-process and normalizes the result to an
|
|
8
11
|
# exit code, for Minitest#run_tests during a single mutant activation.
|
|
9
12
|
class MinitestTestRunner
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
PLUGIN_NAME = "henitai_outcomes"
|
|
14
|
+
|
|
15
|
+
# +preloaded+: the fork server already required the union of the run's
|
|
16
|
+
# test files, so only the ones requested here are kept to run.
|
|
17
|
+
def initialize(preloaded: false)
|
|
18
|
+
@preloaded = preloaded
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Yields once the test files are required and before Minitest runs; a
|
|
22
|
+
# falsy block result skips the run. Per-file outcomes are recorded into
|
|
23
|
+
# +outcomes+ through a Minitest reporter plugin.
|
|
24
|
+
def call(test_files, outcomes = TestFileOutcomes.new, &after_load)
|
|
25
|
+
MinitestAutorun.suppress!
|
|
26
|
+
load_tests(test_files)
|
|
27
|
+
return 1 if after_load && !after_load.call
|
|
28
|
+
|
|
29
|
+
install_outcome_plugin(outcomes)
|
|
14
30
|
# @type var empty_args: Array[String]
|
|
15
31
|
empty_args = []
|
|
16
32
|
status = ::Minitest.run(empty_args)
|
|
@@ -21,18 +37,19 @@ module Henitai
|
|
|
21
37
|
|
|
22
38
|
private
|
|
23
39
|
|
|
24
|
-
def
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
40
|
+
def install_outcome_plugin(outcomes)
|
|
41
|
+
::Minitest.extensions << PLUGIN_NAME unless ::Minitest.extensions.include?(PLUGIN_NAME)
|
|
42
|
+
init = :"plugin_#{PLUGIN_NAME}_init"
|
|
43
|
+
::Minitest.singleton_class.send(:remove_method, init) if ::Minitest.respond_to?(init)
|
|
44
|
+
::Minitest.define_singleton_method(init) do |_options|
|
|
45
|
+
reporter.reporters << MinitestOutcomeReporter.new(outcomes)
|
|
29
46
|
end
|
|
30
|
-
|
|
47
|
+
end
|
|
31
48
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
49
|
+
def load_tests(test_files)
|
|
50
|
+
return MinitestRunnableFilter.new(test_files).apply! if @preloaded
|
|
51
|
+
|
|
52
|
+
test_files.each { |file| require File.expand_path(file) }
|
|
36
53
|
end
|
|
37
54
|
end
|
|
38
55
|
end
|
|
@@ -10,8 +10,11 @@ module Henitai
|
|
|
10
10
|
# The framework-specific test invocation is delegated to #run_tests, which
|
|
11
11
|
# including classes must implement.
|
|
12
12
|
module MutantRunSupport
|
|
13
|
+
# The previous attempt's report is cleared before the fork, so a retried
|
|
14
|
+
# child that dies early can never be classified by stale data.
|
|
13
15
|
def spawn_mutant(mutant:, test_files:)
|
|
14
|
-
log_paths =
|
|
16
|
+
log_paths = mutant_log_paths(mutant)
|
|
17
|
+
child_report_store.clear(log_paths[:report_path])
|
|
15
18
|
RspecProcessRunner.new.spawn_mutant(self, mutant:, test_files:, log_paths:)
|
|
16
19
|
end
|
|
17
20
|
|
|
@@ -29,17 +32,22 @@ module Henitai
|
|
|
29
32
|
}
|
|
30
33
|
end
|
|
31
34
|
|
|
35
|
+
# Mutant runs carry a report path and are classified from the child's
|
|
36
|
+
# report; the baseline suite has none and keeps exit-status semantics.
|
|
37
|
+
def mutant_log_paths(mutant)
|
|
38
|
+
log_paths = scenario_log_paths(mutant_log_name(mutant))
|
|
39
|
+
log_paths.merge(report_path: log_paths[:log_path].sub(/\.log\z/, ".report.json"))
|
|
40
|
+
end
|
|
41
|
+
|
|
32
42
|
def build_result(wait_result, log_paths)
|
|
33
43
|
stdout = scenario_log_support.read_log_file(log_paths[:stdout_path])
|
|
34
44
|
stderr = scenario_log_support.read_log_file(log_paths[:stderr_path])
|
|
35
45
|
scenario_log_support.write_combined_log(log_paths[:log_path], stdout, stderr)
|
|
46
|
+
output = { stdout:, stderr:, log_path: log_paths[:log_path] }
|
|
47
|
+
return ScenarioExecutionResult.build(wait_result:, **output) unless log_paths.key?(:report_path)
|
|
36
48
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
stdout:,
|
|
40
|
-
stderr:,
|
|
41
|
-
log_path: log_paths[:log_path]
|
|
42
|
-
)
|
|
49
|
+
report = child_report_store.read(log_paths[:report_path])
|
|
50
|
+
ScenarioExecutionResult.build_for_mutant(wait_result:, report:, **output)
|
|
43
51
|
end
|
|
44
52
|
|
|
45
53
|
def run_in_child(mutant:, test_files:, log_paths:)
|
|
@@ -61,17 +69,69 @@ module Henitai
|
|
|
61
69
|
private
|
|
62
70
|
|
|
63
71
|
def run_child_activation_and_tests(mutant:, test_files:, log_paths:)
|
|
72
|
+
report = ->(data) { child_report_store.write(log_paths[:report_path], data) }
|
|
64
73
|
scenario_log_support.with_coverage_dir(mutant.id) do
|
|
65
74
|
scenario_log_support.capture_child_output(log_paths) do
|
|
66
|
-
|
|
67
|
-
child_debug_log.activation_start(mutant.id)
|
|
68
|
-
activation_result = Mutant::Activator.activate!(mutant)
|
|
69
|
-
child_debug_log.activation_check
|
|
70
|
-
child_debug_log.activation_end(activation_result, test_files:)
|
|
71
|
-
activation_result == :compile_error ? 2 : run_tests(test_files)
|
|
75
|
+
run_tests_in_child(mutant, test_files, report)
|
|
72
76
|
end
|
|
73
77
|
end
|
|
74
78
|
end
|
|
79
|
+
|
|
80
|
+
# Any failure to put the mutant in place is a harness outcome: the tests
|
|
81
|
+
# never ran against the mutation, so nothing may count as a kill.
|
|
82
|
+
def activate_in_child(mutant, test_files, report)
|
|
83
|
+
return true unless activate_with_debug_log(mutant, test_files) == :compile_error
|
|
84
|
+
|
|
85
|
+
report.call("outcome" => "activation_failed", "reason" => "the activation source does not compile")
|
|
86
|
+
false
|
|
87
|
+
rescue StandardError, ScriptError => e
|
|
88
|
+
warn "henitai: mutant activation failed: #{e.class}: #{e.message}"
|
|
89
|
+
report.call("outcome" => "activation_failed", "reason" => "#{e.class}: #{e.message}")
|
|
90
|
+
false
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def activate_with_debug_log(mutant, test_files)
|
|
94
|
+
log = child_debug_log
|
|
95
|
+
log.mutant_meta(mutant)
|
|
96
|
+
log.activation_start(mutant.id)
|
|
97
|
+
activation_result = Mutant::Activator.activate!(mutant)
|
|
98
|
+
log.activation_check
|
|
99
|
+
log.activation_end(activation_result, test_files:)
|
|
100
|
+
activation_result
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# RSpec and Minitest let SystemExit through by design. Raised by the
|
|
104
|
+
# code under test it ends the run midway, so it is reported instead of
|
|
105
|
+
# becoming the child's exit status.
|
|
106
|
+
#
|
|
107
|
+
# The mutant is activated from #run_tests' after-load hook: once the test
|
|
108
|
+
# files (and the code they require) are loaded, before any test runs.
|
|
109
|
+
# Activating earlier would load the subject file on its own, without the
|
|
110
|
+
# superclasses and constants the rest of the application defines.
|
|
111
|
+
def run_tests_in_child(mutant, test_files, report)
|
|
112
|
+
activated = nil
|
|
113
|
+
outcomes = TestFileOutcomes.new
|
|
114
|
+
exit_code = run_tests(test_files, outcomes) { activated = activate_in_child(mutant, test_files, report) }
|
|
115
|
+
return finish_without_tests(activated, report) unless activated
|
|
116
|
+
|
|
117
|
+
report.call(
|
|
118
|
+
"outcome" => "tests_finished",
|
|
119
|
+
"exit_code" => exit_code,
|
|
120
|
+
"failed_tests" => outcomes.failed_files,
|
|
121
|
+
"unattributed_failures" => outcomes.unattributed_failures?
|
|
122
|
+
)
|
|
123
|
+
exit_code
|
|
124
|
+
rescue SystemExit => e
|
|
125
|
+
report.call("outcome" => "system_exit", "status" => e.status)
|
|
126
|
+
1
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# A nil activation means the hook never ran: the test files failed to
|
|
130
|
+
# load, before the mutant was in place.
|
|
131
|
+
def finish_without_tests(activated, report)
|
|
132
|
+
report.call("outcome" => "environment_failed") if activated.nil?
|
|
133
|
+
2
|
|
134
|
+
end
|
|
75
135
|
end
|
|
76
136
|
end
|
|
77
137
|
end
|
|
@@ -8,7 +8,20 @@ module Henitai
|
|
|
8
8
|
module RspecChildRunner
|
|
9
9
|
private
|
|
10
10
|
|
|
11
|
-
def run_rspec_runner(test_files)
|
|
11
|
+
def run_rspec_runner(test_files, &after_load)
|
|
12
|
+
log = child_debug_log
|
|
13
|
+
runner = @preloaded_runner || prepare_rspec_runner(test_files)
|
|
14
|
+
return 1 unless ready_to_run?(after_load)
|
|
15
|
+
|
|
16
|
+
run_rspec_specs(runner, example_groups_to_run(test_files))
|
|
17
|
+
rescue SystemExit => e
|
|
18
|
+
log.write("[henitai-debug-child] runner_run_system_exit status=#{e.status.inspect}")
|
|
19
|
+
raise
|
|
20
|
+
ensure
|
|
21
|
+
child_debug_log.write("[henitai-debug-child] runner_run_ensure")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def prepare_rspec_runner(test_files)
|
|
12
25
|
log = child_debug_log
|
|
13
26
|
log.write("[henitai-debug-child] build_rspec_runner_start")
|
|
14
27
|
runner = build_rspec_runner
|
|
@@ -17,12 +30,34 @@ module Henitai
|
|
|
17
30
|
configure_rspec_runner(runner)
|
|
18
31
|
log.write("[henitai-debug-child] configure_rspec_runner_return")
|
|
19
32
|
load_rspec_spec_files(test_files)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
33
|
+
runner
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# A fork-server process loaded the union of the run's spec files; each
|
|
37
|
+
# mutant process runs only the groups defined in its own files.
|
|
38
|
+
def example_groups_to_run(test_files)
|
|
39
|
+
groups = ::RSpec.world.ordered_example_groups
|
|
40
|
+
return groups unless @preloaded_runner
|
|
41
|
+
|
|
42
|
+
wanted = test_files.map { |file| File.expand_path(file) }
|
|
43
|
+
groups.select { |group| wanted.include?(example_group_file(group)) }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def example_group_file(group)
|
|
47
|
+
metadata = group.metadata
|
|
48
|
+
metadata[:absolute_file_path] || File.expand_path(metadata[:file_path])
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# The after-load hook (mutant activation) runs only when every spec file
|
|
52
|
+
# loaded; a falsy hook result also skips the run.
|
|
53
|
+
def ready_to_run?(after_load)
|
|
54
|
+
rspec_loaded_cleanly? && (after_load.nil? || after_load.call)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# RSpec records a spec file that failed to load and quits before running.
|
|
58
|
+
def rspec_loaded_cleanly?
|
|
59
|
+
world = ::RSpec.world
|
|
60
|
+
!(world.wants_to_quit || world.rspec_is_quitting)
|
|
26
61
|
end
|
|
27
62
|
|
|
28
63
|
def build_rspec_runner
|
|
@@ -51,9 +86,9 @@ module Henitai
|
|
|
51
86
|
child_debug_log.write("[henitai-debug-child] load_spec_files_return")
|
|
52
87
|
end
|
|
53
88
|
|
|
54
|
-
def run_rspec_specs(runner)
|
|
89
|
+
def run_rspec_specs(runner, example_groups)
|
|
55
90
|
child_debug_log.write("[henitai-debug-child] run_specs_start")
|
|
56
|
-
result = runner.send(:run_specs,
|
|
91
|
+
result = runner.send(:run_specs, example_groups)
|
|
57
92
|
child_debug_log.write("[henitai-debug-child] run_specs_return result=#{result.inspect}")
|
|
58
93
|
result
|
|
59
94
|
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
module Integration
|
|
5
|
+
# Lets the fork server (ADR-15) boot the RSpec environment once. The
|
|
6
|
+
# booted runner stays in the server; every forked mutant process then runs
|
|
7
|
+
# its own subset of the loaded example groups.
|
|
8
|
+
module RspecPreload
|
|
9
|
+
class PreloadError < StandardError; end
|
|
10
|
+
|
|
11
|
+
def fork_server_supported?
|
|
12
|
+
true
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Loads RSpec, its configured requires (spec_helper, rails_helper, ...)
|
|
16
|
+
# and the given spec files, without activating a mutant or running a
|
|
17
|
+
# test. Coverage startup is suppressed before anything loads, as it is
|
|
18
|
+
# in every mutant child.
|
|
19
|
+
#
|
|
20
|
+
# @raise [PreloadError] when a spec file failed to load
|
|
21
|
+
def preload_environment(test_files)
|
|
22
|
+
CoverageRuntimeSuppressors.suppress_simplecov!
|
|
23
|
+
CoverageRuntimeSuppressors.suppress_coverage!
|
|
24
|
+
subprocess_env.each { |key, value| ENV[key] = value }
|
|
25
|
+
require "rspec/core"
|
|
26
|
+
::RSpec.configuration.fail_if_no_examples = true
|
|
27
|
+
runner = prepare_rspec_runner(test_files)
|
|
28
|
+
raise PreloadError, "the spec files failed to load" unless rspec_loaded_cleanly?
|
|
29
|
+
|
|
30
|
+
@preloaded_runner = runner
|
|
31
|
+
nil
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -52,7 +52,10 @@ module Henitai
|
|
|
52
52
|
end
|
|
53
53
|
|
|
54
54
|
# Captures the PID and log paths for a spawned mutant child process.
|
|
55
|
-
|
|
55
|
+
# +timeout+ is set only when the spawner knows better than the scheduler's
|
|
56
|
+
# resolver how long this child may run (a fallback child that boots the
|
|
57
|
+
# suite itself while the fork server is active).
|
|
58
|
+
ChildHandle = Struct.new(:pid, :log_paths, :timeout)
|
|
56
59
|
|
|
57
60
|
# Runs RSpec child and suite processes on behalf of the integration.
|
|
58
61
|
class RspecProcessRunner
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "fileutils"
|
|
4
|
+
require_relative "capped_log_sink"
|
|
4
5
|
|
|
5
6
|
module Henitai
|
|
6
7
|
module Integration
|
|
@@ -12,6 +13,9 @@ module Henitai
|
|
|
12
13
|
MAX_LOG_BYTES_ENV = "HENITAI_MAX_LOG_BYTES"
|
|
13
14
|
DEFAULT_MAX_LOG_BYTES = 5_000_000
|
|
14
15
|
DRAIN_CHUNK_BYTES = 65_536
|
|
16
|
+
# Bound on the reads of the final drain, so a grandchild that keeps
|
|
17
|
+
# writing cannot hold the close up.
|
|
18
|
+
DRAIN_AVAILABLE_READS = 16
|
|
15
19
|
|
|
16
20
|
def capture_child_output(log_paths)
|
|
17
21
|
output_files = open_child_output(log_paths)
|
|
@@ -69,37 +73,46 @@ module Henitai
|
|
|
69
73
|
file = File.new(path, "wb")
|
|
70
74
|
file.sync = true
|
|
71
75
|
reader, writer = IO.pipe
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
sink = CappedLogSink.new(file, cap)
|
|
77
|
+
stop_reader, stop_writer = IO.pipe
|
|
78
|
+
thread = Thread.new { drain_capped(reader, stop_reader, sink) }
|
|
79
|
+
{ file:, reader:, writer:, stop_reader:, stop_writer:, thread: }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Drains until EOF or until +stop_reader+ becomes readable. Never killed:
|
|
83
|
+
# the stop signal ends it cooperatively between sink writes.
|
|
84
|
+
# Uses Kernel#select rather than IO.select: the drain runs beside the
|
|
85
|
+
# tests under mutation, and specs that stub or count IO.select must not
|
|
86
|
+
# see the harness's own waits.
|
|
87
|
+
def drain_capped(reader, stop_reader, sink)
|
|
79
88
|
loop do
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
slice = chunk.byteslice(0, cap - written)
|
|
84
|
-
file.write(slice)
|
|
85
|
-
written += slice.bytesize
|
|
86
|
-
next unless written >= cap && !truncated
|
|
89
|
+
ready, = select([reader, stop_reader])
|
|
90
|
+
break drain_available(reader, sink) if ready.include?(stop_reader)
|
|
87
91
|
|
|
88
|
-
|
|
89
|
-
truncated = true
|
|
92
|
+
sink.write(reader.readpartial(DRAIN_CHUNK_BYTES))
|
|
90
93
|
end
|
|
91
94
|
rescue IOError
|
|
92
95
|
# EOFError (a subclass) on writer close, or a closed pipe: draining done.
|
|
93
96
|
nil
|
|
94
97
|
end
|
|
95
98
|
|
|
99
|
+
# Closing the writer lets the drain thread reach EOF. A grandchild may
|
|
100
|
+
# still hold a copy of the writer, so EOF is not awaited: the stop pipe
|
|
101
|
+
# makes the thread take what is readable right now and exit.
|
|
96
102
|
def finish_capped_stream(stream)
|
|
97
103
|
return unless stream
|
|
98
104
|
|
|
99
105
|
stream[:writer].close
|
|
106
|
+
stream[:stop_writer].close
|
|
100
107
|
stream[:thread].join
|
|
101
|
-
|
|
102
|
-
|
|
108
|
+
%i[reader stop_reader file].each { |key| stream[key].close }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def drain_available(reader, sink)
|
|
112
|
+
DRAIN_AVAILABLE_READS.times { sink.write(reader.read_nonblock(DRAIN_CHUNK_BYTES)) }
|
|
113
|
+
rescue IO::WaitReadable, IOError
|
|
114
|
+
# Nothing more buffered right now, or the pipe is closed.
|
|
115
|
+
nil
|
|
103
116
|
end
|
|
104
117
|
|
|
105
118
|
def redirect_child_output(output_files)
|