henitai 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +296 -1
- data/README.md +109 -16
- data/assets/schema/henitai.schema.json +20 -0
- data/lib/henitai/arid_node_filter.rb +4 -19
- data/lib/henitai/bootstrap_error_message.rb +20 -0
- data/lib/henitai/canonical_report_merger.rb +23 -2
- data/lib/henitai/cli/run_command.rb +25 -7
- data/lib/henitai/cli/run_options.rb +1 -1
- data/lib/henitai/cli.rb +4 -1
- data/lib/henitai/configuration.rb +4 -1
- data/lib/henitai/configuration_validator/rules.rb +8 -0
- data/lib/henitai/configuration_validator/scalars.rb +9 -0
- data/lib/henitai/configuration_validator.rb +2 -1
- data/lib/henitai/constant_path.rb +48 -0
- data/lib/henitai/control_run.rb +73 -0
- data/lib/henitai/coverage_bootstrapper.rb +57 -15
- data/lib/henitai/coverage_formatter.rb +26 -3
- data/lib/henitai/coverage_report_reader.rb +21 -0
- data/lib/henitai/equivalence_detector/operand_predicates.rb +16 -0
- data/lib/henitai/equivalence_detector.rb +46 -38
- data/lib/henitai/example_summary.rb +24 -0
- data/lib/henitai/execution_engine/fork_server_scope.rb +57 -0
- data/lib/henitai/execution_engine.rb +75 -60
- data/lib/henitai/fidelity_check.rb +173 -0
- data/lib/henitai/flaky_detection_baseline.rb +61 -0
- data/lib/henitai/flaky_mutant_retrier.rb +43 -0
- data/lib/henitai/flaky_retry_decision.rb +23 -0
- data/lib/henitai/flaky_test_registry.rb +63 -0
- data/lib/henitai/fork_server/client.rb +231 -0
- data/lib/henitai/fork_server/group_killer.rb +48 -0
- data/lib/henitai/fork_server/group_terminator.rb +47 -0
- data/lib/henitai/fork_server/launcher.rb +38 -0
- data/lib/henitai/fork_server/mutant_runner.rb +141 -0
- data/lib/henitai/fork_server/protocol.rb +73 -0
- data/lib/henitai/fork_server/reaper.rb +24 -0
- data/lib/henitai/fork_server/runtime.rb +109 -0
- data/lib/henitai/fork_server/server.rb +108 -0
- data/lib/henitai/fork_server/server_process.rb +73 -0
- data/lib/henitai/fork_server/status.rb +52 -0
- data/lib/henitai/fork_server.rb +43 -0
- data/lib/henitai/generated_artifacts.rb +3 -0
- data/lib/henitai/git_diff_analyzer.rb +28 -11
- data/lib/henitai/incremental_filter.rb +20 -10
- data/lib/henitai/integration/base.rb +72 -5
- data/lib/henitai/integration/capped_log_sink.rb +34 -0
- data/lib/henitai/integration/child_bootstrap.rb +7 -0
- data/lib/henitai/integration/child_report_store.rb +36 -0
- data/lib/henitai/integration/minitest.rb +17 -7
- data/lib/henitai/integration/minitest_autorun.rb +20 -0
- data/lib/henitai/integration/minitest_outcome_reporter.rb +28 -0
- data/lib/henitai/integration/minitest_preload.rb +30 -0
- data/lib/henitai/integration/minitest_runnable_filter.rb +61 -0
- data/lib/henitai/integration/minitest_test_runner.rb +32 -15
- data/lib/henitai/integration/mutant_run_support.rb +73 -13
- data/lib/henitai/integration/rspec_child_runner.rb +44 -9
- data/lib/henitai/integration/rspec_preload.rb +35 -0
- data/lib/henitai/integration/rspec_process_runner.rb +4 -1
- data/lib/henitai/integration/scenario_log_support.rb +31 -18
- data/lib/henitai/integration.rb +16 -3
- data/lib/henitai/minitest_coverage_reporter.rb +22 -6
- data/lib/henitai/minitest_result_file.rb +62 -0
- data/lib/henitai/mutant/activation.rb +32 -0
- data/lib/henitai/mutant/activator.rb +75 -55
- data/lib/henitai/mutant/def_injection.rb +136 -0
- data/lib/henitai/mutant/lexical_scope_check.rb +120 -0
- data/lib/henitai/mutant/lexical_target.rb +105 -0
- data/lib/henitai/mutant/replacement.rb +52 -0
- data/lib/henitai/mutant.rb +27 -8
- data/lib/henitai/mutant_generator.rb +62 -18
- data/lib/henitai/mutant_history_store/verdict_cache.rb +12 -11
- data/lib/henitai/mutant_history_store.rb +14 -4
- data/lib/henitai/mutant_identity.rb +5 -1
- data/lib/henitai/mutant_verdict.rb +71 -0
- data/lib/henitai/operators/array_declaration.rb +4 -0
- data/lib/henitai/operators/assignment_expression.rb +2 -0
- data/lib/henitai/operators/conditional_expression.rb +21 -1
- data/lib/henitai/operators/hash_literal.rb +7 -4
- data/lib/henitai/operators/method_expression.rb +14 -3
- data/lib/henitai/operators/regex_metacharacters.rb +89 -0
- data/lib/henitai/operators/regex_mutator.rb +23 -28
- data/lib/henitai/operators/return_value.rb +1 -1
- data/lib/henitai/operators.rb +1 -0
- data/lib/henitai/per_test_coverage.rb +11 -7
- data/lib/henitai/per_test_coverage_collector.rb +30 -31
- data/lib/henitai/per_test_coverage_delta.rb +60 -0
- data/lib/henitai/per_test_coverage_selector.rb +26 -13
- data/lib/henitai/process_wakeup.rb +8 -3
- data/lib/henitai/process_worker_runner.rb +19 -5
- data/lib/henitai/reporter.rb +15 -8
- data/lib/henitai/result.rb +32 -16
- data/lib/henitai/reuse_context.rb +50 -0
- data/lib/henitai/rspec_coverage_formatter.rb +4 -0
- data/lib/henitai/rspec_outcomes.rb +72 -0
- data/lib/henitai/runner.rb +58 -19
- data/lib/henitai/runner_dependencies.rb +22 -1
- data/lib/henitai/sampling_strategy.rb +36 -6
- data/lib/henitai/scenario_execution_result.rb +28 -5
- data/lib/henitai/signature_line.rb +21 -0
- data/lib/henitai/slot_scheduler/draining.rb +12 -2
- data/lib/henitai/slot_scheduler/process_control.rb +33 -0
- data/lib/henitai/slot_scheduler/retry_policy.rb +8 -6
- data/lib/henitai/slot_scheduler.rb +47 -8
- data/lib/henitai/source_file_selection.rb +1 -1
- data/lib/henitai/source_parser.rb +29 -9
- data/lib/henitai/static_filter.rb +45 -14
- data/lib/henitai/stillborn_filter.rb +70 -2
- data/lib/henitai/subject.rb +8 -3
- data/lib/henitai/subject_resolver.rb +65 -31
- data/lib/henitai/suite_boot_clock.rb +48 -0
- data/lib/henitai/survivor_activation_cache.rb +18 -11
- data/lib/henitai/survivor_rerun_strategy.rb +47 -43
- data/lib/henitai/test_file_outcomes.rb +79 -0
- data/lib/henitai/timeout_calibrator.rb +31 -9
- data/lib/henitai/timeout_ceiling.rb +37 -0
- data/lib/henitai/verdict_fingerprint.rb +29 -30
- data/lib/henitai/version.rb +1 -1
- data/lib/henitai.rb +17 -1
- data/sig/configuration_validator.rbs +1 -0
- data/sig/henitai.rbs +227 -45
- metadata +46 -3
- data/lib/henitai/survivor_test_filter.rb +0 -72
|
@@ -32,18 +32,22 @@ module Henitai
|
|
|
32
32
|
# @param config [Configuration]
|
|
33
33
|
# @param integration [Integration::Base]
|
|
34
34
|
# @param test_files [Array<String>, nil] test files to run; defaults to
|
|
35
|
-
# all files reported by the integration when nil
|
|
35
|
+
# all files reported by the integration when nil.
|
|
36
|
+
# Either way, `config.test_excludes` matches are dropped.
|
|
37
|
+
# @raise [ConfigurationError] when the excludes leave no test file to run
|
|
36
38
|
def ensure!(source_files:, config:, integration:, test_files: nil)
|
|
37
39
|
return if source_files.empty?
|
|
38
40
|
|
|
39
|
-
resolved_test_files = resolve_test_files(integration, test_files)
|
|
41
|
+
resolved_test_files = resolve_test_files(integration, test_files, config)
|
|
40
42
|
|
|
41
43
|
# Skip the bootstrap only when the coverage artifacts are both newer than
|
|
42
44
|
# all watched files and actually cover the configured sources. A fresh
|
|
43
45
|
# but irrelevant report (e.g. from a different working directory) must
|
|
44
46
|
# still trigger a re-bootstrap rather than silently proceeding with no
|
|
45
47
|
# usable coverage.
|
|
46
|
-
|
|
48
|
+
if flaky_detection?(config)
|
|
49
|
+
run_flaky_baseline(integration, config, resolved_test_files)
|
|
50
|
+
elsif !coverage_ready?(source_files, config, integration, resolved_test_files)
|
|
47
51
|
bootstrap_coverage(integration, config, resolved_test_files)
|
|
48
52
|
end
|
|
49
53
|
|
|
@@ -67,7 +71,25 @@ module Henitai
|
|
|
67
71
|
def coverage_ready?(source_files, config, integration, test_files)
|
|
68
72
|
coverage_fresh?(source_files, config, test_files) &&
|
|
69
73
|
coverage_available?(source_files, config) &&
|
|
70
|
-
per_test_coverage_ready?(source_files, config, integration, test_files)
|
|
74
|
+
per_test_coverage_ready?(source_files, config, integration, test_files) &&
|
|
75
|
+
boot_seconds_ready?(config, integration)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Timeout calibration reads the boot time the per-test collector writes
|
|
79
|
+
# next to the coverage artifacts, so a missing boot file means the
|
|
80
|
+
# baseline must run again to measure it.
|
|
81
|
+
def boot_seconds_ready?(config, integration)
|
|
82
|
+
return true unless calibration_enabled?(config) && per_test_coverage_supported?(integration)
|
|
83
|
+
|
|
84
|
+
File.exist?(boot_seconds_report_path(config))
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def calibration_enabled?(config)
|
|
88
|
+
config.respond_to?(:timeout_configured?) && !config.timeout_configured?
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def boot_seconds_report_path(config)
|
|
92
|
+
File.join(reports_dir(config), PerTestCoverageCollector::BOOT_FILE_NAME)
|
|
71
93
|
end
|
|
72
94
|
|
|
73
95
|
def covered_source_files(source_files, coverage_lines)
|
|
@@ -133,19 +155,25 @@ module Henitai
|
|
|
133
155
|
return
|
|
134
156
|
end
|
|
135
157
|
|
|
136
|
-
raise CoverageError,
|
|
158
|
+
raise CoverageError, BootstrapErrorMessage.build(result)
|
|
137
159
|
end
|
|
138
160
|
end
|
|
139
161
|
end
|
|
140
162
|
|
|
141
|
-
|
|
142
|
-
|
|
163
|
+
# Flaky detection needs two complete baseline runs, so the existing
|
|
164
|
+
# coverage is never reused in this mode.
|
|
165
|
+
def flaky_detection?(config)
|
|
166
|
+
config.respond_to?(:flaky_detection) && config.flaky_detection
|
|
167
|
+
end
|
|
143
168
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
169
|
+
def run_flaky_baseline(integration, config, test_files)
|
|
170
|
+
with_reports_dir(config) do
|
|
171
|
+
with_coverage_dir(config) do
|
|
172
|
+
FlakyDetectionBaseline.new(reports_dir: reports_dir(config))
|
|
173
|
+
.run(integration:, test_files:)
|
|
174
|
+
record_dependency_manifest(config)
|
|
175
|
+
end
|
|
176
|
+
end
|
|
149
177
|
end
|
|
150
178
|
|
|
151
179
|
def with_coverage_dir(config)
|
|
@@ -225,10 +253,24 @@ module Henitai
|
|
|
225
253
|
Array(source_files) + Array(test_files) + VerdictFingerprint.dependency_files
|
|
226
254
|
end
|
|
227
255
|
|
|
228
|
-
|
|
229
|
-
|
|
256
|
+
# Excluded tests are dropped here rather than in `bootstrap_coverage`, so
|
|
257
|
+
# the same filtered list feeds both the suite run and the freshness watch
|
|
258
|
+
# list: a test that no longer contributes to the baseline must not
|
|
259
|
+
# invalidate it either.
|
|
260
|
+
def resolve_test_files(integration, test_files, config)
|
|
261
|
+
candidates = test_files.nil? ? integration.test_files : test_files
|
|
262
|
+
patterns = configured_test_excludes(config)
|
|
263
|
+
retained = ExcludedTestFilter.new(patterns:).reject(candidates)
|
|
264
|
+
return retained unless retained.empty? && candidates.any?
|
|
265
|
+
|
|
266
|
+
raise ConfigurationError,
|
|
267
|
+
"test_excludes #{Array(patterns).inspect} excludes every test file; " \
|
|
268
|
+
"coverage cannot be generated"
|
|
269
|
+
end
|
|
230
270
|
|
|
231
|
-
|
|
271
|
+
# Specs pass bare config doubles exposing only what the example needs.
|
|
272
|
+
def configured_test_excludes(config)
|
|
273
|
+
config.respond_to?(:test_excludes) ? config.test_excludes : nil
|
|
232
274
|
end
|
|
233
275
|
|
|
234
276
|
def reports_dir(config)
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "henitai/per_test_coverage_collector"
|
|
4
|
+
require "henitai/test_file_outcomes"
|
|
5
|
+
require "henitai/rspec_outcomes"
|
|
4
6
|
|
|
5
7
|
module Henitai
|
|
6
8
|
# Collects per-test coverage data for static filtering heuristics.
|
|
@@ -8,8 +10,14 @@ module Henitai
|
|
|
8
10
|
REPORT_DIR_ENV = PerTestCoverageCollector::REPORT_DIR_ENV
|
|
9
11
|
REPORT_FILE_NAME = PerTestCoverageCollector::REPORT_FILE_NAME
|
|
10
12
|
|
|
11
|
-
def initialize(_output)
|
|
12
|
-
@collector = PerTestCoverageCollector.new
|
|
13
|
+
def initialize(_output, boot_clock: SuiteBootClock.from_env(ENV))
|
|
14
|
+
@collector = PerTestCoverageCollector.new(boot_clock:)
|
|
15
|
+
@outcomes = TestFileOutcomes.new
|
|
16
|
+
@outcome_listener = RspecOutcomes::Listener.new(@outcomes)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def example_started(_notification)
|
|
20
|
+
@collector.test_started
|
|
13
21
|
end
|
|
14
22
|
|
|
15
23
|
def example_finished(notification)
|
|
@@ -18,10 +26,25 @@ module Henitai
|
|
|
18
26
|
example.metadata[:file_path],
|
|
19
27
|
duration: example.execution_result.run_time
|
|
20
28
|
)
|
|
29
|
+
@outcome_listener.example_finished(notification)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def example_group_started(notification)
|
|
33
|
+
@outcome_listener.example_group_started(notification)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def example_group_finished(notification)
|
|
37
|
+
@outcome_listener.example_group_finished(notification)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def message(notification)
|
|
41
|
+
@outcome_listener.message(notification)
|
|
21
42
|
end
|
|
22
43
|
|
|
23
|
-
def dump_summary(
|
|
44
|
+
def dump_summary(summary)
|
|
45
|
+
@outcome_listener.dump_summary(summary)
|
|
24
46
|
@collector.write_report
|
|
47
|
+
@outcomes.write(TestFileOutcomes.default_report_path)
|
|
25
48
|
end
|
|
26
49
|
end
|
|
27
50
|
end
|
|
@@ -7,6 +7,7 @@ module Henitai
|
|
|
7
7
|
class CoverageReportReader
|
|
8
8
|
DEFAULT_COVERAGE_REPORT_PATH = "coverage/.resultset.json"
|
|
9
9
|
DEFAULT_PER_TEST_COVERAGE_REPORT_PATH = "coverage/henitai_per_test.json"
|
|
10
|
+
DEFAULT_SUITE_BOOT_REPORT_PATH = "coverage/henitai_suite_boot.json"
|
|
10
11
|
|
|
11
12
|
def coverage_lines_by_file(path = DEFAULT_COVERAGE_REPORT_PATH)
|
|
12
13
|
return {} unless File.exist?(path)
|
|
@@ -37,6 +38,26 @@ module Henitai
|
|
|
37
38
|
end.to_h
|
|
38
39
|
end
|
|
39
40
|
|
|
41
|
+
# Suite boot seconds (loading helpers, spec files and before(:suite)).
|
|
42
|
+
# Missing or malformed reports yield 0.0, the value for "unknown".
|
|
43
|
+
def boot_seconds(path = DEFAULT_SUITE_BOOT_REPORT_PATH)
|
|
44
|
+
return 0.0 unless File.exist?(path)
|
|
45
|
+
|
|
46
|
+
parsed = JSON.parse(File.read(path))
|
|
47
|
+
parsed.is_a?(Hash) ? usable_boot_seconds(parsed["boot_seconds"]) : 0.0
|
|
48
|
+
rescue JSON::ParserError, ArgumentError, TypeError
|
|
49
|
+
0.0
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Only a finite, non-negative number is a usable measurement; anything
|
|
53
|
+
# else is treated as unknown.
|
|
54
|
+
def usable_boot_seconds(value)
|
|
55
|
+
return 0.0 unless value.is_a?(Numeric)
|
|
56
|
+
|
|
57
|
+
seconds = Float(value)
|
|
58
|
+
seconds.finite? && seconds >= 0 ? seconds : 0.0
|
|
59
|
+
end
|
|
60
|
+
|
|
40
61
|
private
|
|
41
62
|
|
|
42
63
|
def per_test_report(path)
|
|
@@ -27,8 +27,24 @@ module Henitai
|
|
|
27
27
|
# Multiplicative identity: `x * 1`, `x / 1` and `x ** 1` all reduce to `x`.
|
|
28
28
|
def one_operand?(node) = numeric_operand?(node, 1)
|
|
29
29
|
|
|
30
|
+
# The identities above only hold when the receiver is a number. For an
|
|
31
|
+
# unknown receiver `"ab" * 1` is `"ab"` while `"ab" / 1` raises, and a
|
|
32
|
+
# custom object may define its own operators. Negative zero is excluded
|
|
33
|
+
# because `-0.0 + 0` is `0.0` while `-0.0 - 0` is `-0.0`.
|
|
34
|
+
def numeric_receiver?(node)
|
|
35
|
+
return false unless node.is_a?(Parser::AST::Node)
|
|
36
|
+
|
|
37
|
+
case node.type
|
|
38
|
+
when :int then true
|
|
39
|
+
when :float then !negative_zero?(node.children.first)
|
|
40
|
+
else false
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
30
44
|
private
|
|
31
45
|
|
|
46
|
+
def negative_zero?(value) = value.zero? && (1.0 / value).negative?
|
|
47
|
+
|
|
32
48
|
# Reads the right-hand operand of a binary send. Guards against a
|
|
33
49
|
# malformed node whose operand is a bare Ruby value rather than an AST
|
|
34
50
|
# node -- that shape does not occur in parsed source, but the detector
|
|
@@ -6,9 +6,25 @@ require_relative "equivalence_detector/operand_predicates"
|
|
|
6
6
|
module Henitai
|
|
7
7
|
# Detects obvious equivalent mutants before execution.
|
|
8
8
|
#
|
|
9
|
-
# The detector is intentionally conservative: it only marks
|
|
10
|
-
# equivalent when the AST shape and the
|
|
11
|
-
#
|
|
9
|
+
# The detector is intentionally conservative: it only marks a mutation as
|
|
10
|
+
# equivalent when the AST shape and the literal operands prove the
|
|
11
|
+
# equivalence for every runtime value. Each rule below states the argument
|
|
12
|
+
# that makes it sound; a rule that cannot state one is not implemented.
|
|
13
|
+
#
|
|
14
|
+
# Rules:
|
|
15
|
+
# * Additive identity, `n + 0` <-> `n - 0`, when `n` is an Integer literal
|
|
16
|
+
# or a non-negative-zero Float literal. Both return `n`.
|
|
17
|
+
# * Multiplicative identity, `n * 1` <-> `n / 1` <-> `n ** 1`, under the
|
|
18
|
+
# same receiver condition. Both return `n`.
|
|
19
|
+
# * Left-literal logical identity, `false || x` -> `x` and `true && x` -> `x`.
|
|
20
|
+
# `false || x` is `x` for every `x`, and `true && x` is `x` for every `x`.
|
|
21
|
+
# The right-literal forms are unsound: `nil || false` is `false` and
|
|
22
|
+
# `5 && true` is `true`.
|
|
23
|
+
# * Singleton equality, `s == v` <-> `s.equal?(v)` when `s` and `v` are
|
|
24
|
+
# Symbol, nil, true or false literals. These values are unique instances,
|
|
25
|
+
# so `==` reduces to identity.
|
|
26
|
+
# * String equality, `"a" == "b"` <-> `"a".eql?("b")` when both operands are
|
|
27
|
+
# String literals. Both compare type and value for two Strings.
|
|
12
28
|
class EquivalenceDetector
|
|
13
29
|
def analyze(mutant)
|
|
14
30
|
return mutant unless equivalent_mutation?(mutant)
|
|
@@ -48,14 +64,16 @@ module Henitai
|
|
|
48
64
|
additive_operator?(original.children[1]) &&
|
|
49
65
|
additive_operator?(mutated.children[1]) &&
|
|
50
66
|
zero_operand?(original) &&
|
|
51
|
-
zero_operand?(mutated)
|
|
67
|
+
zero_operand?(mutated) &&
|
|
68
|
+
numeric_receiver?(original)
|
|
52
69
|
end
|
|
53
70
|
|
|
54
71
|
def multiplicative_equivalent?(original, mutated)
|
|
55
72
|
multiplicative_operator?(original.children[1]) &&
|
|
56
73
|
multiplicative_operator?(mutated.children[1]) &&
|
|
57
74
|
one_operand?(original) &&
|
|
58
|
-
one_operand?(mutated)
|
|
75
|
+
one_operand?(mutated) &&
|
|
76
|
+
numeric_receiver?(original)
|
|
59
77
|
end
|
|
60
78
|
|
|
61
79
|
def equivalent_logical_mutation?(mutant)
|
|
@@ -71,28 +89,23 @@ module Henitai
|
|
|
71
89
|
end
|
|
72
90
|
|
|
73
91
|
def logical_identity_equivalent?(original, mutated)
|
|
74
|
-
|
|
75
|
-
when :or
|
|
92
|
+
if original.type == :or
|
|
76
93
|
false_identity_equivalent?(original, mutated)
|
|
77
|
-
when :and
|
|
78
|
-
true_identity_equivalent?(original, mutated)
|
|
79
94
|
else
|
|
80
|
-
|
|
95
|
+
true_identity_equivalent?(original, mutated)
|
|
81
96
|
end
|
|
82
97
|
end
|
|
83
98
|
|
|
99
|
+
# `false || x` is `x` for every `x`. The right-literal form `x || false` is
|
|
100
|
+
# not: `nil || false` is `false`, not `nil`.
|
|
84
101
|
def false_identity_equivalent?(original, mutated)
|
|
85
|
-
|
|
86
|
-
return true if false_operand?(original.children[1]) && same_node?(mutated, original.children[0])
|
|
87
|
-
|
|
88
|
-
false
|
|
102
|
+
false_operand?(original.children[0]) && same_node?(mutated, original.children[1])
|
|
89
103
|
end
|
|
90
104
|
|
|
105
|
+
# `true && x` is `x` for every `x`. The right-literal form `x && true` is
|
|
106
|
+
# not: `5 && true` is `true`, not `5`.
|
|
91
107
|
def true_identity_equivalent?(original, mutated)
|
|
92
|
-
|
|
93
|
-
return true if true_operand?(original.children[1]) && same_node?(mutated, original.children[0])
|
|
94
|
-
|
|
95
|
-
false
|
|
108
|
+
true_operand?(original.children[0]) && same_node?(mutated, original.children[1])
|
|
96
109
|
end
|
|
97
110
|
|
|
98
111
|
def false_operand?(node)
|
|
@@ -121,6 +134,8 @@ module Henitai
|
|
|
121
134
|
|
|
122
135
|
def one_operand?(node) = operand_predicates.one_operand?(node)
|
|
123
136
|
|
|
137
|
+
def numeric_receiver?(node) = operand_predicates.numeric_receiver?(node.children[0])
|
|
138
|
+
|
|
124
139
|
def operand_predicates = @operand_predicates ||= OperandPredicates.new
|
|
125
140
|
|
|
126
141
|
# Detects `lhs == <singleton>` mutated to `lhs.equal?(<singleton>)` (or the
|
|
@@ -146,28 +161,27 @@ module Henitai
|
|
|
146
161
|
equality_operators?(original.children[1], mutated.children[1])
|
|
147
162
|
end
|
|
148
163
|
|
|
149
|
-
# Detects `lhs == rhs` mutated to `lhs.eql?(rhs)` (or the reverse) when
|
|
150
|
-
#
|
|
164
|
+
# Detects `lhs == rhs` mutated to `lhs.eql?(rhs)` (or the reverse) when both
|
|
165
|
+
# operands are string literals.
|
|
151
166
|
#
|
|
152
|
-
# String
|
|
153
|
-
#
|
|
154
|
-
#
|
|
155
|
-
#
|
|
156
|
-
# to be a String — proven here by the presence of a :str literal on the
|
|
157
|
-
# receiver or the argument side.
|
|
158
|
-
#
|
|
159
|
-
# When no operand is a string literal we conservatively leave the mutant
|
|
160
|
-
# pending: the receiver could be any object whose custom #== diverges from
|
|
161
|
-
# its #eql?.
|
|
167
|
+
# With two String literals, neither `==` nor `eql?` can reach a custom
|
|
168
|
+
# method, so both compare type and value and return the same result. A
|
|
169
|
+
# single string literal is not enough: an unknown receiver can override
|
|
170
|
+
# `==`, and `"a" == obj` calls `obj == "a"` when obj responds to `to_str`.
|
|
162
171
|
def equivalent_string_eql_mutation?(mutant)
|
|
163
172
|
original = mutant.original_node
|
|
164
173
|
mutated = mutant.mutated_node
|
|
165
174
|
|
|
175
|
+
string_eql_swap?(original, mutated) &&
|
|
176
|
+
string_literal?(original.children[0]) &&
|
|
177
|
+
string_literal?(original.children[2])
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def string_eql_swap?(original, mutated)
|
|
166
181
|
string_eql_send?(original) && string_eql_send?(mutated) &&
|
|
167
182
|
same_receiver?(original, mutated) &&
|
|
168
183
|
string_eql_operators?(original.children[1], mutated.children[1]) &&
|
|
169
|
-
same_rhs?(original, mutated)
|
|
170
|
-
string_operand?(original)
|
|
184
|
+
same_rhs?(original, mutated)
|
|
171
185
|
end
|
|
172
186
|
|
|
173
187
|
def string_eql_send?(node)
|
|
@@ -189,12 +203,6 @@ module Henitai
|
|
|
189
203
|
same_node?(original.children[2], mutated.children[2])
|
|
190
204
|
end
|
|
191
205
|
|
|
192
|
-
# Returns true when at least one operand is a string literal, giving static
|
|
193
|
-
# proof that the comparison is string-typed on at least one side.
|
|
194
|
-
def string_operand?(node)
|
|
195
|
-
string_literal?(node.children[0]) || string_literal?(node.children[2])
|
|
196
|
-
end
|
|
197
|
-
|
|
198
206
|
def string_literal?(node)
|
|
199
207
|
node.is_a?(Parser::AST::Node) && node.type == :str
|
|
200
208
|
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
# Reads whether a test run's output says that no example ran.
|
|
5
|
+
#
|
|
6
|
+
# Both texts are matched at the start of a line only, and only the last
|
|
7
|
+
# summary line counts: failure output may quote either text, indented, from
|
|
8
|
+
# the test source, and the tests themselves may print summaries of nested
|
|
9
|
+
# runs before the run's own.
|
|
10
|
+
module ExampleSummary
|
|
11
|
+
NO_EXAMPLES_FOUND = /^No examples found\.$/
|
|
12
|
+
SUMMARY = /^(\d+) (?:examples?, \d+ failures?|runs?, \d+ assertions?)/
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def none_ran?(output)
|
|
17
|
+
text = output.to_s
|
|
18
|
+
summaries = text.scan(SUMMARY)
|
|
19
|
+
return NO_EXAMPLES_FOUND.match?(text) if summaries.empty?
|
|
20
|
+
|
|
21
|
+
summaries.last == ["0"]
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../fork_server"
|
|
4
|
+
|
|
5
|
+
module Henitai
|
|
6
|
+
class ExecutionEngine
|
|
7
|
+
# Opt-in fork-server execution (`mutation.fork_server`, ADR-15) for the
|
|
8
|
+
# engine's linear path. While the block runs, @fork_server_runner (when a
|
|
9
|
+
# server booted) runs the mutants; it is nil when the flag is off, the
|
|
10
|
+
# integration cannot boot once, or the server failed to boot, and the
|
|
11
|
+
# integration then boots per mutant.
|
|
12
|
+
module ForkServerScope
|
|
13
|
+
private
|
|
14
|
+
|
|
15
|
+
def with_fork_server(mutants, integration, config)
|
|
16
|
+
@fork_server_runner = start_fork_server(mutants, integration, config)
|
|
17
|
+
yield
|
|
18
|
+
ensure
|
|
19
|
+
@fork_server_runner&.stop
|
|
20
|
+
@fork_server_runner = nil
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def start_fork_server(mutants, integration, config)
|
|
24
|
+
return unless fork_server_requested?(config)
|
|
25
|
+
return warn_fork_server_unsupported unless fork_server_supported?(integration)
|
|
26
|
+
|
|
27
|
+
test_files = mutants.flat_map { |mutant| prioritized_tests_for(mutant, integration, config) }.uniq
|
|
28
|
+
return if test_files.empty?
|
|
29
|
+
|
|
30
|
+
ForkServer.start(integration:, mutants:, test_files:, log_path: fork_server_log_path(config),
|
|
31
|
+
fallback_timeout: ->(files) { resolved_timeout(files, config) })
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def fork_server_requested?(config)
|
|
35
|
+
config.respond_to?(:fork_server) && config.fork_server ? true : false
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def fork_server_supported?(integration)
|
|
39
|
+
integration.respond_to?(:fork_server_supported?) && integration.fork_server_supported?
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def warn_fork_server_unsupported
|
|
43
|
+
warn("henitai: mutation.fork_server is not supported for this integration; using per-mutant boot")
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# The scheduler learns of served processes' exits through the server.
|
|
48
|
+
def parallel_runtime_options
|
|
49
|
+
@fork_server_runner&.active? ? { runtime: @fork_server_runner.runtime } : {}
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def fork_server_log_path(config)
|
|
53
|
+
File.join(config.reports_dir, "mutation-logs", "fork-server.log")
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "process_worker_runner"
|
|
4
4
|
require_relative "execution_engine/env_scope"
|
|
5
|
+
require_relative "execution_engine/fork_server_scope"
|
|
5
6
|
|
|
6
7
|
module Henitai
|
|
7
8
|
# Runs pending mutants through the selected integration.
|
|
8
9
|
class ExecutionEngine
|
|
9
10
|
include EnvScope
|
|
11
|
+
include ForkServerScope
|
|
10
12
|
|
|
11
13
|
def run(mutants, integration, config, progress_reporter: nil)
|
|
12
14
|
with_reports_dir(config) do
|
|
@@ -23,17 +25,36 @@ module Henitai
|
|
|
23
25
|
private
|
|
24
26
|
|
|
25
27
|
def execute(mutants, integration, config, progress_reporter)
|
|
26
|
-
@flaky_retry_count = 0
|
|
27
28
|
pending_mutants = Array(mutants).select(&:pending?)
|
|
28
29
|
mutex = Mutex.new
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
@flaky_tests = flaky_tests_for(config)
|
|
31
|
+
@retrier = FlakyMutantRetrier.new(flaky_tests: @flaky_tests, max_retries: max_flaky_retries(config))
|
|
32
|
+
run_pending(pending_mutants, integration, config, progress_reporter, mutex)
|
|
33
|
+
warn_flaky_mutants
|
|
34
|
+
verify_controls(pending_mutants, integration, config)
|
|
35
|
+
mutants
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def run_pending(pending_mutants, integration, config, progress_reporter, mutex)
|
|
39
|
+
with_fork_server(pending_mutants, integration, config) do
|
|
40
|
+
if parallel_execution?(config, pending_mutants)
|
|
41
|
+
run_parallel(pending_mutants, integration, config, progress_reporter)
|
|
42
|
+
else
|
|
43
|
+
run_linear(pending_mutants, integration, config, progress_reporter, mutex)
|
|
44
|
+
@flaky_retry_count = @retrier.retried_count
|
|
45
|
+
end
|
|
33
46
|
end
|
|
47
|
+
end
|
|
34
48
|
|
|
35
|
-
|
|
36
|
-
|
|
49
|
+
def verify_controls(mutants, integration, config)
|
|
50
|
+
return unless config.respond_to?(:control_runs) && config.control_runs
|
|
51
|
+
|
|
52
|
+
timeout_for = ->(test_files) { resolved_timeout(test_files, config) }
|
|
53
|
+
stats = ControlRun.new(integration:, timeout_for:).verify(mutants)
|
|
54
|
+
return if stats[:failed].zero?
|
|
55
|
+
|
|
56
|
+
warn format("henitai: control runs failed for %<failed>d of %<verified>d subjects; " \
|
|
57
|
+
"%<reclassified>d detected verdicts reclassified as CompileError", **stats)
|
|
37
58
|
end
|
|
38
59
|
|
|
39
60
|
def parallel_execution?(config, mutants)
|
|
@@ -57,10 +78,14 @@ module Henitai
|
|
|
57
78
|
end
|
|
58
79
|
|
|
59
80
|
def run_parallel(mutants, integration, config, progress_reporter)
|
|
60
|
-
runner = ProcessWorkerRunner.new(
|
|
81
|
+
runner = ProcessWorkerRunner.new(
|
|
82
|
+
worker_count: worker_count(config),
|
|
83
|
+
flaky_tests: @flaky_tests,
|
|
84
|
+
**parallel_runtime_options
|
|
85
|
+
)
|
|
61
86
|
results = runner.run(
|
|
62
87
|
mutants,
|
|
63
|
-
integration,
|
|
88
|
+
@fork_server_runner || integration,
|
|
64
89
|
config,
|
|
65
90
|
progress_reporter,
|
|
66
91
|
test_file_resolver: ->(mutant) { prioritized_tests_for(mutant, integration, config) },
|
|
@@ -75,8 +100,12 @@ module Henitai
|
|
|
75
100
|
record_test_files(mutant, test_files)
|
|
76
101
|
return record_no_coverage(mutant, progress_reporter, mutex) if test_files.empty?
|
|
77
102
|
|
|
78
|
-
scenario_result =
|
|
103
|
+
scenario_result = @retrier.run(
|
|
104
|
+
mutant: mutant, integration: @fork_server_runner || integration, test_files: test_files,
|
|
105
|
+
timeout: resolved_timeout(test_files, config)
|
|
106
|
+
)
|
|
79
107
|
mutant.status = scenario_status(scenario_result)
|
|
108
|
+
ScenarioExecutionResult.copy_reason(scenario_result, mutant)
|
|
80
109
|
|
|
81
110
|
report_progress(mutant, scenario_result, progress_reporter, mutex)
|
|
82
111
|
end
|
|
@@ -92,19 +121,15 @@ module Henitai
|
|
|
92
121
|
end
|
|
93
122
|
|
|
94
123
|
def report_progress(mutant, scenario_result, progress_reporter, mutex)
|
|
95
|
-
|
|
96
|
-
mutex.synchronize { progress_reporter&.progress(mutant, scenario_result:) }
|
|
97
|
-
else
|
|
98
|
-
progress_reporter&.progress(mutant, scenario_result:)
|
|
99
|
-
end
|
|
124
|
+
mutex.synchronize { progress_reporter&.progress(mutant, scenario_result:) }
|
|
100
125
|
end
|
|
101
126
|
|
|
102
127
|
def prioritized_tests_for(mutant, integration, config)
|
|
103
|
-
tests =
|
|
104
|
-
tests = per_test_coverage_selector.filter(
|
|
105
|
-
tests,
|
|
128
|
+
tests = per_test_coverage_selector.select(
|
|
106
129
|
mutant,
|
|
107
|
-
reports_dir: config.reports_dir
|
|
130
|
+
reports_dir: config.reports_dir,
|
|
131
|
+
test_files: -> { reject_excluded_tests(integration.test_files, config) },
|
|
132
|
+
fallback: -> { reject_excluded_tests(integration.select_tests(mutant.subject), config) }
|
|
108
133
|
)
|
|
109
134
|
test_prioritizer(config).sort(tests, mutant, test_history(config))
|
|
110
135
|
end
|
|
@@ -133,11 +158,21 @@ module Henitai
|
|
|
133
158
|
def resolved_timeout(test_files, config)
|
|
134
159
|
return config.timeout unless calibration_enabled?(config)
|
|
135
160
|
|
|
136
|
-
|
|
137
|
-
|
|
161
|
+
calibrator = timeout_calibrator(config)
|
|
162
|
+
calibrated = calibrator.timeout_for(test_files)
|
|
163
|
+
unless calibrated
|
|
164
|
+
warn_timeout_fallback_once
|
|
165
|
+
return Configuration::DEFAULT_TIMEOUT
|
|
166
|
+
end
|
|
138
167
|
|
|
139
|
-
|
|
140
|
-
|
|
168
|
+
timeout_ceiling(config).cap(calibrated, calibrator.baseline_for(test_files))
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def timeout_ceiling(config)
|
|
172
|
+
@timeout_ceiling ||= TimeoutCeiling.new(
|
|
173
|
+
ceiling: max_timeout(config),
|
|
174
|
+
warner: ->(message) { warn(message) }
|
|
175
|
+
)
|
|
141
176
|
end
|
|
142
177
|
|
|
143
178
|
# Ceiling on the auto-calibrated timeout so a runaway mutant is killed in
|
|
@@ -153,13 +188,21 @@ module Henitai
|
|
|
153
188
|
config.respond_to?(:timeout_configured?) && !config.timeout_configured?
|
|
154
189
|
end
|
|
155
190
|
|
|
191
|
+
# Boot stays in the timeout even with a fork server: served mutants still
|
|
192
|
+
# run the suite's before(:suite) hooks, which the boot measurement includes.
|
|
156
193
|
def timeout_calibrator(config)
|
|
157
194
|
@timeout_calibrator ||= TimeoutCalibrator.new(
|
|
158
195
|
timing_source: timing_source(config),
|
|
159
|
-
multiplier: config.timeout_multiplier
|
|
196
|
+
multiplier: config.timeout_multiplier,
|
|
197
|
+
boot_seconds: -> { boot_seconds(config) }
|
|
160
198
|
)
|
|
161
199
|
end
|
|
162
200
|
|
|
201
|
+
def boot_seconds(config)
|
|
202
|
+
path = File.join(config.reports_dir, PerTestCoverageCollector::BOOT_FILE_NAME)
|
|
203
|
+
CoverageReportReader.new.boot_seconds(path)
|
|
204
|
+
end
|
|
205
|
+
|
|
163
206
|
def warn_timeout_fallback_once
|
|
164
207
|
return if @warned_timeout_fallback
|
|
165
208
|
|
|
@@ -178,34 +221,9 @@ module Henitai
|
|
|
178
221
|
config.history || {}
|
|
179
222
|
end
|
|
180
223
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
# runtime on real CI workloads.
|
|
184
|
-
# rubocop:disable Metrics/MethodLength
|
|
185
|
-
def run_with_flaky_retry(mutant, integration, config, test_files, mutex)
|
|
186
|
-
timeout = resolved_timeout(test_files, config)
|
|
187
|
-
scenario_result = integration.run_mutant(
|
|
188
|
-
mutant:,
|
|
189
|
-
test_files:,
|
|
190
|
-
timeout: timeout
|
|
191
|
-
)
|
|
192
|
-
return scenario_result unless scenario_status(scenario_result) == :survived
|
|
193
|
-
|
|
194
|
-
retries = 0
|
|
195
|
-
max_flaky_retries(config).times do
|
|
196
|
-
retries += 1
|
|
197
|
-
scenario_result = integration.run_mutant(
|
|
198
|
-
mutant:,
|
|
199
|
-
test_files:,
|
|
200
|
-
timeout: timeout
|
|
201
|
-
)
|
|
202
|
-
break unless scenario_status(scenario_result) == :survived
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
mutex.synchronize { @flaky_retry_count += 1 } if retries.positive?
|
|
206
|
-
scenario_result
|
|
224
|
+
def flaky_tests_for(config)
|
|
225
|
+
FlakyTestRegistry.for_config(config)
|
|
207
226
|
end
|
|
208
|
-
# rubocop:enable Metrics/MethodLength
|
|
209
227
|
|
|
210
228
|
def scenario_status(result)
|
|
211
229
|
return result if result.is_a?(Symbol)
|
|
@@ -213,17 +231,14 @@ module Henitai
|
|
|
213
231
|
result.status
|
|
214
232
|
end
|
|
215
233
|
|
|
216
|
-
def warn_flaky_mutants
|
|
217
|
-
return if
|
|
218
|
-
|
|
219
|
-
flaky_ratio = @flaky_retry_count.to_f / total_mutants
|
|
220
|
-
return unless flaky_ratio > 0.05
|
|
234
|
+
def warn_flaky_mutants
|
|
235
|
+
return if @flaky_retry_count.to_i.zero?
|
|
221
236
|
|
|
222
237
|
warn format(
|
|
223
|
-
"Flaky-test mitigation: %<
|
|
224
|
-
flaky
|
|
225
|
-
|
|
226
|
-
|
|
238
|
+
"Flaky-test mitigation: %<count>d killed mutant(s) were re-checked because their killing tests " \
|
|
239
|
+
"are flaky (%<files>s)",
|
|
240
|
+
count: @flaky_retry_count,
|
|
241
|
+
files: @flaky_tests.all.join(", ")
|
|
227
242
|
)
|
|
228
243
|
end
|
|
229
244
|
|