henitai 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +296 -1
- data/README.md +109 -16
- data/assets/schema/henitai.schema.json +20 -0
- data/lib/henitai/arid_node_filter.rb +4 -19
- data/lib/henitai/bootstrap_error_message.rb +20 -0
- data/lib/henitai/canonical_report_merger.rb +23 -2
- data/lib/henitai/cli/run_command.rb +25 -7
- data/lib/henitai/cli/run_options.rb +1 -1
- data/lib/henitai/cli.rb +4 -1
- data/lib/henitai/configuration.rb +4 -1
- data/lib/henitai/configuration_validator/rules.rb +8 -0
- data/lib/henitai/configuration_validator/scalars.rb +9 -0
- data/lib/henitai/configuration_validator.rb +2 -1
- data/lib/henitai/constant_path.rb +48 -0
- data/lib/henitai/control_run.rb +73 -0
- data/lib/henitai/coverage_bootstrapper.rb +57 -15
- data/lib/henitai/coverage_formatter.rb +26 -3
- data/lib/henitai/coverage_report_reader.rb +21 -0
- data/lib/henitai/equivalence_detector/operand_predicates.rb +16 -0
- data/lib/henitai/equivalence_detector.rb +46 -38
- data/lib/henitai/example_summary.rb +24 -0
- data/lib/henitai/execution_engine/fork_server_scope.rb +57 -0
- data/lib/henitai/execution_engine.rb +75 -60
- data/lib/henitai/fidelity_check.rb +173 -0
- data/lib/henitai/flaky_detection_baseline.rb +61 -0
- data/lib/henitai/flaky_mutant_retrier.rb +43 -0
- data/lib/henitai/flaky_retry_decision.rb +23 -0
- data/lib/henitai/flaky_test_registry.rb +63 -0
- data/lib/henitai/fork_server/client.rb +231 -0
- data/lib/henitai/fork_server/group_killer.rb +48 -0
- data/lib/henitai/fork_server/group_terminator.rb +47 -0
- data/lib/henitai/fork_server/launcher.rb +38 -0
- data/lib/henitai/fork_server/mutant_runner.rb +141 -0
- data/lib/henitai/fork_server/protocol.rb +73 -0
- data/lib/henitai/fork_server/reaper.rb +24 -0
- data/lib/henitai/fork_server/runtime.rb +109 -0
- data/lib/henitai/fork_server/server.rb +108 -0
- data/lib/henitai/fork_server/server_process.rb +73 -0
- data/lib/henitai/fork_server/status.rb +52 -0
- data/lib/henitai/fork_server.rb +43 -0
- data/lib/henitai/generated_artifacts.rb +3 -0
- data/lib/henitai/git_diff_analyzer.rb +28 -11
- data/lib/henitai/incremental_filter.rb +20 -10
- data/lib/henitai/integration/base.rb +72 -5
- data/lib/henitai/integration/capped_log_sink.rb +34 -0
- data/lib/henitai/integration/child_bootstrap.rb +7 -0
- data/lib/henitai/integration/child_report_store.rb +36 -0
- data/lib/henitai/integration/minitest.rb +17 -7
- data/lib/henitai/integration/minitest_autorun.rb +20 -0
- data/lib/henitai/integration/minitest_outcome_reporter.rb +28 -0
- data/lib/henitai/integration/minitest_preload.rb +30 -0
- data/lib/henitai/integration/minitest_runnable_filter.rb +61 -0
- data/lib/henitai/integration/minitest_test_runner.rb +32 -15
- data/lib/henitai/integration/mutant_run_support.rb +73 -13
- data/lib/henitai/integration/rspec_child_runner.rb +44 -9
- data/lib/henitai/integration/rspec_preload.rb +35 -0
- data/lib/henitai/integration/rspec_process_runner.rb +4 -1
- data/lib/henitai/integration/scenario_log_support.rb +31 -18
- data/lib/henitai/integration.rb +16 -3
- data/lib/henitai/minitest_coverage_reporter.rb +22 -6
- data/lib/henitai/minitest_result_file.rb +62 -0
- data/lib/henitai/mutant/activation.rb +32 -0
- data/lib/henitai/mutant/activator.rb +75 -55
- data/lib/henitai/mutant/def_injection.rb +136 -0
- data/lib/henitai/mutant/lexical_scope_check.rb +120 -0
- data/lib/henitai/mutant/lexical_target.rb +105 -0
- data/lib/henitai/mutant/replacement.rb +52 -0
- data/lib/henitai/mutant.rb +27 -8
- data/lib/henitai/mutant_generator.rb +62 -18
- data/lib/henitai/mutant_history_store/verdict_cache.rb +12 -11
- data/lib/henitai/mutant_history_store.rb +14 -4
- data/lib/henitai/mutant_identity.rb +5 -1
- data/lib/henitai/mutant_verdict.rb +71 -0
- data/lib/henitai/operators/array_declaration.rb +4 -0
- data/lib/henitai/operators/assignment_expression.rb +2 -0
- data/lib/henitai/operators/conditional_expression.rb +21 -1
- data/lib/henitai/operators/hash_literal.rb +7 -4
- data/lib/henitai/operators/method_expression.rb +14 -3
- data/lib/henitai/operators/regex_metacharacters.rb +89 -0
- data/lib/henitai/operators/regex_mutator.rb +23 -28
- data/lib/henitai/operators/return_value.rb +1 -1
- data/lib/henitai/operators.rb +1 -0
- data/lib/henitai/per_test_coverage.rb +11 -7
- data/lib/henitai/per_test_coverage_collector.rb +30 -31
- data/lib/henitai/per_test_coverage_delta.rb +60 -0
- data/lib/henitai/per_test_coverage_selector.rb +26 -13
- data/lib/henitai/process_wakeup.rb +8 -3
- data/lib/henitai/process_worker_runner.rb +19 -5
- data/lib/henitai/reporter.rb +15 -8
- data/lib/henitai/result.rb +32 -16
- data/lib/henitai/reuse_context.rb +50 -0
- data/lib/henitai/rspec_coverage_formatter.rb +4 -0
- data/lib/henitai/rspec_outcomes.rb +72 -0
- data/lib/henitai/runner.rb +58 -19
- data/lib/henitai/runner_dependencies.rb +22 -1
- data/lib/henitai/sampling_strategy.rb +36 -6
- data/lib/henitai/scenario_execution_result.rb +28 -5
- data/lib/henitai/signature_line.rb +21 -0
- data/lib/henitai/slot_scheduler/draining.rb +12 -2
- data/lib/henitai/slot_scheduler/process_control.rb +33 -0
- data/lib/henitai/slot_scheduler/retry_policy.rb +8 -6
- data/lib/henitai/slot_scheduler.rb +47 -8
- data/lib/henitai/source_file_selection.rb +1 -1
- data/lib/henitai/source_parser.rb +29 -9
- data/lib/henitai/static_filter.rb +45 -14
- data/lib/henitai/stillborn_filter.rb +70 -2
- data/lib/henitai/subject.rb +8 -3
- data/lib/henitai/subject_resolver.rb +65 -31
- data/lib/henitai/suite_boot_clock.rb +48 -0
- data/lib/henitai/survivor_activation_cache.rb +18 -11
- data/lib/henitai/survivor_rerun_strategy.rb +47 -43
- data/lib/henitai/test_file_outcomes.rb +79 -0
- data/lib/henitai/timeout_calibrator.rb +31 -9
- data/lib/henitai/timeout_ceiling.rb +37 -0
- data/lib/henitai/verdict_fingerprint.rb +29 -30
- data/lib/henitai/version.rb +1 -1
- data/lib/henitai.rb +17 -1
- data/sig/configuration_validator.rbs +1 -0
- data/sig/henitai.rbs +227 -45
- metadata +46 -3
- data/lib/henitai/survivor_test_filter.rb +0 -72
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "henitai/test_file_outcomes"
|
|
4
|
+
|
|
5
|
+
module Henitai
|
|
6
|
+
# Records the outcome of each finished RSpec example per spec file.
|
|
7
|
+
#
|
|
8
|
+
# Failures outside an example produce no failed example. A hook error in
|
|
9
|
+
# `before(:context)` fails the group's examples, so it is attributed
|
|
10
|
+
# already; an `after(:context)` error is announced as a message while its
|
|
11
|
+
# group is open and is attributed to that group's file. Everything else
|
|
12
|
+
# counted by `errors_outside_of_examples_count` (a spec file that fails to
|
|
13
|
+
# load, `before(:suite)`) has no file and is recorded as unattributed.
|
|
14
|
+
module RspecOutcomes
|
|
15
|
+
EVENTS = %i[example_finished example_group_started example_group_finished message dump_summary].freeze
|
|
16
|
+
HOOK_ERROR = "An error occurred in "
|
|
17
|
+
|
|
18
|
+
# RSpec drops finished examples from its groups after a run, so
|
|
19
|
+
# outcomes are collected as the reporter announces them.
|
|
20
|
+
class Listener
|
|
21
|
+
def initialize(outcomes)
|
|
22
|
+
@outcomes = outcomes
|
|
23
|
+
@open_group_files = []
|
|
24
|
+
@attributed_errors = 0
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def example_finished(notification)
|
|
28
|
+
RspecOutcomes.record(@outcomes, notification.example)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def example_group_started(notification)
|
|
32
|
+
@open_group_files.push(RspecOutcomes.group_file(notification.group))
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def example_group_finished(_notification)
|
|
36
|
+
@open_group_files.pop
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def message(notification)
|
|
40
|
+
file = @open_group_files.last
|
|
41
|
+
return unless file && notification.message.strip.start_with?(HOOK_ERROR)
|
|
42
|
+
|
|
43
|
+
@outcomes.record(file, failed: true)
|
|
44
|
+
@attributed_errors += 1
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def dump_summary(summary)
|
|
48
|
+
return unless summary.errors_outside_of_examples_count > @attributed_errors
|
|
49
|
+
|
|
50
|
+
@outcomes.record_unattributed_failure
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
module_function
|
|
55
|
+
|
|
56
|
+
def register(outcomes, reporter)
|
|
57
|
+
reporter.register_listener(Listener.new(outcomes), *EVENTS)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def group_file(group)
|
|
61
|
+
metadata = group.metadata
|
|
62
|
+
metadata[:absolute_file_path] || metadata[:file_path]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def record(outcomes, example)
|
|
66
|
+
status = example.execution_result.status
|
|
67
|
+
return if status.nil?
|
|
68
|
+
|
|
69
|
+
outcomes.record(example.metadata[:file_path], failed: status == :failed)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
data/lib/henitai/runner.rb
CHANGED
|
@@ -11,16 +11,20 @@ module Henitai
|
|
|
11
11
|
#
|
|
12
12
|
# Gate 2 — Mutant generation
|
|
13
13
|
# Apply operators to each Subject's AST. Filter arid (non-productive)
|
|
14
|
-
# nodes
|
|
14
|
+
# nodes from the built-in catalog. Produces the initial mutant list.
|
|
15
15
|
#
|
|
16
16
|
# Gate 3 — Static filtering
|
|
17
|
-
#
|
|
17
|
+
# Mark mutants whose original start line matches an ignore_patterns
|
|
18
|
+
# entry as :ignored, and remove compile-time errors.
|
|
18
19
|
# Apply per-test coverage data: mark :no_coverage for uncovered mutants.
|
|
19
20
|
#
|
|
20
21
|
# Gate 4 — Mutant execution
|
|
21
22
|
# Run surviving mutants in isolated child processes (fork isolation).
|
|
22
|
-
# Each child process loads the test suite
|
|
23
|
-
#
|
|
23
|
+
# Each child process loads the test suite, then re-evaluates the
|
|
24
|
+
# subject's own `def` with the mutation spliced in, inside its original
|
|
25
|
+
# module/class openers (ADR-13). Module#define_method is only the fallback
|
|
26
|
+
# for subjects that cannot be reopened by name. Collect kill/survive/
|
|
27
|
+
# timeout results.
|
|
24
28
|
#
|
|
25
29
|
# Gate 5 — Reporting
|
|
26
30
|
# Write results to configured reporters (terminal, html, json, dashboard).
|
|
@@ -29,7 +33,8 @@ module Henitai
|
|
|
29
33
|
attr_reader :config, :result
|
|
30
34
|
|
|
31
35
|
# @param mode [Hash] execution-mode flags: +dry_run:+ stops before Gate 4,
|
|
32
|
-
# +incremental:+ reuses still-valid Killed verdicts from history
|
|
36
|
+
# +incremental:+ reuses still-valid Killed verdicts from history,
|
|
37
|
+
# +force:+ disables the verdict reuse that survivor reruns apply.
|
|
33
38
|
# +deps+ is assigned in the body, not defaulted in the signature: the
|
|
34
39
|
# default needs @config, and `config:` itself defaults to a load.
|
|
35
40
|
#
|
|
@@ -45,6 +50,7 @@ module Henitai
|
|
|
45
50
|
@survivors_from = survivors_from
|
|
46
51
|
@dry_run = mode.fetch(:dry_run, false)
|
|
47
52
|
@incremental = mode.fetch(:incremental, false)
|
|
53
|
+
@force = mode.fetch(:force, false)
|
|
48
54
|
end
|
|
49
55
|
|
|
50
56
|
# Entry point — runs the full pipeline and returns a Result.
|
|
@@ -76,6 +82,9 @@ module Henitai
|
|
|
76
82
|
# static/skip/arid/stillborn filtering — everything short of execution.
|
|
77
83
|
def pipeline_mutants
|
|
78
84
|
if survivor_rerun? && (fast_mutants = survivor_strategy.try_recipe_run)
|
|
85
|
+
# Gate 0 still runs: test selection reads the per-test map, which must
|
|
86
|
+
# include new or edited tests.
|
|
87
|
+
bootstrap_coverage(fast_mutants.filter_map { |mutant| mutant.subject.source_file }.uniq)
|
|
79
88
|
return fast_mutants
|
|
80
89
|
end
|
|
81
90
|
|
|
@@ -104,8 +113,9 @@ module Henitai
|
|
|
104
113
|
)
|
|
105
114
|
end
|
|
106
115
|
|
|
116
|
+
# A survivor rerun must see every prior survivor, so it never samples.
|
|
107
117
|
def generate_mutants(subjects)
|
|
108
|
-
mutant_generator.generate(subjects, operators, config:)
|
|
118
|
+
mutant_generator.generate(subjects, operators, config:, sample: !survivor_rerun?)
|
|
109
119
|
end
|
|
110
120
|
|
|
111
121
|
def filter_mutants(mutants)
|
|
@@ -116,25 +126,26 @@ module Henitai
|
|
|
116
126
|
bootstrap_thread = bootstrap_mutants(source_files)
|
|
117
127
|
mutants = generate_mutants(subjects)
|
|
118
128
|
bootstrap_thread.value
|
|
119
|
-
filtered =
|
|
120
|
-
return filtered unless survivor_rerun?
|
|
129
|
+
filtered = filter_mutants(mutants)
|
|
130
|
+
return apply_incremental_filter(filtered) unless survivor_rerun?
|
|
121
131
|
|
|
122
|
-
survivor_strategy.apply_selection(filtered)
|
|
132
|
+
survivor_strategy.apply_selection(filtered, reuse_filter: @force ? nil : reuse_filter)
|
|
123
133
|
end
|
|
124
134
|
|
|
125
|
-
#
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
#
|
|
129
|
-
# mutants_for, so the live per-test map it reads is never
|
|
135
|
+
# Verdict reuse (opt-in `--incremental`; always for `--survivors-from`
|
|
136
|
+
# reruns, where it runs on the selected survivors): still-valid Killed and
|
|
137
|
+
# Survived verdicts from the history store are marked with their stored
|
|
138
|
+
# status + from_cache before execution. It runs after the coverage
|
|
139
|
+
# bootstrap join in mutants_for, so the live per-test map it reads is never
|
|
140
|
+
# mid-write.
|
|
130
141
|
def apply_incremental_filter(mutants)
|
|
131
142
|
return mutants unless @incremental
|
|
132
143
|
|
|
133
|
-
|
|
134
|
-
dependency_fingerprint: VerdictFingerprint.dependency_fingerprint)
|
|
135
|
-
.apply(mutants)
|
|
144
|
+
reuse_filter.apply(mutants)
|
|
136
145
|
end
|
|
137
146
|
|
|
147
|
+
def reuse_filter = deps.reuse_filter
|
|
148
|
+
|
|
138
149
|
def bootstrap_mutants(source_files) = Thread.new { bootstrap_coverage(source_files) }
|
|
139
150
|
|
|
140
151
|
def execute_mutants(mutants)
|
|
@@ -150,12 +161,16 @@ module Henitai
|
|
|
150
161
|
Reporter.run_all(names: config.reporters, result:, config:, history_store:)
|
|
151
162
|
end
|
|
152
163
|
|
|
164
|
+
# History is an auxiliary trend artifact: a corrupt or locked database
|
|
165
|
+
# must not discard the run's verdicts, so the reports are still written.
|
|
153
166
|
def persist_history(result, recorded_at)
|
|
154
167
|
history_store.record(
|
|
155
168
|
result,
|
|
156
169
|
version: Henitai::VERSION,
|
|
157
170
|
recorded_at:
|
|
158
171
|
)
|
|
172
|
+
rescue StandardError => e
|
|
173
|
+
warn "henitai: could not record mutation history (#{e.class}: #{e.message}); reports are still written"
|
|
159
174
|
end
|
|
160
175
|
|
|
161
176
|
def build_result(mutants, started_at, finished_at)
|
|
@@ -177,10 +192,33 @@ module Henitai
|
|
|
177
192
|
git_sha: safe_head_sha,
|
|
178
193
|
source_provider: source_provider,
|
|
179
194
|
authoritative: full_run?,
|
|
180
|
-
since: @since
|
|
195
|
+
since: @since,
|
|
196
|
+
complete_files: complete_files,
|
|
197
|
+
sampling: result_sampling
|
|
181
198
|
)
|
|
182
199
|
end
|
|
183
200
|
|
|
201
|
+
# Files whose every subject this run mutated: no subject patterns and no
|
|
202
|
+
# survivor rerun (a --since run still selects whole changed files).
|
|
203
|
+
def complete_files
|
|
204
|
+
return [] if Array(@subjects).any? || survivor_rerun?
|
|
205
|
+
|
|
206
|
+
source_files
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Records the sampling that actually shaped the mutant set, with the
|
|
210
|
+
# default seed filled in so the report is reproducible on its own.
|
|
211
|
+
def result_sampling
|
|
212
|
+
sampling = optional_config(:sampling)
|
|
213
|
+
return nil if survivor_rerun? || !(sampling && sampling[:ratio])
|
|
214
|
+
|
|
215
|
+
{
|
|
216
|
+
strategy: (sampling[:strategy] || "stratified").to_s,
|
|
217
|
+
ratio: sampling[:ratio],
|
|
218
|
+
seed: sampling[:seed] || SamplingStrategy::DEFAULT_SEED
|
|
219
|
+
}
|
|
220
|
+
end
|
|
221
|
+
|
|
184
222
|
def safe_head_sha
|
|
185
223
|
git_diff_analyzer.head_sha
|
|
186
224
|
rescue StandardError
|
|
@@ -205,6 +243,7 @@ module Henitai
|
|
|
205
243
|
def operators = deps.operators
|
|
206
244
|
def history_store = deps.history_store
|
|
207
245
|
def per_test_coverage = deps.per_test_coverage
|
|
246
|
+
def source_selection_coverage = deps.source_selection_coverage
|
|
208
247
|
def source_provider = deps.source_provider
|
|
209
248
|
|
|
210
249
|
def progress_reporter = deps.progress_reporter(full_run: full_run?)
|
|
@@ -216,7 +255,7 @@ module Henitai
|
|
|
216
255
|
def source_file_selection
|
|
217
256
|
@source_file_selection ||= SourceFileSelection.new(
|
|
218
257
|
config: config, since: @since,
|
|
219
|
-
git_diff_analyzer: git_diff_analyzer, per_test_coverage:
|
|
258
|
+
git_diff_analyzer: git_diff_analyzer, per_test_coverage: source_selection_coverage
|
|
220
259
|
)
|
|
221
260
|
end
|
|
222
261
|
|
|
@@ -12,6 +12,10 @@ module Henitai
|
|
|
12
12
|
# {#per_test_coverage} and {#history_store}: the incremental filter proves
|
|
13
13
|
# survivor reuse against the same live coverage map the history store records
|
|
14
14
|
# its intersection set from. Two instances would be two snapshots.
|
|
15
|
+
#
|
|
16
|
+
# {#source_selection_coverage} is deliberately a *different* instance: Gate 1
|
|
17
|
+
# reads the map as it was before this run's coverage bootstrap rewrote it, and
|
|
18
|
+
# a shared instance would memoize that stale snapshot for the reuse decisions.
|
|
15
19
|
class RunnerDependencies
|
|
16
20
|
def initialize(config:)
|
|
17
21
|
@config = config
|
|
@@ -36,10 +40,27 @@ module Henitai
|
|
|
36
40
|
@per_test_coverage ||= PerTestCoverage.new(reports_dir: @config.reports_dir)
|
|
37
41
|
end
|
|
38
42
|
|
|
43
|
+
def source_selection_coverage
|
|
44
|
+
@source_selection_coverage ||= PerTestCoverage.new(reports_dir: @config.reports_dir)
|
|
45
|
+
end
|
|
46
|
+
|
|
39
47
|
def history_store
|
|
40
48
|
@history_store ||= MutantHistoryStore.new(
|
|
41
49
|
path: File.join(@config.reports_dir, Henitai::HISTORY_STORE_FILENAME),
|
|
42
|
-
per_test_coverage: per_test_coverage
|
|
50
|
+
per_test_coverage: per_test_coverage,
|
|
51
|
+
reuse_context: reuse_context
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Shared by the incremental filter and the history store: verdicts are
|
|
56
|
+
# recorded under exactly the context they are later checked against.
|
|
57
|
+
def reuse_context
|
|
58
|
+
@reuse_context ||= ReuseContext.new(config: @config)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def reuse_filter
|
|
62
|
+
IncrementalFilter.new(
|
|
63
|
+
history_store: history_store, per_test_coverage: per_test_coverage, reuse_context: reuse_context
|
|
43
64
|
)
|
|
44
65
|
end
|
|
45
66
|
|
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "digest"
|
|
3
4
|
module Henitai
|
|
4
|
-
# Samples mutants
|
|
5
|
+
# Samples mutants deterministically, stratified by operator within each subject.
|
|
6
|
+
#
|
|
7
|
+
# Per subject, the sample size is ceil(size * ratio), at least one and at most
|
|
8
|
+
# size. Mutants are grouped into strata by operator. Each stratum is ordered by
|
|
9
|
+
# a seeded SHA256 key derived from the mutant's stable id, so a mutant's rank
|
|
10
|
+
# does not depend on other mutants. Strata are then visited round-robin, which
|
|
11
|
+
# spreads the sample across operators and positions. Kept mutants are returned
|
|
12
|
+
# in their input order.
|
|
5
13
|
class SamplingStrategy
|
|
6
|
-
|
|
14
|
+
DEFAULT_SEED = 0
|
|
15
|
+
|
|
16
|
+
def sample(mutants, ratio:, strategy: :stratified, seed: DEFAULT_SEED)
|
|
7
17
|
strategy = strategy.to_sym if strategy.respond_to?(:to_sym)
|
|
8
18
|
|
|
9
19
|
case strategy
|
|
10
20
|
when :stratified
|
|
11
|
-
stratified_sample(Array(mutants), ratio:)
|
|
21
|
+
stratified_sample(Array(mutants), ratio:, seed:)
|
|
12
22
|
else
|
|
13
23
|
raise ArgumentError, "Unsupported sampling strategy: #{strategy}"
|
|
14
24
|
end
|
|
@@ -16,12 +26,32 @@ module Henitai
|
|
|
16
26
|
|
|
17
27
|
private
|
|
18
28
|
|
|
19
|
-
def stratified_sample(mutants, ratio:)
|
|
29
|
+
def stratified_sample(mutants, ratio:, seed:)
|
|
20
30
|
return [] if ratio.to_f <= 0.0
|
|
21
31
|
|
|
22
|
-
|
|
23
|
-
|
|
32
|
+
kept = Set.new.compare_by_identity
|
|
33
|
+
mutants.group_by { |mutant| mutant.subject.expression }.each_value do |group|
|
|
34
|
+
select_within_subject(group, ratio:, seed:).each { |mutant| kept << mutant }
|
|
24
35
|
end
|
|
36
|
+
mutants.select { |mutant| kept.include?(mutant) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def select_within_subject(group, ratio:, seed:)
|
|
40
|
+
strata = group.group_by { |mutant| mutant.operator.to_s }.sort.map do |_operator, members|
|
|
41
|
+
members.sort_by { |mutant| sort_key(mutant, seed) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
round_robin(strata, sample_count(group.size, ratio))
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def round_robin(strata, count)
|
|
48
|
+
depth = strata.map(&:size).max || 0
|
|
49
|
+
|
|
50
|
+
(0...depth).flat_map { |index| strata.filter_map { |stratum| stratum[index] } }.first(count)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def sort_key(mutant, seed)
|
|
54
|
+
Digest::SHA256.hexdigest("#{seed}\0#{mutant.stable_id}")
|
|
25
55
|
end
|
|
26
56
|
|
|
27
57
|
def sample_count(size, ratio)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Henitai
|
|
4
4
|
# Captures the result of one baseline or mutant test run.
|
|
5
5
|
class ScenarioExecutionResult
|
|
6
|
-
attr_reader :status, :stdout, :stderr, :exit_status, :log_path
|
|
6
|
+
attr_reader :status, :status_reason, :stdout, :stderr, :exit_status, :log_path, :killing_tests
|
|
7
7
|
|
|
8
8
|
def self.build(wait_result:, stdout:, stderr:, log_path:)
|
|
9
9
|
new(
|
|
@@ -15,12 +15,32 @@ module Henitai
|
|
|
15
15
|
)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
# Copies the reason behind a harness status onto the mutant the result was
|
|
19
|
+
# produced for. Tolerates results and mutants that carry no reason.
|
|
20
|
+
def self.copy_reason(result, mutant)
|
|
21
|
+
return unless result.respond_to?(:status_reason) && mutant.respond_to?(:status_reason=)
|
|
22
|
+
|
|
23
|
+
mutant.status_reason = result.status_reason
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Classifies a mutant run from the child's report (see MutantVerdict).
|
|
27
|
+
def self.build_for_mutant(wait_result:, report:, stdout:, stderr:, log_path:)
|
|
28
|
+
verdict = MutantVerdict.new(wait_result:, report:, stdout:, stderr:)
|
|
29
|
+
status, status_reason = verdict.to_a
|
|
30
|
+
new(status:, status_reason:, stdout:, stderr:, log_path:, exit_status: exit_status_for(wait_result),
|
|
31
|
+
killing_tests: verdict.killing_tests)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# rubocop:disable Metrics/ParameterLists -- one keyword per captured field
|
|
35
|
+
def initialize(status:, stdout:, stderr:, log_path:, exit_status: nil, status_reason: nil, killing_tests: [])
|
|
36
|
+
# rubocop:enable Metrics/ParameterLists
|
|
19
37
|
@status = status
|
|
38
|
+
@status_reason = status_reason
|
|
20
39
|
@stdout = stdout.to_s
|
|
21
40
|
@stderr = stderr.to_s
|
|
22
41
|
@log_path = log_path
|
|
23
42
|
@exit_status = exit_status
|
|
43
|
+
@killing_tests = killing_tests
|
|
24
44
|
end
|
|
25
45
|
|
|
26
46
|
# Frees the captured child output once it has been consumed (e.g. by the
|
|
@@ -107,14 +127,17 @@ module Henitai
|
|
|
107
127
|
end
|
|
108
128
|
|
|
109
129
|
def no_examples_found?(stdout, stderr)
|
|
110
|
-
stdout.to_s
|
|
130
|
+
output = [stdout.to_s, stderr.to_s].join("\n")
|
|
131
|
+
return false unless ExampleSummary::NO_EXAMPLES_FOUND.match?(output)
|
|
132
|
+
|
|
133
|
+
last_summary = output.scan(ExampleSummary::SUMMARY).last
|
|
134
|
+
last_summary.nil? || last_summary == ["0"]
|
|
111
135
|
end
|
|
112
136
|
|
|
113
137
|
def zero_examples_failure?(wait_result, stdout, stderr)
|
|
114
138
|
return false unless exit_status_for(wait_result) == 1
|
|
115
139
|
|
|
116
|
-
|
|
117
|
-
output.include?("0 examples, 0 failures")
|
|
140
|
+
ExampleSummary.none_ran?([stdout.to_s, stderr.to_s].join("\n"))
|
|
118
141
|
end
|
|
119
142
|
end
|
|
120
143
|
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
# Whether a mutant sits on its subject's def (signature) line. Line coverage
|
|
5
|
+
# credits that line only to the test that loaded the file, so the tests that
|
|
6
|
+
# exercise it are not knowable from the per-test map alone.
|
|
7
|
+
module SignatureLine
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# A subject without a source range (a recipe stub) has an unknown def
|
|
11
|
+
# line, so it is treated as one: that only widens test selection and
|
|
12
|
+
# blocks verdict reuse.
|
|
13
|
+
def on?(mutant)
|
|
14
|
+
subject = mutant.respond_to?(:subject) ? mutant.subject : nil
|
|
15
|
+
return false unless subject.respond_to?(:source_range)
|
|
16
|
+
|
|
17
|
+
range = subject.source_range
|
|
18
|
+
range.nil? || range.begin == mutant.location[:start_line]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -80,9 +80,17 @@ module Henitai
|
|
|
80
80
|
end
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
+
# Unrelated events (CHLD of another slot, server traffic) wake the select
|
|
84
|
+
# early; the SIGTERM grace window must still run its full length.
|
|
83
85
|
def wait_for_drain_window
|
|
84
|
-
wakeup
|
|
85
|
-
|
|
86
|
+
return unless wakeup
|
|
87
|
+
|
|
88
|
+
deadline = monotonic_time + PROCESS_DRAIN_WINDOW
|
|
89
|
+
while (remaining = deadline - monotonic_time).positive?
|
|
90
|
+
woken = wakeup.wait(remaining)
|
|
91
|
+
wakeup.drain
|
|
92
|
+
break unless woken
|
|
93
|
+
end
|
|
86
94
|
end
|
|
87
95
|
|
|
88
96
|
def signal_draining_slots(draining)
|
|
@@ -115,6 +123,7 @@ module Henitai
|
|
|
115
123
|
def reap_and_finalize_slot(slot)
|
|
116
124
|
_, final_status = wnohang_reap(slot.pid)
|
|
117
125
|
reap_pid(slot.pid) unless final_status
|
|
126
|
+
tear_down_process_group(slot.pid)
|
|
118
127
|
|
|
119
128
|
slot_table.release_pid(slot.pid)
|
|
120
129
|
slot_table.delete(slot.slot_id)
|
|
@@ -128,6 +137,7 @@ module Henitai
|
|
|
128
137
|
def record_drain_result(slot, final_status)
|
|
129
138
|
result = drain_verdict.build(slot, final_status)
|
|
130
139
|
slot.mutant.status = result.status
|
|
140
|
+
ScenarioExecutionResult.copy_reason(result, slot.mutant)
|
|
131
141
|
results << result
|
|
132
142
|
progress_reporter&.progress(slot.mutant, scenario_result: result)
|
|
133
143
|
end
|
|
@@ -8,6 +8,9 @@ module Henitai
|
|
|
8
8
|
# scheduler remains the single caller of the process table. Mixed into
|
|
9
9
|
# {SlotScheduler}; relies on its +runtime+ reader.
|
|
10
10
|
module ProcessControl
|
|
11
|
+
GROUP_POLL_INTERVAL = 0.02
|
|
12
|
+
GROUP_POLL_COUNT = 10
|
|
13
|
+
|
|
11
14
|
private
|
|
12
15
|
|
|
13
16
|
def monotonic_time
|
|
@@ -33,6 +36,36 @@ module Henitai
|
|
|
33
36
|
end
|
|
34
37
|
end
|
|
35
38
|
|
|
39
|
+
# Terminates whatever a finished child left behind in its process group
|
|
40
|
+
# (daemons, grandchildren). Waits at most PROCESS_DRAIN_WINDOW for them to
|
|
41
|
+
# exit on SIGTERM before escalating to SIGKILL.
|
|
42
|
+
def tear_down_process_group(pid)
|
|
43
|
+
signal_process_group(pid, :SIGTERM)
|
|
44
|
+
return unless group_members_remain?(pid)
|
|
45
|
+
|
|
46
|
+
signal_process_group(pid, :SIGKILL)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def group_members_remain?(pid)
|
|
50
|
+
GROUP_POLL_COUNT.times do
|
|
51
|
+
return false unless process_group_alive?(pid)
|
|
52
|
+
|
|
53
|
+
runtime.pause(GROUP_POLL_INTERVAL)
|
|
54
|
+
end
|
|
55
|
+
process_group_alive?(pid)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Signal 0 probes existence only. ESRCH means the group is empty; EPERM
|
|
59
|
+
# means members exist but belong to someone else.
|
|
60
|
+
def process_group_alive?(pid)
|
|
61
|
+
runtime.kill(0, -pid)
|
|
62
|
+
true
|
|
63
|
+
rescue Errno::ESRCH
|
|
64
|
+
false
|
|
65
|
+
rescue Errno::EPERM
|
|
66
|
+
true
|
|
67
|
+
end
|
|
68
|
+
|
|
36
69
|
def reap_pid(pid)
|
|
37
70
|
runtime.wait(pid)
|
|
38
71
|
rescue Errno::ECHILD, Errno::ESRCH
|
|
@@ -4,17 +4,19 @@ module Henitai
|
|
|
4
4
|
class SlotScheduler
|
|
5
5
|
# Decides whether a finished slot earns another attempt.
|
|
6
6
|
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
# run down would leak processes past the drain window.
|
|
7
|
+
# The rule itself lives in FlakyRetryDecision; this class adds the shutdown
|
|
8
|
+
# veto. A requested shutdown vetoes retries outright — respawning children
|
|
9
|
+
# while tearing the run down would leak processes past the drain window.
|
|
11
10
|
class RetryPolicy
|
|
12
|
-
def initialize(max_retries:)
|
|
11
|
+
def initialize(max_retries:, flaky_tests: FlakyTestRegistry.new([]))
|
|
13
12
|
@max_retries = max_retries.to_i
|
|
13
|
+
@decision = FlakyRetryDecision.new(flaky_tests: flaky_tests, max_retries: @max_retries)
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def retry?(slot:, result:, shutdown:)
|
|
17
|
-
|
|
17
|
+
return false if shutdown || result.status != :killed
|
|
18
|
+
|
|
19
|
+
@decision.retry?(status: :killed, killing_tests: result.killing_tests, attempts: slot.retry_count)
|
|
18
20
|
end
|
|
19
21
|
end
|
|
20
22
|
end
|
|
@@ -51,7 +51,8 @@ module Henitai
|
|
|
51
51
|
# rubocop:disable Metrics/ParameterLists -- every one of these is a distinct
|
|
52
52
|
# collaborator supplied by ProcessWorkerRunner; bundling them into a context
|
|
53
53
|
# object would only move the list.
|
|
54
|
-
def initialize(integration:, config:, progress_reporter:, options:, host:, slot_table: SlotTable.new
|
|
54
|
+
def initialize(integration:, config:, progress_reporter:, options:, host:, slot_table: SlotTable.new,
|
|
55
|
+
flaky_tests: nil)
|
|
55
56
|
# rubocop:enable Metrics/ParameterLists
|
|
56
57
|
@integration = integration
|
|
57
58
|
@config = config
|
|
@@ -59,6 +60,7 @@ module Henitai
|
|
|
59
60
|
@options = options
|
|
60
61
|
@host = host
|
|
61
62
|
@slot_table = slot_table
|
|
63
|
+
@injected_flaky_tests = flaky_tests
|
|
62
64
|
@pending = []
|
|
63
65
|
@results = []
|
|
64
66
|
@flaky_retry_count = 0
|
|
@@ -119,7 +121,7 @@ module Henitai
|
|
|
119
121
|
worker_index = slot_table.next_free_worker_index(worker_count)
|
|
120
122
|
with_worker_slot(worker_index) do
|
|
121
123
|
handle = integration.spawn_mutant(mutant: mutant, test_files: test_files)
|
|
122
|
-
register_slot(handle, mutant, worker_index, slot_timeout(mutant, test_files))
|
|
124
|
+
register_slot(handle, mutant, worker_index, handle.timeout || slot_timeout(mutant, test_files))
|
|
123
125
|
end
|
|
124
126
|
rescue StandardError => e
|
|
125
127
|
record_spawn_failure(mutant, e)
|
|
@@ -154,14 +156,21 @@ module Henitai
|
|
|
154
156
|
resolver ? resolver.call(mutant, test_files) : config.timeout
|
|
155
157
|
end
|
|
156
158
|
|
|
159
|
+
# Every path that reaps a mutant child ends here, so this is the single
|
|
160
|
+
# place that tears down the child's process group. A reaped pid that owns
|
|
161
|
+
# no slot (wait2(-1) can reap any helper child) is never signalled.
|
|
157
162
|
def complete_slot(pid, wait_result)
|
|
158
163
|
slot_id = slot_table.release_pid(pid)
|
|
159
164
|
return unless slot_id
|
|
160
165
|
|
|
166
|
+
tear_down_process_group(pid)
|
|
167
|
+
|
|
161
168
|
slot = slot_table.fetch(slot_id)
|
|
162
169
|
return unless slot
|
|
163
170
|
|
|
164
171
|
Integration::SchedulerDiagnostics.child_ended(pid)
|
|
172
|
+
return rerun_lost_slot(slot) if lost?(wait_result)
|
|
173
|
+
|
|
165
174
|
result = integration.build_result(wait_result, slot.log_paths)
|
|
166
175
|
dispatch_slot_result(slot, result)
|
|
167
176
|
end
|
|
@@ -175,6 +184,7 @@ module Henitai
|
|
|
175
184
|
def finalize_slot(slot, result)
|
|
176
185
|
slot_table.delete(slot.slot_id)
|
|
177
186
|
slot.mutant.status = result.status
|
|
187
|
+
ScenarioExecutionResult.copy_reason(result, slot.mutant)
|
|
178
188
|
results << result
|
|
179
189
|
progress_reporter&.progress(slot.mutant, scenario_result: result)
|
|
180
190
|
result.release_output! if result.respond_to?(:release_output!)
|
|
@@ -184,7 +194,13 @@ module Henitai
|
|
|
184
194
|
# the runner builds when nothing is pending) pass `config: nil`, and this
|
|
185
195
|
# is only reached once a slot has actually finished.
|
|
186
196
|
def retry_policy
|
|
187
|
-
@retry_policy ||= RetryPolicy.new(max_retries: config.max_flaky_retries)
|
|
197
|
+
@retry_policy ||= RetryPolicy.new(max_retries: config.max_flaky_retries, flaky_tests: flaky_tests)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def flaky_tests
|
|
201
|
+
return @injected_flaky_tests if @injected_flaky_tests
|
|
202
|
+
|
|
203
|
+
FlakyTestRegistry.for_config(config)
|
|
188
204
|
end
|
|
189
205
|
|
|
190
206
|
def drain_verdict
|
|
@@ -199,20 +215,43 @@ module Henitai
|
|
|
199
215
|
@test_file_selection ||= TestFileSelection.new(options: options, integration: integration)
|
|
200
216
|
end
|
|
201
217
|
|
|
218
|
+
# The fork server died while the mutant ran, so nothing is known about
|
|
219
|
+
# it. It is run again on the local path and never becomes a verdict.
|
|
220
|
+
def lost?(wait_result)
|
|
221
|
+
wait_result.respond_to?(:lost?) && wait_result.lost?
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def rerun_lost_slot(slot)
|
|
225
|
+
return slot_table.delete(slot.slot_id) if shutdown?
|
|
226
|
+
|
|
227
|
+
respawn_slot(slot) { |handle| restart_slot(slot, handle) }
|
|
228
|
+
end
|
|
229
|
+
|
|
202
230
|
def retry_slot(slot)
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
231
|
+
respawn_slot(slot) { |handle| finish_retry(slot, handle) }
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def respawn_slot(slot)
|
|
235
|
+
with_worker_slot(slot.worker_index) { yield respawn_handle(slot) }
|
|
208
236
|
rescue StandardError => e
|
|
209
237
|
slot_table.delete(slot.slot_id)
|
|
210
238
|
record_spawn_failure(slot.mutant, e)
|
|
211
239
|
end
|
|
212
240
|
|
|
241
|
+
def respawn_handle(slot)
|
|
242
|
+
test_files = test_file_selection.for(slot.mutant)
|
|
243
|
+
handle = integration.spawn_mutant(mutant: slot.mutant, test_files: test_files)
|
|
244
|
+
slot.timeout = handle.timeout || slot_timeout(slot.mutant, test_files)
|
|
245
|
+
handle
|
|
246
|
+
end
|
|
247
|
+
|
|
213
248
|
def finish_retry(slot, handle)
|
|
214
249
|
@flaky_retry_count += 1 if slot.retry_count.zero?
|
|
215
250
|
slot.retry_count += 1
|
|
251
|
+
restart_slot(slot, handle)
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def restart_slot(slot, handle)
|
|
216
255
|
reset_slot_for_retry(slot, handle)
|
|
217
256
|
slot_table.register_pid(handle.pid, slot.slot_id)
|
|
218
257
|
Integration::SchedulerDiagnostics.child_started(handle.pid)
|
|
@@ -58,7 +58,7 @@ module Henitai
|
|
|
58
58
|
# dirty and untracked files): the working tree is what gets tested, so it is
|
|
59
59
|
# always part of "changed since REF".
|
|
60
60
|
def changed_paths_since
|
|
61
|
-
@git_diff_analyzer.
|
|
61
|
+
@git_diff_analyzer.changed_files_since(@since) +
|
|
62
62
|
@git_diff_analyzer.working_tree_changed_files
|
|
63
63
|
end
|
|
64
64
|
|