henitai 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +222 -1
- data/README.md +100 -13
- data/assets/schema/henitai.schema.json +20 -0
- data/lib/henitai/arid_node_filter.rb +4 -19
- data/lib/henitai/bootstrap_error_message.rb +20 -0
- data/lib/henitai/canonical_report_merger.rb +23 -2
- data/lib/henitai/cli/run_command.rb +25 -7
- data/lib/henitai/cli/run_options.rb +1 -1
- data/lib/henitai/cli.rb +4 -1
- data/lib/henitai/configuration.rb +4 -1
- data/lib/henitai/configuration_validator/rules.rb +8 -0
- data/lib/henitai/configuration_validator/scalars.rb +9 -0
- data/lib/henitai/configuration_validator.rb +2 -1
- data/lib/henitai/constant_path.rb +48 -0
- data/lib/henitai/control_run.rb +73 -0
- data/lib/henitai/coverage_bootstrapper.rb +36 -10
- data/lib/henitai/coverage_formatter.rb +26 -3
- data/lib/henitai/coverage_report_reader.rb +21 -0
- data/lib/henitai/equivalence_detector/operand_predicates.rb +16 -0
- data/lib/henitai/equivalence_detector.rb +46 -38
- data/lib/henitai/example_summary.rb +24 -0
- data/lib/henitai/execution_engine/fork_server_scope.rb +57 -0
- data/lib/henitai/execution_engine.rb +75 -60
- data/lib/henitai/fidelity_check.rb +173 -0
- data/lib/henitai/flaky_detection_baseline.rb +61 -0
- data/lib/henitai/flaky_mutant_retrier.rb +43 -0
- data/lib/henitai/flaky_retry_decision.rb +23 -0
- data/lib/henitai/flaky_test_registry.rb +63 -0
- data/lib/henitai/fork_server/client.rb +231 -0
- data/lib/henitai/fork_server/group_killer.rb +48 -0
- data/lib/henitai/fork_server/group_terminator.rb +47 -0
- data/lib/henitai/fork_server/launcher.rb +38 -0
- data/lib/henitai/fork_server/mutant_runner.rb +141 -0
- data/lib/henitai/fork_server/protocol.rb +73 -0
- data/lib/henitai/fork_server/reaper.rb +24 -0
- data/lib/henitai/fork_server/runtime.rb +109 -0
- data/lib/henitai/fork_server/server.rb +108 -0
- data/lib/henitai/fork_server/server_process.rb +73 -0
- data/lib/henitai/fork_server/status.rb +52 -0
- data/lib/henitai/fork_server.rb +43 -0
- data/lib/henitai/generated_artifacts.rb +3 -0
- data/lib/henitai/git_diff_analyzer.rb +28 -11
- data/lib/henitai/incremental_filter.rb +20 -10
- data/lib/henitai/integration/base.rb +72 -5
- data/lib/henitai/integration/capped_log_sink.rb +34 -0
- data/lib/henitai/integration/child_bootstrap.rb +7 -0
- data/lib/henitai/integration/child_report_store.rb +36 -0
- data/lib/henitai/integration/minitest.rb +7 -5
- data/lib/henitai/integration/minitest_autorun.rb +20 -0
- data/lib/henitai/integration/minitest_outcome_reporter.rb +28 -0
- data/lib/henitai/integration/minitest_preload.rb +30 -0
- data/lib/henitai/integration/minitest_runnable_filter.rb +61 -0
- data/lib/henitai/integration/minitest_test_runner.rb +32 -15
- data/lib/henitai/integration/mutant_run_support.rb +73 -13
- data/lib/henitai/integration/rspec_child_runner.rb +44 -9
- data/lib/henitai/integration/rspec_preload.rb +35 -0
- data/lib/henitai/integration/rspec_process_runner.rb +4 -1
- data/lib/henitai/integration/scenario_log_support.rb +31 -18
- data/lib/henitai/integration.rb +16 -3
- data/lib/henitai/minitest_coverage_reporter.rb +22 -6
- data/lib/henitai/minitest_result_file.rb +62 -0
- data/lib/henitai/mutant/activation.rb +32 -0
- data/lib/henitai/mutant/activator.rb +75 -55
- data/lib/henitai/mutant/def_injection.rb +136 -0
- data/lib/henitai/mutant/lexical_scope_check.rb +120 -0
- data/lib/henitai/mutant/lexical_target.rb +105 -0
- data/lib/henitai/mutant/replacement.rb +52 -0
- data/lib/henitai/mutant.rb +27 -8
- data/lib/henitai/mutant_generator.rb +62 -18
- data/lib/henitai/mutant_history_store/verdict_cache.rb +12 -11
- data/lib/henitai/mutant_history_store.rb +14 -4
- data/lib/henitai/mutant_identity.rb +5 -1
- data/lib/henitai/mutant_verdict.rb +71 -0
- data/lib/henitai/operators/array_declaration.rb +4 -0
- data/lib/henitai/operators/assignment_expression.rb +2 -0
- data/lib/henitai/operators/conditional_expression.rb +21 -1
- data/lib/henitai/operators/hash_literal.rb +7 -4
- data/lib/henitai/operators/method_expression.rb +14 -3
- data/lib/henitai/operators/regex_metacharacters.rb +89 -0
- data/lib/henitai/operators/regex_mutator.rb +23 -28
- data/lib/henitai/operators/return_value.rb +1 -1
- data/lib/henitai/operators.rb +1 -0
- data/lib/henitai/per_test_coverage.rb +11 -7
- data/lib/henitai/per_test_coverage_collector.rb +30 -31
- data/lib/henitai/per_test_coverage_delta.rb +60 -0
- data/lib/henitai/per_test_coverage_selector.rb +26 -13
- data/lib/henitai/process_wakeup.rb +8 -3
- data/lib/henitai/process_worker_runner.rb +19 -5
- data/lib/henitai/reporter.rb +15 -8
- data/lib/henitai/result.rb +32 -16
- data/lib/henitai/reuse_context.rb +50 -0
- data/lib/henitai/rspec_coverage_formatter.rb +4 -0
- data/lib/henitai/rspec_outcomes.rb +72 -0
- data/lib/henitai/runner.rb +58 -19
- data/lib/henitai/runner_dependencies.rb +22 -1
- data/lib/henitai/sampling_strategy.rb +36 -6
- data/lib/henitai/scenario_execution_result.rb +28 -5
- data/lib/henitai/signature_line.rb +21 -0
- data/lib/henitai/slot_scheduler/draining.rb +12 -2
- data/lib/henitai/slot_scheduler/process_control.rb +33 -0
- data/lib/henitai/slot_scheduler/retry_policy.rb +8 -6
- data/lib/henitai/slot_scheduler.rb +47 -8
- data/lib/henitai/source_file_selection.rb +1 -1
- data/lib/henitai/source_parser.rb +29 -9
- data/lib/henitai/static_filter.rb +45 -14
- data/lib/henitai/stillborn_filter.rb +70 -2
- data/lib/henitai/subject.rb +8 -3
- data/lib/henitai/subject_resolver.rb +65 -31
- data/lib/henitai/suite_boot_clock.rb +48 -0
- data/lib/henitai/survivor_activation_cache.rb +18 -11
- data/lib/henitai/survivor_rerun_strategy.rb +47 -43
- data/lib/henitai/test_file_outcomes.rb +79 -0
- data/lib/henitai/timeout_calibrator.rb +31 -9
- data/lib/henitai/timeout_ceiling.rb +37 -0
- data/lib/henitai/verdict_fingerprint.rb +29 -30
- data/lib/henitai/version.rb +1 -1
- data/lib/henitai.rb +17 -1
- data/sig/configuration_validator.rbs +1 -0
- data/sig/henitai.rbs +225 -45
- metadata +46 -3
- data/lib/henitai/survivor_test_filter.rb +0 -72
data/lib/henitai/result.rb
CHANGED
|
@@ -26,9 +26,8 @@ module Henitai
|
|
|
26
26
|
process_abort: true
|
|
27
27
|
}.freeze
|
|
28
28
|
|
|
29
|
-
# Maps each criterion to the mutant status it governs.
|
|
30
|
-
#
|
|
31
|
-
# so that entry is forward-looking wiring rather than a live code path.
|
|
29
|
+
# Maps each criterion to the mutant status it governs. MutantVerdict emits
|
|
30
|
+
# :runtime_error for process aborts, signals, and unknown child outcomes.
|
|
32
31
|
CRITERION_STATUSES = {
|
|
33
32
|
test_result: :killed,
|
|
34
33
|
timeout: :timeout,
|
|
@@ -36,7 +35,7 @@ module Henitai
|
|
|
36
35
|
}.freeze
|
|
37
36
|
|
|
38
37
|
attr_reader :mutants, :started_at, :finished_at, :thresholds, :survivor_stats,
|
|
39
|
-
:session_id, :git_sha, :since, :coverage_criteria
|
|
38
|
+
:session_id, :git_sha, :since, :coverage_criteria, :complete_files, :sampling
|
|
40
39
|
|
|
41
40
|
# @param source_provider [#call] maps a file path to its source string.
|
|
42
41
|
# Injected so the domain object performs no disk IO; the caller (which
|
|
@@ -48,12 +47,16 @@ module Henitai
|
|
|
48
47
|
# canonical report; non-authoritative (subject-scoped, --since,
|
|
49
48
|
# --survivors-from) runs are merged into it instead — see
|
|
50
49
|
# CanonicalReportMerger.
|
|
50
|
+
# @param complete_files [Array<String>, nil] source files this run mutated
|
|
51
|
+
# completely (every subject). The merger replaces a prior report's entry
|
|
52
|
+
# for such a file wholesale instead of merging by stableId, so stale
|
|
53
|
+
# mutants of changed or deleted code do not linger.
|
|
51
54
|
# rubocop:disable Metrics/ParameterLists
|
|
52
55
|
def initialize(mutants:, started_at:, finished_at:, thresholds: nil,
|
|
53
56
|
partial_rerun: false, survivor_stats: nil,
|
|
54
57
|
session_id: SecureRandom.uuid, git_sha: nil,
|
|
55
58
|
source_provider: ->(_file) { "" }, authoritative: true,
|
|
56
|
-
since: nil, coverage_criteria: nil)
|
|
59
|
+
since: nil, coverage_criteria: nil, complete_files: nil, sampling: nil)
|
|
57
60
|
@mutants = mutants
|
|
58
61
|
@started_at = started_at
|
|
59
62
|
@finished_at = finished_at
|
|
@@ -65,7 +68,9 @@ module Henitai
|
|
|
65
68
|
@source_provider = source_provider
|
|
66
69
|
@authoritative = authoritative
|
|
67
70
|
@since = since
|
|
71
|
+
@complete_files = Array(complete_files)
|
|
68
72
|
@coverage_criteria = DEFAULT_COVERAGE_CRITERIA.merge(coverage_criteria || {})
|
|
73
|
+
@sampling = sampling
|
|
69
74
|
end
|
|
70
75
|
# rubocop:enable Metrics/ParameterLists
|
|
71
76
|
|
|
@@ -89,11 +94,12 @@ module Henitai
|
|
|
89
94
|
|
|
90
95
|
# Mutation Score (MS) — Architektur-Formel aus Abschnitt 6.1:
|
|
91
96
|
#
|
|
92
|
-
# MS = detected / (total − ignored − no_coverage − compile_error − equivalent)
|
|
97
|
+
# MS = detected / (total − ignored − no_coverage − compile_error − equivalent − pending)
|
|
93
98
|
#
|
|
94
99
|
# Confirmed equivalent mutants werden aus BEIDEN Seiten der Gleichung entfernt:
|
|
95
100
|
# Sie sind weder im Zähler (nicht detektierbar) noch im Nenner (nicht testbar).
|
|
96
|
-
# Das ist der entscheidende Unterschied zum MSI.
|
|
101
|
+
# Das ist der entscheidende Unterschied zum MSI. Pending mutants (left over from
|
|
102
|
+
# an interrupted run, without a verdict) are excluded from both sides as well.
|
|
97
103
|
#
|
|
98
104
|
# @return [Float, nil] 0.0–100.0, nil wenn kein valider Mutant vorhanden
|
|
99
105
|
def mutation_score
|
|
@@ -102,8 +108,9 @@ module Henitai
|
|
|
102
108
|
|
|
103
109
|
# Mutation Score Indicator (MSI) — naive Berechnung ohne Äquivalenz-Bereinigung:
|
|
104
110
|
#
|
|
105
|
-
# MSI = killed /
|
|
111
|
+
# MSI = killed / (total − pending)
|
|
106
112
|
#
|
|
113
|
+
# Only mutants with a verdict count; pending mutants have none.
|
|
107
114
|
# MSI ist immer ≤ MS. Der Unterschied kommuniziert die Äquivalenz-Unsicherheit.
|
|
108
115
|
# Beide Werte MÜSSEN im Report zusammen ausgewiesen werden (Anti-Pattern: nur MS).
|
|
109
116
|
#
|
|
@@ -151,6 +158,8 @@ module Henitai
|
|
|
151
158
|
schema = base_schema
|
|
152
159
|
sha = @git_sha
|
|
153
160
|
schema[:gitSha] = sha if sha
|
|
161
|
+
schema[:completeFiles] = complete_files if complete_files.any? && !authoritative?
|
|
162
|
+
schema[:sampling] = @sampling if @sampling
|
|
154
163
|
return schema unless partial_rerun?
|
|
155
164
|
|
|
156
165
|
schema[:partialRerun] = true
|
|
@@ -171,7 +180,7 @@ module Henitai
|
|
|
171
180
|
end
|
|
172
181
|
|
|
173
182
|
def mutation_score_for(list)
|
|
174
|
-
excluded = %i[ignored no_coverage compile_error equivalent]
|
|
183
|
+
excluded = %i[ignored no_coverage compile_error equivalent pending]
|
|
175
184
|
valid = list.reject { |m| excluded.include?(m.status) }
|
|
176
185
|
return nil if valid.empty?
|
|
177
186
|
|
|
@@ -179,9 +188,10 @@ module Henitai
|
|
|
179
188
|
end
|
|
180
189
|
|
|
181
190
|
def mutation_score_indicator_for(list)
|
|
182
|
-
|
|
191
|
+
decided = list.reject { |m| m.status == :pending }
|
|
192
|
+
return nil if decided.empty?
|
|
183
193
|
|
|
184
|
-
((
|
|
194
|
+
((decided.count(&:killed?).to_f / decided.count) * 100.0).round(2).to_f
|
|
185
195
|
end
|
|
186
196
|
|
|
187
197
|
def base_schema
|
|
@@ -205,10 +215,16 @@ module Henitai
|
|
|
205
215
|
language: "ruby",
|
|
206
216
|
source: @source_provider.call(file_mutants.first.location[:file]),
|
|
207
217
|
mutants: file_mutants.map { |m| mutant_to_schema(m) }
|
|
208
|
-
}
|
|
218
|
+
}.merge(file_provenance)
|
|
209
219
|
end
|
|
210
220
|
end
|
|
211
221
|
|
|
222
|
+
# Vendor fields: which run produced this file entry. Kept per file so a
|
|
223
|
+
# merged report can tell which session snapshot holds its mutants.
|
|
224
|
+
def file_provenance
|
|
225
|
+
{ sessionId: @session_id, gitSha: @git_sha }.compact
|
|
226
|
+
end
|
|
227
|
+
|
|
212
228
|
def mutant_to_schema(mutant)
|
|
213
229
|
{
|
|
214
230
|
id: mutant.id,
|
|
@@ -261,11 +277,11 @@ module Henitai
|
|
|
261
277
|
end
|
|
262
278
|
|
|
263
279
|
# Serialized as the schema's statusReason so directive reasons show up
|
|
264
|
-
# next to the Ignored status
|
|
280
|
+
# next to the Ignored status, and harness reasons (a failed fidelity
|
|
281
|
+
# check, for example) next to CompileError, in the HTML report.
|
|
265
282
|
def status_reason_for(mutant)
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
mutant.ignore_reason
|
|
283
|
+
reason = mutant.status_reason if mutant.respond_to?(:status_reason)
|
|
284
|
+
reason || (mutant.ignore_reason if mutant.respond_to?(:ignore_reason))
|
|
269
285
|
end
|
|
270
286
|
|
|
271
287
|
# Vendored extension field (like stableId): marks verdicts reused from
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
require_relative "verdict_fingerprint"
|
|
7
|
+
|
|
8
|
+
module Henitai
|
|
9
|
+
# Run-level fingerprint of everything outside a mutant and its covering
|
|
10
|
+
# tests that can change a verdict: the test dependency files, the
|
|
11
|
+
# verdict-relevant configuration and the henitai version. Recorded with every
|
|
12
|
+
# cached Killed/Survived verdict and required to match for reuse.
|
|
13
|
+
#
|
|
14
|
+
# Computed once per instance, so the filter that reads verdicts and the
|
|
15
|
+
# history store that writes them always agree. nil when the dependency files
|
|
16
|
+
# cannot be read — nothing is then reusable.
|
|
17
|
+
class ReuseContext
|
|
18
|
+
def initialize(config:, version: Henitai::VERSION, root: Dir.pwd)
|
|
19
|
+
@config = config
|
|
20
|
+
@version = version
|
|
21
|
+
@root = root
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def fingerprint
|
|
25
|
+
return @fingerprint if defined?(@fingerprint)
|
|
26
|
+
|
|
27
|
+
@fingerprint = compute
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def compute
|
|
33
|
+
dependencies = VerdictFingerprint.dependency_fingerprint(@root)
|
|
34
|
+
return nil unless dependencies
|
|
35
|
+
|
|
36
|
+
Digest::SHA256.hexdigest(JSON.generate(components(dependencies)))
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def components(dependencies)
|
|
40
|
+
{
|
|
41
|
+
"dependencies" => dependencies,
|
|
42
|
+
"version" => @version,
|
|
43
|
+
"integration" => @config.integration.to_s,
|
|
44
|
+
"operators" => @config.operators.to_s,
|
|
45
|
+
"test_excludes" => Array(@config.test_excludes).map(&:to_s).sort,
|
|
46
|
+
"coverage_criteria" => @config.coverage_criteria.to_h.sort_by { |key, _| key.to_s }
|
|
47
|
+
}
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "henitai/test_file_outcomes"
|
|
4
|
+
|
|
5
|
+
module Henitai
|
|
6
|
+
# Records the outcome of each finished RSpec example per spec file.
|
|
7
|
+
#
|
|
8
|
+
# Failures outside an example produce no failed example. A hook error in
|
|
9
|
+
# `before(:context)` fails the group's examples, so it is attributed
|
|
10
|
+
# already; an `after(:context)` error is announced as a message while its
|
|
11
|
+
# group is open and is attributed to that group's file. Everything else
|
|
12
|
+
# counted by `errors_outside_of_examples_count` (a spec file that fails to
|
|
13
|
+
# load, `before(:suite)`) has no file and is recorded as unattributed.
|
|
14
|
+
module RspecOutcomes
|
|
15
|
+
EVENTS = %i[example_finished example_group_started example_group_finished message dump_summary].freeze
|
|
16
|
+
HOOK_ERROR = "An error occurred in "
|
|
17
|
+
|
|
18
|
+
# RSpec drops finished examples from its groups after a run, so
|
|
19
|
+
# outcomes are collected as the reporter announces them.
|
|
20
|
+
class Listener
|
|
21
|
+
def initialize(outcomes)
|
|
22
|
+
@outcomes = outcomes
|
|
23
|
+
@open_group_files = []
|
|
24
|
+
@attributed_errors = 0
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def example_finished(notification)
|
|
28
|
+
RspecOutcomes.record(@outcomes, notification.example)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def example_group_started(notification)
|
|
32
|
+
@open_group_files.push(RspecOutcomes.group_file(notification.group))
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def example_group_finished(_notification)
|
|
36
|
+
@open_group_files.pop
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def message(notification)
|
|
40
|
+
file = @open_group_files.last
|
|
41
|
+
return unless file && notification.message.strip.start_with?(HOOK_ERROR)
|
|
42
|
+
|
|
43
|
+
@outcomes.record(file, failed: true)
|
|
44
|
+
@attributed_errors += 1
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def dump_summary(summary)
|
|
48
|
+
return unless summary.errors_outside_of_examples_count > @attributed_errors
|
|
49
|
+
|
|
50
|
+
@outcomes.record_unattributed_failure
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
module_function
|
|
55
|
+
|
|
56
|
+
def register(outcomes, reporter)
|
|
57
|
+
reporter.register_listener(Listener.new(outcomes), *EVENTS)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def group_file(group)
|
|
61
|
+
metadata = group.metadata
|
|
62
|
+
metadata[:absolute_file_path] || metadata[:file_path]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def record(outcomes, example)
|
|
66
|
+
status = example.execution_result.status
|
|
67
|
+
return if status.nil?
|
|
68
|
+
|
|
69
|
+
outcomes.record(example.metadata[:file_path], failed: status == :failed)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
data/lib/henitai/runner.rb
CHANGED
|
@@ -11,16 +11,20 @@ module Henitai
|
|
|
11
11
|
#
|
|
12
12
|
# Gate 2 — Mutant generation
|
|
13
13
|
# Apply operators to each Subject's AST. Filter arid (non-productive)
|
|
14
|
-
# nodes
|
|
14
|
+
# nodes from the built-in catalog. Produces the initial mutant list.
|
|
15
15
|
#
|
|
16
16
|
# Gate 3 — Static filtering
|
|
17
|
-
#
|
|
17
|
+
# Mark mutants whose original start line matches an ignore_patterns
|
|
18
|
+
# entry as :ignored, and remove compile-time errors.
|
|
18
19
|
# Apply per-test coverage data: mark :no_coverage for uncovered mutants.
|
|
19
20
|
#
|
|
20
21
|
# Gate 4 — Mutant execution
|
|
21
22
|
# Run surviving mutants in isolated child processes (fork isolation).
|
|
22
|
-
# Each child process loads the test suite
|
|
23
|
-
#
|
|
23
|
+
# Each child process loads the test suite, then re-evaluates the
|
|
24
|
+
# subject's own `def` with the mutation spliced in, inside its original
|
|
25
|
+
# module/class openers (ADR-13). Module#define_method is only the fallback
|
|
26
|
+
# for subjects that cannot be reopened by name. Collect kill/survive/
|
|
27
|
+
# timeout results.
|
|
24
28
|
#
|
|
25
29
|
# Gate 5 — Reporting
|
|
26
30
|
# Write results to configured reporters (terminal, html, json, dashboard).
|
|
@@ -29,7 +33,8 @@ module Henitai
|
|
|
29
33
|
attr_reader :config, :result
|
|
30
34
|
|
|
31
35
|
# @param mode [Hash] execution-mode flags: +dry_run:+ stops before Gate 4,
|
|
32
|
-
# +incremental:+ reuses still-valid Killed verdicts from history
|
|
36
|
+
# +incremental:+ reuses still-valid Killed verdicts from history,
|
|
37
|
+
# +force:+ disables the verdict reuse that survivor reruns apply.
|
|
33
38
|
# +deps+ is assigned in the body, not defaulted in the signature: the
|
|
34
39
|
# default needs @config, and `config:` itself defaults to a load.
|
|
35
40
|
#
|
|
@@ -45,6 +50,7 @@ module Henitai
|
|
|
45
50
|
@survivors_from = survivors_from
|
|
46
51
|
@dry_run = mode.fetch(:dry_run, false)
|
|
47
52
|
@incremental = mode.fetch(:incremental, false)
|
|
53
|
+
@force = mode.fetch(:force, false)
|
|
48
54
|
end
|
|
49
55
|
|
|
50
56
|
# Entry point — runs the full pipeline and returns a Result.
|
|
@@ -76,6 +82,9 @@ module Henitai
|
|
|
76
82
|
# static/skip/arid/stillborn filtering — everything short of execution.
|
|
77
83
|
def pipeline_mutants
|
|
78
84
|
if survivor_rerun? && (fast_mutants = survivor_strategy.try_recipe_run)
|
|
85
|
+
# Gate 0 still runs: test selection reads the per-test map, which must
|
|
86
|
+
# include new or edited tests.
|
|
87
|
+
bootstrap_coverage(fast_mutants.filter_map { |mutant| mutant.subject.source_file }.uniq)
|
|
79
88
|
return fast_mutants
|
|
80
89
|
end
|
|
81
90
|
|
|
@@ -104,8 +113,9 @@ module Henitai
|
|
|
104
113
|
)
|
|
105
114
|
end
|
|
106
115
|
|
|
116
|
+
# A survivor rerun must see every prior survivor, so it never samples.
|
|
107
117
|
def generate_mutants(subjects)
|
|
108
|
-
mutant_generator.generate(subjects, operators, config:)
|
|
118
|
+
mutant_generator.generate(subjects, operators, config:, sample: !survivor_rerun?)
|
|
109
119
|
end
|
|
110
120
|
|
|
111
121
|
def filter_mutants(mutants)
|
|
@@ -116,25 +126,26 @@ module Henitai
|
|
|
116
126
|
bootstrap_thread = bootstrap_mutants(source_files)
|
|
117
127
|
mutants = generate_mutants(subjects)
|
|
118
128
|
bootstrap_thread.value
|
|
119
|
-
filtered =
|
|
120
|
-
return filtered unless survivor_rerun?
|
|
129
|
+
filtered = filter_mutants(mutants)
|
|
130
|
+
return apply_incremental_filter(filtered) unless survivor_rerun?
|
|
121
131
|
|
|
122
|
-
survivor_strategy.apply_selection(filtered)
|
|
132
|
+
survivor_strategy.apply_selection(filtered, reuse_filter: @force ? nil : reuse_filter)
|
|
123
133
|
end
|
|
124
134
|
|
|
125
|
-
#
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
#
|
|
129
|
-
# mutants_for, so the live per-test map it reads is never
|
|
135
|
+
# Verdict reuse (opt-in `--incremental`; always for `--survivors-from`
|
|
136
|
+
# reruns, where it runs on the selected survivors): still-valid Killed and
|
|
137
|
+
# Survived verdicts from the history store are marked with their stored
|
|
138
|
+
# status + from_cache before execution. It runs after the coverage
|
|
139
|
+
# bootstrap join in mutants_for, so the live per-test map it reads is never
|
|
140
|
+
# mid-write.
|
|
130
141
|
def apply_incremental_filter(mutants)
|
|
131
142
|
return mutants unless @incremental
|
|
132
143
|
|
|
133
|
-
|
|
134
|
-
dependency_fingerprint: VerdictFingerprint.dependency_fingerprint)
|
|
135
|
-
.apply(mutants)
|
|
144
|
+
reuse_filter.apply(mutants)
|
|
136
145
|
end
|
|
137
146
|
|
|
147
|
+
def reuse_filter = deps.reuse_filter
|
|
148
|
+
|
|
138
149
|
def bootstrap_mutants(source_files) = Thread.new { bootstrap_coverage(source_files) }
|
|
139
150
|
|
|
140
151
|
def execute_mutants(mutants)
|
|
@@ -150,12 +161,16 @@ module Henitai
|
|
|
150
161
|
Reporter.run_all(names: config.reporters, result:, config:, history_store:)
|
|
151
162
|
end
|
|
152
163
|
|
|
164
|
+
# History is an auxiliary trend artifact: a corrupt or locked database
|
|
165
|
+
# must not discard the run's verdicts, so the reports are still written.
|
|
153
166
|
def persist_history(result, recorded_at)
|
|
154
167
|
history_store.record(
|
|
155
168
|
result,
|
|
156
169
|
version: Henitai::VERSION,
|
|
157
170
|
recorded_at:
|
|
158
171
|
)
|
|
172
|
+
rescue StandardError => e
|
|
173
|
+
warn "henitai: could not record mutation history (#{e.class}: #{e.message}); reports are still written"
|
|
159
174
|
end
|
|
160
175
|
|
|
161
176
|
def build_result(mutants, started_at, finished_at)
|
|
@@ -177,10 +192,33 @@ module Henitai
|
|
|
177
192
|
git_sha: safe_head_sha,
|
|
178
193
|
source_provider: source_provider,
|
|
179
194
|
authoritative: full_run?,
|
|
180
|
-
since: @since
|
|
195
|
+
since: @since,
|
|
196
|
+
complete_files: complete_files,
|
|
197
|
+
sampling: result_sampling
|
|
181
198
|
)
|
|
182
199
|
end
|
|
183
200
|
|
|
201
|
+
# Files whose every subject this run mutated: no subject patterns and no
|
|
202
|
+
# survivor rerun (a --since run still selects whole changed files).
|
|
203
|
+
def complete_files
|
|
204
|
+
return [] if Array(@subjects).any? || survivor_rerun?
|
|
205
|
+
|
|
206
|
+
source_files
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Records the sampling that actually shaped the mutant set, with the
|
|
210
|
+
# default seed filled in so the report is reproducible on its own.
|
|
211
|
+
def result_sampling
|
|
212
|
+
sampling = optional_config(:sampling)
|
|
213
|
+
return nil if survivor_rerun? || !(sampling && sampling[:ratio])
|
|
214
|
+
|
|
215
|
+
{
|
|
216
|
+
strategy: (sampling[:strategy] || "stratified").to_s,
|
|
217
|
+
ratio: sampling[:ratio],
|
|
218
|
+
seed: sampling[:seed] || SamplingStrategy::DEFAULT_SEED
|
|
219
|
+
}
|
|
220
|
+
end
|
|
221
|
+
|
|
184
222
|
def safe_head_sha
|
|
185
223
|
git_diff_analyzer.head_sha
|
|
186
224
|
rescue StandardError
|
|
@@ -205,6 +243,7 @@ module Henitai
|
|
|
205
243
|
def operators = deps.operators
|
|
206
244
|
def history_store = deps.history_store
|
|
207
245
|
def per_test_coverage = deps.per_test_coverage
|
|
246
|
+
def source_selection_coverage = deps.source_selection_coverage
|
|
208
247
|
def source_provider = deps.source_provider
|
|
209
248
|
|
|
210
249
|
def progress_reporter = deps.progress_reporter(full_run: full_run?)
|
|
@@ -216,7 +255,7 @@ module Henitai
|
|
|
216
255
|
def source_file_selection
|
|
217
256
|
@source_file_selection ||= SourceFileSelection.new(
|
|
218
257
|
config: config, since: @since,
|
|
219
|
-
git_diff_analyzer: git_diff_analyzer, per_test_coverage:
|
|
258
|
+
git_diff_analyzer: git_diff_analyzer, per_test_coverage: source_selection_coverage
|
|
220
259
|
)
|
|
221
260
|
end
|
|
222
261
|
|
|
@@ -12,6 +12,10 @@ module Henitai
|
|
|
12
12
|
# {#per_test_coverage} and {#history_store}: the incremental filter proves
|
|
13
13
|
# survivor reuse against the same live coverage map the history store records
|
|
14
14
|
# its intersection set from. Two instances would be two snapshots.
|
|
15
|
+
#
|
|
16
|
+
# {#source_selection_coverage} is deliberately a *different* instance: Gate 1
|
|
17
|
+
# reads the map as it was before this run's coverage bootstrap rewrote it, and
|
|
18
|
+
# a shared instance would memoize that stale snapshot for the reuse decisions.
|
|
15
19
|
class RunnerDependencies
|
|
16
20
|
def initialize(config:)
|
|
17
21
|
@config = config
|
|
@@ -36,10 +40,27 @@ module Henitai
|
|
|
36
40
|
@per_test_coverage ||= PerTestCoverage.new(reports_dir: @config.reports_dir)
|
|
37
41
|
end
|
|
38
42
|
|
|
43
|
+
def source_selection_coverage
|
|
44
|
+
@source_selection_coverage ||= PerTestCoverage.new(reports_dir: @config.reports_dir)
|
|
45
|
+
end
|
|
46
|
+
|
|
39
47
|
def history_store
|
|
40
48
|
@history_store ||= MutantHistoryStore.new(
|
|
41
49
|
path: File.join(@config.reports_dir, Henitai::HISTORY_STORE_FILENAME),
|
|
42
|
-
per_test_coverage: per_test_coverage
|
|
50
|
+
per_test_coverage: per_test_coverage,
|
|
51
|
+
reuse_context: reuse_context
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Shared by the incremental filter and the history store: verdicts are
|
|
56
|
+
# recorded under exactly the context they are later checked against.
|
|
57
|
+
def reuse_context
|
|
58
|
+
@reuse_context ||= ReuseContext.new(config: @config)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def reuse_filter
|
|
62
|
+
IncrementalFilter.new(
|
|
63
|
+
history_store: history_store, per_test_coverage: per_test_coverage, reuse_context: reuse_context
|
|
43
64
|
)
|
|
44
65
|
end
|
|
45
66
|
|
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "digest"
|
|
3
4
|
module Henitai
|
|
4
|
-
# Samples mutants
|
|
5
|
+
# Samples mutants deterministically, stratified by operator within each subject.
|
|
6
|
+
#
|
|
7
|
+
# Per subject, the sample size is ceil(size * ratio), at least one and at most
|
|
8
|
+
# size. Mutants are grouped into strata by operator. Each stratum is ordered by
|
|
9
|
+
# a seeded SHA256 key derived from the mutant's stable id, so a mutant's rank
|
|
10
|
+
# does not depend on other mutants. Strata are then visited round-robin, which
|
|
11
|
+
# spreads the sample across operators and positions. Kept mutants are returned
|
|
12
|
+
# in their input order.
|
|
5
13
|
class SamplingStrategy
|
|
6
|
-
|
|
14
|
+
DEFAULT_SEED = 0
|
|
15
|
+
|
|
16
|
+
def sample(mutants, ratio:, strategy: :stratified, seed: DEFAULT_SEED)
|
|
7
17
|
strategy = strategy.to_sym if strategy.respond_to?(:to_sym)
|
|
8
18
|
|
|
9
19
|
case strategy
|
|
10
20
|
when :stratified
|
|
11
|
-
stratified_sample(Array(mutants), ratio:)
|
|
21
|
+
stratified_sample(Array(mutants), ratio:, seed:)
|
|
12
22
|
else
|
|
13
23
|
raise ArgumentError, "Unsupported sampling strategy: #{strategy}"
|
|
14
24
|
end
|
|
@@ -16,12 +26,32 @@ module Henitai
|
|
|
16
26
|
|
|
17
27
|
private
|
|
18
28
|
|
|
19
|
-
def stratified_sample(mutants, ratio:)
|
|
29
|
+
def stratified_sample(mutants, ratio:, seed:)
|
|
20
30
|
return [] if ratio.to_f <= 0.0
|
|
21
31
|
|
|
22
|
-
|
|
23
|
-
|
|
32
|
+
kept = Set.new.compare_by_identity
|
|
33
|
+
mutants.group_by { |mutant| mutant.subject.expression }.each_value do |group|
|
|
34
|
+
select_within_subject(group, ratio:, seed:).each { |mutant| kept << mutant }
|
|
24
35
|
end
|
|
36
|
+
mutants.select { |mutant| kept.include?(mutant) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def select_within_subject(group, ratio:, seed:)
|
|
40
|
+
strata = group.group_by { |mutant| mutant.operator.to_s }.sort.map do |_operator, members|
|
|
41
|
+
members.sort_by { |mutant| sort_key(mutant, seed) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
round_robin(strata, sample_count(group.size, ratio))
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def round_robin(strata, count)
|
|
48
|
+
depth = strata.map(&:size).max || 0
|
|
49
|
+
|
|
50
|
+
(0...depth).flat_map { |index| strata.filter_map { |stratum| stratum[index] } }.first(count)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def sort_key(mutant, seed)
|
|
54
|
+
Digest::SHA256.hexdigest("#{seed}\0#{mutant.stable_id}")
|
|
25
55
|
end
|
|
26
56
|
|
|
27
57
|
def sample_count(size, ratio)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Henitai
|
|
4
4
|
# Captures the result of one baseline or mutant test run.
|
|
5
5
|
class ScenarioExecutionResult
|
|
6
|
-
attr_reader :status, :stdout, :stderr, :exit_status, :log_path
|
|
6
|
+
attr_reader :status, :status_reason, :stdout, :stderr, :exit_status, :log_path, :killing_tests
|
|
7
7
|
|
|
8
8
|
def self.build(wait_result:, stdout:, stderr:, log_path:)
|
|
9
9
|
new(
|
|
@@ -15,12 +15,32 @@ module Henitai
|
|
|
15
15
|
)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
# Copies the reason behind a harness status onto the mutant the result was
|
|
19
|
+
# produced for. Tolerates results and mutants that carry no reason.
|
|
20
|
+
def self.copy_reason(result, mutant)
|
|
21
|
+
return unless result.respond_to?(:status_reason) && mutant.respond_to?(:status_reason=)
|
|
22
|
+
|
|
23
|
+
mutant.status_reason = result.status_reason
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Classifies a mutant run from the child's report (see MutantVerdict).
|
|
27
|
+
def self.build_for_mutant(wait_result:, report:, stdout:, stderr:, log_path:)
|
|
28
|
+
verdict = MutantVerdict.new(wait_result:, report:, stdout:, stderr:)
|
|
29
|
+
status, status_reason = verdict.to_a
|
|
30
|
+
new(status:, status_reason:, stdout:, stderr:, log_path:, exit_status: exit_status_for(wait_result),
|
|
31
|
+
killing_tests: verdict.killing_tests)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# rubocop:disable Metrics/ParameterLists -- one keyword per captured field
|
|
35
|
+
def initialize(status:, stdout:, stderr:, log_path:, exit_status: nil, status_reason: nil, killing_tests: [])
|
|
36
|
+
# rubocop:enable Metrics/ParameterLists
|
|
19
37
|
@status = status
|
|
38
|
+
@status_reason = status_reason
|
|
20
39
|
@stdout = stdout.to_s
|
|
21
40
|
@stderr = stderr.to_s
|
|
22
41
|
@log_path = log_path
|
|
23
42
|
@exit_status = exit_status
|
|
43
|
+
@killing_tests = killing_tests
|
|
24
44
|
end
|
|
25
45
|
|
|
26
46
|
# Frees the captured child output once it has been consumed (e.g. by the
|
|
@@ -107,14 +127,17 @@ module Henitai
|
|
|
107
127
|
end
|
|
108
128
|
|
|
109
129
|
def no_examples_found?(stdout, stderr)
|
|
110
|
-
stdout.to_s
|
|
130
|
+
output = [stdout.to_s, stderr.to_s].join("\n")
|
|
131
|
+
return false unless ExampleSummary::NO_EXAMPLES_FOUND.match?(output)
|
|
132
|
+
|
|
133
|
+
last_summary = output.scan(ExampleSummary::SUMMARY).last
|
|
134
|
+
last_summary.nil? || last_summary == ["0"]
|
|
111
135
|
end
|
|
112
136
|
|
|
113
137
|
def zero_examples_failure?(wait_result, stdout, stderr)
|
|
114
138
|
return false unless exit_status_for(wait_result) == 1
|
|
115
139
|
|
|
116
|
-
|
|
117
|
-
output.include?("0 examples, 0 failures")
|
|
140
|
+
ExampleSummary.none_ran?([stdout.to_s, stderr.to_s].join("\n"))
|
|
118
141
|
end
|
|
119
142
|
end
|
|
120
143
|
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
# Whether a mutant sits on its subject's def (signature) line. Line coverage
|
|
5
|
+
# credits that line only to the test that loaded the file, so the tests that
|
|
6
|
+
# exercise it are not knowable from the per-test map alone.
|
|
7
|
+
module SignatureLine
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# A subject without a source range (a recipe stub) has an unknown def
|
|
11
|
+
# line, so it is treated as one: that only widens test selection and
|
|
12
|
+
# blocks verdict reuse.
|
|
13
|
+
def on?(mutant)
|
|
14
|
+
subject = mutant.respond_to?(:subject) ? mutant.subject : nil
|
|
15
|
+
return false unless subject.respond_to?(:source_range)
|
|
16
|
+
|
|
17
|
+
range = subject.source_range
|
|
18
|
+
range.nil? || range.begin == mutant.location[:start_line]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -80,9 +80,17 @@ module Henitai
|
|
|
80
80
|
end
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
+
# Unrelated events (CHLD of another slot, server traffic) wake the select
|
|
84
|
+
# early; the SIGTERM grace window must still run its full length.
|
|
83
85
|
def wait_for_drain_window
|
|
84
|
-
wakeup
|
|
85
|
-
|
|
86
|
+
return unless wakeup
|
|
87
|
+
|
|
88
|
+
deadline = monotonic_time + PROCESS_DRAIN_WINDOW
|
|
89
|
+
while (remaining = deadline - monotonic_time).positive?
|
|
90
|
+
woken = wakeup.wait(remaining)
|
|
91
|
+
wakeup.drain
|
|
92
|
+
break unless woken
|
|
93
|
+
end
|
|
86
94
|
end
|
|
87
95
|
|
|
88
96
|
def signal_draining_slots(draining)
|
|
@@ -115,6 +123,7 @@ module Henitai
|
|
|
115
123
|
def reap_and_finalize_slot(slot)
|
|
116
124
|
_, final_status = wnohang_reap(slot.pid)
|
|
117
125
|
reap_pid(slot.pid) unless final_status
|
|
126
|
+
tear_down_process_group(slot.pid)
|
|
118
127
|
|
|
119
128
|
slot_table.release_pid(slot.pid)
|
|
120
129
|
slot_table.delete(slot.slot_id)
|
|
@@ -128,6 +137,7 @@ module Henitai
|
|
|
128
137
|
def record_drain_result(slot, final_status)
|
|
129
138
|
result = drain_verdict.build(slot, final_status)
|
|
130
139
|
slot.mutant.status = result.status
|
|
140
|
+
ScenarioExecutionResult.copy_reason(result, slot.mutant)
|
|
131
141
|
results << result
|
|
132
142
|
progress_reporter&.progress(slot.mutant, scenario_result: result)
|
|
133
143
|
end
|