henitai 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +222 -1
- data/README.md +100 -13
- data/assets/schema/henitai.schema.json +20 -0
- data/lib/henitai/arid_node_filter.rb +4 -19
- data/lib/henitai/bootstrap_error_message.rb +20 -0
- data/lib/henitai/canonical_report_merger.rb +23 -2
- data/lib/henitai/cli/run_command.rb +25 -7
- data/lib/henitai/cli/run_options.rb +1 -1
- data/lib/henitai/cli.rb +4 -1
- data/lib/henitai/configuration.rb +4 -1
- data/lib/henitai/configuration_validator/rules.rb +8 -0
- data/lib/henitai/configuration_validator/scalars.rb +9 -0
- data/lib/henitai/configuration_validator.rb +2 -1
- data/lib/henitai/constant_path.rb +48 -0
- data/lib/henitai/control_run.rb +73 -0
- data/lib/henitai/coverage_bootstrapper.rb +36 -10
- data/lib/henitai/coverage_formatter.rb +26 -3
- data/lib/henitai/coverage_report_reader.rb +21 -0
- data/lib/henitai/equivalence_detector/operand_predicates.rb +16 -0
- data/lib/henitai/equivalence_detector.rb +46 -38
- data/lib/henitai/example_summary.rb +24 -0
- data/lib/henitai/execution_engine/fork_server_scope.rb +57 -0
- data/lib/henitai/execution_engine.rb +75 -60
- data/lib/henitai/fidelity_check.rb +173 -0
- data/lib/henitai/flaky_detection_baseline.rb +61 -0
- data/lib/henitai/flaky_mutant_retrier.rb +43 -0
- data/lib/henitai/flaky_retry_decision.rb +23 -0
- data/lib/henitai/flaky_test_registry.rb +63 -0
- data/lib/henitai/fork_server/client.rb +231 -0
- data/lib/henitai/fork_server/group_killer.rb +48 -0
- data/lib/henitai/fork_server/group_terminator.rb +47 -0
- data/lib/henitai/fork_server/launcher.rb +38 -0
- data/lib/henitai/fork_server/mutant_runner.rb +141 -0
- data/lib/henitai/fork_server/protocol.rb +73 -0
- data/lib/henitai/fork_server/reaper.rb +24 -0
- data/lib/henitai/fork_server/runtime.rb +109 -0
- data/lib/henitai/fork_server/server.rb +108 -0
- data/lib/henitai/fork_server/server_process.rb +73 -0
- data/lib/henitai/fork_server/status.rb +52 -0
- data/lib/henitai/fork_server.rb +43 -0
- data/lib/henitai/generated_artifacts.rb +3 -0
- data/lib/henitai/git_diff_analyzer.rb +28 -11
- data/lib/henitai/incremental_filter.rb +20 -10
- data/lib/henitai/integration/base.rb +72 -5
- data/lib/henitai/integration/capped_log_sink.rb +34 -0
- data/lib/henitai/integration/child_bootstrap.rb +7 -0
- data/lib/henitai/integration/child_report_store.rb +36 -0
- data/lib/henitai/integration/minitest.rb +7 -5
- data/lib/henitai/integration/minitest_autorun.rb +20 -0
- data/lib/henitai/integration/minitest_outcome_reporter.rb +28 -0
- data/lib/henitai/integration/minitest_preload.rb +30 -0
- data/lib/henitai/integration/minitest_runnable_filter.rb +61 -0
- data/lib/henitai/integration/minitest_test_runner.rb +32 -15
- data/lib/henitai/integration/mutant_run_support.rb +73 -13
- data/lib/henitai/integration/rspec_child_runner.rb +44 -9
- data/lib/henitai/integration/rspec_preload.rb +35 -0
- data/lib/henitai/integration/rspec_process_runner.rb +4 -1
- data/lib/henitai/integration/scenario_log_support.rb +31 -18
- data/lib/henitai/integration.rb +16 -3
- data/lib/henitai/minitest_coverage_reporter.rb +22 -6
- data/lib/henitai/minitest_result_file.rb +62 -0
- data/lib/henitai/mutant/activation.rb +32 -0
- data/lib/henitai/mutant/activator.rb +75 -55
- data/lib/henitai/mutant/def_injection.rb +136 -0
- data/lib/henitai/mutant/lexical_scope_check.rb +120 -0
- data/lib/henitai/mutant/lexical_target.rb +105 -0
- data/lib/henitai/mutant/replacement.rb +52 -0
- data/lib/henitai/mutant.rb +27 -8
- data/lib/henitai/mutant_generator.rb +62 -18
- data/lib/henitai/mutant_history_store/verdict_cache.rb +12 -11
- data/lib/henitai/mutant_history_store.rb +14 -4
- data/lib/henitai/mutant_identity.rb +5 -1
- data/lib/henitai/mutant_verdict.rb +71 -0
- data/lib/henitai/operators/array_declaration.rb +4 -0
- data/lib/henitai/operators/assignment_expression.rb +2 -0
- data/lib/henitai/operators/conditional_expression.rb +21 -1
- data/lib/henitai/operators/hash_literal.rb +7 -4
- data/lib/henitai/operators/method_expression.rb +14 -3
- data/lib/henitai/operators/regex_metacharacters.rb +89 -0
- data/lib/henitai/operators/regex_mutator.rb +23 -28
- data/lib/henitai/operators/return_value.rb +1 -1
- data/lib/henitai/operators.rb +1 -0
- data/lib/henitai/per_test_coverage.rb +11 -7
- data/lib/henitai/per_test_coverage_collector.rb +30 -31
- data/lib/henitai/per_test_coverage_delta.rb +60 -0
- data/lib/henitai/per_test_coverage_selector.rb +26 -13
- data/lib/henitai/process_wakeup.rb +8 -3
- data/lib/henitai/process_worker_runner.rb +19 -5
- data/lib/henitai/reporter.rb +15 -8
- data/lib/henitai/result.rb +32 -16
- data/lib/henitai/reuse_context.rb +50 -0
- data/lib/henitai/rspec_coverage_formatter.rb +4 -0
- data/lib/henitai/rspec_outcomes.rb +72 -0
- data/lib/henitai/runner.rb +58 -19
- data/lib/henitai/runner_dependencies.rb +22 -1
- data/lib/henitai/sampling_strategy.rb +36 -6
- data/lib/henitai/scenario_execution_result.rb +28 -5
- data/lib/henitai/signature_line.rb +21 -0
- data/lib/henitai/slot_scheduler/draining.rb +12 -2
- data/lib/henitai/slot_scheduler/process_control.rb +33 -0
- data/lib/henitai/slot_scheduler/retry_policy.rb +8 -6
- data/lib/henitai/slot_scheduler.rb +47 -8
- data/lib/henitai/source_file_selection.rb +1 -1
- data/lib/henitai/source_parser.rb +29 -9
- data/lib/henitai/static_filter.rb +45 -14
- data/lib/henitai/stillborn_filter.rb +70 -2
- data/lib/henitai/subject.rb +8 -3
- data/lib/henitai/subject_resolver.rb +65 -31
- data/lib/henitai/suite_boot_clock.rb +48 -0
- data/lib/henitai/survivor_activation_cache.rb +18 -11
- data/lib/henitai/survivor_rerun_strategy.rb +47 -43
- data/lib/henitai/test_file_outcomes.rb +79 -0
- data/lib/henitai/timeout_calibrator.rb +31 -9
- data/lib/henitai/timeout_ceiling.rb +37 -0
- data/lib/henitai/verdict_fingerprint.rb +29 -30
- data/lib/henitai/version.rb +1 -1
- data/lib/henitai.rb +17 -1
- data/sig/configuration_validator.rbs +1 -0
- data/sig/henitai.rbs +225 -45
- metadata +46 -3
- data/lib/henitai/survivor_test_filter.rb +0 -72
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
# Measures how long a baseline suite process spent before its first test
|
|
5
|
+
# started. The parent passes its spawn wall-clock time through
|
|
6
|
+
# STARTED_AT_ENV; the boot time covers loading helpers and spec files and
|
|
7
|
+
# running before(:suite).
|
|
8
|
+
class SuiteBootClock
|
|
9
|
+
STARTED_AT_ENV = "HENITAI_SUITE_STARTED_AT"
|
|
10
|
+
|
|
11
|
+
# @param started_at [Float, nil] epoch seconds of the suite spawn
|
|
12
|
+
# @param clock [#call] returns the current epoch seconds
|
|
13
|
+
def initialize(started_at:, clock: -> { Time.now.to_f })
|
|
14
|
+
@started_at = started_at
|
|
15
|
+
@clock = clock
|
|
16
|
+
@boot_seconds = nil
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# @param env [Hash] environment of the suite process
|
|
20
|
+
# @return [SuiteBootClock]
|
|
21
|
+
def self.from_env(env, clock: -> { Time.now.to_f })
|
|
22
|
+
new(started_at: parse_timestamp(env[STARTED_AT_ENV]), clock:)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.parse_timestamp(value)
|
|
26
|
+
Float(value)
|
|
27
|
+
rescue ArgumentError, TypeError
|
|
28
|
+
nil
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Records boot time at the first test start only; later calls are ignored.
|
|
32
|
+
def mark_first_test_started
|
|
33
|
+
return if @boot_seconds || @started_at.nil?
|
|
34
|
+
|
|
35
|
+
@boot_seconds = [@clock.call - @started_at, 0.0].max
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @return [Float] boot seconds, 0.0 when unknown
|
|
39
|
+
def boot_seconds
|
|
40
|
+
@boot_seconds || 0.0
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# @return [Boolean] true once a boot time has been recorded
|
|
44
|
+
def measured?
|
|
45
|
+
!@boot_seconds.nil?
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -4,7 +4,7 @@ require "json"
|
|
|
4
4
|
require "fileutils"
|
|
5
5
|
|
|
6
6
|
module Henitai
|
|
7
|
-
# Stores and retrieves pre-computed
|
|
7
|
+
# Stores and retrieves pre-computed activations (see Mutant::Activation) for
|
|
8
8
|
# survived mutants, enabling survivor reruns to skip the full mutant-generation
|
|
9
9
|
# pipeline.
|
|
10
10
|
#
|
|
@@ -14,7 +14,7 @@ module Henitai
|
|
|
14
14
|
# objects directly and execute them without re-parsing source files.
|
|
15
15
|
#
|
|
16
16
|
# A recipe entry encodes everything needed to activate and re-report a mutant:
|
|
17
|
-
# the
|
|
17
|
+
# the activation (source, mode, file, line), subject coordinates, operator, description,
|
|
18
18
|
# location, and the coveredBy test list.
|
|
19
19
|
class SurvivorActivationCache
|
|
20
20
|
FILENAME = "activation-recipes.json"
|
|
@@ -26,10 +26,10 @@ module Henitai
|
|
|
26
26
|
# @return [Hash<String, Hash>] stableId → recipe
|
|
27
27
|
def self.compute(survived_mutants)
|
|
28
28
|
survived_mutants.each_with_object({}) do |mutant, cache|
|
|
29
|
-
|
|
30
|
-
next unless
|
|
29
|
+
activation = Mutant::Activator.activation_for(mutant)
|
|
30
|
+
next unless activation
|
|
31
31
|
|
|
32
|
-
cache[mutant.stable_id] = build_recipe(mutant,
|
|
32
|
+
cache[mutant.stable_id] = build_recipe(mutant, activation)
|
|
33
33
|
end
|
|
34
34
|
end
|
|
35
35
|
|
|
@@ -53,13 +53,11 @@ module Henitai
|
|
|
53
53
|
class << self
|
|
54
54
|
private
|
|
55
55
|
|
|
56
|
-
def build_recipe(mutant,
|
|
56
|
+
def build_recipe(mutant, activation)
|
|
57
57
|
{
|
|
58
|
-
"
|
|
59
|
-
"
|
|
60
|
-
|
|
61
|
-
"methodType" => mutant.subject.method_type.to_s,
|
|
62
|
-
"sourceFile" => mutant.subject.source_file,
|
|
58
|
+
"activation" => activation.to_h,
|
|
59
|
+
"replacement" => mutant.mutated_source,
|
|
60
|
+
**subject_fields(mutant.subject),
|
|
63
61
|
"operator" => mutant.operator,
|
|
64
62
|
"description" => mutant.description,
|
|
65
63
|
"location" => serialize_location(mutant.location),
|
|
@@ -67,6 +65,15 @@ module Henitai
|
|
|
67
65
|
}
|
|
68
66
|
end
|
|
69
67
|
|
|
68
|
+
def subject_fields(subject)
|
|
69
|
+
{
|
|
70
|
+
"namespace" => subject.namespace,
|
|
71
|
+
"methodName" => subject.method_name,
|
|
72
|
+
"methodType" => subject.method_type.to_s,
|
|
73
|
+
"sourceFile" => subject.source_file
|
|
74
|
+
}
|
|
75
|
+
end
|
|
76
|
+
|
|
70
77
|
def serialize_location(location)
|
|
71
78
|
{
|
|
72
79
|
"file" => location[:file],
|
|
@@ -12,8 +12,15 @@ module Henitai
|
|
|
12
12
|
# fast path, bypassing source parsing and mutant generation), or
|
|
13
13
|
# * filters a freshly generated mutant list down to the prior survivors.
|
|
14
14
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
15
|
+
# Survivors are never trusted on the strength of the prior report alone: a
|
|
16
|
+
# selected survivor is skipped only when the {IncrementalFilter} content
|
|
17
|
+
# fingerprint rules prove its stored verdict still valid (new or edited
|
|
18
|
+
# tests, helpers and configuration all force a re-run). Recipe stubs have no
|
|
19
|
+
# source range to fingerprint, so they always re-run.
|
|
20
|
+
#
|
|
21
|
+
# In both cases it records {#survivor_stats} (matched/unmatched counts, the
|
|
22
|
+
# reused-verdict count and the drift warning) for the Runner to attach to
|
|
23
|
+
# its Result.
|
|
17
24
|
class SurvivorRerunStrategy
|
|
18
25
|
attr_reader :survivor_stats
|
|
19
26
|
|
|
@@ -32,26 +39,19 @@ module Henitai
|
|
|
32
39
|
# bypassing source parsing and mutant generation entirely.
|
|
33
40
|
# Returns the mutant array on success, or nil if recipes are unavailable.
|
|
34
41
|
def try_recipe_run
|
|
35
|
-
dirty_worktree_files = dirty_worktree_changed_files
|
|
36
42
|
loaded = load_survivor_report
|
|
37
|
-
return nil unless recipe_fast_path_safe?(loaded
|
|
43
|
+
return nil unless recipe_fast_path_safe?(loaded)
|
|
38
44
|
|
|
39
|
-
run_from_recipes(loaded
|
|
45
|
+
run_from_recipes(loaded)
|
|
40
46
|
end
|
|
41
47
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
selector = SurvivorSelector.new(survivor_ids:
|
|
48
|
+
# @param reuse_filter [IncrementalFilter, nil] marks survivors whose
|
|
49
|
+
# stored verdict is provably still valid as cache hits.
|
|
50
|
+
def apply_selection(mutants, reuse_filter: nil)
|
|
51
|
+
selector = SurvivorSelector.new(survivor_ids: load_survivor_report.survivor_ids)
|
|
46
52
|
selected = selector.select(mutants)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
selected,
|
|
50
|
-
test_filter(
|
|
51
|
-
loaded,
|
|
52
|
-
dirty_source_files: dirty_source_files?(dirty_worktree_files, git_sha: loaded.git_sha)
|
|
53
|
-
).apply(selected)
|
|
54
|
-
)
|
|
53
|
+
reuse_filter&.apply(selected)
|
|
54
|
+
finalize_selection(selector, selected)
|
|
55
55
|
end
|
|
56
56
|
|
|
57
57
|
private
|
|
@@ -60,20 +60,16 @@ module Henitai
|
|
|
60
60
|
SurvivorLoader.new(@survivors_from, include_paths: Array(@config.includes)).load
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
def run_from_recipes(loaded
|
|
63
|
+
def run_from_recipes(loaded)
|
|
64
64
|
recipes = load_activation_recipes(loaded.survivor_ids)
|
|
65
65
|
return nil if recipes.nil?
|
|
66
66
|
|
|
67
67
|
selector, stubs = recipe_selector_and_stubs(loaded.survivor_ids, recipes)
|
|
68
|
-
|
|
69
|
-
loaded,
|
|
70
|
-
dirty_source_files: dirty_source_files?(dirty_worktree_files, git_sha: loaded.git_sha)
|
|
71
|
-
).apply(stubs)
|
|
72
|
-
finalize_survivor_split(selector, stubs, split)
|
|
68
|
+
finalize_selection(selector, stubs)
|
|
73
69
|
end
|
|
74
70
|
|
|
75
|
-
def recipe_fast_path_safe?(loaded
|
|
76
|
-
!dirty_source_files?(
|
|
71
|
+
def recipe_fast_path_safe?(loaded)
|
|
72
|
+
!dirty_source_files?(dirty_worktree_changed_files, git_sha: loaded.git_sha)
|
|
77
73
|
end
|
|
78
74
|
|
|
79
75
|
# Builds stub Mutants from recipes and a SurvivorSelector primed with the
|
|
@@ -92,11 +88,29 @@ module Henitai
|
|
|
92
88
|
path = File.join(File.dirname(@survivors_from), SurvivorActivationCache::FILENAME)
|
|
93
89
|
recipes = SurvivorActivationCache.load(path)
|
|
94
90
|
return nil if recipes.nil?
|
|
95
|
-
return nil unless survivor_ids.all? { |id|
|
|
91
|
+
return nil unless survivor_ids.all? { |id| replayable_recipe?(recipes[id]) }
|
|
96
92
|
|
|
97
93
|
recipes
|
|
98
94
|
end
|
|
99
95
|
|
|
96
|
+
# Recipes without an "activation" entry predate ADR-13: their
|
|
97
|
+
# define_method sources lose lexical scope, so they are not replayed.
|
|
98
|
+
# A :toplevel activation needs the persisted scope and owner, because the
|
|
99
|
+
# stub subject has no AST to resolve them from.
|
|
100
|
+
def replayable_recipe?(recipe)
|
|
101
|
+
return false unless recipe.is_a?(Hash) && recipe["activation"].is_a?(Hash)
|
|
102
|
+
|
|
103
|
+
activation = recipe["activation"]
|
|
104
|
+
activation["mode"] != "toplevel" || toplevel_fields_valid?(activation)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def toplevel_fields_valid?(activation)
|
|
108
|
+
scope = activation["lexicalScope"]
|
|
109
|
+
scope.is_a?(Array) && scope.all?(String) &&
|
|
110
|
+
activation["owner"].is_a?(String) &&
|
|
111
|
+
[true, false].include?(activation["singleton"])
|
|
112
|
+
end
|
|
113
|
+
|
|
100
114
|
def build_stub_mutant(stable_id, recipe)
|
|
101
115
|
mutant = Mutant.new(
|
|
102
116
|
subject: stub_subject_from_recipe(recipe),
|
|
@@ -105,7 +119,8 @@ module Henitai
|
|
|
105
119
|
description: recipe.fetch("description"),
|
|
106
120
|
location: recipe_location(recipe["location"]),
|
|
107
121
|
precomputed_stable_id: stable_id,
|
|
108
|
-
|
|
122
|
+
precomputed_activation: Mutant::Activation.from_h(recipe.fetch("activation")),
|
|
123
|
+
precomputed_replacement: recipe["replacement"]
|
|
109
124
|
)
|
|
110
125
|
mutant.covered_by = recipe["coveredBy"]
|
|
111
126
|
mutant
|
|
@@ -132,21 +147,10 @@ module Henitai
|
|
|
132
147
|
}.compact
|
|
133
148
|
end
|
|
134
149
|
|
|
135
|
-
def
|
|
136
|
-
split[:stable].each { |m| m.status = :survived }
|
|
150
|
+
def finalize_selection(selector, selected)
|
|
137
151
|
warn_survivor_drift(selector) if selector.drift_warning?
|
|
138
|
-
@survivor_stats = build_survivor_stats(selector, selected
|
|
139
|
-
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
def test_filter(loaded, dirty_source_files:)
|
|
143
|
-
SurvivorTestFilter.new(
|
|
144
|
-
coverage_map: loaded.coverage_map,
|
|
145
|
-
git_sha: loaded.git_sha,
|
|
146
|
-
dirty_source_files:,
|
|
147
|
-
worktree_changed_files: Array(dirty_worktree_changed_files),
|
|
148
|
-
diff_analyzer: @git_diff_analyzer
|
|
149
|
-
)
|
|
152
|
+
@survivor_stats = build_survivor_stats(selector, selected)
|
|
153
|
+
selected
|
|
150
154
|
end
|
|
151
155
|
|
|
152
156
|
def dirty_worktree_changed_files
|
|
@@ -170,12 +174,12 @@ module Henitai
|
|
|
170
174
|
"could not be matched; the source may have drifted - consider a full run"
|
|
171
175
|
end
|
|
172
176
|
|
|
173
|
-
def build_survivor_stats(selector, selected
|
|
177
|
+
def build_survivor_stats(selector, selected)
|
|
174
178
|
{
|
|
175
179
|
matched: selected.size,
|
|
176
180
|
unmatched_count: selector.unmatched_ids.size,
|
|
177
181
|
unmatched_ids: selector.unmatched_ids,
|
|
178
|
-
skipped_count:
|
|
182
|
+
skipped_count: selected.count(&:from_cache?),
|
|
179
183
|
drift_warning: selector.drift_warning?
|
|
180
184
|
}
|
|
181
185
|
end
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
require_relative "per_test_coverage_collector"
|
|
7
|
+
|
|
8
|
+
module Henitai
|
|
9
|
+
# Records per-test-file pass/fail outcomes from a test run.
|
|
10
|
+
#
|
|
11
|
+
# A failure that happens outside any example (a spec file that fails to
|
|
12
|
+
# load, a `before(:suite)` hook, a Minitest result without a usable file)
|
|
13
|
+
# cannot be pinned to a test file. It is recorded separately as an
|
|
14
|
+
# unattributed failure so consumers never mistake the run for a clean one.
|
|
15
|
+
class TestFileOutcomes
|
|
16
|
+
REPORT_FILE_NAME = "henitai_suite_outcomes.json"
|
|
17
|
+
|
|
18
|
+
def self.report_path(reports_dir)
|
|
19
|
+
File.join(reports_dir, REPORT_FILE_NAME)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.default_report_path
|
|
23
|
+
report_path(ENV.fetch(PerTestCoverageCollector::REPORT_DIR_ENV, "coverage"))
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def self.load(path)
|
|
27
|
+
return new unless File.exist?(path)
|
|
28
|
+
|
|
29
|
+
data = JSON.parse(File.read(path))
|
|
30
|
+
new.tap do |outcomes|
|
|
31
|
+
outcomes.record_unattributed_failure if data["unattributed_failures"]
|
|
32
|
+
data.fetch("outcomes", {}).each do |file, outcome|
|
|
33
|
+
outcomes.record(file, failed: outcome == "failed")
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
rescue JSON::ParserError
|
|
37
|
+
new
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def initialize
|
|
41
|
+
@failed = {}
|
|
42
|
+
@unattributed_failures = false
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def record(file, failed:)
|
|
46
|
+
path = File.expand_path(file)
|
|
47
|
+
@failed[path] = (@failed[path] || false) || failed
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def record_unattributed_failure
|
|
51
|
+
@unattributed_failures = true
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def unattributed_failures?
|
|
55
|
+
@unattributed_failures
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def failed_files
|
|
59
|
+
@failed.select { |_file, failed| failed }.keys.sort
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def outcomes
|
|
63
|
+
@failed.transform_values { |failed| failed ? "failed" : "passed" }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def write(path)
|
|
67
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
68
|
+
File.write(path, JSON.pretty_generate(payload))
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def payload
|
|
74
|
+
data = { "outcomes" => outcomes }
|
|
75
|
+
data["unattributed_failures"] = true if @unattributed_failures
|
|
76
|
+
data
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -3,36 +3,58 @@
|
|
|
3
3
|
module Henitai
|
|
4
4
|
# Derives a per-mutant timeout from measured per-test-file durations.
|
|
5
5
|
#
|
|
6
|
-
# The
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
6
|
+
# The measured baseline is `sum(durations of the selected test files) +
|
|
7
|
+
# boot seconds`, where boot covers loading helpers and spec files and
|
|
8
|
+
# running before(:suite). The calibrated value is `multiplier × baseline`,
|
|
9
|
+
# clamped to a floor so a near-instant baseline doesn't flag normal jitter
|
|
10
|
+
# as a timeout. Returns nil — caller falls back to the static default —
|
|
11
|
+
# when any selected test lacks timing data, so a partially measured run is
|
|
12
|
+
# treated as uncalibratable rather than under-estimated.
|
|
11
13
|
class TimeoutCalibrator
|
|
12
14
|
FLOOR_SECONDS = 2.0
|
|
13
15
|
|
|
14
16
|
# @param timing_source [#call] returns a Hash of test file path to
|
|
15
17
|
# wall-clock seconds; resolved lazily on first use.
|
|
16
18
|
# @param multiplier [Numeric]
|
|
17
|
-
|
|
19
|
+
# @param boot_seconds [#call] returns the measured suite boot time.
|
|
20
|
+
# @param include_boot [Boolean] false when boot is paid once elsewhere
|
|
21
|
+
# (for example by a preloaded fork server) and must not be repeated.
|
|
22
|
+
def initialize(timing_source:, multiplier:, boot_seconds: -> { 0.0 }, include_boot: true)
|
|
18
23
|
@timing_source = timing_source
|
|
19
24
|
@multiplier = multiplier
|
|
25
|
+
@boot_seconds = boot_seconds
|
|
26
|
+
@include_boot = include_boot
|
|
20
27
|
end
|
|
21
28
|
|
|
22
29
|
# @param test_files [Array<String>]
|
|
23
30
|
# @return [Float, nil]
|
|
24
31
|
def timeout_for(test_files)
|
|
32
|
+
baseline = baseline_for(test_files)
|
|
33
|
+
return nil if baseline.nil?
|
|
34
|
+
|
|
35
|
+
[@multiplier * baseline, FLOOR_SECONDS].max
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @param test_files [Array<String>]
|
|
39
|
+
# @return [Float, nil] measured baseline seconds, nil without timing data
|
|
40
|
+
def baseline_for(test_files)
|
|
25
41
|
files = Array(test_files)
|
|
26
42
|
return nil if files.empty? || durations.empty?
|
|
27
43
|
|
|
28
|
-
|
|
29
|
-
return nil if
|
|
44
|
+
measured = files.map { |file| durations[normalize(file)] }
|
|
45
|
+
return nil if measured.any?(&:nil?)
|
|
30
46
|
|
|
31
|
-
|
|
47
|
+
measured.sum + boot_component
|
|
32
48
|
end
|
|
33
49
|
|
|
34
50
|
private
|
|
35
51
|
|
|
52
|
+
def boot_component
|
|
53
|
+
return 0.0 unless @include_boot
|
|
54
|
+
|
|
55
|
+
@boot_component ||= @boot_seconds.call.to_f
|
|
56
|
+
end
|
|
57
|
+
|
|
36
58
|
def durations
|
|
37
59
|
@durations ||= (@timing_source.call || {}).transform_keys { |key| normalize(key) }
|
|
38
60
|
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Henitai
|
|
4
|
+
# Applies the max_timeout ceiling to a calibrated timeout. The ceiling stops a
|
|
5
|
+
# runaway mutant from running for minutes, but it must never push the timeout
|
|
6
|
+
# below the measured test baseline, or every mutant would be reported as a
|
|
7
|
+
# timeout instead of being tested.
|
|
8
|
+
class TimeoutCeiling
|
|
9
|
+
def initialize(ceiling:, warner:)
|
|
10
|
+
@ceiling = ceiling
|
|
11
|
+
@warner = warner
|
|
12
|
+
@warned = false
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def cap(calibrated, baseline)
|
|
16
|
+
return calibrated if calibrated <= @ceiling
|
|
17
|
+
|
|
18
|
+
warn_baseline_exceeds_ceiling(baseline) if baseline > @ceiling
|
|
19
|
+
[@ceiling, baseline + TimeoutCalibrator::FLOOR_SECONDS].max
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def warn_baseline_exceeds_ceiling(baseline)
|
|
25
|
+
return if @warned
|
|
26
|
+
|
|
27
|
+
@warned = true
|
|
28
|
+
@warner.call(
|
|
29
|
+
format(
|
|
30
|
+
"henitai: measured test baseline of %<baseline>.1fs exceeds max_timeout (%<ceiling>.1fs); " \
|
|
31
|
+
"using %<timeout>.1fs instead",
|
|
32
|
+
baseline:, ceiling: @ceiling, timeout: baseline + TimeoutCalibrator::FLOOR_SECONDS
|
|
33
|
+
)
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -10,9 +10,12 @@ module Henitai
|
|
|
10
10
|
# Content fingerprints that decide whether a stored verdict is still
|
|
11
11
|
# trustworthy: the subject's source (method body) and the covering test
|
|
12
12
|
# files must both be byte-identical to what was recorded. Survived verdicts
|
|
13
|
-
# additionally record the full-map covering-set membership
|
|
14
|
-
#
|
|
15
|
-
#
|
|
13
|
+
# additionally record the full-map covering-set membership (ADR-11). Both
|
|
14
|
+
# verdict kinds carry the run's {ReuseContext} fingerprint (test
|
|
15
|
+
# dependencies, verdict-relevant configuration, henitai version) and are
|
|
16
|
+
# reusable only under an identical one. Any read failure yields nil / a
|
|
17
|
+
# mismatch — conservative, never reuse on doubt; rows recorded without a
|
|
18
|
+
# context are never reusable.
|
|
16
19
|
module VerdictFingerprint
|
|
17
20
|
# Files that influence test behavior but never appear in any covering
|
|
18
21
|
# set: helpers, support code, fixtures/factories, lockfile and tool
|
|
@@ -49,24 +52,29 @@ module Henitai
|
|
|
49
52
|
nil
|
|
50
53
|
end
|
|
51
54
|
|
|
52
|
-
# JSON fingerprint of the covering test files: their sorted paths
|
|
53
|
-
# combined content hash. nil when no tests are known
|
|
54
|
-
|
|
55
|
+
# JSON fingerprint of the covering test files: their sorted paths, a
|
|
56
|
+
# combined content hash and the reuse context. nil when no tests are known,
|
|
57
|
+
# any is unreadable, or there is no context to bind the verdict to.
|
|
58
|
+
def tests_fingerprint(test_files, context_sha:)
|
|
59
|
+
return nil if context_sha.nil?
|
|
60
|
+
|
|
55
61
|
paths = Array(test_files).map(&:to_s).sort
|
|
56
62
|
return nil if paths.empty?
|
|
57
63
|
|
|
58
64
|
sha = combined_content_sha(paths)
|
|
59
65
|
return nil unless sha
|
|
60
66
|
|
|
61
|
-
JSON.generate("paths" => paths, "sha" => sha)
|
|
67
|
+
JSON.generate("paths" => paths, "sha" => sha, "context" => context_sha)
|
|
62
68
|
end
|
|
63
69
|
|
|
64
|
-
# True when every recorded test file still exists with identical content
|
|
65
|
-
|
|
66
|
-
|
|
70
|
+
# True when every recorded test file still exists with identical content
|
|
71
|
+
# and the recorded reuse context equals the current one.
|
|
72
|
+
def tests_fingerprint_current?(fingerprint_json, context_sha:)
|
|
73
|
+
return false if fingerprint_json.nil? || context_sha.nil?
|
|
67
74
|
|
|
68
75
|
fingerprint = JSON.parse(fingerprint_json)
|
|
69
|
-
|
|
76
|
+
fingerprint.fetch("context") == context_sha &&
|
|
77
|
+
combined_content_sha(fingerprint.fetch("paths")) == fingerprint.fetch("sha")
|
|
70
78
|
rescue StandardError
|
|
71
79
|
false
|
|
72
80
|
end
|
|
@@ -101,38 +109,29 @@ module Henitai
|
|
|
101
109
|
def dependency_fingerprint(root = Dir.pwd) = combined_content_sha(dependency_files(root))
|
|
102
110
|
|
|
103
111
|
# Fingerprint recorded for a Survived verdict: the sorted full-map
|
|
104
|
-
# intersection set (paths + combined content sha) plus the
|
|
105
|
-
#
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
return nil if dependency_sha.nil?
|
|
109
|
-
|
|
110
|
-
paths = Array(test_files).map(&:to_s).sort
|
|
111
|
-
return nil if paths.empty?
|
|
112
|
-
|
|
113
|
-
sha = combined_content_sha(paths)
|
|
114
|
-
return nil unless sha
|
|
115
|
-
|
|
116
|
-
JSON.generate("paths" => paths, "sha" => sha, "dependencies" => dependency_sha)
|
|
112
|
+
# intersection set (paths + combined content sha) plus the reuse context.
|
|
113
|
+
# nil when any input is unavailable — the verdict is then never reusable.
|
|
114
|
+
def survivor_tests_fingerprint(test_files, context_sha:)
|
|
115
|
+
tests_fingerprint(test_files, context_sha:)
|
|
117
116
|
end
|
|
118
117
|
|
|
119
118
|
# True when the recorded covering set still equals the live one in
|
|
120
|
-
# membership AND content, and the
|
|
119
|
+
# membership AND content, and the reuse context is unchanged.
|
|
121
120
|
# Any parse/read failure or missing field resolves to stale.
|
|
122
|
-
def survivor_fingerprint_current?(fingerprint_json, live_paths:,
|
|
123
|
-
return false if fingerprint_json.nil? ||
|
|
121
|
+
def survivor_fingerprint_current?(fingerprint_json, live_paths:, context_sha:)
|
|
122
|
+
return false if fingerprint_json.nil? || context_sha.nil?
|
|
124
123
|
|
|
125
|
-
survivor_fingerprint_matches?(JSON.parse(fingerprint_json), live_paths,
|
|
124
|
+
survivor_fingerprint_matches?(JSON.parse(fingerprint_json), live_paths, context_sha)
|
|
126
125
|
rescue StandardError
|
|
127
126
|
false
|
|
128
127
|
end
|
|
129
128
|
|
|
130
|
-
def survivor_fingerprint_matches?(fingerprint, live_paths,
|
|
129
|
+
def survivor_fingerprint_matches?(fingerprint, live_paths, context_sha)
|
|
131
130
|
recorded_paths = fingerprint.fetch("paths")
|
|
132
131
|
return false if recorded_paths.empty?
|
|
133
132
|
|
|
134
133
|
recorded_paths == Array(live_paths).map(&:to_s).sort &&
|
|
135
|
-
fingerprint.fetch("
|
|
134
|
+
fingerprint.fetch("context") == context_sha &&
|
|
136
135
|
combined_content_sha(recorded_paths) == fingerprint.fetch("sha")
|
|
137
136
|
end
|
|
138
137
|
|
data/lib/henitai/version.rb
CHANGED
data/lib/henitai.rb
CHANGED
|
@@ -28,12 +28,14 @@ module Henitai
|
|
|
28
28
|
autoload :CoverageReportReader, "henitai/coverage_report_reader"
|
|
29
29
|
autoload :PerTestCoverage, "henitai/per_test_coverage"
|
|
30
30
|
autoload :PerTestCoverageSelector, "henitai/per_test_coverage_selector"
|
|
31
|
+
autoload :SignatureLine, "henitai/signature_line"
|
|
31
32
|
autoload :Subject, "henitai/subject"
|
|
32
33
|
autoload :Mutant, "henitai/mutant"
|
|
33
34
|
autoload :MutantIdentity, "henitai/mutant_identity"
|
|
34
35
|
autoload :Operator, "henitai/operator"
|
|
35
36
|
autoload :Operators, "henitai/operators"
|
|
36
37
|
autoload :SourceParser, "henitai/source_parser"
|
|
38
|
+
autoload :ConstantPath, "henitai/constant_path"
|
|
37
39
|
autoload :SubjectResolver, "henitai/subject_resolver"
|
|
38
40
|
autoload :GeneratedArtifacts, "henitai/generated_artifacts"
|
|
39
41
|
autoload :GitDiffAnalyzer, "henitai/git_diff_analyzer"
|
|
@@ -46,31 +48,45 @@ module Henitai
|
|
|
46
48
|
autoload :EquivalenceDetector, "henitai/equivalence_detector"
|
|
47
49
|
autoload :StaticFilter, "henitai/static_filter"
|
|
48
50
|
autoload :IncrementalFilter, "henitai/incremental_filter"
|
|
51
|
+
autoload :ReuseContext, "henitai/reuse_context"
|
|
49
52
|
autoload :VerdictFingerprint, "henitai/verdict_fingerprint"
|
|
50
53
|
autoload :StillbornFilter, "henitai/stillborn_filter"
|
|
54
|
+
autoload :FidelityCheck, "henitai/fidelity_check"
|
|
55
|
+
autoload :ControlRun, "henitai/control_run"
|
|
51
56
|
autoload :CanonicalReportMerger, "henitai/canonical_report_merger"
|
|
52
57
|
autoload :CanonicalReportWriter, "henitai/canonical_report_writer"
|
|
53
58
|
autoload :CheckpointReporter, "henitai/checkpoint_reporter"
|
|
54
59
|
autoload :CompositeProgressReporter, "henitai/composite_progress_reporter"
|
|
60
|
+
autoload :BootstrapErrorMessage, "henitai/bootstrap_error_message"
|
|
55
61
|
autoload :ReportsDirectoryLock, "henitai/reports_directory_lock"
|
|
56
62
|
autoload :SurvivorLoader, "henitai/survivor_loader"
|
|
57
63
|
autoload :SurvivorSelector, "henitai/survivor_selector"
|
|
58
|
-
autoload :SurvivorTestFilter, "henitai/survivor_test_filter"
|
|
59
64
|
autoload :SurvivorActivationCache, "henitai/survivor_activation_cache"
|
|
60
65
|
autoload :SurvivorRerunStrategy, "henitai/survivor_rerun_strategy"
|
|
61
66
|
autoload :DirtySourceDetector, "henitai/dirty_source_detector"
|
|
62
67
|
autoload :SourceFileSelection, "henitai/source_file_selection"
|
|
63
68
|
autoload :SubjectSelection, "henitai/subject_selection"
|
|
64
69
|
autoload :ScenarioExecutionResult, "henitai/scenario_execution_result"
|
|
70
|
+
autoload :FlakyTestRegistry, "henitai/flaky_test_registry"
|
|
71
|
+
autoload :FlakyDetectionBaseline, "henitai/flaky_detection_baseline"
|
|
72
|
+
autoload :FlakyMutantRetrier, "henitai/flaky_mutant_retrier"
|
|
73
|
+
autoload :TestFileOutcomes, "henitai/test_file_outcomes"
|
|
74
|
+
autoload :FlakyRetryDecision, "henitai/flaky_retry_decision"
|
|
75
|
+
autoload :ExampleSummary, "henitai/example_summary"
|
|
76
|
+
autoload :MutantVerdict, "henitai/mutant_verdict"
|
|
65
77
|
autoload :CoverageFormatter, "henitai/coverage_formatter"
|
|
66
78
|
autoload :MinitestCoverageReporter, "henitai/minitest_coverage_reporter"
|
|
67
79
|
autoload :PerTestCoverageCollector, "henitai/per_test_coverage_collector"
|
|
80
|
+
autoload :PerTestCoverageDelta, "henitai/per_test_coverage_delta"
|
|
68
81
|
autoload :SyntaxValidator, "henitai/syntax_validator"
|
|
69
82
|
autoload :SamplingStrategy, "henitai/sampling_strategy"
|
|
70
83
|
autoload :TestPrioritizer, "henitai/test_prioritizer"
|
|
71
84
|
autoload :ExcludedTestFilter, "henitai/excluded_test_filter"
|
|
85
|
+
autoload :SuiteBootClock, "henitai/suite_boot_clock"
|
|
72
86
|
autoload :TimeoutCalibrator, "henitai/timeout_calibrator"
|
|
87
|
+
autoload :TimeoutCeiling, "henitai/timeout_ceiling"
|
|
73
88
|
autoload :ExecutionEngine, "henitai/execution_engine"
|
|
89
|
+
autoload :ForkServer, "henitai/fork_server"
|
|
74
90
|
autoload :ProcessWorkerRunner, "henitai/process_worker_runner"
|
|
75
91
|
autoload :SlotScheduler, "henitai/slot_scheduler"
|
|
76
92
|
autoload :ProcessWakeup, "henitai/process_wakeup"
|
|
@@ -46,6 +46,7 @@ module Henitai
|
|
|
46
46
|
def self?.validate_max_flaky_retries: (untyped) -> void
|
|
47
47
|
def self?.validate_sampling_ratio: (untyped) -> void
|
|
48
48
|
def self?.validate_sampling_strategy: (untyped) -> void
|
|
49
|
+
def self?.validate_sampling_seed: (untyped) -> void
|
|
49
50
|
def self?.validate_sampling_completeness: (Hash[Symbol, untyped]) -> void
|
|
50
51
|
def self?.describe_array_type: (untyped) -> String
|
|
51
52
|
end
|