evilution 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +1 -0
- data/.rubocop_todo.yml +30 -0
- data/CHANGELOG.md +59 -0
- data/README.md +285 -45
- data/docs/architecture.md +110 -12
- data/docs/ast_pattern_syntax.md +35 -7
- data/docs/combined_runs_spike.md +407 -0
- data/docs/integrations.md +1 -1
- data/docs/isolation.md +63 -2
- data/docs/migration-from-mutant.md +16 -11
- data/docs/mutant_schemata_spike.md +387 -0
- data/docs/operator_yield_study.md +576 -0
- data/docs/probe_first_spike.md +264 -0
- data/lib/evilution/ast/ancestor_lookup.rb +36 -0
- data/lib/evilution/ast/callback_declaration.rb +36 -3
- data/lib/evilution/ast/class_body.rb +62 -16
- data/lib/evilution/ast/def_region.rb +130 -0
- data/lib/evilution/ast/local_declaration.rb +37 -0
- data/lib/evilution/ast/parent_index.rb +50 -0
- data/lib/evilution/ast/parse_cache.rb +27 -0
- data/lib/evilution/ast/parser.rb +8 -3
- data/lib/evilution/ast/pattern/filter.rb +53 -12
- data/lib/evilution/ast/skip_filter.rb +83 -0
- data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
- data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
- data/lib/evilution/ast/skip_sites/logging.rb +93 -0
- data/lib/evilution/ast/skip_sites/message.rb +44 -0
- data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
- data/lib/evilution/ast/skip_sites/presets.rb +31 -0
- data/lib/evilution/ast/skip_sites/purity.rb +69 -0
- data/lib/evilution/ast/skip_sites/requires.rb +40 -0
- data/lib/evilution/ast/skip_sites.rb +9 -0
- data/lib/evilution/ast/source_surgeon.rb +20 -2
- data/lib/evilution/ast/syntax_check.rb +10 -8
- data/lib/evilution/baseline/report.rb +11 -2
- data/lib/evilution/baseline.rb +31 -6
- data/lib/evilution/cache/keys.rb +140 -0
- data/lib/evilution/cache/run_fingerprint.rb +51 -0
- data/lib/evilution/cache/skeleton.rb +57 -0
- data/lib/evilution/cache/spec_digest.rb +39 -0
- data/lib/evilution/cache.rb +145 -42
- data/lib/evilution/class_tree.rb +38 -0
- data/lib/evilution/cli/command.rb +12 -4
- data/lib/evilution/cli/commands/run.rb +12 -3
- data/lib/evilution/cli/commands/subjects.rb +5 -2
- data/lib/evilution/cli/parser/options_builder.rb +67 -3
- data/lib/evilution/cli/printers/environment.rb +34 -4
- data/lib/evilution/cli/printers/session_detail.rb +20 -3
- data/lib/evilution/compare/categorizer.rb +2 -1
- data/lib/evilution/compare/normalizer.rb +3 -2
- data/lib/evilution/config/sources.rb +3 -0
- data/lib/evilution/config/validators/base.rb +9 -0
- data/lib/evilution/config/validators/since.rb +22 -0
- data/lib/evilution/config/validators/skip_sites.rb +24 -0
- data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
- data/lib/evilution/config/validators/timeout_factor.rb +10 -0
- data/lib/evilution/config/validators/timeout_floor.rb +10 -0
- data/lib/evilution/config.rb +122 -6
- data/lib/evilution/coverage/load_tracker.rb +50 -0
- data/lib/evilution/coverage/map.rb +42 -5
- data/lib/evilution/coverage/map_builder.rb +123 -22
- data/lib/evilution/coverage/map_store.rb +15 -5
- data/lib/evilution/coverage/recorder.rb +42 -3
- data/lib/evilution/coverage_example_filter.rb +35 -0
- data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
- data/lib/evilution/example_filter.rb +20 -0
- data/lib/evilution/git/changed_lines.rb +103 -0
- data/lib/evilution/git/since_scope.rb +77 -0
- data/lib/evilution/git/unified_diff.rb +68 -0
- data/lib/evilution/infra_crash.rb +26 -0
- data/lib/evilution/integration/base.rb +97 -6
- data/lib/evilution/integration/crash_detector.rb +46 -6
- data/lib/evilution/integration/known_failures.rb +4 -0
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
- data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
- data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
- data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
- data/lib/evilution/integration/loading/concern_registry.rb +87 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
- data/lib/evilution/integration/loading/created_modules.rb +56 -0
- data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
- data/lib/evilution/integration/loading/neutralization.rb +41 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
- data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
- data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
- data/lib/evilution/integration/minitest.rb +22 -2
- data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
- data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
- data/lib/evilution/integration/rspec/example_ids.rb +23 -0
- data/lib/evilution/integration/rspec/example_order.rb +86 -0
- data/lib/evilution/integration/rspec/killer_first.rb +72 -0
- data/lib/evilution/integration/rspec/result_builder.rb +7 -0
- data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
- data/lib/evilution/integration/rspec/state_guard.rb +2 -2
- data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
- data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
- data/lib/evilution/integration/rspec.rb +98 -47
- data/lib/evilution/integration/stop_policy.rb +49 -0
- data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
- data/lib/evilution/integration/test_unit.rb +28 -2
- data/lib/evilution/isolation/fork.rb +58 -13
- data/lib/evilution/isolation/in_process.rb +12 -1
- data/lib/evilution/line_set.rb +72 -0
- data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
- data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
- data/lib/evilution/mcp/info_tool.rb +24 -17
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
- data/lib/evilution/mcp/mutate_tool.rb +32 -7
- data/lib/evilution/mcp/session_tool.rb +1 -2
- data/lib/evilution/mcp.rb +4 -0
- data/lib/evilution/mutation/line_diff.rb +111 -0
- data/lib/evilution/mutation.rb +17 -13
- data/lib/evilution/mutator/base.rb +121 -98
- data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
- data/lib/evilution/mutator/line_range_filter.rb +27 -0
- data/lib/evilution/mutator/load_sources.rb +88 -0
- data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
- data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
- data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
- data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
- data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
- data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
- data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
- data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
- data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
- data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
- data/lib/evilution/mutator/registry.rb +11 -5
- data/lib/evilution/mutator/restore_source_cache.rb +41 -0
- data/lib/evilution/parallel/pool.rb +11 -4
- data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
- data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
- data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
- data/lib/evilution/parallel/work_queue/worker.rb +10 -3
- data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
- data/lib/evilution/parallel/work_queue.rb +69 -12
- data/lib/evilution/parallel_db_warning.rb +39 -12
- data/lib/evilution/process_supervisor.rb +22 -2
- data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
- data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
- data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
- data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +4 -4
- data/lib/evilution/reporter/cli.rb +9 -2
- data/lib/evilution/reporter/html/assets/style.css +8 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
- data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
- data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
- data/lib/evilution/reporter/json/subjects.rb +1 -0
- data/lib/evilution/reporter/json/timings.rb +42 -0
- data/lib/evilution/reporter/json.rb +13 -50
- data/lib/evilution/result/coverage_gap.rb +5 -0
- data/lib/evilution/result/mutation_result.rb +55 -3
- data/lib/evilution/result/mutation_timings.rb +54 -0
- data/lib/evilution/result/run_timings.rb +20 -0
- data/lib/evilution/result/subject_score.rb +8 -1
- data/lib/evilution/result/subject_scorer.rb +5 -3
- data/lib/evilution/result/summary.rb +49 -12
- data/lib/evilution/runner/baseline_runner.rb +47 -5
- data/lib/evilution/runner/diagnostics.rb +0 -13
- data/lib/evilution/runner/isolation_resolver.rb +16 -4
- data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
- data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
- data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
- data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
- data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
- data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
- data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
- data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
- data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
- data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
- data/lib/evilution/runner/mutation_executor.rb +75 -10
- data/lib/evilution/runner/mutation_planner.rb +29 -21
- data/lib/evilution/runner/phase_clock.rb +22 -0
- data/lib/evilution/runner/report_publisher.rb +1 -1
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +87 -26
- data/lib/evilution/session/diff.rb +9 -2
- data/lib/evilution/session/store.rb +17 -2
- data/lib/evilution/source_ast_cache.rb +10 -0
- data/lib/evilution/spec_ast_cache.rb +15 -0
- data/lib/evilution/spec_resolver/constant_references.rb +66 -0
- data/lib/evilution/spec_resolver/project_files.rb +50 -0
- data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
- data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
- data/lib/evilution/spec_resolver.rb +76 -65
- data/lib/evilution/spec_selector.rb +17 -13
- data/lib/evilution/subject.rb +7 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution/worker_database/copies.rb +73 -0
- data/lib/evilution/worker_database/schema.rb +39 -0
- data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
- data/lib/evilution/worker_database.rb +214 -0
- data/schema/evilution.config.schema.json +74 -2
- metadata +84 -13
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
- data/scripts/benchmark_density +0 -262
- data/scripts/benchmark_density.yml +0 -19
- data/scripts/canary_manifest.yml +0 -47
- data/scripts/compare_mutations +0 -421
- data/scripts/compare_mutations.yml +0 -24
- data/scripts/compare_targeting +0 -279
- data/scripts/compare_targeting.example.yml +0 -24
- data/scripts/mutant_json_adapter +0 -227
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../mutation_executor"
|
|
4
|
+
|
|
5
|
+
# A run in two batches: first every subject's sentinel (see ProbeFirst), then
|
|
6
|
+
# the other mutations, once the probe knows which subjects no test exercises
|
|
7
|
+
# and the runless verdict can report their mutations without a run. Each
|
|
8
|
+
# batch is run by the block, as a strategy runs a whole run, with a notifier
|
|
9
|
+
# that numbers its results after the first batch's, so progress, fail-fast and
|
|
10
|
+
# the logs see one run. The results come back in the order the mutations came
|
|
11
|
+
# in.
|
|
12
|
+
#
|
|
13
|
+
# The second batch waits for the slowest sentinel: a parallel run's workers
|
|
14
|
+
# idle at the end of the first batch.
|
|
15
|
+
class Evilution::Runner::MutationExecutor::ProbeFirstRun
|
|
16
|
+
# The run's notifier as one batch sees it: started and finished once for the
|
|
17
|
+
# whole run, here, and counting on from the results reported before.
|
|
18
|
+
class Continuation
|
|
19
|
+
def initialize(notifier, offset)
|
|
20
|
+
@notifier = notifier
|
|
21
|
+
@offset = offset
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def start(_total); end
|
|
25
|
+
|
|
26
|
+
def finish; end
|
|
27
|
+
|
|
28
|
+
def notify(result, index)
|
|
29
|
+
@notifier.notify(result, @offset + index)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(probe:, notifier:)
|
|
34
|
+
@probe = probe
|
|
35
|
+
@notifier = notifier
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def call(mutations, &run_batch)
|
|
39
|
+
sentinels, rest = @probe.split(mutations)
|
|
40
|
+
@notifier.start(mutations.length)
|
|
41
|
+
first = run_batch.call(sentinels, Continuation.new(@notifier, 0))
|
|
42
|
+
first.results.each { |result| @probe.record(result) }
|
|
43
|
+
second = first.truncated ? nil : run_batch.call(rest, Continuation.new(@notifier, first.results.length))
|
|
44
|
+
@notifier.finish
|
|
45
|
+
merge(mutations, [first, second].compact)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def merge(mutations, executions)
|
|
51
|
+
position = mutations.each_with_index.to_h
|
|
52
|
+
Evilution::Runner::MutationExecutor::ExecutionResult.new(
|
|
53
|
+
results: executions.flat_map(&:results).sort_by { |result| position.fetch(result.mutation) },
|
|
54
|
+
truncated: executions.any?(&:truncated),
|
|
55
|
+
infra_retried: executions.sum(&:infra_retried),
|
|
56
|
+
worker_stats: executions.flat_map(&:worker_stats)
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../mutation_executor"
|
|
4
4
|
require_relative "../../result/mutation_result"
|
|
5
|
-
require_relative "
|
|
5
|
+
require_relative "../../infra_crash"
|
|
6
|
+
require_relative "runless_verdict"
|
|
6
7
|
|
|
7
8
|
class Evilution::Runner::MutationExecutor::ResultCache
|
|
8
|
-
|
|
9
|
+
# The verdicts about a mutation. A survivor is kept too: its key changes
|
|
10
|
+
# with the subject and its spec files, so a test that would now kill it
|
|
11
|
+
# re-runs it.
|
|
12
|
+
CACHEABLE_STATUSES = %i[killed survived timeout].freeze
|
|
9
13
|
private_constant :CACHEABLE_STATUSES
|
|
10
14
|
|
|
11
15
|
Partition = Data.define(:uncached_indices, :cached_results)
|
|
@@ -14,25 +18,28 @@ class Evilution::Runner::MutationExecutor::ResultCache
|
|
|
14
18
|
@backend = backend
|
|
15
19
|
end
|
|
16
20
|
|
|
17
|
-
|
|
21
|
+
# time_limit: the longest the mutation's test run would be allowed now; nil
|
|
22
|
+
# when it would run without a limit.
|
|
23
|
+
def fetch(mutation, time_limit:)
|
|
18
24
|
return nil unless @backend
|
|
19
25
|
|
|
20
26
|
data = @backend.fetch(mutation)
|
|
21
27
|
return nil unless data
|
|
22
|
-
return nil unless
|
|
28
|
+
return nil unless reusable?(data, time_limit)
|
|
23
29
|
|
|
24
30
|
Evilution::Result::MutationResult.new(
|
|
25
31
|
mutation: mutation,
|
|
26
32
|
status: data[:status],
|
|
27
33
|
duration: data[:duration],
|
|
28
34
|
killing_test: data[:killing_test],
|
|
29
|
-
test_command: data[:test_command]
|
|
35
|
+
test_command: data[:test_command],
|
|
36
|
+
time_limit: data[:time_limit]
|
|
30
37
|
)
|
|
31
38
|
end
|
|
32
39
|
|
|
33
40
|
def store(mutation, result)
|
|
34
41
|
return unless @backend
|
|
35
|
-
return unless
|
|
42
|
+
return unless CACHEABLE_STATUSES.include?(result.status)
|
|
36
43
|
return if infra_crash?(result)
|
|
37
44
|
# Nor is one where only already-failing examples failed: the baseline
|
|
38
45
|
# that says so belongs to this run, and the cache keeps no trace of it.
|
|
@@ -42,7 +49,15 @@ class Evilution::Runner::MutationExecutor::ResultCache
|
|
|
42
49
|
status: result.status,
|
|
43
50
|
duration: result.duration,
|
|
44
51
|
killing_test: result.killing_test,
|
|
45
|
-
test_command: result.test_command
|
|
52
|
+
test_command: result.test_command,
|
|
53
|
+
time_limit: result.time_limit)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Writes out what the run stored (see Evilution::Cache#flush).
|
|
57
|
+
def flush
|
|
58
|
+
return unless @backend
|
|
59
|
+
|
|
60
|
+
@backend.flush
|
|
46
61
|
end
|
|
47
62
|
|
|
48
63
|
# A kill that came from an infrastructure crash is not a verdict about the
|
|
@@ -51,20 +66,24 @@ class Evilution::Runner::MutationExecutor::ResultCache
|
|
|
51
66
|
# would defeat both the demotion to neutral and the serial retry
|
|
52
67
|
# (EV-j0bv / GH #1607).
|
|
53
68
|
def infra_crash?(result)
|
|
54
|
-
Evilution::
|
|
69
|
+
Evilution::InfraCrash.class_name?(result.error_class)
|
|
55
70
|
end
|
|
56
71
|
|
|
57
|
-
|
|
72
|
+
# time_limit_for: answers the time limit #fetch is to compare against for
|
|
73
|
+
# each mutation. A mutation the runless verdict answers goes in with the
|
|
74
|
+
# cached results: it needs no worker, and its verdict is not stored.
|
|
75
|
+
def partition(batch, packer:, time_limit_for:, runless: Evilution::Runner::MutationExecutor::RunlessVerdict.new)
|
|
58
76
|
uncached_indices = []
|
|
59
77
|
cached_results = {}
|
|
60
78
|
|
|
61
79
|
batch.each_with_index do |mutation, i|
|
|
62
|
-
|
|
63
|
-
|
|
80
|
+
verdict = runless.call(mutation)
|
|
81
|
+
if verdict
|
|
82
|
+
cached_results[i] = packer.compact(verdict)
|
|
64
83
|
next
|
|
65
84
|
end
|
|
66
85
|
|
|
67
|
-
cached = fetch(mutation)
|
|
86
|
+
cached = fetch(mutation, time_limit: time_limit_for.call(mutation))
|
|
68
87
|
if cached
|
|
69
88
|
cached_results[i] = packer.compact(cached)
|
|
70
89
|
else
|
|
@@ -77,7 +96,19 @@ class Evilution::Runner::MutationExecutor::ResultCache
|
|
|
77
96
|
|
|
78
97
|
private
|
|
79
98
|
|
|
80
|
-
|
|
81
|
-
|
|
99
|
+
# A kill or a survivor stands whatever the limit is now. A timeout says only
|
|
100
|
+
# that the run needed more than the limit it had, so it answers for a run
|
|
101
|
+
# that would get no more time than that; one that does not record its limit
|
|
102
|
+
# answers for none.
|
|
103
|
+
def reusable?(data, time_limit)
|
|
104
|
+
case data[:status]
|
|
105
|
+
when :killed, :survived then true
|
|
106
|
+
when :timeout then timeout_reusable?(data[:time_limit], time_limit)
|
|
107
|
+
else false
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def timeout_reusable?(cached_limit, time_limit)
|
|
112
|
+
cached_limit.is_a?(Numeric) && time_limit.is_a?(Numeric) && time_limit <= cached_limit
|
|
82
113
|
end
|
|
83
114
|
end
|
|
@@ -24,7 +24,7 @@ class Evilution::Runner::MutationExecutor::ResultNotifier
|
|
|
24
24
|
@progress_bar.tick(status: result.status) if @progress_bar
|
|
25
25
|
@diagnostics.log_progress(index, result.status)
|
|
26
26
|
@diagnostics.log_mutation_diagnostics(result)
|
|
27
|
-
@survived_count += 1 if result.survived?
|
|
27
|
+
@survived_count += 1 if result.survived? || result.uncovered?
|
|
28
28
|
truncate? ? :truncate : :continue
|
|
29
29
|
end
|
|
30
30
|
|
|
@@ -5,19 +5,7 @@ require_relative "../../result/mutation_result"
|
|
|
5
5
|
|
|
6
6
|
class Evilution::Runner::MutationExecutor::ResultPacker
|
|
7
7
|
def compact(result)
|
|
8
|
-
|
|
9
|
-
status: result.status,
|
|
10
|
-
duration: result.duration,
|
|
11
|
-
killing_test: result.killing_test,
|
|
12
|
-
test_command: result.test_command,
|
|
13
|
-
child_rss_kb: result.child_rss_kb,
|
|
14
|
-
memory_delta_kb: result.memory_delta_kb,
|
|
15
|
-
parent_rss_kb: result.parent_rss_kb,
|
|
16
|
-
error_message: result.error_message,
|
|
17
|
-
error_class: result.error_class,
|
|
18
|
-
error_backtrace: result.error_backtrace,
|
|
19
|
-
known_failures_only: result.known_failures_only?
|
|
20
|
-
}
|
|
8
|
+
run_fields(result).merge(memory_fields(result), error_fields(result))
|
|
21
9
|
end
|
|
22
10
|
|
|
23
11
|
def rebuild(mutation, data)
|
|
@@ -27,17 +15,59 @@ class Evilution::Runner::MutationExecutor::ResultPacker
|
|
|
27
15
|
duration: data[:duration],
|
|
28
16
|
killing_test: data[:killing_test],
|
|
29
17
|
test_command: data[:test_command],
|
|
30
|
-
memory:
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
18
|
+
memory: rebuild_memory(data),
|
|
19
|
+
error: rebuild_error(data),
|
|
20
|
+
known_failures_only: data[:known_failures_only],
|
|
21
|
+
timings: rebuild_timings(data),
|
|
22
|
+
time_limit: data[:time_limit],
|
|
23
|
+
examples_run: data[:examples_run],
|
|
24
|
+
implied_by: data[:implied_by]
|
|
25
|
+
)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def run_fields(result)
|
|
31
|
+
{
|
|
32
|
+
status: result.status,
|
|
33
|
+
duration: result.duration,
|
|
34
|
+
killing_test: result.killing_test,
|
|
35
|
+
test_command: result.test_command,
|
|
36
|
+
known_failures_only: result.known_failures_only?,
|
|
37
|
+
timings: result.timings ? result.timings.to_h : nil,
|
|
38
|
+
time_limit: result.time_limit,
|
|
39
|
+
examples_run: result.examples_run,
|
|
40
|
+
implied_by: result.implied_by
|
|
41
|
+
}
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def memory_fields(result)
|
|
45
|
+
{ child_rss_kb: result.child_rss_kb, memory_delta_kb: result.memory_delta_kb, parent_rss_kb: result.parent_rss_kb }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def error_fields(result)
|
|
49
|
+
{ error_message: result.error_message, error_class: result.error_class, error_backtrace: result.error_backtrace }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def rebuild_memory(data)
|
|
53
|
+
Evilution::Result::MemoryStats.from_fields(
|
|
54
|
+
child_rss_kb: data[:child_rss_kb], memory_delta_kb: data[:memory_delta_kb], parent_rss_kb: data[:parent_rss_kb]
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def rebuild_error(data)
|
|
59
|
+
Evilution::Result::ErrorInfo.from_fields(
|
|
60
|
+
message: data[:error_message], klass: data[:error_class], backtrace: data[:error_backtrace]
|
|
41
61
|
)
|
|
42
62
|
end
|
|
63
|
+
|
|
64
|
+
# queue_wait is measured in the worker around the isolated run, so it
|
|
65
|
+
# arrives beside the result's own timings rather than inside them.
|
|
66
|
+
def rebuild_timings(data)
|
|
67
|
+
timings = Evilution::Result::MutationTimings.from_h(data[:timings])
|
|
68
|
+
wait = data[:queue_wait]
|
|
69
|
+
return timings unless wait.is_a?(Numeric)
|
|
70
|
+
|
|
71
|
+
timings ? timings.with_queue_wait(wait) : Evilution::Result::MutationTimings.new(queue_wait: wait)
|
|
72
|
+
end
|
|
43
73
|
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../mutation_executor"
|
|
4
|
+
require_relative "../../result/mutation_result"
|
|
5
|
+
require_relative "probe_first"
|
|
6
|
+
|
|
7
|
+
# The verdict a mutation gets without a test run, when one can be given:
|
|
8
|
+
# its mutated source does not parse, no test executes any line it changes
|
|
9
|
+
# (asked of `coverage`, an integration backed by a coverage map), or its
|
|
10
|
+
# subject's sentinel survived (asked of `probe`, see ProbeFirst), which
|
|
11
|
+
# implies it survives too. nil for every other mutation, which runs.
|
|
12
|
+
#
|
|
13
|
+
# A mutation no test executes could only survive, and a survivor costs the
|
|
14
|
+
# most of any verdict: the whole resolved spec file, then a confirmation run.
|
|
15
|
+
class Evilution::Runner::MutationExecutor::RunlessVerdict
|
|
16
|
+
def initialize(coverage: nil, probe: Evilution::Runner::MutationExecutor::ProbeFirst::NONE)
|
|
17
|
+
@coverage = coverage
|
|
18
|
+
@probe = probe
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def call(mutation)
|
|
22
|
+
return result(mutation, :unparseable) if mutation.unparseable?
|
|
23
|
+
return result(mutation, :uncovered) if !@coverage.nil? && @coverage.uncovered?(mutation)
|
|
24
|
+
|
|
25
|
+
implied(mutation, @probe.implied_by(mutation))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def result(mutation, status)
|
|
31
|
+
Evilution::Result::MutationResult.new(mutation: mutation, status: status)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def implied(mutation, sentinel_operator)
|
|
35
|
+
return nil if sentinel_operator.nil?
|
|
36
|
+
|
|
37
|
+
Evilution::Result::MutationResult.new(mutation: mutation, status: :survived, implied_by: sentinel_operator)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../parallel"
|
|
4
|
+
|
|
5
|
+
# The mutations of a parallel run as the pool sends them out: each paired
|
|
6
|
+
# with the tests that killed its neighbours (see KillHistory), looked up when
|
|
7
|
+
# the pool takes it, not when the run began, so it carries every kill reported
|
|
8
|
+
# by then.
|
|
9
|
+
class Evilution::Runner::MutationExecutor::Strategy::Parallel::HintedItems
|
|
10
|
+
def initialize(mutations, kill_history)
|
|
11
|
+
@mutations = mutations
|
|
12
|
+
@kill_history = kill_history
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def length
|
|
16
|
+
@mutations.length
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def empty?
|
|
20
|
+
@mutations.empty?
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def [](index)
|
|
24
|
+
mutation = @mutations[index]
|
|
25
|
+
[mutation, @kill_history.killers_for(mutation)]
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../parallel"
|
|
4
|
+
require_relative "../../../../infra_crash"
|
|
5
|
+
|
|
6
|
+
# Turns the compact results of a parallel run, which arrive in whatever order
|
|
7
|
+
# the workers finish, into the run's results in mutation order. A result is
|
|
8
|
+
# held back until every earlier mutation has one, then rebuilt, cached when the
|
|
9
|
+
# workers produced it in this run, has its mutation's sources dropped, goes
|
|
10
|
+
# through the neutralisation pipeline and on to the notifier, so what the
|
|
11
|
+
# notifier sees, where fail-fast cuts the run and what the reports list are the
|
|
12
|
+
# same as one mutation after another would give.
|
|
13
|
+
class Evilution::Runner::MutationExecutor::Strategy::Parallel::OrderedDelivery
|
|
14
|
+
MEMORY_LOG_LABEL = "after batch"
|
|
15
|
+
|
|
16
|
+
attr_reader :results
|
|
17
|
+
|
|
18
|
+
# `log_every` is how many results go by between two memory logs.
|
|
19
|
+
def initialize(mutations:, packer:, cache:, pipeline:, notifier:, baseline_result:, diagnostics:, log_every:)
|
|
20
|
+
@mutations = mutations
|
|
21
|
+
@packer = packer
|
|
22
|
+
@cache = cache
|
|
23
|
+
@pipeline = pipeline
|
|
24
|
+
@notifier = notifier
|
|
25
|
+
@baseline_result = baseline_result
|
|
26
|
+
@diagnostics = diagnostics
|
|
27
|
+
@log_every = log_every
|
|
28
|
+
@pending = {}
|
|
29
|
+
@results = []
|
|
30
|
+
@truncated = false
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def truncated?
|
|
34
|
+
@truncated
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# `fresh` says the workers produced the result in this run, as opposed to it
|
|
38
|
+
# coming from the cache.
|
|
39
|
+
# Results held back or arriving once the run is truncated are neither listed
|
|
40
|
+
# nor notified. The fresh ones still go to the cache: they are verdicts the
|
|
41
|
+
# workers spent the time on, and a later --incremental run need not repeat
|
|
42
|
+
# them.
|
|
43
|
+
def accept(index, compact, fresh:)
|
|
44
|
+
@pending[index] = [compact, fresh]
|
|
45
|
+
release while !@truncated && @pending.key?(@results.length)
|
|
46
|
+
cache_held_back if @truncated
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def cache_held_back
|
|
52
|
+
@pending.each do |index, (compact, fresh)|
|
|
53
|
+
next unless fresh
|
|
54
|
+
|
|
55
|
+
mutation = @mutations[index]
|
|
56
|
+
@cache.store(mutation, @packer.rebuild(mutation, compact))
|
|
57
|
+
end
|
|
58
|
+
@pending.clear
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def release
|
|
62
|
+
mutation = @mutations[@results.length]
|
|
63
|
+
compact, fresh = @pending.delete(@results.length)
|
|
64
|
+
result = @packer.rebuild(mutation, compact)
|
|
65
|
+
@cache.store(mutation, result) if fresh
|
|
66
|
+
release_sources(mutation, result)
|
|
67
|
+
record(@pipeline.call(result, baseline_result: @baseline_result))
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Sources are dropped as soon as a mutation has been judged, to keep a long
|
|
71
|
+
# run's memory flat. A mutation the workers could not judge because of
|
|
72
|
+
# infrastructure contention is re-run serially once the pool is done
|
|
73
|
+
# (InfraRetry), which needs its sources, so it keeps them until then. The
|
|
74
|
+
# neutralisation pipeline has not run yet here, so the crash is still a kill.
|
|
75
|
+
def release_sources(mutation, result)
|
|
76
|
+
return if Evilution::InfraCrash.class_name?(result.error_class)
|
|
77
|
+
|
|
78
|
+
mutation.strip_sources!
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def record(result)
|
|
82
|
+
@results << result
|
|
83
|
+
@truncated = @notifier.notify(result, @results.length) == :truncate
|
|
84
|
+
log_memory
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def log_memory
|
|
88
|
+
return unless @diagnostics
|
|
89
|
+
return unless log_due?
|
|
90
|
+
|
|
91
|
+
@diagnostics.log_memory(MEMORY_LOG_LABEL, "#{@results.length} complete")
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def log_due?
|
|
95
|
+
@truncated || (@results.length % @log_every).zero? || @results.length == @mutations.length
|
|
96
|
+
end
|
|
97
|
+
end
|