evilution 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +1 -0
- data/.rubocop_todo.yml +30 -0
- data/CHANGELOG.md +59 -0
- data/README.md +285 -45
- data/docs/architecture.md +110 -12
- data/docs/ast_pattern_syntax.md +35 -7
- data/docs/combined_runs_spike.md +407 -0
- data/docs/integrations.md +1 -1
- data/docs/isolation.md +63 -2
- data/docs/migration-from-mutant.md +16 -11
- data/docs/mutant_schemata_spike.md +387 -0
- data/docs/operator_yield_study.md +576 -0
- data/docs/probe_first_spike.md +264 -0
- data/lib/evilution/ast/ancestor_lookup.rb +36 -0
- data/lib/evilution/ast/callback_declaration.rb +36 -3
- data/lib/evilution/ast/class_body.rb +62 -16
- data/lib/evilution/ast/def_region.rb +130 -0
- data/lib/evilution/ast/local_declaration.rb +37 -0
- data/lib/evilution/ast/parent_index.rb +50 -0
- data/lib/evilution/ast/parse_cache.rb +27 -0
- data/lib/evilution/ast/parser.rb +8 -3
- data/lib/evilution/ast/pattern/filter.rb +53 -12
- data/lib/evilution/ast/skip_filter.rb +83 -0
- data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
- data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
- data/lib/evilution/ast/skip_sites/logging.rb +93 -0
- data/lib/evilution/ast/skip_sites/message.rb +44 -0
- data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
- data/lib/evilution/ast/skip_sites/presets.rb +31 -0
- data/lib/evilution/ast/skip_sites/purity.rb +69 -0
- data/lib/evilution/ast/skip_sites/requires.rb +40 -0
- data/lib/evilution/ast/skip_sites.rb +9 -0
- data/lib/evilution/ast/source_surgeon.rb +20 -2
- data/lib/evilution/ast/syntax_check.rb +10 -8
- data/lib/evilution/baseline/report.rb +11 -2
- data/lib/evilution/baseline.rb +31 -6
- data/lib/evilution/cache/keys.rb +140 -0
- data/lib/evilution/cache/run_fingerprint.rb +51 -0
- data/lib/evilution/cache/skeleton.rb +57 -0
- data/lib/evilution/cache/spec_digest.rb +39 -0
- data/lib/evilution/cache.rb +145 -42
- data/lib/evilution/class_tree.rb +38 -0
- data/lib/evilution/cli/command.rb +12 -4
- data/lib/evilution/cli/commands/run.rb +12 -3
- data/lib/evilution/cli/commands/subjects.rb +5 -2
- data/lib/evilution/cli/parser/options_builder.rb +67 -3
- data/lib/evilution/cli/printers/environment.rb +34 -4
- data/lib/evilution/cli/printers/session_detail.rb +20 -3
- data/lib/evilution/compare/categorizer.rb +2 -1
- data/lib/evilution/compare/normalizer.rb +3 -2
- data/lib/evilution/config/sources.rb +3 -0
- data/lib/evilution/config/validators/base.rb +9 -0
- data/lib/evilution/config/validators/since.rb +22 -0
- data/lib/evilution/config/validators/skip_sites.rb +24 -0
- data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
- data/lib/evilution/config/validators/timeout_factor.rb +10 -0
- data/lib/evilution/config/validators/timeout_floor.rb +10 -0
- data/lib/evilution/config.rb +122 -6
- data/lib/evilution/coverage/load_tracker.rb +50 -0
- data/lib/evilution/coverage/map.rb +42 -5
- data/lib/evilution/coverage/map_builder.rb +123 -22
- data/lib/evilution/coverage/map_store.rb +15 -5
- data/lib/evilution/coverage/recorder.rb +42 -3
- data/lib/evilution/coverage_example_filter.rb +35 -0
- data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
- data/lib/evilution/example_filter.rb +20 -0
- data/lib/evilution/git/changed_lines.rb +103 -0
- data/lib/evilution/git/since_scope.rb +77 -0
- data/lib/evilution/git/unified_diff.rb +68 -0
- data/lib/evilution/infra_crash.rb +26 -0
- data/lib/evilution/integration/base.rb +97 -6
- data/lib/evilution/integration/crash_detector.rb +46 -6
- data/lib/evilution/integration/known_failures.rb +4 -0
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
- data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
- data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
- data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
- data/lib/evilution/integration/loading/concern_registry.rb +87 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
- data/lib/evilution/integration/loading/created_modules.rb +56 -0
- data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
- data/lib/evilution/integration/loading/neutralization.rb +41 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
- data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
- data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
- data/lib/evilution/integration/minitest.rb +22 -2
- data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
- data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
- data/lib/evilution/integration/rspec/example_ids.rb +23 -0
- data/lib/evilution/integration/rspec/example_order.rb +86 -0
- data/lib/evilution/integration/rspec/killer_first.rb +72 -0
- data/lib/evilution/integration/rspec/result_builder.rb +7 -0
- data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
- data/lib/evilution/integration/rspec/state_guard.rb +2 -2
- data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
- data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
- data/lib/evilution/integration/rspec.rb +98 -47
- data/lib/evilution/integration/stop_policy.rb +49 -0
- data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
- data/lib/evilution/integration/test_unit.rb +28 -2
- data/lib/evilution/isolation/fork.rb +58 -13
- data/lib/evilution/isolation/in_process.rb +12 -1
- data/lib/evilution/line_set.rb +72 -0
- data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
- data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
- data/lib/evilution/mcp/info_tool.rb +24 -17
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
- data/lib/evilution/mcp/mutate_tool.rb +32 -7
- data/lib/evilution/mcp/session_tool.rb +1 -2
- data/lib/evilution/mcp.rb +4 -0
- data/lib/evilution/mutation/line_diff.rb +111 -0
- data/lib/evilution/mutation.rb +17 -13
- data/lib/evilution/mutator/base.rb +121 -98
- data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
- data/lib/evilution/mutator/line_range_filter.rb +27 -0
- data/lib/evilution/mutator/load_sources.rb +88 -0
- data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
- data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
- data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
- data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
- data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
- data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
- data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
- data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
- data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
- data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
- data/lib/evilution/mutator/registry.rb +11 -5
- data/lib/evilution/mutator/restore_source_cache.rb +41 -0
- data/lib/evilution/parallel/pool.rb +11 -4
- data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
- data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
- data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
- data/lib/evilution/parallel/work_queue/worker.rb +10 -3
- data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
- data/lib/evilution/parallel/work_queue.rb +69 -12
- data/lib/evilution/parallel_db_warning.rb +39 -12
- data/lib/evilution/process_supervisor.rb +22 -2
- data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
- data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
- data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
- data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +4 -4
- data/lib/evilution/reporter/cli.rb +9 -2
- data/lib/evilution/reporter/html/assets/style.css +8 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
- data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
- data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
- data/lib/evilution/reporter/json/subjects.rb +1 -0
- data/lib/evilution/reporter/json/timings.rb +42 -0
- data/lib/evilution/reporter/json.rb +13 -50
- data/lib/evilution/result/coverage_gap.rb +5 -0
- data/lib/evilution/result/mutation_result.rb +55 -3
- data/lib/evilution/result/mutation_timings.rb +54 -0
- data/lib/evilution/result/run_timings.rb +20 -0
- data/lib/evilution/result/subject_score.rb +8 -1
- data/lib/evilution/result/subject_scorer.rb +5 -3
- data/lib/evilution/result/summary.rb +49 -12
- data/lib/evilution/runner/baseline_runner.rb +47 -5
- data/lib/evilution/runner/diagnostics.rb +0 -13
- data/lib/evilution/runner/isolation_resolver.rb +16 -4
- data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
- data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
- data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
- data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
- data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
- data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
- data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
- data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
- data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
- data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
- data/lib/evilution/runner/mutation_executor.rb +75 -10
- data/lib/evilution/runner/mutation_planner.rb +29 -21
- data/lib/evilution/runner/phase_clock.rb +22 -0
- data/lib/evilution/runner/report_publisher.rb +1 -1
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +87 -26
- data/lib/evilution/session/diff.rb +9 -2
- data/lib/evilution/session/store.rb +17 -2
- data/lib/evilution/source_ast_cache.rb +10 -0
- data/lib/evilution/spec_ast_cache.rb +15 -0
- data/lib/evilution/spec_resolver/constant_references.rb +66 -0
- data/lib/evilution/spec_resolver/project_files.rb +50 -0
- data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
- data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
- data/lib/evilution/spec_resolver.rb +76 -65
- data/lib/evilution/spec_selector.rb +17 -13
- data/lib/evilution/subject.rb +7 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution/worker_database/copies.rb +73 -0
- data/lib/evilution/worker_database/schema.rb +39 -0
- data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
- data/lib/evilution/worker_database.rb +214 -0
- data/schema/evilution.config.schema.json +74 -2
- metadata +84 -13
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
- data/scripts/benchmark_density +0 -262
- data/scripts/benchmark_density.yml +0 -19
- data/scripts/canary_manifest.yml +0 -47
- data/scripts/compare_mutations +0 -421
- data/scripts/compare_mutations.yml +0 -24
- data/scripts/compare_targeting +0 -279
- data/scripts/compare_targeting.example.yml +0 -24
- data/scripts/mutant_json_adapter +0 -227
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../integration"
|
|
4
|
+
require_relative "stop_policy"
|
|
4
5
|
|
|
5
6
|
class Evilution::Integration::MinitestCrashDetector
|
|
7
|
+
# The run's stop policy. Left alone, the detector never stops a run: only a
|
|
8
|
+
# mutation's own run is handed a policy that can.
|
|
9
|
+
attr_writer :policy
|
|
10
|
+
|
|
6
11
|
def initialize
|
|
12
|
+
@policy = Evilution::Integration::StopPolicy::DISABLED
|
|
7
13
|
reset
|
|
8
14
|
end
|
|
9
15
|
|
|
@@ -28,12 +34,17 @@ class Evilution::Integration::MinitestCrashDetector
|
|
|
28
34
|
@crashes = []
|
|
29
35
|
end
|
|
30
36
|
|
|
37
|
+
# UnexpectedError and Skip both subclass Assertion. A crash hands its
|
|
38
|
+
# exception to the policy and an assertion failure nothing; a skip says
|
|
39
|
+
# nothing about the mutant, so the policy never hears of it.
|
|
31
40
|
def record(result)
|
|
32
41
|
result.failures.each do |failure|
|
|
33
42
|
if failure.is_a?(::Minitest::UnexpectedError)
|
|
34
43
|
@crashes << failure.error
|
|
44
|
+
@policy.record(id: test_id(result), exception: failure.error)
|
|
35
45
|
elsif failure.is_a?(::Minitest::Assertion)
|
|
36
46
|
@assertion_failures += 1
|
|
47
|
+
@policy.record(id: test_id(result)) unless failure.is_a?(::Minitest::Skip)
|
|
37
48
|
end
|
|
38
49
|
end
|
|
39
50
|
end
|
|
@@ -59,4 +70,12 @@ class Evilution::Integration::MinitestCrashDetector
|
|
|
59
70
|
def unique_crash_classes
|
|
60
71
|
@crashes.map { |e| e.class.name }.uniq
|
|
61
72
|
end
|
|
73
|
+
|
|
74
|
+
private
|
|
75
|
+
|
|
76
|
+
# TestIds is loaded by the Minitest integration, the detector's only user.
|
|
77
|
+
# Requiring it here would load that integration from this file, mid-load.
|
|
78
|
+
def test_id(result)
|
|
79
|
+
Evilution::Integration::Minitest::TestIds.of(result)
|
|
80
|
+
end
|
|
62
81
|
end
|
|
@@ -33,12 +33,13 @@ class Evilution::Integration::RSpec::BaselineRunner
|
|
|
33
33
|
# What RSpec printed is the only account of a failure outside any example --
|
|
34
34
|
# a file that does not load, a suite hook that raises.
|
|
35
35
|
def report(passed, output)
|
|
36
|
-
|
|
36
|
+
durations = Evilution::Integration::RSpec::ExampleIds.durations(::RSpec.world)
|
|
37
|
+
return Evilution::Baseline::Report.build(passed: true, example_durations: durations) if passed
|
|
37
38
|
|
|
38
39
|
failures = failed_examples
|
|
39
40
|
Evilution::Baseline::Report.build(
|
|
40
41
|
passed: false, failures: failures, error: failures.empty? ? output : nil,
|
|
41
|
-
failed_ids: Evilution::Integration::RSpec::ExampleIds.failed(::RSpec.world)
|
|
42
|
+
failed_ids: Evilution::Integration::RSpec::ExampleIds.failed(::RSpec.world), example_durations: durations
|
|
42
43
|
)
|
|
43
44
|
end
|
|
44
45
|
|
|
@@ -17,8 +17,13 @@ class Evilution::Integration::RSpec::CrashDetectorLifecycle
|
|
|
17
17
|
# The run rebuilds RSpec's formatter loader so its formatters are bound to the
|
|
18
18
|
# run's own streams (EV-m6xc / GH #1627), which drops any formatter registered
|
|
19
19
|
# before it — the detector included. It is registered per run instead.
|
|
20
|
-
|
|
20
|
+
#
|
|
21
|
+
# The policy is reset with it: one latched by the previous run would stop
|
|
22
|
+
# this one at its first failure, decisive or not.
|
|
23
|
+
def register(policy)
|
|
24
|
+
policy.reset
|
|
21
25
|
detector = current
|
|
26
|
+
detector.policy = policy
|
|
22
27
|
::RSpec.configuration.add_formatter(detector)
|
|
23
28
|
detector
|
|
24
29
|
end
|
|
@@ -7,6 +7,12 @@ module Evilution::Integration::RSpec::ExampleFilterApplier
|
|
|
7
7
|
def call(_mutation, files)
|
|
8
8
|
files
|
|
9
9
|
end
|
|
10
|
+
|
|
11
|
+
def warm(_mutation, _files); end
|
|
12
|
+
|
|
13
|
+
def uncovered?(_mutation, _files)
|
|
14
|
+
false
|
|
15
|
+
end
|
|
10
16
|
end
|
|
11
17
|
|
|
12
18
|
class Custom
|
|
@@ -17,5 +23,15 @@ module Evilution::Integration::RSpec::ExampleFilterApplier
|
|
|
17
23
|
def call(mutation, files)
|
|
18
24
|
@filter.call(mutation, files)
|
|
19
25
|
end
|
|
26
|
+
|
|
27
|
+
# Only a filter that caches what it parses has anything to warm.
|
|
28
|
+
def warm(mutation, files)
|
|
29
|
+
@filter.warm(mutation, files) if @filter.respond_to?(:warm)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Only a filter backed by line coverage knows which lines no example runs.
|
|
33
|
+
def uncovered?(mutation, files)
|
|
34
|
+
@filter.respond_to?(:uncovered?) && @filter.uncovered?(mutation, files)
|
|
35
|
+
end
|
|
20
36
|
end
|
|
21
37
|
end
|
|
@@ -11,6 +11,11 @@ require_relative "../rspec"
|
|
|
11
11
|
# resolved before the position within the file is added.
|
|
12
12
|
module Evilution::Integration::RSpec::ExampleIds
|
|
13
13
|
FAILED = :failed
|
|
14
|
+
# An example with any of these statuses was run; one RSpec skipped, or never
|
|
15
|
+
# reached because the run stopped, has no status at all.
|
|
16
|
+
EXECUTED = %i[passed failed pending].freeze
|
|
17
|
+
# A pending example's time says nothing about what running it costs.
|
|
18
|
+
TIMED = %i[passed failed].freeze
|
|
14
19
|
|
|
15
20
|
def self.of(example)
|
|
16
21
|
metadata = example.metadata
|
|
@@ -26,6 +31,24 @@ module Evilution::Integration::RSpec::ExampleIds
|
|
|
26
31
|
.map { |example| of(example) }
|
|
27
32
|
end
|
|
28
33
|
|
|
34
|
+
# How long each example that ran took, in seconds, by id.
|
|
35
|
+
def self.durations(world)
|
|
36
|
+
return {} unless world.respond_to?(:all_examples)
|
|
37
|
+
|
|
38
|
+
world.all_examples.each_with_object({}) do |example, durations|
|
|
39
|
+
result = example.execution_result
|
|
40
|
+
durations[of(example)] = result.run_time if TIMED.include?(result.status)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# How many examples the run in RSpec.world executed; nil when the world
|
|
45
|
+
# cannot list them.
|
|
46
|
+
def self.executed_count(world)
|
|
47
|
+
return nil unless world.respond_to?(:all_examples)
|
|
48
|
+
|
|
49
|
+
world.all_examples.count { |example| EXECUTED.include?(example.execution_result.status) }
|
|
50
|
+
end
|
|
51
|
+
|
|
29
52
|
def self.canonical(path)
|
|
30
53
|
File.realpath(path)
|
|
31
54
|
rescue SystemCallError
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../rspec"
|
|
4
|
+
require_relative "example_ids"
|
|
5
|
+
|
|
6
|
+
# The order a mutation's run tries its examples in, when it stops at the
|
|
7
|
+
# first kill: the cost of a kill is whatever runs before it.
|
|
8
|
+
#
|
|
9
|
+
# First the examples that killed this mutation's neighbours earlier in the
|
|
10
|
+
# run, in the order they were given; then the rest, cheapest first by the time
|
|
11
|
+
# each took in the baseline. Examples without a time come last, and examples
|
|
12
|
+
# that compare equal keep the order they were defined in.
|
|
13
|
+
#
|
|
14
|
+
# RSpec asks an ordering for the examples of one group and, separately, for
|
|
15
|
+
# its child groups, so examples never leave their group and a group's
|
|
16
|
+
# `before(:context)` hooks still run once, around all of its examples. A group
|
|
17
|
+
# goes first when any example under it is a killer, otherwise by the time its
|
|
18
|
+
# timed examples take in all.
|
|
19
|
+
#
|
|
20
|
+
# Registered with RSpec under NAME and chosen with `--order NAME`. Outside a
|
|
21
|
+
# run (between #start and #finish) it keeps the order it is given, so a
|
|
22
|
+
# process whose global ordering it is left as runs everything as defined.
|
|
23
|
+
class Evilution::Integration::RSpec::ExampleOrder
|
|
24
|
+
NAME = :evilution_killer_first
|
|
25
|
+
UNKNOWN = Float::INFINITY
|
|
26
|
+
private_constant :UNKNOWN
|
|
27
|
+
|
|
28
|
+
# Whether RSpec runs an ordering named on the command line. An older one
|
|
29
|
+
# ignores a name it does not know and keeps whatever order it had.
|
|
30
|
+
def self.supported?
|
|
31
|
+
defined?(::RSpec::Core::Ordering::Delayed) ? true : false
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# durations: seconds per example id (see ExampleIds) from the baseline.
|
|
35
|
+
def initialize(durations:)
|
|
36
|
+
@durations = durations
|
|
37
|
+
finish
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Whether there is anything to order by: without killers or durations every
|
|
41
|
+
# example compares equal and the defined order stands.
|
|
42
|
+
def informed?(killers)
|
|
43
|
+
!(@durations.empty? && killers.empty?)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# killers: the ids of the examples to run first, the most telling first.
|
|
47
|
+
def start(killers)
|
|
48
|
+
@ranks = {}
|
|
49
|
+
killers.each_with_index { |id, rank| @ranks[id] ||= rank }
|
|
50
|
+
@ids = {}.compare_by_identity
|
|
51
|
+
@active = true
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def finish
|
|
55
|
+
@active = false
|
|
56
|
+
@ranks = {}
|
|
57
|
+
@ids = {}.compare_by_identity
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def order(items)
|
|
61
|
+
return items unless @active
|
|
62
|
+
|
|
63
|
+
items.each_with_index.sort_by { |item, index| [*key(item), index] }.map(&:first)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def key(item)
|
|
69
|
+
item.respond_to?(:descendant_filtered_examples) ? group_key(item) : example_key(item)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def example_key(example)
|
|
73
|
+
id = (@ids[example] ||= Evilution::Integration::RSpec::ExampleIds.of(example))
|
|
74
|
+
[@ranks.fetch(id, UNKNOWN), @durations.fetch(id, UNKNOWN)]
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# A group's cost is what its timed examples take: an example without a time
|
|
78
|
+
# (pending, or past what a report keeps per file) says nothing of the others.
|
|
79
|
+
# A group with examples but no time for any of them comes after the timed.
|
|
80
|
+
def group_key(group)
|
|
81
|
+
keys = group.descendant_filtered_examples.map { |example| example_key(example) }
|
|
82
|
+
known = keys.map(&:last).reject { |seconds| seconds == UNKNOWN }
|
|
83
|
+
cost = known.empty? && !keys.empty? ? UNKNOWN : known.sum
|
|
84
|
+
[keys.map(&:first).min || UNKNOWN, cost]
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../rspec"
|
|
4
|
+
require_relative "example_order"
|
|
5
|
+
|
|
6
|
+
# Whether a mutation's run asks RSpec for the killer-first order (see
|
|
7
|
+
# ExampleOrder), and that order put in place for the run.
|
|
8
|
+
#
|
|
9
|
+
# Only when enabled: the user asked for it, and the run stops at its first
|
|
10
|
+
# kill (one that goes through every example costs the same in any order). A
|
|
11
|
+
# suite whose examples lean on the ones defined before them fails in another
|
|
12
|
+
# order whatever the mutant, and would report kills nobody can reproduce, so
|
|
13
|
+
# the defined order is the default. Without killers or baseline times there is
|
|
14
|
+
# nothing to order by, and the defined order stands as well.
|
|
15
|
+
class Evilution::Integration::RSpec::KillerFirst
|
|
16
|
+
def initialize(enabled:, durations:)
|
|
17
|
+
@enabled = enabled
|
|
18
|
+
@order = Evilution::Integration::RSpec::ExampleOrder.new(durations: durations)
|
|
19
|
+
@killers = []
|
|
20
|
+
@suspended = false
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# The ids of the examples to run first, the most telling first; kept until
|
|
24
|
+
# the next call.
|
|
25
|
+
def prioritize(test_ids)
|
|
26
|
+
@killers = Array(test_ids)
|
|
27
|
+
nil
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def applies?
|
|
31
|
+
reorders? && Evilution::Integration::RSpec::ExampleOrder.supported?
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Whether the next run is meant to go in the killer-first order, as far as a
|
|
35
|
+
# process that has not loaded RSpec can tell: the parent asks, to know
|
|
36
|
+
# whether a run that timed out may owe that to the order.
|
|
37
|
+
def reorders?
|
|
38
|
+
@enabled && !@suspended && @order.informed?(@killers)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Runs the block with every run in it left in the defined order.
|
|
42
|
+
def suspended
|
|
43
|
+
@suspended = true
|
|
44
|
+
yield
|
|
45
|
+
ensure
|
|
46
|
+
@suspended = false
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# RSpec applies `--order` only to the first run in a process that names
|
|
50
|
+
# one: from then on the order counts as forced, and naming another changes
|
|
51
|
+
# nothing. A process that has already run RSpec -- the canary, a run before
|
|
52
|
+
# this one -- would keep the defined order. So the ordering is made the
|
|
53
|
+
# global one directly, as well as registered under the name the run's
|
|
54
|
+
# arguments give, which is what decides when this run is the first.
|
|
55
|
+
#
|
|
56
|
+
# Afterwards the global ordering is the defined one, as the `--order
|
|
57
|
+
# defined` of every other evilution run leaves it.
|
|
58
|
+
def during(ordered)
|
|
59
|
+
return yield unless ordered
|
|
60
|
+
|
|
61
|
+
registry = ::RSpec.configuration.ordering_registry
|
|
62
|
+
registry.register(Evilution::Integration::RSpec::ExampleOrder::NAME, @order)
|
|
63
|
+
registry.register(:global, @order)
|
|
64
|
+
@order.start(@killers)
|
|
65
|
+
yield
|
|
66
|
+
ensure
|
|
67
|
+
if ordered
|
|
68
|
+
@order.finish
|
|
69
|
+
registry.register(:global, registry.fetch(:defined))
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -61,4 +61,11 @@ class Evilution::Integration::RSpec::ResultBuilder
|
|
|
61
61
|
|
|
62
62
|
{ passed: false, test_command: command }
|
|
63
63
|
end
|
|
64
|
+
|
|
65
|
+
# killing_example: the id of the example whose failure killed, nil when
|
|
66
|
+
# none did. examples_run: how many examples the run executed, nil when
|
|
67
|
+
# RSpec could not say. Either is left out when nil.
|
|
68
|
+
def with_run_facts(result, killing_example:, examples_run:)
|
|
69
|
+
result.merge({ killing_example: killing_example, examples_run: examples_run }.compact)
|
|
70
|
+
end
|
|
64
71
|
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../state_guard"
|
|
4
|
+
require_relative "../../rspec"
|
|
5
|
+
require_relative "../../../class_tree"
|
|
6
|
+
|
|
7
|
+
# Empties the example groups a run defined of their constants and instance
|
|
8
|
+
# variables, so that what they hold does not outlive the run in a process
|
|
9
|
+
# that runs more than one.
|
|
10
|
+
#
|
|
11
|
+
# The groups are found below ExampleGroup in the class tree rather than among
|
|
12
|
+
# every class on the heap: a heap walk, twice a run, touched every object the
|
|
13
|
+
# process holds.
|
|
14
|
+
class Evilution::Integration::RSpec::StateGuard::ExampleGroupDescendants
|
|
15
|
+
def snapshot
|
|
16
|
+
Set.new(descendants.map(&:object_id))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def release(eg_before)
|
|
20
|
+
return unless eg_before
|
|
21
|
+
|
|
22
|
+
descendants.each do |klass|
|
|
23
|
+
next if eg_before.include?(klass.object_id)
|
|
24
|
+
|
|
25
|
+
klass.constants(false).each do |const|
|
|
26
|
+
klass.send(:remove_const, const)
|
|
27
|
+
rescue NameError
|
|
28
|
+
# Constant may have been removed concurrently (e.g. via autoload
|
|
29
|
+
# reload) between #constants(false) and #remove_const. Best-effort
|
|
30
|
+
# cleanup — nothing to do if it's already gone.
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
klass.instance_variables.each do |ivar|
|
|
34
|
+
klass.remove_instance_variable(ivar)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def descendants
|
|
42
|
+
Evilution::ClassTree.descendants(::RSpec::Core::ExampleGroup)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -7,7 +7,7 @@ require_relative "../rspec"
|
|
|
7
7
|
# children loaded, so the namespace must exist first.
|
|
8
8
|
class Evilution::Integration::RSpec::StateGuard; end # rubocop:disable Lint/EmptyClass
|
|
9
9
|
|
|
10
|
-
require_relative "state_guard/
|
|
10
|
+
require_relative "state_guard/example_group_descendants"
|
|
11
11
|
require_relative "state_guard/world_example_groups"
|
|
12
12
|
require_relative "state_guard/world_sources_by_path"
|
|
13
13
|
require_relative "state_guard/world_filtered_examples"
|
|
@@ -19,7 +19,7 @@ require_relative "../../diagnostic"
|
|
|
19
19
|
|
|
20
20
|
class Evilution::Integration::RSpec::StateGuard
|
|
21
21
|
DEFAULT_STRATEGIES = [
|
|
22
|
-
|
|
22
|
+
ExampleGroupDescendants.new,
|
|
23
23
|
WorldExampleGroups.new,
|
|
24
24
|
WorldSourcesByPath.new,
|
|
25
25
|
WorldFilteredExamples.new,
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../rspec"
|
|
4
|
+
|
|
5
|
+
# Targets are passed straight through when they resolve against Dir.pwd,
|
|
6
|
+
# preserving the relative-path contract for callers/tests. When the CWD
|
|
7
|
+
# cannot find them — workers chdir'd into a per-mutation sandbox by the
|
|
8
|
+
# isolator — the targets are expanded against
|
|
9
|
+
# Evilution::PROJECT_ROOT so RSpec::Core::Runner can still load the files.
|
|
10
|
+
class Evilution::Integration::RSpec::TargetResolver
|
|
11
|
+
# Splits a target into (path, line). RSpec example locations end in
|
|
12
|
+
# `:LINE` (or `:LINE:LINE...` for nested groups), so the suffix is
|
|
13
|
+
# anchored to the END of the string — that way path components which
|
|
14
|
+
# themselves contain colons (e.g. a Windows-style `C:/proj/spec/x.rb`)
|
|
15
|
+
# are not mis-split by a naive `partition(":")`.
|
|
16
|
+
TARGET_LINE_SUFFIX = /\A(.+?)(:\d+(?::\d+)*)\z/
|
|
17
|
+
private_constant :TARGET_LINE_SUFFIX
|
|
18
|
+
|
|
19
|
+
def call(targets)
|
|
20
|
+
targets.map { |target| resolve(target) }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def resolve(target)
|
|
24
|
+
return target if target.start_with?("/")
|
|
25
|
+
|
|
26
|
+
if (match = TARGET_LINE_SUFFIX.match(target))
|
|
27
|
+
path = match[1]
|
|
28
|
+
line_suffix = match[2]
|
|
29
|
+
else
|
|
30
|
+
path = target
|
|
31
|
+
line_suffix = ""
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
return target if File.exist?(path)
|
|
35
|
+
return target unless Evilution.in_isolated_worker?
|
|
36
|
+
|
|
37
|
+
absolute = File.expand_path(path, Evilution::PROJECT_ROOT)
|
|
38
|
+
return target unless File.exist?(absolute)
|
|
39
|
+
|
|
40
|
+
"#{absolute}#{line_suffix}"
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -13,12 +13,14 @@ class Evilution::Integration::RSpec::TestFileResolver
|
|
|
13
13
|
@warner = warner
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
# warn: false resolves without telling the user about a file nothing
|
|
17
|
+
# resolves for, as when the files are only looked up ahead of the run.
|
|
18
|
+
def call(mutation, warn: true)
|
|
17
19
|
return @test_files if @test_files
|
|
18
20
|
|
|
19
21
|
resolved = Array(@spec_selector.call(mutation.file_path))
|
|
20
22
|
if resolved.empty?
|
|
21
|
-
@warner.call(mutation.file_path, fallback_to_full_suite: @fallback_to_full_suite)
|
|
23
|
+
@warner.call(mutation.file_path, fallback_to_full_suite: @fallback_to_full_suite) if warn
|
|
22
24
|
return @fallback_to_full_suite ? ["spec"] : nil
|
|
23
25
|
end
|
|
24
26
|
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "forwardable"
|
|
4
|
+
require "shellwords"
|
|
3
5
|
require "stringio"
|
|
4
6
|
require_relative "base"
|
|
5
7
|
require_relative "known_failures"
|
|
8
|
+
require_relative "stop_policy"
|
|
6
9
|
require_relative "../spec_resolver"
|
|
7
10
|
require_relative "../spec_selector"
|
|
8
11
|
require_relative "../related_spec_heuristic"
|
|
@@ -10,6 +13,8 @@ require_relative "../related_spec_heuristic"
|
|
|
10
13
|
require_relative "../integration"
|
|
11
14
|
|
|
12
15
|
class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
16
|
+
extend Forwardable
|
|
17
|
+
|
|
13
18
|
def self.baseline_runner
|
|
14
19
|
BaselineRunner.new
|
|
15
20
|
end
|
|
@@ -26,6 +31,9 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
26
31
|
spec_selector: nil,
|
|
27
32
|
example_filter: nil,
|
|
28
33
|
known_failures: [],
|
|
34
|
+
stop_on_kill: false,
|
|
35
|
+
killer_first: false,
|
|
36
|
+
example_durations: {},
|
|
29
37
|
framework_loader: FrameworkLoader.new,
|
|
30
38
|
test_file_resolver: nil,
|
|
31
39
|
example_filter_applier: nil,
|
|
@@ -47,9 +55,42 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
47
55
|
@result_builder = result_builder
|
|
48
56
|
@state_guard = state_guard
|
|
49
57
|
@known_failures = Evilution::Integration::KnownFailures.new(known_failures)
|
|
58
|
+
@stop_policy = Evilution::Integration::StopPolicy.new(enabled: stop_on_kill, known_failures: @known_failures)
|
|
59
|
+
@killer_first = KillerFirst.new(enabled: stop_on_kill && killer_first, durations: example_durations)
|
|
60
|
+
@target_resolver = TargetResolver.new
|
|
50
61
|
super(hooks: hooks)
|
|
51
62
|
end
|
|
52
63
|
|
|
64
|
+
# Taken by the runs that follow, until the next call (see KillerFirst).
|
|
65
|
+
def prioritize(test_ids)
|
|
66
|
+
@killer_first.prioritize(test_ids)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def_delegator :@killer_first, :reorders?
|
|
70
|
+
def_delegator :@killer_first, :suspended, :in_defined_order
|
|
71
|
+
|
|
72
|
+
# One mutation per source file: every mutation in a file shares its source
|
|
73
|
+
# and the spec files the selector resolves for it. The related-spec
|
|
74
|
+
# heuristic, when on, adds files per mutation from its diff; those are
|
|
75
|
+
# warmed only if the file's first mutation adds them, and are otherwise
|
|
76
|
+
# parsed by the child that runs the mutation, as without warming.
|
|
77
|
+
def warm_targeting(mutations)
|
|
78
|
+
mutations.uniq(&:file_path).each do |mutation|
|
|
79
|
+
files = @test_file_resolver.call(mutation, warn: false)
|
|
80
|
+
@example_filter_applier.warm(mutation, files) unless files.nil?
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Whether no example of the spec files this mutation would run executes the
|
|
85
|
+
# lines it changes (see Evilution::CoverageExampleFilter#uncovered?). Asked in
|
|
86
|
+
# the parent, ahead of the run, like #warm_targeting.
|
|
87
|
+
def uncovered?(mutation)
|
|
88
|
+
files = @test_file_resolver.call(mutation, warn: false)
|
|
89
|
+
return false if files.nil?
|
|
90
|
+
|
|
91
|
+
@example_filter_applier.uncovered?(mutation, files)
|
|
92
|
+
end
|
|
93
|
+
|
|
53
94
|
private
|
|
54
95
|
|
|
55
96
|
def build_example_filter_applier(example_filter)
|
|
@@ -73,10 +114,20 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
73
114
|
targets = @example_filter_applier.call(mutation, files)
|
|
74
115
|
return @result_builder.unresolved_example(mutation) if targets.nil?
|
|
75
116
|
|
|
76
|
-
result = run_targets(targets)
|
|
117
|
+
result = run_targets(targets, ordered: @killer_first.applies?)
|
|
77
118
|
return result unless confirm?(result, targets, files)
|
|
78
119
|
|
|
79
|
-
|
|
120
|
+
confirm_run(result, files)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# A survivor's confirming run is left in the defined order, so its verdict
|
|
124
|
+
# comes from the file as it always ran. The examples both runs ran count.
|
|
125
|
+
def confirm_run(first, files)
|
|
126
|
+
started = monotonic_now
|
|
127
|
+
confirmed = run_targets(files)
|
|
128
|
+
examples_run = [first[:examples_run], confirmed[:examples_run]].compact
|
|
129
|
+
confirmed = confirmed.merge(examples_run: examples_run.sum) unless examples_run.empty?
|
|
130
|
+
confirmed.merge(timings: { confirmation: monotonic_now - started })
|
|
80
131
|
end
|
|
81
132
|
|
|
82
133
|
# Targeting runs a subset of the examples in a spec file, chosen by name. A
|
|
@@ -96,68 +147,65 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
96
147
|
(result[:passed] || result[:known_failures_only]) && targets != files
|
|
97
148
|
end
|
|
98
149
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
150
|
+
# The command reported is the run in the defined order: it is what a reader
|
|
151
|
+
# can run to reproduce the verdict, and the killer-first ordering exists
|
|
152
|
+
# only inside evilution. It is shell-escaped so it pastes as is: an example
|
|
153
|
+
# id ("spec/x_spec.rb[1:2]") is a glob to zsh.
|
|
154
|
+
def run_targets(targets, ordered: false)
|
|
155
|
+
files = @target_resolver.call(targets)
|
|
156
|
+
command = "rspec #{Shellwords.join(run_args("defined", files))}"
|
|
157
|
+
args = ordered ? run_args(ExampleOrder::NAME, files) : run_args("defined", files)
|
|
102
158
|
|
|
103
159
|
reset_examples
|
|
104
|
-
execute_run(args, command)
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
# Targets are passed straight through when they resolve against Dir.pwd,
|
|
108
|
-
# preserving the relative-path contract for callers/tests. When the CWD
|
|
109
|
-
# cannot find them — workers chdir'd into a per-mutation sandbox by the
|
|
110
|
-
# isolator (EV-wqxu / GH #1278) — the targets are expanded against
|
|
111
|
-
# Evilution::PROJECT_ROOT so RSpec::Core::Runner can still load the files.
|
|
112
|
-
def resolve_targets(targets)
|
|
113
|
-
targets.map { |target| resolve_target(target) }
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
# Splits a target into (path, line). RSpec example locations end in
|
|
117
|
-
# `:LINE` (or `:LINE:LINE...` for nested groups), so the suffix is
|
|
118
|
-
# anchored to the END of the string — that way path components which
|
|
119
|
-
# themselves contain colons (e.g. a Windows-style `C:/proj/spec/x.rb`)
|
|
120
|
-
# are not mis-split by a naive `partition(":")`.
|
|
121
|
-
TARGET_LINE_SUFFIX = /\A(.+?)(:\d+(?::\d+)*)\z/
|
|
122
|
-
private_constant :TARGET_LINE_SUFFIX
|
|
123
|
-
|
|
124
|
-
def resolve_target(target)
|
|
125
|
-
return target if target.start_with?("/")
|
|
126
|
-
|
|
127
|
-
if (match = TARGET_LINE_SUFFIX.match(target))
|
|
128
|
-
path = match[1]
|
|
129
|
-
line_suffix = match[2]
|
|
130
|
-
else
|
|
131
|
-
path = target
|
|
132
|
-
line_suffix = ""
|
|
133
|
-
end
|
|
134
|
-
|
|
135
|
-
return target if File.exist?(path)
|
|
136
|
-
return target unless Evilution.in_isolated_worker?
|
|
137
|
-
|
|
138
|
-
absolute = File.expand_path(path, Evilution::PROJECT_ROOT)
|
|
139
|
-
return target unless File.exist?(absolute)
|
|
160
|
+
execute_run(args, command, ordered: ordered)
|
|
161
|
+
end
|
|
140
162
|
|
|
141
|
-
|
|
163
|
+
def run_args(order, files)
|
|
164
|
+
["--format", "progress", "--no-color", "--order", order.to_s, *files]
|
|
142
165
|
end
|
|
143
166
|
|
|
144
|
-
def execute_run(args, command)
|
|
167
|
+
def execute_run(args, command, ordered: false)
|
|
145
168
|
snapshot = @state_guard.snapshot
|
|
169
|
+
quit_before = ::RSpec.world.wants_to_quit
|
|
170
|
+
detector = nil
|
|
146
171
|
begin
|
|
147
172
|
out = StringIO.new
|
|
148
173
|
err = StringIO.new
|
|
149
174
|
claim_rspec_streams(out, err)
|
|
150
|
-
detector = @crash_detector_lifecycle.register
|
|
151
|
-
status = ::RSpec::Core::Runner.run(args, err, out)
|
|
152
|
-
|
|
153
|
-
@result_builder.from_run(status, command, detector, examples_loaded:, known_failures_only: flagged)
|
|
175
|
+
detector = @crash_detector_lifecycle.register(@stop_policy)
|
|
176
|
+
status = @killer_first.during(ordered) { ::RSpec::Core::Runner.run(args, err, out) }
|
|
177
|
+
build_result(status, command, detector)
|
|
154
178
|
rescue StandardError => e
|
|
155
179
|
{ passed: false, error: e.message, test_command: command }
|
|
156
180
|
ensure
|
|
181
|
+
lower_own_stop(quit_before, detector)
|
|
157
182
|
@state_guard.release(snapshot)
|
|
158
183
|
end
|
|
159
184
|
end
|
|
160
185
|
|
|
186
|
+
# Which example killed, if one did, and how many ran: the parent runs the
|
|
187
|
+
# killer first for the mutations next to this one, and the count is what
|
|
188
|
+
# the ordering saves.
|
|
189
|
+
def build_result(status, command, detector)
|
|
190
|
+
flagged = known_failures_only?(status)
|
|
191
|
+
result = @result_builder.from_run(status, command, detector, examples_loaded:, known_failures_only: flagged)
|
|
192
|
+
@result_builder.with_run_facts(result, killing_example: detector.killing_id,
|
|
193
|
+
examples_run: ExampleIds.executed_count(::RSpec.world))
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# A run that stops at its kill raises RSpec.world.wants_to_quit, and left up
|
|
197
|
+
# it would make the next RSpec::Core::Runner.run in this process return a
|
|
198
|
+
# failure status without running anything -- a false kill. So the flag goes
|
|
199
|
+
# back to what it was, but only when the crash detector raised it: raised by
|
|
200
|
+
# anything else, such as RSpec's own handler on the user's first Ctrl-C, it
|
|
201
|
+
# stays up even if the run later stopped at a kill, so the next Ctrl-C still
|
|
202
|
+
# exits.
|
|
203
|
+
def lower_own_stop(quit_before, detector)
|
|
204
|
+
return if detector.nil?
|
|
205
|
+
|
|
206
|
+
::RSpec.world.wants_to_quit = quit_before if detector.raised_quit?
|
|
207
|
+
end
|
|
208
|
+
|
|
161
209
|
# RSpec redirects its own output to the streams it is handed only when the
|
|
162
210
|
# stream its configuration already holds is the current `$stdout`. In-process
|
|
163
211
|
# isolation swaps `$stdout` for a null IO around every mutation, and a
|
|
@@ -212,6 +260,9 @@ end
|
|
|
212
260
|
|
|
213
261
|
require_relative "rspec/framework_loader"
|
|
214
262
|
require_relative "rspec/example_ids"
|
|
263
|
+
require_relative "rspec/example_order"
|
|
264
|
+
require_relative "rspec/killer_first"
|
|
265
|
+
require_relative "rspec/target_resolver"
|
|
215
266
|
require_relative "rspec/test_file_resolver"
|
|
216
267
|
require_relative "rspec/unresolved_spec_warner"
|
|
217
268
|
require_relative "rspec/example_filter_applier"
|