evilution 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +1 -0
- data/.rubocop_todo.yml +30 -0
- data/CHANGELOG.md +59 -0
- data/README.md +285 -45
- data/docs/architecture.md +110 -12
- data/docs/ast_pattern_syntax.md +35 -7
- data/docs/combined_runs_spike.md +407 -0
- data/docs/integrations.md +1 -1
- data/docs/isolation.md +63 -2
- data/docs/migration-from-mutant.md +16 -11
- data/docs/mutant_schemata_spike.md +387 -0
- data/docs/operator_yield_study.md +576 -0
- data/docs/probe_first_spike.md +264 -0
- data/lib/evilution/ast/ancestor_lookup.rb +36 -0
- data/lib/evilution/ast/callback_declaration.rb +36 -3
- data/lib/evilution/ast/class_body.rb +62 -16
- data/lib/evilution/ast/def_region.rb +130 -0
- data/lib/evilution/ast/local_declaration.rb +37 -0
- data/lib/evilution/ast/parent_index.rb +50 -0
- data/lib/evilution/ast/parse_cache.rb +27 -0
- data/lib/evilution/ast/parser.rb +8 -3
- data/lib/evilution/ast/pattern/filter.rb +53 -12
- data/lib/evilution/ast/skip_filter.rb +83 -0
- data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
- data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
- data/lib/evilution/ast/skip_sites/logging.rb +93 -0
- data/lib/evilution/ast/skip_sites/message.rb +44 -0
- data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
- data/lib/evilution/ast/skip_sites/presets.rb +31 -0
- data/lib/evilution/ast/skip_sites/purity.rb +69 -0
- data/lib/evilution/ast/skip_sites/requires.rb +40 -0
- data/lib/evilution/ast/skip_sites.rb +9 -0
- data/lib/evilution/ast/source_surgeon.rb +20 -2
- data/lib/evilution/ast/syntax_check.rb +10 -8
- data/lib/evilution/baseline/report.rb +11 -2
- data/lib/evilution/baseline.rb +31 -6
- data/lib/evilution/cache/keys.rb +140 -0
- data/lib/evilution/cache/run_fingerprint.rb +51 -0
- data/lib/evilution/cache/skeleton.rb +57 -0
- data/lib/evilution/cache/spec_digest.rb +39 -0
- data/lib/evilution/cache.rb +145 -42
- data/lib/evilution/class_tree.rb +38 -0
- data/lib/evilution/cli/command.rb +12 -4
- data/lib/evilution/cli/commands/run.rb +12 -3
- data/lib/evilution/cli/commands/subjects.rb +5 -2
- data/lib/evilution/cli/parser/options_builder.rb +67 -3
- data/lib/evilution/cli/printers/environment.rb +34 -4
- data/lib/evilution/cli/printers/session_detail.rb +20 -3
- data/lib/evilution/compare/categorizer.rb +2 -1
- data/lib/evilution/compare/normalizer.rb +3 -2
- data/lib/evilution/config/sources.rb +3 -0
- data/lib/evilution/config/validators/base.rb +9 -0
- data/lib/evilution/config/validators/since.rb +22 -0
- data/lib/evilution/config/validators/skip_sites.rb +24 -0
- data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
- data/lib/evilution/config/validators/timeout_factor.rb +10 -0
- data/lib/evilution/config/validators/timeout_floor.rb +10 -0
- data/lib/evilution/config.rb +122 -6
- data/lib/evilution/coverage/load_tracker.rb +50 -0
- data/lib/evilution/coverage/map.rb +42 -5
- data/lib/evilution/coverage/map_builder.rb +123 -22
- data/lib/evilution/coverage/map_store.rb +15 -5
- data/lib/evilution/coverage/recorder.rb +42 -3
- data/lib/evilution/coverage_example_filter.rb +35 -0
- data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
- data/lib/evilution/example_filter.rb +20 -0
- data/lib/evilution/git/changed_lines.rb +103 -0
- data/lib/evilution/git/since_scope.rb +77 -0
- data/lib/evilution/git/unified_diff.rb +68 -0
- data/lib/evilution/infra_crash.rb +26 -0
- data/lib/evilution/integration/base.rb +97 -6
- data/lib/evilution/integration/crash_detector.rb +46 -6
- data/lib/evilution/integration/known_failures.rb +4 -0
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
- data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
- data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
- data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
- data/lib/evilution/integration/loading/concern_registry.rb +87 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
- data/lib/evilution/integration/loading/created_modules.rb +56 -0
- data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
- data/lib/evilution/integration/loading/neutralization.rb +41 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
- data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
- data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
- data/lib/evilution/integration/minitest.rb +22 -2
- data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
- data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
- data/lib/evilution/integration/rspec/example_ids.rb +23 -0
- data/lib/evilution/integration/rspec/example_order.rb +86 -0
- data/lib/evilution/integration/rspec/killer_first.rb +72 -0
- data/lib/evilution/integration/rspec/result_builder.rb +7 -0
- data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
- data/lib/evilution/integration/rspec/state_guard.rb +2 -2
- data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
- data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
- data/lib/evilution/integration/rspec.rb +98 -47
- data/lib/evilution/integration/stop_policy.rb +49 -0
- data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
- data/lib/evilution/integration/test_unit.rb +28 -2
- data/lib/evilution/isolation/fork.rb +58 -13
- data/lib/evilution/isolation/in_process.rb +12 -1
- data/lib/evilution/line_set.rb +72 -0
- data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
- data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
- data/lib/evilution/mcp/info_tool.rb +24 -17
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
- data/lib/evilution/mcp/mutate_tool.rb +32 -7
- data/lib/evilution/mcp/session_tool.rb +1 -2
- data/lib/evilution/mcp.rb +4 -0
- data/lib/evilution/mutation/line_diff.rb +111 -0
- data/lib/evilution/mutation.rb +17 -13
- data/lib/evilution/mutator/base.rb +121 -98
- data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
- data/lib/evilution/mutator/line_range_filter.rb +27 -0
- data/lib/evilution/mutator/load_sources.rb +88 -0
- data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
- data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
- data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
- data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
- data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
- data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
- data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
- data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
- data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
- data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
- data/lib/evilution/mutator/registry.rb +11 -5
- data/lib/evilution/mutator/restore_source_cache.rb +41 -0
- data/lib/evilution/parallel/pool.rb +11 -4
- data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
- data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
- data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
- data/lib/evilution/parallel/work_queue/worker.rb +10 -3
- data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
- data/lib/evilution/parallel/work_queue.rb +69 -12
- data/lib/evilution/parallel_db_warning.rb +39 -12
- data/lib/evilution/process_supervisor.rb +22 -2
- data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
- data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
- data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
- data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +4 -4
- data/lib/evilution/reporter/cli.rb +9 -2
- data/lib/evilution/reporter/html/assets/style.css +8 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
- data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
- data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
- data/lib/evilution/reporter/json/subjects.rb +1 -0
- data/lib/evilution/reporter/json/timings.rb +42 -0
- data/lib/evilution/reporter/json.rb +13 -50
- data/lib/evilution/result/coverage_gap.rb +5 -0
- data/lib/evilution/result/mutation_result.rb +55 -3
- data/lib/evilution/result/mutation_timings.rb +54 -0
- data/lib/evilution/result/run_timings.rb +20 -0
- data/lib/evilution/result/subject_score.rb +8 -1
- data/lib/evilution/result/subject_scorer.rb +5 -3
- data/lib/evilution/result/summary.rb +49 -12
- data/lib/evilution/runner/baseline_runner.rb +47 -5
- data/lib/evilution/runner/diagnostics.rb +0 -13
- data/lib/evilution/runner/isolation_resolver.rb +16 -4
- data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
- data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
- data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
- data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
- data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
- data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
- data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
- data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
- data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
- data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
- data/lib/evilution/runner/mutation_executor.rb +75 -10
- data/lib/evilution/runner/mutation_planner.rb +29 -21
- data/lib/evilution/runner/phase_clock.rb +22 -0
- data/lib/evilution/runner/report_publisher.rb +1 -1
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +87 -26
- data/lib/evilution/session/diff.rb +9 -2
- data/lib/evilution/session/store.rb +17 -2
- data/lib/evilution/source_ast_cache.rb +10 -0
- data/lib/evilution/spec_ast_cache.rb +15 -0
- data/lib/evilution/spec_resolver/constant_references.rb +66 -0
- data/lib/evilution/spec_resolver/project_files.rb +50 -0
- data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
- data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
- data/lib/evilution/spec_resolver.rb +76 -65
- data/lib/evilution/spec_selector.rb +17 -13
- data/lib/evilution/subject.rb +7 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution/worker_database/copies.rb +73 -0
- data/lib/evilution/worker_database/schema.rb +39 -0
- data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
- data/lib/evilution/worker_database.rb +214 -0
- data/schema/evilution.config.schema.json +74 -2
- metadata +84 -13
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
- data/scripts/benchmark_density +0 -262
- data/scripts/benchmark_density.yml +0 -19
- data/scripts/canary_manifest.yml +0 -47
- data/scripts/compare_mutations +0 -421
- data/scripts/compare_mutations.yml +0 -24
- data/scripts/compare_targeting +0 -279
- data/scripts/compare_targeting.example.yml +0 -24
- data/scripts/mutant_json_adapter +0 -227
|
@@ -8,13 +8,17 @@ require_relative "../pct"
|
|
|
8
8
|
# `FAIL` and exit 0 (EV-39t1 / GH #1604); with no minimum configured the line
|
|
9
9
|
# states the score and says so, rather than implying a gate that is not armed.
|
|
10
10
|
class Evilution::Reporter::CLI::LineFormatters::ResultLine
|
|
11
|
-
|
|
11
|
+
# since: the git ref a --since run was limited to the changes since. Such a
|
|
12
|
+
# run that mutated nothing passes: its changes hold nothing to score.
|
|
13
|
+
def initialize(pct: Evilution::Reporter::CLI::Pct.new, min_score: nil, since: nil)
|
|
12
14
|
@pct = pct
|
|
13
15
|
@min_score = min_score
|
|
16
|
+
@since = since
|
|
14
17
|
end
|
|
15
18
|
|
|
16
19
|
def format(summary)
|
|
17
20
|
return unresolved_targets_line(summary) if summary.unresolved_targets?
|
|
21
|
+
return "Result: PASS (nothing to mutate in the lines changed since #{@since})" if nothing_changed?(summary)
|
|
18
22
|
return no_threshold_line(summary) unless gate?
|
|
19
23
|
|
|
20
24
|
pass_fail = summary.success?(min_score: @min_score) ? "PASS" : "FAIL"
|
|
@@ -25,6 +29,10 @@ class Evilution::Reporter::CLI::LineFormatters::ResultLine
|
|
|
25
29
|
|
|
26
30
|
private
|
|
27
31
|
|
|
32
|
+
def nothing_changed?(summary)
|
|
33
|
+
!@since.nil? && summary.total.zero?
|
|
34
|
+
end
|
|
35
|
+
|
|
28
36
|
# A minimum of zero passes every score, so it is not a gate either.
|
|
29
37
|
def gate?
|
|
30
38
|
!@min_score.nil? && @min_score.positive?
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../line_formatters"
|
|
4
|
+
|
|
5
|
+
# The mutations each skip_sites preset left out, under the mutation count
|
|
6
|
+
# whose "skipped" figure they are part of.
|
|
7
|
+
class Evilution::Reporter::CLI::LineFormatters::SkippedSites
|
|
8
|
+
def format(summary)
|
|
9
|
+
sites = summary.skipped_sites
|
|
10
|
+
return nil if sites.empty?
|
|
11
|
+
|
|
12
|
+
"Skipped sites: #{sites.map { |name, count| "#{name} #{count}" }.join(", ")}"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../line_formatters"
|
|
4
|
+
|
|
5
|
+
# A run aimed at lines that hold no subject generates nothing for them, and
|
|
6
|
+
# its report alone reads like a run that found nothing to mutate. The warning
|
|
7
|
+
# printed while the run starts says why, but it is easily lost above the
|
|
8
|
+
# progress output; this line repeats where in the report itself, next to the
|
|
9
|
+
# mutation count it explains.
|
|
10
|
+
class Evilution::Reporter::CLI::LineFormatters::UncoveredCode
|
|
11
|
+
def format(summary)
|
|
12
|
+
return nil unless summary.uncovered_code?
|
|
13
|
+
|
|
14
|
+
listing = summary.uncovered_code.map { |entry| " #{entry[:file]}:#{entry[:lines].join(", ")}" }.join("\n")
|
|
15
|
+
"! Targeted lines hold code outside every subject — no mutations were generated for them:\n#{listing}"
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../cli"
|
|
4
4
|
require_relative "line_formatters/mutations"
|
|
5
|
+
require_relative "line_formatters/skipped_sites"
|
|
5
6
|
require_relative "line_formatters/score"
|
|
6
7
|
require_relative "line_formatters/baseline_neutralized_notice"
|
|
7
8
|
require_relative "line_formatters/error_rate_warning"
|
|
8
9
|
require_relative "line_formatters/unresolved_rate_warning"
|
|
9
10
|
require_relative "line_formatters/unresolved_targets"
|
|
11
|
+
require_relative "line_formatters/uncovered_code"
|
|
10
12
|
require_relative "line_formatters/infra_retry_notice"
|
|
11
13
|
require_relative "line_formatters/duration"
|
|
12
14
|
require_relative "line_formatters/efficiency"
|
|
@@ -15,11 +17,13 @@ require_relative "line_formatters/peak_memory"
|
|
|
15
17
|
class Evilution::Reporter::CLI::MetricsBlock
|
|
16
18
|
DEFAULT_LINES = [
|
|
17
19
|
Evilution::Reporter::CLI::LineFormatters::Mutations.new,
|
|
20
|
+
Evilution::Reporter::CLI::LineFormatters::SkippedSites.new,
|
|
18
21
|
Evilution::Reporter::CLI::LineFormatters::Score.new,
|
|
19
22
|
Evilution::Reporter::CLI::LineFormatters::BaselineNeutralizedNotice.new,
|
|
20
23
|
Evilution::Reporter::CLI::LineFormatters::ErrorRateWarning.new,
|
|
21
24
|
Evilution::Reporter::CLI::LineFormatters::UnresolvedRateWarning.new,
|
|
22
25
|
Evilution::Reporter::CLI::LineFormatters::UnresolvedTargets.new,
|
|
26
|
+
Evilution::Reporter::CLI::LineFormatters::UncoveredCode.new,
|
|
23
27
|
Evilution::Reporter::CLI::LineFormatters::InfraRetryNotice.new,
|
|
24
28
|
Evilution::Reporter::CLI::LineFormatters::Duration.new,
|
|
25
29
|
Evilution::Reporter::CLI::LineFormatters::Efficiency.new,
|
|
@@ -6,18 +6,18 @@ require_relative "line_formatters/result_line"
|
|
|
6
6
|
require_relative "line_formatters/feedback_footer"
|
|
7
7
|
|
|
8
8
|
class Evilution::Reporter::CLI::Trailer
|
|
9
|
-
def self.default_lines(min_score: nil)
|
|
9
|
+
def self.default_lines(min_score: nil, since: nil)
|
|
10
10
|
[
|
|
11
11
|
Evilution::Reporter::CLI::LineFormatters::TruncationNotice.new,
|
|
12
|
-
Evilution::Reporter::CLI::LineFormatters::ResultLine.new(min_score: min_score),
|
|
12
|
+
Evilution::Reporter::CLI::LineFormatters::ResultLine.new(min_score: min_score, since: since),
|
|
13
13
|
Evilution::Reporter::CLI::LineFormatters::FeedbackFooter.new
|
|
14
14
|
]
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
DEFAULT_LINES = default_lines.freeze
|
|
18
18
|
|
|
19
|
-
def initialize(lines: nil, min_score: nil)
|
|
20
|
-
@lines = lines || self.class.default_lines(min_score: min_score)
|
|
19
|
+
def initialize(lines: nil, min_score: nil, since: nil)
|
|
20
|
+
@lines = lines || self.class.default_lines(min_score: min_score, since: since)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
def call(summary)
|
|
@@ -7,11 +7,12 @@ class Evilution::Reporter::CLI
|
|
|
7
7
|
|
|
8
8
|
def initialize(
|
|
9
9
|
min_score: nil,
|
|
10
|
+
since: nil,
|
|
10
11
|
header: LineFormatters::Header.new,
|
|
11
12
|
metrics_block: MetricsBlock.new,
|
|
12
13
|
section_renderer: SectionRenderer.new,
|
|
13
14
|
sections: DEFAULT_SECTIONS,
|
|
14
|
-
trailer: Trailer.new(min_score: min_score)
|
|
15
|
+
trailer: Trailer.new(min_score: min_score, since: since)
|
|
15
16
|
)
|
|
16
17
|
@header = header
|
|
17
18
|
@metrics_block = metrics_block
|
|
@@ -38,6 +39,7 @@ require_relative "cli/section"
|
|
|
38
39
|
require_relative "cli/section_renderer"
|
|
39
40
|
require_relative "cli/line_formatters/header"
|
|
40
41
|
require_relative "cli/line_formatters/mutations"
|
|
42
|
+
require_relative "cli/line_formatters/skipped_sites"
|
|
41
43
|
require_relative "cli/line_formatters/score"
|
|
42
44
|
require_relative "cli/line_formatters/duration"
|
|
43
45
|
require_relative "cli/line_formatters/efficiency"
|
|
@@ -60,9 +62,14 @@ Evilution::Reporter::CLI.const_set(
|
|
|
60
62
|
[
|
|
61
63
|
Evilution::Reporter::CLI::Section.new(
|
|
62
64
|
title: ->(gaps) { "Survived mutations (#{gaps.length} coverage gap#{"s" unless gaps.length == 1}):" },
|
|
63
|
-
fetcher: lambda(&:
|
|
65
|
+
fetcher: lambda(&:survived_gaps),
|
|
64
66
|
formatter: Evilution::Reporter::CLI::ItemFormatters::CoverageGap.new
|
|
65
67
|
),
|
|
68
|
+
Evilution::Reporter::CLI::Section.new(
|
|
69
|
+
title: "Uncovered mutations (no test executes the lines they change, not run):",
|
|
70
|
+
fetcher: lambda(&:uncovered_results),
|
|
71
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::ResultLocation.new
|
|
72
|
+
),
|
|
66
73
|
Evilution::Reporter::CLI::Section.new(
|
|
67
74
|
title: lambda { |groups|
|
|
68
75
|
subjects = groups.sum(&:length)
|
|
@@ -28,6 +28,7 @@ h1 { font-size: 1.5rem; color: #f0f6fc; }
|
|
|
28
28
|
.map-line.equivalent { color: #8b949e; }
|
|
29
29
|
.map-line.unresolved { color: #8b949e; }
|
|
30
30
|
.map-line.unparseable { color: #8b949e; }
|
|
31
|
+
.map-line.uncovered { color: #f85149; }
|
|
31
32
|
.line-number { min-width: 60px; color: #8b949e; }
|
|
32
33
|
.operator { flex: 1; }
|
|
33
34
|
.status-badge { font-size: 0.7rem; padding: 0.1rem 0.5rem; border-radius: 10px; text-transform: uppercase; font-weight: bold; }
|
|
@@ -39,6 +40,7 @@ h1 { font-size: 1.5rem; color: #f0f6fc; }
|
|
|
39
40
|
.status-badge.error { background: #4a3a10; }
|
|
40
41
|
.status-badge.unresolved { background: #21262d; }
|
|
41
42
|
.status-badge.unparseable { background: #21262d; }
|
|
43
|
+
.status-badge.uncovered { background: #4a1a1a; }
|
|
42
44
|
.survived-details { border-top: 1px solid #30363d; padding: 1rem; }
|
|
43
45
|
.survived-details h3 { color: #f85149; font-size: 0.9rem; margin-bottom: 0.75rem; }
|
|
44
46
|
.survived-entry { background: #1c1a1a; border: 1px solid #4a1a1a; border-radius: 6px; padding: 0.75rem; margin-bottom: 0.75rem; }
|
|
@@ -83,3 +85,9 @@ h1 { font-size: 1.5rem; color: #f0f6fc; }
|
|
|
83
85
|
.survived-entry.regression { border-color: #f85149; background: #2a1010; }
|
|
84
86
|
.regression-badge { background: #da3633; color: #fff; font-size: 0.65rem; padding: 0.1rem 0.4rem; border-radius: 4px; margin-left: 0.5rem; text-transform: uppercase; font-weight: bold; }
|
|
85
87
|
footer { text-align: center; color: #484f58; font-size: 0.75rem; margin-top: 2rem; padding-top: 1rem; border-top: 1px solid #21262d; }
|
|
88
|
+
.uncovered-details { border-top: 1px solid #30363d; padding: 1rem; }
|
|
89
|
+
.uncovered-details h3 { color: #f85149; font-size: 0.9rem; margin-bottom: 0.75rem; }
|
|
90
|
+
.uncovered-details ul { list-style: none; }
|
|
91
|
+
.uncovered-details li { font-size: 0.85rem; padding: 0.2rem 0; color: #8b949e; }
|
|
92
|
+
.uncovered-details .operator { color: #c9d1d9; font-family: monospace; margin-right: 0.75rem; }
|
|
93
|
+
.uncovered-details .location { font-family: monospace; }
|
|
@@ -7,6 +7,7 @@ require_relative "error_details"
|
|
|
7
7
|
require_relative "neutral_details"
|
|
8
8
|
require_relative "unresolved_details"
|
|
9
9
|
require_relative "unparseable_details"
|
|
10
|
+
require_relative "uncovered_details"
|
|
10
11
|
|
|
11
12
|
class Evilution::Reporter::HTML::Sections::FileSection < Evilution::Reporter::HTML::Section
|
|
12
13
|
template "file_section"
|
|
@@ -28,6 +29,10 @@ class Evilution::Reporter::HTML::Sections::FileSection < Evilution::Reporter::HT
|
|
|
28
29
|
@results.count(&:survived?)
|
|
29
30
|
end
|
|
30
31
|
|
|
32
|
+
def uncovered_count
|
|
33
|
+
@results.count(&:uncovered?)
|
|
34
|
+
end
|
|
35
|
+
|
|
31
36
|
def total
|
|
32
37
|
@results.length
|
|
33
38
|
end
|
|
@@ -44,6 +49,10 @@ class Evilution::Reporter::HTML::Sections::FileSection < Evilution::Reporter::HT
|
|
|
44
49
|
)
|
|
45
50
|
end
|
|
46
51
|
|
|
52
|
+
def uncovered_html
|
|
53
|
+
Evilution::Reporter::HTML::Sections::UncoveredDetails.render_if(@results.select(&:uncovered?))
|
|
54
|
+
end
|
|
55
|
+
|
|
47
56
|
def error_html
|
|
48
57
|
Evilution::Reporter::HTML::Sections::ErrorDetails.render_if(@results.select(&:error?))
|
|
49
58
|
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../sections"
|
|
4
|
+
|
|
5
|
+
# The mutations no test executes any changed line of: never run, scored as
|
|
6
|
+
# survivors.
|
|
7
|
+
class Evilution::Reporter::HTML::Sections::UncoveredDetails < Evilution::Reporter::HTML::Section
|
|
8
|
+
template "uncovered_details"
|
|
9
|
+
|
|
10
|
+
def self.render_if(uncovered)
|
|
11
|
+
return "" if uncovered.empty?
|
|
12
|
+
|
|
13
|
+
new(uncovered).render
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def initialize(uncovered)
|
|
17
|
+
@uncovered = uncovered
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
attr_reader :uncovered
|
|
23
|
+
|
|
24
|
+
def sorted
|
|
25
|
+
uncovered.sort_by { |r| [r.mutation.line, r.mutation.operator_name] }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
<section class="file-section">
|
|
2
2
|
<h2 class="file-header">
|
|
3
3
|
<span class="file-path"><%= h(@path) %></span>
|
|
4
|
-
<span class="file-stats"><%= killed_count %> killed / <%= survived_count %> survived /
|
|
4
|
+
<span class="file-stats"><%= killed_count %> killed / <%= survived_count %> survived / <%- if uncovered_count.positive? -%><%= uncovered_count %> uncovered / <%- end -%><%= total %> total</span>
|
|
5
5
|
</h2>
|
|
6
6
|
<div class="mutation-map"><%= map_html %></div>
|
|
7
7
|
<%= survived_html %>
|
|
8
|
+
<%= uncovered_html %>
|
|
8
9
|
<%= error_html %>
|
|
9
10
|
<%= neutral_html %>
|
|
10
11
|
<%= unresolved_html %>
|
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
<div class="card"><span class="card-value"><%= @summary.total %></span><span class="card-label">Total</span></div>
|
|
3
3
|
<div class="card card-killed"><span class="card-value"><%= @summary.killed %></span><span class="card-label">Killed</span></div>
|
|
4
4
|
<div class="card card-survived"><span class="card-value"><%= @summary.survived %></span><span class="card-label">Survived</span></div>
|
|
5
|
+
<%- if @summary.uncovered.positive? -%>
|
|
6
|
+
<div class="card card-survived"><span class="card-value"><%= @summary.uncovered %></span><span class="card-label">Uncovered</span></div>
|
|
7
|
+
<%- end -%>
|
|
5
8
|
<div class="card"><span class="card-value"><%= @summary.timed_out %></span><span class="card-label">Timed Out</span></div>
|
|
6
9
|
<div class="card"><span class="card-value"><%= @summary.errors %></span><span class="card-label">Errors</span></div>
|
|
7
10
|
<div class="card"><span class="card-value"><%= @summary.neutral %></span><span class="card-label">Neutral</span></div>
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
<div class="uncovered-details">
|
|
2
|
+
<h3>Uncovered (<%= uncovered.length %>) — no test executes the lines they change, not run</h3>
|
|
3
|
+
<ul>
|
|
4
|
+
<%- sorted.each do |r| -%>
|
|
5
|
+
<li>
|
|
6
|
+
<span class="operator"><%= h(r.mutation.operator_name) %></span>
|
|
7
|
+
<span class="location"><%= h(r.mutation.file_path) %>:<%= r.mutation.line %></span>
|
|
8
|
+
</li>
|
|
9
|
+
<%- end -%>
|
|
10
|
+
</ul>
|
|
11
|
+
</div>
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../json"
|
|
4
|
+
|
|
5
|
+
# One mutation's entry in a JSON report, whichever bucket it is listed under.
|
|
6
|
+
class Evilution::Reporter::JSON::MutationDetail
|
|
7
|
+
# The mutations that point at a missing test, and get a suggestion for one:
|
|
8
|
+
# those that ran and survived, and those no test executes at all.
|
|
9
|
+
UNTESTED_STATUSES = %i[survived uncovered].freeze
|
|
10
|
+
private_constant :UNTESTED_STATUSES
|
|
11
|
+
|
|
12
|
+
def initialize(suggestion:, timings:)
|
|
13
|
+
@suggestion = suggestion
|
|
14
|
+
@timings = timings
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call(result)
|
|
18
|
+
mutation = result.mutation
|
|
19
|
+
detail = base_mutation_fields(mutation, result)
|
|
20
|
+
append_survived_fields(detail, mutation) if UNTESTED_STATUSES.include?(result.status)
|
|
21
|
+
detail[:test_command] = result.test_command if result.test_command
|
|
22
|
+
append_run_fields(detail, result)
|
|
23
|
+
append_memory_fields(detail, result)
|
|
24
|
+
append_error_fields(detail, result)
|
|
25
|
+
append_neutral_reason(detail, result)
|
|
26
|
+
detail.merge!(@timings.mutation_fields(result))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# Why a neutral was recorded — a red baseline and an infrastructure crash want
|
|
32
|
+
# opposite responses, and both land in the same bucket.
|
|
33
|
+
def append_neutral_reason(detail, result)
|
|
34
|
+
reason = result.neutral_reason
|
|
35
|
+
return unless reason
|
|
36
|
+
|
|
37
|
+
detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def base_mutation_fields(mutation, result)
|
|
41
|
+
{
|
|
42
|
+
operator: mutation.operator_name,
|
|
43
|
+
file: mutation.file_path,
|
|
44
|
+
line: mutation.line,
|
|
45
|
+
status: result.status.to_s,
|
|
46
|
+
duration: result.duration.round(4),
|
|
47
|
+
diff: mutation.diff
|
|
48
|
+
}
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def append_survived_fields(detail, mutation)
|
|
52
|
+
detail[:suggestion] = @suggestion.suggestion_for(mutation)
|
|
53
|
+
unified = mutation.unified_diff
|
|
54
|
+
detail[:unified_diff] = unified if unified
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Which test killed the mutation and how many tests its run took, where the
|
|
58
|
+
# integration tells.
|
|
59
|
+
# A survivor reported without a run names the sentinel that implied it.
|
|
60
|
+
def append_run_fields(detail, result)
|
|
61
|
+
detail[:killing_test] = result.killing_test if result.killing_test
|
|
62
|
+
detail[:examples_run] = result.examples_run if result.examples_run
|
|
63
|
+
detail[:implied_by] = result.implied_by if result.implied_by
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def append_memory_fields(detail, result)
|
|
67
|
+
detail[:parent_rss_kb] = result.parent_rss_kb if result.parent_rss_kb
|
|
68
|
+
detail[:child_rss_kb] = result.child_rss_kb if result.child_rss_kb
|
|
69
|
+
detail[:memory_delta_kb] = result.memory_delta_kb if result.memory_delta_kb
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def append_error_fields(detail, result)
|
|
73
|
+
detail[:error_message] = result.error_message if result.error_message
|
|
74
|
+
detail[:error_class] = result.error_class if result.error_class
|
|
75
|
+
detail[:error_backtrace] = result.error_backtrace if result.error_backtrace
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../json"
|
|
4
|
+
|
|
5
|
+
# Where the run's time went: each phase of the run, each spec file of the
|
|
6
|
+
# baseline, each mutation phase summed over the mutations, and each worker of
|
|
7
|
+
# a parallel run with the share of its life it spent busy. Every section is
|
|
8
|
+
# always present, empty when the run did not measure it, so a consumer reads
|
|
9
|
+
# one shape.
|
|
10
|
+
class Evilution::Reporter::JSON::Timings
|
|
11
|
+
def call(summary)
|
|
12
|
+
timings = summary.timings
|
|
13
|
+
{
|
|
14
|
+
phases: timings.phases.transform_values { |seconds| seconds.round(4) },
|
|
15
|
+
baseline: timings.baseline_spec_files.map { |file, seconds| { spec_file: file, duration: seconds.round(4) } },
|
|
16
|
+
mutations: summary.mutation_timing_totals.transform_values { |seconds| seconds.round(4) },
|
|
17
|
+
workers: timings.workers.map { |worker| worker_row(worker) }
|
|
18
|
+
}
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The fields a mutation's row carries about its own time: where it went,
|
|
22
|
+
# when the run measured it, and the limit it ran under, when it used it up.
|
|
23
|
+
def mutation_fields(result)
|
|
24
|
+
fields = {}
|
|
25
|
+
fields[:time_limit] = result.time_limit.round(4) if result.time_limit
|
|
26
|
+
timings = result.timings
|
|
27
|
+
fields[:timings] = timings.to_h.transform_values { |seconds| seconds.round(4) } if timings
|
|
28
|
+
fields
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def worker_row(worker)
|
|
34
|
+
{
|
|
35
|
+
pid: worker.pid,
|
|
36
|
+
items: worker.items_completed,
|
|
37
|
+
busy: worker.busy_time.round(4),
|
|
38
|
+
wall: worker.wall_time.round(4),
|
|
39
|
+
utilization: worker.utilization.round(4)
|
|
40
|
+
}
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -8,10 +8,12 @@ require_relative "../reporter"
|
|
|
8
8
|
require_relative "../session/schema"
|
|
9
9
|
|
|
10
10
|
class Evilution::Reporter::JSON
|
|
11
|
-
def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new, baseline: Baseline.new)
|
|
12
|
-
|
|
11
|
+
def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new, baseline: Baseline.new, timings: Timings.new)
|
|
12
|
+
suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: suggest_tests, integration: integration)
|
|
13
|
+
@mutation_detail = MutationDetail.new(suggestion: suggestion, timings: timings)
|
|
13
14
|
@subjects = subjects
|
|
14
15
|
@baseline = baseline
|
|
16
|
+
@timings = timings
|
|
15
17
|
end
|
|
16
18
|
|
|
17
19
|
def call(summary)
|
|
@@ -28,6 +30,7 @@ class Evilution::Reporter::JSON
|
|
|
28
30
|
summary: build_summary(summary),
|
|
29
31
|
coverage_gaps: build_coverage_gaps(summary),
|
|
30
32
|
subjects: @subjects.call(summary),
|
|
33
|
+
timings: @timings.call(summary),
|
|
31
34
|
**result_categories(summary)
|
|
32
35
|
}
|
|
33
36
|
append_disabled_to_report(report, summary)
|
|
@@ -45,7 +48,8 @@ class Evilution::Reporter::JSON
|
|
|
45
48
|
neutral: map_details(summary.neutral_results),
|
|
46
49
|
equivalent: map_details(summary.equivalent_results),
|
|
47
50
|
unresolved: map_details(summary.unresolved_results),
|
|
48
|
-
unparseable: map_details(summary.unparseable_results)
|
|
51
|
+
unparseable: map_details(summary.unparseable_results),
|
|
52
|
+
uncovered: map_details(summary.uncovered_results)
|
|
49
53
|
}
|
|
50
54
|
end
|
|
51
55
|
|
|
@@ -82,7 +86,8 @@ class Evilution::Reporter::JSON
|
|
|
82
86
|
neutral: summary.neutral,
|
|
83
87
|
equivalent: summary.equivalent,
|
|
84
88
|
unresolved: summary.unresolved,
|
|
85
|
-
unparseable: summary.unparseable
|
|
89
|
+
unparseable: summary.unparseable,
|
|
90
|
+
uncovered: summary.uncovered
|
|
86
91
|
}
|
|
87
92
|
end
|
|
88
93
|
|
|
@@ -100,6 +105,7 @@ class Evilution::Reporter::JSON
|
|
|
100
105
|
append_diagnostic_summary_fields(data, summary)
|
|
101
106
|
data[:truncated] = true if summary.truncated?
|
|
102
107
|
data[:skipped] = summary.skipped if summary.skipped.positive?
|
|
108
|
+
data[:skipped_sites] = summary.skipped_sites unless summary.skipped_sites.empty?
|
|
103
109
|
peak = summary.peak_memory_mb
|
|
104
110
|
data[:peak_memory_mb] = peak.round(1) if peak
|
|
105
111
|
end
|
|
@@ -115,52 +121,7 @@ class Evilution::Reporter::JSON
|
|
|
115
121
|
end
|
|
116
122
|
|
|
117
123
|
def build_mutation_detail(result)
|
|
118
|
-
|
|
119
|
-
detail = base_mutation_fields(mutation, result)
|
|
120
|
-
append_survived_fields(detail, mutation) if result.status == :survived
|
|
121
|
-
detail[:test_command] = result.test_command if result.test_command
|
|
122
|
-
append_memory_fields(detail, result)
|
|
123
|
-
append_error_fields(detail, result)
|
|
124
|
-
append_neutral_reason(detail, result)
|
|
125
|
-
detail
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
# Why a neutral was recorded — a red baseline and an infrastructure crash want
|
|
129
|
-
# opposite responses, and both land in the same bucket (EV-5pob / GH #1606).
|
|
130
|
-
def append_neutral_reason(detail, result)
|
|
131
|
-
reason = result.neutral_reason
|
|
132
|
-
return unless reason
|
|
133
|
-
|
|
134
|
-
detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
def base_mutation_fields(mutation, result)
|
|
138
|
-
{
|
|
139
|
-
operator: mutation.operator_name,
|
|
140
|
-
file: mutation.file_path,
|
|
141
|
-
line: mutation.line,
|
|
142
|
-
status: result.status.to_s,
|
|
143
|
-
duration: result.duration.round(4),
|
|
144
|
-
diff: mutation.diff
|
|
145
|
-
}
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
def append_survived_fields(detail, mutation)
|
|
149
|
-
detail[:suggestion] = @suggestion.suggestion_for(mutation)
|
|
150
|
-
unified = mutation.unified_diff
|
|
151
|
-
detail[:unified_diff] = unified if unified
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
def append_memory_fields(detail, result)
|
|
155
|
-
detail[:parent_rss_kb] = result.parent_rss_kb if result.parent_rss_kb
|
|
156
|
-
detail[:child_rss_kb] = result.child_rss_kb if result.child_rss_kb
|
|
157
|
-
detail[:memory_delta_kb] = result.memory_delta_kb if result.memory_delta_kb
|
|
158
|
-
end
|
|
159
|
-
|
|
160
|
-
def append_error_fields(detail, result)
|
|
161
|
-
detail[:error_message] = result.error_message if result.error_message
|
|
162
|
-
detail[:error_class] = result.error_class if result.error_class
|
|
163
|
-
detail[:error_backtrace] = result.error_backtrace if result.error_backtrace
|
|
124
|
+
@mutation_detail.call(result)
|
|
164
125
|
end
|
|
165
126
|
|
|
166
127
|
def build_coverage_gaps(summary)
|
|
@@ -188,3 +149,5 @@ end
|
|
|
188
149
|
|
|
189
150
|
require_relative "json/subjects"
|
|
190
151
|
require_relative "json/baseline"
|
|
152
|
+
require_relative "json/timings"
|
|
153
|
+
require_relative "json/mutation_detail"
|
|
@@ -25,6 +25,11 @@ class Evilution::Result::CoverageGap
|
|
|
25
25
|
mutation_results.first.mutation.diff
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
+
# The survivors a probe-first sentinel implied, which did not run.
|
|
29
|
+
def implied_results
|
|
30
|
+
mutation_results.select(&:implied_by)
|
|
31
|
+
end
|
|
32
|
+
|
|
28
33
|
def count
|
|
29
34
|
mutation_results.length
|
|
30
35
|
end
|
|
@@ -3,17 +3,31 @@
|
|
|
3
3
|
require_relative "../result"
|
|
4
4
|
require_relative "error_info"
|
|
5
5
|
require_relative "memory_stats"
|
|
6
|
+
require_relative "mutation_timings"
|
|
6
7
|
|
|
7
8
|
class Evilution::Result::MutationResult
|
|
8
|
-
STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable].freeze
|
|
9
|
+
STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable uncovered].freeze
|
|
9
10
|
|
|
10
|
-
attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
|
|
11
|
+
attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason,
|
|
12
|
+
:time_limit, :examples_run, :implied_by
|
|
11
13
|
|
|
12
14
|
# known_failures_only: the tests failed, but every example that did was
|
|
13
15
|
# already failing before any mutation ran, so the failure says nothing
|
|
14
16
|
# about this one.
|
|
17
|
+
#
|
|
18
|
+
# time_limit: the seconds the test run was allowed, recorded on a run that
|
|
19
|
+
# used them up so a reader can tell a short adaptive limit from a long one.
|
|
20
|
+
#
|
|
21
|
+
# killing_test: the id of the test whose failure killed the mutation, where
|
|
22
|
+
# the integration names it. examples_run: how many tests ran against the
|
|
23
|
+
# mutation, where the integration counts them.
|
|
24
|
+
#
|
|
25
|
+
# implied_by: the operator of the sentinel mutation whose survival implied
|
|
26
|
+
# this verdict, given without a run (see
|
|
27
|
+
# Runner::MutationExecutor::ProbeFirst); nil for a verdict of its own.
|
|
15
28
|
def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
|
|
16
|
-
test_command: nil, memory: nil, error: nil, neutral_reason: nil, known_failures_only: false
|
|
29
|
+
test_command: nil, memory: nil, error: nil, neutral_reason: nil, known_failures_only: false,
|
|
30
|
+
timings: nil, time_limit: nil, examples_run: nil, implied_by: nil)
|
|
17
31
|
raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
|
|
18
32
|
|
|
19
33
|
@mutation = mutation
|
|
@@ -25,9 +39,30 @@ class Evilution::Result::MutationResult
|
|
|
25
39
|
@error = error
|
|
26
40
|
@neutral_reason = neutral_reason
|
|
27
41
|
@known_failures_only = known_failures_only == true
|
|
42
|
+
@timings = timings
|
|
43
|
+
@time_limit = time_limit
|
|
44
|
+
@examples_run = examples_run
|
|
45
|
+
@implied_by = implied_by
|
|
28
46
|
freeze
|
|
29
47
|
end
|
|
30
48
|
|
|
49
|
+
def with_time_limit(seconds)
|
|
50
|
+
copy(time_limit: seconds)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# This result came from running the mutation again after a first run used up
|
|
54
|
+
# a shorter limit; that run is part of what the mutation cost.
|
|
55
|
+
def after_adaptive_timeout(seconds)
|
|
56
|
+
base = timings || Evilution::Result::MutationTimings.new
|
|
57
|
+
copy(duration: @duration + seconds, timings: base.with_adaptive_timeout(seconds))
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Where the duration went (Evilution::Result::MutationTimings), when the
|
|
61
|
+
# test process measured it.
|
|
62
|
+
def timings
|
|
63
|
+
@timings.is_a?(Evilution::Result::MutationTimings) ? @timings : nil
|
|
64
|
+
end
|
|
65
|
+
|
|
31
66
|
# Positive type checks, not nil checks. EV-s5br / GH #1174: the
|
|
32
67
|
# nil_replacement mutator can swap a nil default into `false` (or some other
|
|
33
68
|
# non-typed value), and `nil?` then returns false, sending a missing method
|
|
@@ -93,4 +128,21 @@ class Evilution::Result::MutationResult
|
|
|
93
128
|
def unparseable?
|
|
94
129
|
status == :unparseable
|
|
95
130
|
end
|
|
131
|
+
|
|
132
|
+
# No test executes any line the mutation changes, so it was never run: it
|
|
133
|
+
# could only have survived. Scored as a survivor, but kept apart from them.
|
|
134
|
+
def uncovered?
|
|
135
|
+
status == :uncovered
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
private
|
|
139
|
+
|
|
140
|
+
def copy(**changes)
|
|
141
|
+
self.class.new(
|
|
142
|
+
mutation: @mutation, status: @status, duration: @duration, killing_test: @killing_test,
|
|
143
|
+
test_command: @test_command, memory: @memory, error: @error, neutral_reason: @neutral_reason,
|
|
144
|
+
known_failures_only: @known_failures_only, timings: @timings, time_limit: @time_limit, examples_run: @examples_run,
|
|
145
|
+
implied_by: @implied_by, **changes
|
|
146
|
+
)
|
|
147
|
+
end
|
|
96
148
|
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../result"
|
|
4
|
+
|
|
5
|
+
# Where one mutation's time went, in seconds:
|
|
6
|
+
#
|
|
7
|
+
# - apply: parsing and evaluating the mutated source
|
|
8
|
+
# - test_run: the first test run against it
|
|
9
|
+
# - confirmation: the re-run of a survivor against its whole spec files
|
|
10
|
+
# - queue_wait: in a parallel run, the time from the dispatcher sending it to a
|
|
11
|
+
# worker to that worker starting it
|
|
12
|
+
# - adaptive_timeout: a first run that used up a limit shorter than the fixed
|
|
13
|
+
# one, after which the mutation was run again under the fixed
|
|
14
|
+
# limit; the other phases then belong to that second run
|
|
15
|
+
#
|
|
16
|
+
# A phase that did not happen is nil. What the mutation's duration holds beyond
|
|
17
|
+
# these is the cost of the isolation around it: fork, sandbox, reap.
|
|
18
|
+
class Evilution::Result::MutationTimings
|
|
19
|
+
PHASES = %i[apply test_run confirmation queue_wait adaptive_timeout].freeze
|
|
20
|
+
|
|
21
|
+
attr_reader(*PHASES)
|
|
22
|
+
|
|
23
|
+
# The timings a test process sent back, which may be missing or not a Hash at
|
|
24
|
+
# all when the mutation broke what produced them.
|
|
25
|
+
def self.from_h(hash)
|
|
26
|
+
return nil unless hash.is_a?(Hash)
|
|
27
|
+
|
|
28
|
+
fields = PHASES.to_h { |phase| [phase, hash[phase]] }.select { |_, value| value.is_a?(Numeric) }
|
|
29
|
+
return nil if fields.empty?
|
|
30
|
+
|
|
31
|
+
new(**fields)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def initialize(apply: nil, test_run: nil, confirmation: nil, queue_wait: nil, adaptive_timeout: nil)
|
|
35
|
+
@apply = apply
|
|
36
|
+
@test_run = test_run
|
|
37
|
+
@confirmation = confirmation
|
|
38
|
+
@queue_wait = queue_wait
|
|
39
|
+
@adaptive_timeout = adaptive_timeout
|
|
40
|
+
freeze
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def with_queue_wait(seconds)
|
|
44
|
+
self.class.new(**to_h, queue_wait: seconds)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def with_adaptive_timeout(seconds)
|
|
48
|
+
self.class.new(**to_h, adaptive_timeout: seconds)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def to_h
|
|
52
|
+
PHASES.to_h { |phase| [phase, public_send(phase)] }.compact
|
|
53
|
+
end
|
|
54
|
+
end
|