evilution 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +30 -0
- data/CHANGELOG.md +49 -0
- data/README.md +285 -45
- data/docs/architecture.md +110 -12
- data/docs/ast_pattern_syntax.md +35 -7
- data/docs/combined_runs_spike.md +407 -0
- data/docs/integrations.md +1 -1
- data/docs/isolation.md +63 -2
- data/docs/migration-from-mutant.md +16 -11
- data/docs/mutant_schemata_spike.md +387 -0
- data/docs/operator_yield_study.md +576 -0
- data/docs/probe_first_spike.md +264 -0
- data/lib/evilution/ast/ancestor_lookup.rb +36 -0
- data/lib/evilution/ast/callback_declaration.rb +36 -3
- data/lib/evilution/ast/class_body.rb +62 -16
- data/lib/evilution/ast/def_region.rb +130 -0
- data/lib/evilution/ast/local_declaration.rb +37 -0
- data/lib/evilution/ast/parent_index.rb +50 -0
- data/lib/evilution/ast/parse_cache.rb +27 -0
- data/lib/evilution/ast/parser.rb +8 -3
- data/lib/evilution/ast/pattern/filter.rb +53 -12
- data/lib/evilution/ast/skip_filter.rb +83 -0
- data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
- data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
- data/lib/evilution/ast/skip_sites/logging.rb +93 -0
- data/lib/evilution/ast/skip_sites/message.rb +44 -0
- data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
- data/lib/evilution/ast/skip_sites/presets.rb +31 -0
- data/lib/evilution/ast/skip_sites/purity.rb +69 -0
- data/lib/evilution/ast/skip_sites/requires.rb +40 -0
- data/lib/evilution/ast/skip_sites.rb +9 -0
- data/lib/evilution/ast/source_surgeon.rb +20 -2
- data/lib/evilution/ast/syntax_check.rb +10 -8
- data/lib/evilution/baseline/report.rb +11 -2
- data/lib/evilution/baseline.rb +31 -6
- data/lib/evilution/cache/keys.rb +140 -0
- data/lib/evilution/cache/run_fingerprint.rb +51 -0
- data/lib/evilution/cache/skeleton.rb +57 -0
- data/lib/evilution/cache/spec_digest.rb +39 -0
- data/lib/evilution/cache.rb +143 -50
- data/lib/evilution/class_tree.rb +38 -0
- data/lib/evilution/cli/command.rb +12 -4
- data/lib/evilution/cli/commands/run.rb +12 -3
- data/lib/evilution/cli/commands/subjects.rb +5 -2
- data/lib/evilution/cli/parser/options_builder.rb +67 -3
- data/lib/evilution/cli/printers/environment.rb +34 -4
- data/lib/evilution/cli/printers/session_detail.rb +20 -3
- data/lib/evilution/compare/categorizer.rb +2 -1
- data/lib/evilution/compare/normalizer.rb +3 -2
- data/lib/evilution/config/sources.rb +3 -0
- data/lib/evilution/config/validators/base.rb +9 -0
- data/lib/evilution/config/validators/since.rb +22 -0
- data/lib/evilution/config/validators/skip_sites.rb +24 -0
- data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
- data/lib/evilution/config/validators/timeout_factor.rb +10 -0
- data/lib/evilution/config/validators/timeout_floor.rb +10 -0
- data/lib/evilution/config.rb +122 -6
- data/lib/evilution/coverage/load_tracker.rb +50 -0
- data/lib/evilution/coverage/map.rb +42 -5
- data/lib/evilution/coverage/map_builder.rb +123 -22
- data/lib/evilution/coverage/map_store.rb +15 -5
- data/lib/evilution/coverage/recorder.rb +42 -3
- data/lib/evilution/coverage_example_filter.rb +35 -0
- data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
- data/lib/evilution/example_filter.rb +20 -0
- data/lib/evilution/git/changed_lines.rb +103 -0
- data/lib/evilution/git/since_scope.rb +77 -0
- data/lib/evilution/git/unified_diff.rb +68 -0
- data/lib/evilution/infra_crash.rb +26 -0
- data/lib/evilution/integration/base.rb +97 -6
- data/lib/evilution/integration/crash_detector.rb +46 -6
- data/lib/evilution/integration/known_failures.rb +4 -0
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
- data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
- data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
- data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
- data/lib/evilution/integration/loading/concern_registry.rb +87 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
- data/lib/evilution/integration/loading/created_modules.rb +56 -0
- data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
- data/lib/evilution/integration/loading/neutralization.rb +41 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
- data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
- data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
- data/lib/evilution/integration/minitest.rb +22 -2
- data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
- data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
- data/lib/evilution/integration/rspec/example_ids.rb +23 -0
- data/lib/evilution/integration/rspec/example_order.rb +86 -0
- data/lib/evilution/integration/rspec/killer_first.rb +72 -0
- data/lib/evilution/integration/rspec/result_builder.rb +7 -0
- data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
- data/lib/evilution/integration/rspec/state_guard.rb +2 -2
- data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
- data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
- data/lib/evilution/integration/rspec.rb +98 -47
- data/lib/evilution/integration/stop_policy.rb +49 -0
- data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
- data/lib/evilution/integration/test_unit.rb +28 -2
- data/lib/evilution/isolation/fork.rb +58 -13
- data/lib/evilution/isolation/in_process.rb +12 -1
- data/lib/evilution/line_set.rb +72 -0
- data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
- data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
- data/lib/evilution/mcp/info_tool.rb +24 -17
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
- data/lib/evilution/mcp/mutate_tool.rb +32 -7
- data/lib/evilution/mcp/session_tool.rb +1 -2
- data/lib/evilution/mcp.rb +4 -0
- data/lib/evilution/mutation/line_diff.rb +111 -0
- data/lib/evilution/mutation.rb +17 -13
- data/lib/evilution/mutator/base.rb +121 -98
- data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
- data/lib/evilution/mutator/line_range_filter.rb +27 -0
- data/lib/evilution/mutator/load_sources.rb +88 -0
- data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
- data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
- data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
- data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
- data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
- data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
- data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
- data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
- data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
- data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
- data/lib/evilution/mutator/registry.rb +11 -5
- data/lib/evilution/mutator/restore_source_cache.rb +41 -0
- data/lib/evilution/parallel/pool.rb +11 -4
- data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
- data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
- data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
- data/lib/evilution/parallel/work_queue/worker.rb +10 -3
- data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
- data/lib/evilution/parallel/work_queue.rb +69 -12
- data/lib/evilution/parallel_db_warning.rb +39 -12
- data/lib/evilution/process_supervisor.rb +22 -2
- data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
- data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
- data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
- data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +4 -4
- data/lib/evilution/reporter/cli.rb +9 -2
- data/lib/evilution/reporter/html/assets/style.css +8 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
- data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
- data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
- data/lib/evilution/reporter/json/subjects.rb +1 -0
- data/lib/evilution/reporter/json/timings.rb +42 -0
- data/lib/evilution/reporter/json.rb +13 -50
- data/lib/evilution/result/coverage_gap.rb +5 -0
- data/lib/evilution/result/mutation_result.rb +55 -3
- data/lib/evilution/result/mutation_timings.rb +54 -0
- data/lib/evilution/result/run_timings.rb +20 -0
- data/lib/evilution/result/subject_score.rb +8 -1
- data/lib/evilution/result/subject_scorer.rb +5 -3
- data/lib/evilution/result/summary.rb +49 -12
- data/lib/evilution/runner/baseline_runner.rb +47 -5
- data/lib/evilution/runner/diagnostics.rb +0 -13
- data/lib/evilution/runner/isolation_resolver.rb +16 -4
- data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
- data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
- data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
- data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
- data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
- data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
- data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
- data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
- data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
- data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
- data/lib/evilution/runner/mutation_executor.rb +75 -10
- data/lib/evilution/runner/mutation_planner.rb +29 -21
- data/lib/evilution/runner/phase_clock.rb +22 -0
- data/lib/evilution/runner/report_publisher.rb +1 -1
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +87 -26
- data/lib/evilution/session/diff.rb +9 -2
- data/lib/evilution/session/store.rb +17 -2
- data/lib/evilution/source_ast_cache.rb +10 -0
- data/lib/evilution/spec_ast_cache.rb +15 -0
- data/lib/evilution/spec_resolver/constant_references.rb +66 -0
- data/lib/evilution/spec_resolver/project_files.rb +50 -0
- data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
- data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
- data/lib/evilution/spec_resolver.rb +76 -65
- data/lib/evilution/spec_selector.rb +17 -13
- data/lib/evilution/subject.rb +7 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution/worker_database/copies.rb +73 -0
- data/lib/evilution/worker_database/schema.rb +39 -0
- data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
- data/lib/evilution/worker_database.rb +214 -0
- data/schema/evilution.config.schema.json +74 -2
- metadata +84 -13
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
- data/scripts/benchmark_density +0 -262
- data/scripts/benchmark_density.yml +0 -19
- data/scripts/canary_manifest.yml +0 -47
- data/scripts/compare_mutations +0 -421
- data/scripts/compare_mutations.yml +0 -24
- data/scripts/compare_targeting +0 -279
- data/scripts/compare_targeting.example.yml +0 -24
- data/scripts/mutant_json_adapter +0 -227
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative "../state_guard"
|
|
4
|
-
require_relative "../../rspec"
|
|
5
|
-
|
|
6
|
-
class Evilution::Integration::RSpec::StateGuard::ObjectSpaceExampleGroups
|
|
7
|
-
def snapshot
|
|
8
|
-
groups = Set.new
|
|
9
|
-
ObjectSpace.each_object(Class) do |klass|
|
|
10
|
-
groups << klass.object_id if klass < ::RSpec::Core::ExampleGroup
|
|
11
|
-
rescue TypeError
|
|
12
|
-
# ObjectSpace iteration may surface partially-initialized or anonymous
|
|
13
|
-
# classes whose `<` comparison raises. Skipping them is safe — they
|
|
14
|
-
# cannot be ExampleGroup descendants we need to track.
|
|
15
|
-
end
|
|
16
|
-
groups
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def release(eg_before)
|
|
20
|
-
return unless eg_before
|
|
21
|
-
|
|
22
|
-
ObjectSpace.each_object(Class) do |klass|
|
|
23
|
-
next unless klass < ::RSpec::Core::ExampleGroup
|
|
24
|
-
next if eg_before.include?(klass.object_id)
|
|
25
|
-
|
|
26
|
-
klass.constants(false).each do |const|
|
|
27
|
-
klass.send(:remove_const, const)
|
|
28
|
-
rescue NameError
|
|
29
|
-
# Constant may have been removed concurrently (e.g. via autoload
|
|
30
|
-
# reload) between #constants(false) and #remove_const. Best-effort
|
|
31
|
-
# cleanup — nothing to do if it's already gone.
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
klass.instance_variables.each do |ivar|
|
|
35
|
-
klass.remove_instance_variable(ivar)
|
|
36
|
-
end
|
|
37
|
-
rescue TypeError
|
|
38
|
-
# Same defensive case as #snapshot: skip classes whose `<` raises
|
|
39
|
-
# mid-iteration.
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
end
|
data/scripts/benchmark_density
DELETED
|
@@ -1,262 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
# Mutation Density Benchmark
|
|
5
|
-
#
|
|
6
|
-
# Compares mutation counts between evilution and a reference mutation testing
|
|
7
|
-
# tool on a corpus of files.
|
|
8
|
-
# See docs/mutation_density_benchmark.md for methodology.
|
|
9
|
-
#
|
|
10
|
-
# Usage:
|
|
11
|
-
# scripts/benchmark_density CONFIG_FILE [--verbose]
|
|
12
|
-
#
|
|
13
|
-
# The config file is a YAML file listing the benchmark corpus.
|
|
14
|
-
# See scripts/benchmark_density.yml for the format.
|
|
15
|
-
|
|
16
|
-
require "yaml"
|
|
17
|
-
require "open3"
|
|
18
|
-
require "optparse"
|
|
19
|
-
|
|
20
|
-
module BenchmarkDensity
|
|
21
|
-
class ConfigError < StandardError; end
|
|
22
|
-
|
|
23
|
-
class Config
|
|
24
|
-
attr_reader :project_root, :target_ratio, :reference_cmd, :files
|
|
25
|
-
|
|
26
|
-
def initialize(path)
|
|
27
|
-
data = YAML.safe_load_file(path, symbolize_names: true, permitted_classes: [Symbol])
|
|
28
|
-
raise ConfigError, "config file must contain a YAML mapping" unless data.is_a?(Hash)
|
|
29
|
-
|
|
30
|
-
@project_root = data[:project_root]
|
|
31
|
-
@target_ratio = data.key?(:target_ratio) ? data[:target_ratio] : 1.5
|
|
32
|
-
@reference_cmd = data.key?(:reference_cmd) ? data[:reference_cmd] : []
|
|
33
|
-
@files = data.key?(:files) ? data[:files] : []
|
|
34
|
-
|
|
35
|
-
validate!
|
|
36
|
-
rescue Psych::Exception => e
|
|
37
|
-
raise ConfigError, "failed to parse config file #{path}: #{e.message}"
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
private
|
|
41
|
-
|
|
42
|
-
def validate!
|
|
43
|
-
raise ConfigError, "project_root is required in config" if @project_root.nil?
|
|
44
|
-
raise ConfigError, "project_root does not exist: #{@project_root}" unless Dir.exist?(@project_root)
|
|
45
|
-
raise ConfigError, "reference_cmd is required in config" if @reference_cmd.empty?
|
|
46
|
-
raise ConfigError, "files list is empty — add benchmark files to the config" if @files.empty?
|
|
47
|
-
|
|
48
|
-
@files.each_with_index { |entry, index| validate_file_entry!(entry, index) }
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def validate_file_entry!(entry, index)
|
|
52
|
-
raise ConfigError, "files[#{index}] must be a mapping" unless entry.is_a?(Hash)
|
|
53
|
-
|
|
54
|
-
path = entry[:path]
|
|
55
|
-
target = entry[:reference_target]
|
|
56
|
-
raise ConfigError, "files[#{index}] is missing required path" if path.nil? || path.to_s.strip.empty?
|
|
57
|
-
raise ConfigError, "files[#{index}] is missing required reference_target" if target.nil? || target.to_s.strip.empty?
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
class EvilutionCounter
|
|
62
|
-
def initialize(project_root)
|
|
63
|
-
@project_root = project_root
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def count_mutations(file_path)
|
|
67
|
-
full_path = File.join(@project_root, file_path)
|
|
68
|
-
cmd = ["bundle", "exec", "evilution", "subjects", full_path]
|
|
69
|
-
stdout, stderr, status = Open3.capture3(*cmd, chdir: @project_root)
|
|
70
|
-
|
|
71
|
-
unless status.success?
|
|
72
|
-
warn " evilution failed for #{file_path}: #{stderr.strip}"
|
|
73
|
-
return nil
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
count = parse_subject_count(stdout)
|
|
77
|
-
if count.nil?
|
|
78
|
-
warn " evilution produced unparseable output for #{file_path}: #{stdout.lines.first&.strip}"
|
|
79
|
-
return nil
|
|
80
|
-
end
|
|
81
|
-
count
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
private
|
|
85
|
-
|
|
86
|
-
def parse_subject_count(output)
|
|
87
|
-
match = output.match(/(\d+)\s+subjects,\s+(\d+)\s+mutations/)
|
|
88
|
-
match ? match[2].to_i : nil
|
|
89
|
-
end
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
class ReferenceCounter
|
|
93
|
-
def initialize(project_root, reference_cmd)
|
|
94
|
-
@project_root = project_root
|
|
95
|
-
@reference_cmd = reference_cmd
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
def count_mutations(reference_target)
|
|
99
|
-
cmd = @reference_cmd + [reference_target]
|
|
100
|
-
stdout, stderr, _status = Open3.capture3(*cmd, chdir: @project_root)
|
|
101
|
-
|
|
102
|
-
# Reference tool may exit non-zero when mutants survive, which is expected.
|
|
103
|
-
# Only treat it as failure if there's no parseable output.
|
|
104
|
-
count = parse_mutation_count(stdout)
|
|
105
|
-
if count.nil?
|
|
106
|
-
warn " reference tool failed for #{reference_target}: #{stderr.lines.first&.strip}"
|
|
107
|
-
return nil
|
|
108
|
-
end
|
|
109
|
-
count
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
private
|
|
113
|
-
|
|
114
|
-
def parse_mutation_count(output)
|
|
115
|
-
match = output.match(/mutations:\s*(\d+)/i)
|
|
116
|
-
match ? match[1].to_i : nil
|
|
117
|
-
end
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
class Runner
|
|
121
|
-
HEADER_FMT = "%-45<file>s %10<evilution>s %10<reference>s %8<ratio>s"
|
|
122
|
-
ROW_FMT = "%-45<file>s %10<evilution>s %10<reference>s %8<ratio>s"
|
|
123
|
-
TOTAL_FMT = "%-45<file>s %10<evilution>d %10<reference>d %7.2<ratio>fx"
|
|
124
|
-
|
|
125
|
-
def initialize(config:, verbose: false)
|
|
126
|
-
@config = config
|
|
127
|
-
@verbose = verbose
|
|
128
|
-
@evilution = EvilutionCounter.new(config.project_root)
|
|
129
|
-
@reference = ReferenceCounter.new(config.project_root, config.reference_cmd)
|
|
130
|
-
end
|
|
131
|
-
|
|
132
|
-
def call
|
|
133
|
-
results = collect_results
|
|
134
|
-
print_table(results)
|
|
135
|
-
print_summary(results)
|
|
136
|
-
passing?(results) ? 0 : 1
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
private
|
|
140
|
-
|
|
141
|
-
def collect_results
|
|
142
|
-
@config.files.map do |entry|
|
|
143
|
-
path = entry[:path]
|
|
144
|
-
reference_target = entry[:reference_target]
|
|
145
|
-
|
|
146
|
-
warn " Measuring #{path}..." if @verbose
|
|
147
|
-
ev_count = @evilution.count_mutations(path)
|
|
148
|
-
ref_count = @reference.count_mutations(reference_target)
|
|
149
|
-
warn " done" if @verbose
|
|
150
|
-
|
|
151
|
-
{ path: path, evilution: ev_count, reference: ref_count }
|
|
152
|
-
end
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
def print_table(results)
|
|
156
|
-
puts format(HEADER_FMT, file: "File", evilution: "Evilution", reference: "Reference", ratio: "Ratio")
|
|
157
|
-
puts "-" * 75
|
|
158
|
-
results.each { |r| puts format_result_row(r) }
|
|
159
|
-
puts "-" * 75
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
def format_result_row(result)
|
|
163
|
-
ratio = compute_ratio(result[:evilution], result[:reference])
|
|
164
|
-
format(ROW_FMT,
|
|
165
|
-
file: result[:path],
|
|
166
|
-
evilution: result[:evilution]&.to_s || "ERR",
|
|
167
|
-
reference: result[:reference]&.to_s || "ERR",
|
|
168
|
-
ratio: ratio ? format("%.2fx", ratio) : "N/A")
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
def print_summary(results)
|
|
172
|
-
totals = compute_totals(results)
|
|
173
|
-
print_total_line(totals)
|
|
174
|
-
puts ""
|
|
175
|
-
print_status_line(totals)
|
|
176
|
-
print_error_warning(totals[:error_count])
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
def print_total_line(totals)
|
|
180
|
-
if totals[:invalid]
|
|
181
|
-
puts format(ROW_FMT,
|
|
182
|
-
file: "TOTAL", evilution: totals[:ev].to_s, reference: totals[:ref].to_s, ratio: "N/A")
|
|
183
|
-
else
|
|
184
|
-
puts format(TOTAL_FMT,
|
|
185
|
-
file: "TOTAL", evilution: totals[:ev], reference: totals[:ref], ratio: totals[:display_ratio])
|
|
186
|
-
end
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
def compute_totals(results)
|
|
190
|
-
valid, errored = partition_results(results)
|
|
191
|
-
ev_total = valid.sum { |r| r[:evilution] }
|
|
192
|
-
ref_total = valid.sum { |r| r[:reference] }
|
|
193
|
-
invalid = errored.any? || ev_total.zero?
|
|
194
|
-
ratio = ev_total.positive? ? ref_total.to_f / ev_total : 0.0
|
|
195
|
-
|
|
196
|
-
{ ev: ev_total, ref: ref_total, ratio: ratio, display_ratio: ratio.round(2),
|
|
197
|
-
invalid: invalid, error_count: errored.length }
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
def partition_results(results)
|
|
201
|
-
valid = results.select { |r| r[:evilution] && r[:reference] }
|
|
202
|
-
errored = results - valid
|
|
203
|
-
[valid, errored]
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
def print_status_line(totals)
|
|
207
|
-
target = @config.target_ratio
|
|
208
|
-
if totals[:invalid]
|
|
209
|
-
puts "Density ratio: INVALID (target: < #{target}x) — FAIL"
|
|
210
|
-
return
|
|
211
|
-
end
|
|
212
|
-
|
|
213
|
-
status = totals[:ratio] < target ? "PASS" : "FAIL"
|
|
214
|
-
puts "Density ratio: #{totals[:display_ratio]}x (target: < #{target}x) — #{status}"
|
|
215
|
-
end
|
|
216
|
-
|
|
217
|
-
def print_error_warning(error_count)
|
|
218
|
-
puts "WARNING: #{error_count} file(s) had errors — benchmark marked invalid" if error_count.positive?
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
def passing?(results)
|
|
222
|
-
totals = compute_totals(results)
|
|
223
|
-
return false if totals[:invalid]
|
|
224
|
-
|
|
225
|
-
totals[:ratio] < @config.target_ratio
|
|
226
|
-
end
|
|
227
|
-
|
|
228
|
-
def compute_ratio(evilution, reference)
|
|
229
|
-
return nil if evilution.nil? || reference.nil? || evilution.zero?
|
|
230
|
-
|
|
231
|
-
(reference.to_f / evilution).round(2)
|
|
232
|
-
end
|
|
233
|
-
end
|
|
234
|
-
end
|
|
235
|
-
|
|
236
|
-
if __FILE__ == $PROGRAM_NAME
|
|
237
|
-
verbose = false
|
|
238
|
-
parser = OptionParser.new do |opts|
|
|
239
|
-
opts.banner = "Usage: #{$PROGRAM_NAME} CONFIG_FILE [--verbose]"
|
|
240
|
-
opts.on("-v", "--verbose", "Show progress") { verbose = true }
|
|
241
|
-
opts.on("-h", "--help", "Show help") do
|
|
242
|
-
puts opts
|
|
243
|
-
exit 0
|
|
244
|
-
end
|
|
245
|
-
end
|
|
246
|
-
parser.parse!
|
|
247
|
-
|
|
248
|
-
config_path = ARGV.first
|
|
249
|
-
unless config_path
|
|
250
|
-
warn parser.banner
|
|
251
|
-
exit 2
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
begin
|
|
255
|
-
config = BenchmarkDensity::Config.new(config_path)
|
|
256
|
-
runner = BenchmarkDensity::Runner.new(config: config, verbose: verbose)
|
|
257
|
-
exit runner.call
|
|
258
|
-
rescue BenchmarkDensity::ConfigError => e
|
|
259
|
-
warn "Config error: #{e.message}"
|
|
260
|
-
exit 2
|
|
261
|
-
end
|
|
262
|
-
end
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
# Benchmark corpus configuration
|
|
2
|
-
#
|
|
3
|
-
# See docs/mutation_density_benchmark.md for selection criteria and methodology.
|
|
4
|
-
#
|
|
5
|
-
# Example:
|
|
6
|
-
#
|
|
7
|
-
# project_root: /path/to/rails/app
|
|
8
|
-
# target_ratio: 1.5
|
|
9
|
-
# reference_cmd: ["bundle", "exec", "mutant", "run", "--integration", "rspec", "--fail-fast"]
|
|
10
|
-
# files:
|
|
11
|
-
# - path: app/models/user.rb
|
|
12
|
-
# reference_target: "User"
|
|
13
|
-
# - path: app/services/payment_processor.rb
|
|
14
|
-
# reference_target: "PaymentProcessor"
|
|
15
|
-
|
|
16
|
-
project_root: null # Set to the absolute path of the target project
|
|
17
|
-
target_ratio: 1.5
|
|
18
|
-
reference_cmd: [] # Command prefix for the reference tool
|
|
19
|
-
files: []
|
data/scripts/canary_manifest.yml
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
# Canary manifest for the EV-7ydn validation run of scripts/compare_targeting.
|
|
2
|
-
#
|
|
3
|
-
# scripts/compare_targeting scripts/canary_manifest.yml --out .artifacts/targeting_validation.md
|
|
4
|
-
#
|
|
5
|
-
# Each repo is run through evilution three times (full_file / lexical / coverage)
|
|
6
|
-
# over the SAME mutation set; the gate is total lost_kills == 0 (coverage must
|
|
7
|
-
# never lose a kill full-file caught) before the default flips to coverage.
|
|
8
|
-
#
|
|
9
|
-
# `dir:` assumes the EV-rxob canary checkout layout (/tmp/ev-canaries/<repo>,
|
|
10
|
-
# already `bundle install`ed). Adjust to your checkout. Per-repo `args` mirror
|
|
11
|
-
# the EV-rxob R2 findings: repos whose specs are NOT lib-mirrored need an
|
|
12
|
-
# explicit --spec (EV-z7f5 / GH #1325), or auto spec-resolution yields 0.0 and
|
|
13
|
-
# the comparison is meaningless.
|
|
14
|
-
#
|
|
15
|
-
# Start with the R2 infra=none tier that ran clean; extend with the DB-tier
|
|
16
|
-
# repos once their services are up.
|
|
17
|
-
|
|
18
|
-
repos:
|
|
19
|
-
# --- R2 PASS, lib-mirrored specs (auto-resolution works) ---
|
|
20
|
-
- name: thoughtbot/factory_bot
|
|
21
|
-
dir: /tmp/ev-canaries/factory_bot
|
|
22
|
-
args: ["lib", "--jobs", "4"]
|
|
23
|
-
|
|
24
|
-
- name: jnunemaker/httparty
|
|
25
|
-
dir: /tmp/ev-canaries/httparty
|
|
26
|
-
args: ["lib", "--jobs", "4"]
|
|
27
|
-
|
|
28
|
-
- name: rubocop/rubocop
|
|
29
|
-
dir: /tmp/ev-canaries/rubocop
|
|
30
|
-
args: ["lib", "--jobs", "4"]
|
|
31
|
-
|
|
32
|
-
- name: rack/rack
|
|
33
|
-
dir: /tmp/ev-canaries/rack
|
|
34
|
-
args: ["lib", "--jobs", "4"]
|
|
35
|
-
|
|
36
|
-
# --- Non-lib-mirrored specs: explicit --spec required (EV-z7f5 / GH #1325) ---
|
|
37
|
-
- name: bblimke/webmock
|
|
38
|
-
dir: /tmp/ev-canaries/webmock
|
|
39
|
-
args: ["lib", "--spec", "spec/unit", "--jobs", "4"]
|
|
40
|
-
|
|
41
|
-
- name: doorkeeper-gem/doorkeeper
|
|
42
|
-
dir: /tmp/ev-canaries/doorkeeper
|
|
43
|
-
args: ["lib", "--spec", "spec", "--jobs", "4"]
|
|
44
|
-
|
|
45
|
-
- name: ruby-concurrency/concurrent-ruby
|
|
46
|
-
dir: /tmp/ev-canaries/concurrent-ruby
|
|
47
|
-
args: ["lib", "--spec", "spec", "--jobs", "4"]
|