evilution 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +1 -0
- data/.rubocop_todo.yml +30 -0
- data/CHANGELOG.md +59 -0
- data/README.md +285 -45
- data/docs/architecture.md +110 -12
- data/docs/ast_pattern_syntax.md +35 -7
- data/docs/combined_runs_spike.md +407 -0
- data/docs/integrations.md +1 -1
- data/docs/isolation.md +63 -2
- data/docs/migration-from-mutant.md +16 -11
- data/docs/mutant_schemata_spike.md +387 -0
- data/docs/operator_yield_study.md +576 -0
- data/docs/probe_first_spike.md +264 -0
- data/lib/evilution/ast/ancestor_lookup.rb +36 -0
- data/lib/evilution/ast/callback_declaration.rb +36 -3
- data/lib/evilution/ast/class_body.rb +62 -16
- data/lib/evilution/ast/def_region.rb +130 -0
- data/lib/evilution/ast/local_declaration.rb +37 -0
- data/lib/evilution/ast/parent_index.rb +50 -0
- data/lib/evilution/ast/parse_cache.rb +27 -0
- data/lib/evilution/ast/parser.rb +8 -3
- data/lib/evilution/ast/pattern/filter.rb +53 -12
- data/lib/evilution/ast/skip_filter.rb +83 -0
- data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
- data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
- data/lib/evilution/ast/skip_sites/logging.rb +93 -0
- data/lib/evilution/ast/skip_sites/message.rb +44 -0
- data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
- data/lib/evilution/ast/skip_sites/presets.rb +31 -0
- data/lib/evilution/ast/skip_sites/purity.rb +69 -0
- data/lib/evilution/ast/skip_sites/requires.rb +40 -0
- data/lib/evilution/ast/skip_sites.rb +9 -0
- data/lib/evilution/ast/source_surgeon.rb +20 -2
- data/lib/evilution/ast/syntax_check.rb +10 -8
- data/lib/evilution/baseline/report.rb +11 -2
- data/lib/evilution/baseline.rb +31 -6
- data/lib/evilution/cache/keys.rb +140 -0
- data/lib/evilution/cache/run_fingerprint.rb +51 -0
- data/lib/evilution/cache/skeleton.rb +57 -0
- data/lib/evilution/cache/spec_digest.rb +39 -0
- data/lib/evilution/cache.rb +145 -42
- data/lib/evilution/class_tree.rb +38 -0
- data/lib/evilution/cli/command.rb +12 -4
- data/lib/evilution/cli/commands/run.rb +12 -3
- data/lib/evilution/cli/commands/subjects.rb +5 -2
- data/lib/evilution/cli/parser/options_builder.rb +67 -3
- data/lib/evilution/cli/printers/environment.rb +34 -4
- data/lib/evilution/cli/printers/session_detail.rb +20 -3
- data/lib/evilution/compare/categorizer.rb +2 -1
- data/lib/evilution/compare/normalizer.rb +3 -2
- data/lib/evilution/config/sources.rb +3 -0
- data/lib/evilution/config/validators/base.rb +9 -0
- data/lib/evilution/config/validators/since.rb +22 -0
- data/lib/evilution/config/validators/skip_sites.rb +24 -0
- data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
- data/lib/evilution/config/validators/timeout_factor.rb +10 -0
- data/lib/evilution/config/validators/timeout_floor.rb +10 -0
- data/lib/evilution/config.rb +122 -6
- data/lib/evilution/coverage/load_tracker.rb +50 -0
- data/lib/evilution/coverage/map.rb +42 -5
- data/lib/evilution/coverage/map_builder.rb +123 -22
- data/lib/evilution/coverage/map_store.rb +15 -5
- data/lib/evilution/coverage/recorder.rb +42 -3
- data/lib/evilution/coverage_example_filter.rb +35 -0
- data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
- data/lib/evilution/example_filter.rb +20 -0
- data/lib/evilution/git/changed_lines.rb +103 -0
- data/lib/evilution/git/since_scope.rb +77 -0
- data/lib/evilution/git/unified_diff.rb +68 -0
- data/lib/evilution/infra_crash.rb +26 -0
- data/lib/evilution/integration/base.rb +97 -6
- data/lib/evilution/integration/crash_detector.rb +46 -6
- data/lib/evilution/integration/known_failures.rb +4 -0
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
- data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
- data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
- data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
- data/lib/evilution/integration/loading/concern_registry.rb +87 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
- data/lib/evilution/integration/loading/created_modules.rb +56 -0
- data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
- data/lib/evilution/integration/loading/neutralization.rb +41 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
- data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
- data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
- data/lib/evilution/integration/minitest.rb +22 -2
- data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
- data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
- data/lib/evilution/integration/rspec/example_ids.rb +23 -0
- data/lib/evilution/integration/rspec/example_order.rb +86 -0
- data/lib/evilution/integration/rspec/killer_first.rb +72 -0
- data/lib/evilution/integration/rspec/result_builder.rb +7 -0
- data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
- data/lib/evilution/integration/rspec/state_guard.rb +2 -2
- data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
- data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
- data/lib/evilution/integration/rspec.rb +98 -47
- data/lib/evilution/integration/stop_policy.rb +49 -0
- data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
- data/lib/evilution/integration/test_unit.rb +28 -2
- data/lib/evilution/isolation/fork.rb +58 -13
- data/lib/evilution/isolation/in_process.rb +12 -1
- data/lib/evilution/line_set.rb +72 -0
- data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
- data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
- data/lib/evilution/mcp/info_tool.rb +24 -17
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
- data/lib/evilution/mcp/mutate_tool.rb +32 -7
- data/lib/evilution/mcp/session_tool.rb +1 -2
- data/lib/evilution/mcp.rb +4 -0
- data/lib/evilution/mutation/line_diff.rb +111 -0
- data/lib/evilution/mutation.rb +17 -13
- data/lib/evilution/mutator/base.rb +121 -98
- data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
- data/lib/evilution/mutator/line_range_filter.rb +27 -0
- data/lib/evilution/mutator/load_sources.rb +88 -0
- data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
- data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
- data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
- data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
- data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
- data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
- data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
- data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
- data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
- data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
- data/lib/evilution/mutator/registry.rb +11 -5
- data/lib/evilution/mutator/restore_source_cache.rb +41 -0
- data/lib/evilution/parallel/pool.rb +11 -4
- data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
- data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
- data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
- data/lib/evilution/parallel/work_queue/worker.rb +10 -3
- data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
- data/lib/evilution/parallel/work_queue.rb +69 -12
- data/lib/evilution/parallel_db_warning.rb +39 -12
- data/lib/evilution/process_supervisor.rb +22 -2
- data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
- data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
- data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
- data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +4 -4
- data/lib/evilution/reporter/cli.rb +9 -2
- data/lib/evilution/reporter/html/assets/style.css +8 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
- data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
- data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
- data/lib/evilution/reporter/json/subjects.rb +1 -0
- data/lib/evilution/reporter/json/timings.rb +42 -0
- data/lib/evilution/reporter/json.rb +13 -50
- data/lib/evilution/result/coverage_gap.rb +5 -0
- data/lib/evilution/result/mutation_result.rb +55 -3
- data/lib/evilution/result/mutation_timings.rb +54 -0
- data/lib/evilution/result/run_timings.rb +20 -0
- data/lib/evilution/result/subject_score.rb +8 -1
- data/lib/evilution/result/subject_scorer.rb +5 -3
- data/lib/evilution/result/summary.rb +49 -12
- data/lib/evilution/runner/baseline_runner.rb +47 -5
- data/lib/evilution/runner/diagnostics.rb +0 -13
- data/lib/evilution/runner/isolation_resolver.rb +16 -4
- data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
- data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
- data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
- data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
- data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
- data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
- data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
- data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
- data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
- data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
- data/lib/evilution/runner/mutation_executor.rb +75 -10
- data/lib/evilution/runner/mutation_planner.rb +29 -21
- data/lib/evilution/runner/phase_clock.rb +22 -0
- data/lib/evilution/runner/report_publisher.rb +1 -1
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +87 -26
- data/lib/evilution/session/diff.rb +9 -2
- data/lib/evilution/session/store.rb +17 -2
- data/lib/evilution/source_ast_cache.rb +10 -0
- data/lib/evilution/spec_ast_cache.rb +15 -0
- data/lib/evilution/spec_resolver/constant_references.rb +66 -0
- data/lib/evilution/spec_resolver/project_files.rb +50 -0
- data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
- data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
- data/lib/evilution/spec_resolver.rb +76 -65
- data/lib/evilution/spec_selector.rb +17 -13
- data/lib/evilution/subject.rb +7 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution/worker_database/copies.rb +73 -0
- data/lib/evilution/worker_database/schema.rb +39 -0
- data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
- data/lib/evilution/worker_database.rb +214 -0
- data/schema/evilution.config.schema.json +74 -2
- metadata +84 -13
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
- data/scripts/benchmark_density +0 -262
- data/scripts/benchmark_density.yml +0 -19
- data/scripts/canary_manifest.yml +0 -47
- data/scripts/compare_mutations +0 -421
- data/scripts/compare_mutations.yml +0 -24
- data/scripts/compare_targeting +0 -279
- data/scripts/compare_targeting.example.yml +0 -24
- data/scripts/mutant_json_adapter +0 -227
data/scripts/compare_mutations
DELETED
|
@@ -1,421 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
# Head-to-Head Mutation Comparison
|
|
5
|
-
#
|
|
6
|
-
# Runs evilution and a reference mutation testing tool on a set of files,
|
|
7
|
-
# catalogs every mutation the reference tool produces that evilution doesn't,
|
|
8
|
-
# and outputs a structured comparison report.
|
|
9
|
-
#
|
|
10
|
-
# Usage:
|
|
11
|
-
# scripts/compare_mutations CONFIG_FILE [--verbose]
|
|
12
|
-
#
|
|
13
|
-
# See scripts/compare_mutations.yml for config format.
|
|
14
|
-
|
|
15
|
-
require "yaml"
|
|
16
|
-
require "json"
|
|
17
|
-
require "open3"
|
|
18
|
-
require "optparse"
|
|
19
|
-
require "fileutils"
|
|
20
|
-
|
|
21
|
-
module CompareMutations
|
|
22
|
-
class ConfigError < StandardError; end
|
|
23
|
-
class ToolError < StandardError; end
|
|
24
|
-
|
|
25
|
-
class Config
|
|
26
|
-
attr_reader :project_root, :reference_cmd, :output_dir, :files
|
|
27
|
-
|
|
28
|
-
def initialize(path)
|
|
29
|
-
data = YAML.safe_load_file(path, symbolize_names: true, permitted_classes: [Symbol])
|
|
30
|
-
raise ConfigError, "config file must contain a YAML mapping" unless data.is_a?(Hash)
|
|
31
|
-
|
|
32
|
-
@project_root = data[:project_root]
|
|
33
|
-
@reference_cmd = data.key?(:reference_cmd) ? data[:reference_cmd] : []
|
|
34
|
-
@output_dir = data[:output_dir] || "comparison_results"
|
|
35
|
-
@files = data.key?(:files) ? data[:files] : []
|
|
36
|
-
|
|
37
|
-
validate!
|
|
38
|
-
rescue Psych::Exception => e
|
|
39
|
-
raise ConfigError, "failed to parse config: #{e.message}"
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
private
|
|
43
|
-
|
|
44
|
-
def validate!
|
|
45
|
-
raise ConfigError, "project_root is required in config" if @project_root.nil?
|
|
46
|
-
raise ConfigError, "project_root does not exist: #{@project_root}" unless Dir.exist?(@project_root)
|
|
47
|
-
raise ConfigError, "reference_cmd is required in config" if @reference_cmd.empty?
|
|
48
|
-
raise ConfigError, "files list is empty — add files to the config" if @files.empty?
|
|
49
|
-
|
|
50
|
-
@files.each_with_index { |entry, index| validate_file_entry!(entry, index) }
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def validate_file_entry!(entry, index)
|
|
54
|
-
raise ConfigError, "files[#{index}] must be a mapping" unless entry.is_a?(Hash)
|
|
55
|
-
|
|
56
|
-
path = entry[:path]
|
|
57
|
-
target = entry[:reference_target]
|
|
58
|
-
raise ConfigError, "files[#{index}] is missing required path" if path.nil? || path.to_s.strip.empty?
|
|
59
|
-
raise ConfigError, "files[#{index}] is missing required reference_target" if target.nil? || target.to_s.strip.empty?
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
class MutationSet
|
|
64
|
-
attr_reader :mutations, :fingerprints
|
|
65
|
-
|
|
66
|
-
def self.from_json(data)
|
|
67
|
-
mutations = data.map do |entry|
|
|
68
|
-
entry = entry.transform_keys(&:to_s)
|
|
69
|
-
entry["fingerprint"] = compute_fingerprint(entry)
|
|
70
|
-
entry
|
|
71
|
-
end
|
|
72
|
-
new(mutations)
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
def self.compute_fingerprint(entry)
|
|
76
|
-
normalized_diff = entry["diff"].to_s.gsub(/[[:space:]]/, "")
|
|
77
|
-
"#{entry["line"]}:#{normalized_diff}"
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def initialize(mutations)
|
|
81
|
-
@mutations = mutations
|
|
82
|
-
@fingerprints = mutations.map { |m| m["fingerprint"] }
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def size
|
|
86
|
-
@mutations.size
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
def empty?
|
|
90
|
-
@mutations.empty?
|
|
91
|
-
end
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
class Comparison
|
|
95
|
-
attr_reader :evilution_set, :reference_set
|
|
96
|
-
|
|
97
|
-
def initialize(evilution:, reference:)
|
|
98
|
-
@evilution_set = evilution
|
|
99
|
-
@reference_set = reference
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
def extra_in_reference
|
|
103
|
-
ev_fps = Set.new(@evilution_set.fingerprints)
|
|
104
|
-
@reference_set.mutations.reject { |m| ev_fps.include?(m["fingerprint"]) }
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
def extra_in_evilution
|
|
108
|
-
ref_fps = Set.new(@reference_set.fingerprints)
|
|
109
|
-
@evilution_set.mutations.reject { |m| ref_fps.include?(m["fingerprint"]) }
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
def density_ratio
|
|
113
|
-
return 0.0 if @evilution_set.empty?
|
|
114
|
-
|
|
115
|
-
@reference_set.size.to_f / @evilution_set.size
|
|
116
|
-
end
|
|
117
|
-
end
|
|
118
|
-
|
|
119
|
-
class Catalog
|
|
120
|
-
def initialize(extras)
|
|
121
|
-
@extras = extras
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
def by_operator
|
|
125
|
-
@extras.group_by { |m| m["operator"] }
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
def summary
|
|
129
|
-
by_operator.map { |op, mutations| { operator: op, count: mutations.size } }
|
|
130
|
-
.sort_by { |entry| -entry[:count] }
|
|
131
|
-
end
|
|
132
|
-
end
|
|
133
|
-
|
|
134
|
-
class EvilutionCollector
|
|
135
|
-
def initialize(project_root)
|
|
136
|
-
@project_root = project_root
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
def collect(file_path)
|
|
140
|
-
full_path = File.join(@project_root, file_path)
|
|
141
|
-
cmd = ["bundle", "exec", "evilution", "util", "mutation", full_path, "--format", "json"]
|
|
142
|
-
stdout, stderr, status = Open3.capture3(*cmd, chdir: @project_root)
|
|
143
|
-
|
|
144
|
-
raise ToolError, "evilution failed for #{file_path}: #{stderr.strip}" unless status.success?
|
|
145
|
-
|
|
146
|
-
JSON.parse(stdout)
|
|
147
|
-
rescue JSON::ParserError => e
|
|
148
|
-
raise ToolError, "evilution produced invalid JSON for #{file_path}: #{e.message}"
|
|
149
|
-
end
|
|
150
|
-
end
|
|
151
|
-
|
|
152
|
-
class ReferenceCollector
|
|
153
|
-
def initialize(project_root, reference_cmd)
|
|
154
|
-
@project_root = project_root
|
|
155
|
-
@reference_cmd = reference_cmd
|
|
156
|
-
end
|
|
157
|
-
|
|
158
|
-
def collect(reference_target)
|
|
159
|
-
cmd = @reference_cmd + [reference_target]
|
|
160
|
-
stdout, stderr, status = Open3.capture3(*cmd, chdir: @project_root)
|
|
161
|
-
|
|
162
|
-
raise ToolError, "reference tool failed for #{reference_target}: #{stderr.lines.first&.strip}" unless status.success?
|
|
163
|
-
|
|
164
|
-
JSON.parse(stdout)
|
|
165
|
-
rescue JSON::ParserError => e
|
|
166
|
-
raise ToolError, "reference tool produced invalid JSON for #{reference_target}: #{e.message}"
|
|
167
|
-
end
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
class Reporter
|
|
171
|
-
def initialize(output_dir)
|
|
172
|
-
@output_dir = output_dir
|
|
173
|
-
end
|
|
174
|
-
|
|
175
|
-
def write_file_report(file_path, comparison, catalog)
|
|
176
|
-
basename = File.basename(file_path, ".rb")
|
|
177
|
-
write_json_report(basename, file_path, comparison, catalog)
|
|
178
|
-
write_text_report(basename, file_path, comparison, catalog)
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def write_summary(file_results)
|
|
182
|
-
write_summary_json(file_results)
|
|
183
|
-
write_summary_text(file_results)
|
|
184
|
-
end
|
|
185
|
-
|
|
186
|
-
private
|
|
187
|
-
|
|
188
|
-
def write_json_report(basename, file_path, comparison, catalog)
|
|
189
|
-
data = {
|
|
190
|
-
file: file_path,
|
|
191
|
-
evilution_count: comparison.evilution_set.size,
|
|
192
|
-
reference_count: comparison.reference_set.size,
|
|
193
|
-
density_ratio: comparison.density_ratio.round(2),
|
|
194
|
-
extra_in_reference: comparison.extra_in_reference,
|
|
195
|
-
extra_in_evilution: comparison.extra_in_evilution,
|
|
196
|
-
operator_summary: catalog.summary
|
|
197
|
-
}
|
|
198
|
-
File.write(File.join(@output_dir, "#{basename}.json"), JSON.pretty_generate(data))
|
|
199
|
-
end
|
|
200
|
-
|
|
201
|
-
def write_text_report(basename, file_path, comparison, catalog)
|
|
202
|
-
lines = build_file_header(file_path, comparison)
|
|
203
|
-
lines.concat(build_extra_reference_section(comparison, catalog))
|
|
204
|
-
lines.concat(build_extra_evilution_section(comparison))
|
|
205
|
-
File.write(File.join(@output_dir, "#{basename}.txt"), lines.join("\n"))
|
|
206
|
-
end
|
|
207
|
-
|
|
208
|
-
def build_file_header(file_path, comparison)
|
|
209
|
-
[
|
|
210
|
-
"# Mutation Comparison: #{file_path}",
|
|
211
|
-
"",
|
|
212
|
-
"Evilution: #{comparison.evilution_set.size} mutations",
|
|
213
|
-
"Reference: #{comparison.reference_set.size} mutations",
|
|
214
|
-
"Ratio: #{comparison.density_ratio.round(2)}x",
|
|
215
|
-
""
|
|
216
|
-
]
|
|
217
|
-
end
|
|
218
|
-
|
|
219
|
-
def build_extra_reference_section(comparison, catalog)
|
|
220
|
-
extra = comparison.extra_in_reference
|
|
221
|
-
lines = ["## Extra in reference (#{extra.size})", ""]
|
|
222
|
-
|
|
223
|
-
catalog.summary.each do |entry|
|
|
224
|
-
lines.concat(format_operator_group(entry, catalog.by_operator[entry[:operator]]))
|
|
225
|
-
end
|
|
226
|
-
|
|
227
|
-
lines
|
|
228
|
-
end
|
|
229
|
-
|
|
230
|
-
def format_operator_group(entry, mutations)
|
|
231
|
-
lines = ["### #{entry[:operator]} (#{entry[:count]})"]
|
|
232
|
-
mutations.each { |m| lines.concat(format_extra_mutation(m)) }
|
|
233
|
-
lines
|
|
234
|
-
end
|
|
235
|
-
|
|
236
|
-
def format_extra_mutation(mutation)
|
|
237
|
-
lines = [" Line #{mutation["line"]}:"]
|
|
238
|
-
mutation["diff"].to_s.each_line { |l| lines << " #{l.chomp}" }
|
|
239
|
-
lines << ""
|
|
240
|
-
lines
|
|
241
|
-
end
|
|
242
|
-
|
|
243
|
-
def build_extra_evilution_section(comparison)
|
|
244
|
-
ev_extra = comparison.extra_in_evilution
|
|
245
|
-
return [] if ev_extra.empty?
|
|
246
|
-
|
|
247
|
-
lines = ["## Extra in evilution (#{ev_extra.size})", ""]
|
|
248
|
-
ev_extra.each do |m|
|
|
249
|
-
lines << " #{m["operator"]} line #{m["line"]}: #{m["diff"].to_s.lines.first&.chomp}"
|
|
250
|
-
end
|
|
251
|
-
lines
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
def write_summary_json(file_results)
|
|
255
|
-
data = {
|
|
256
|
-
timestamp: Time.now.iso8601,
|
|
257
|
-
files: file_results,
|
|
258
|
-
totals: compute_summary_totals(file_results)
|
|
259
|
-
}
|
|
260
|
-
File.write(File.join(@output_dir, "summary.json"), JSON.pretty_generate(data))
|
|
261
|
-
end
|
|
262
|
-
|
|
263
|
-
def write_summary_text(file_results)
|
|
264
|
-
totals = compute_summary_totals(file_results)
|
|
265
|
-
lines = build_summary_header
|
|
266
|
-
lines.concat(build_summary_rows(file_results))
|
|
267
|
-
lines.concat(build_summary_totals(totals))
|
|
268
|
-
lines.concat(build_operator_breakdown(totals))
|
|
269
|
-
File.write(File.join(@output_dir, "summary.txt"), lines.join("\n"))
|
|
270
|
-
end
|
|
271
|
-
|
|
272
|
-
def build_summary_header
|
|
273
|
-
header = "File " \
|
|
274
|
-
"Evilution Reference Ratio Extra"
|
|
275
|
-
["# Head-to-Head Mutation Comparison Summary", "", header, ("-" * 83)]
|
|
276
|
-
end
|
|
277
|
-
|
|
278
|
-
def build_summary_rows(file_results)
|
|
279
|
-
file_results.map do |r|
|
|
280
|
-
format("%-45<file>s %10<ev>d %10<ref>d %7.2<ratio>fx %8<extra>d",
|
|
281
|
-
file: r[:file], ev: r[:evilution_count], ref: r[:reference_count],
|
|
282
|
-
ratio: r[:density_ratio], extra: r[:extra_count])
|
|
283
|
-
end
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
def build_summary_totals(totals)
|
|
287
|
-
total_line = format("%-45<file>s %10<ev>d %10<ref>d %7.2<ratio>fx %8<extra>d",
|
|
288
|
-
file: "TOTAL", ev: totals[:evilution_total], ref: totals[:reference_total],
|
|
289
|
-
ratio: totals[:density_ratio], extra: totals[:extra_total])
|
|
290
|
-
[("-" * 83), total_line, ""]
|
|
291
|
-
end
|
|
292
|
-
|
|
293
|
-
def build_operator_breakdown(totals)
|
|
294
|
-
lines = ["## Operator breakdown (reference extras)", ""]
|
|
295
|
-
totals[:operator_summary].each do |entry|
|
|
296
|
-
lines << " #{entry[:operator]}: #{entry[:count]}"
|
|
297
|
-
end
|
|
298
|
-
lines
|
|
299
|
-
end
|
|
300
|
-
|
|
301
|
-
def compute_summary_totals(file_results)
|
|
302
|
-
ev_total = file_results.sum { |r| r[:evilution_count] }
|
|
303
|
-
ref_total = file_results.sum { |r| r[:reference_count] }
|
|
304
|
-
extra_total = file_results.sum { |r| r[:extra_count] }
|
|
305
|
-
ratio = ev_total.positive? ? (ref_total.to_f / ev_total).round(2) : 0.0
|
|
306
|
-
merged = merge_operator_summaries(file_results)
|
|
307
|
-
|
|
308
|
-
{ evilution_total: ev_total, reference_total: ref_total,
|
|
309
|
-
density_ratio: ratio, extra_total: extra_total, operator_summary: merged }
|
|
310
|
-
end
|
|
311
|
-
|
|
312
|
-
def merge_operator_summaries(file_results)
|
|
313
|
-
all_operators = file_results.flat_map { |r| r[:operator_summary] }
|
|
314
|
-
all_operators.group_by { |e| e[:operator] }
|
|
315
|
-
.map { |op, entries| { operator: op, count: entries.sum { |e| e[:count] } } }
|
|
316
|
-
.sort_by { |e| -e[:count] }
|
|
317
|
-
end
|
|
318
|
-
end
|
|
319
|
-
|
|
320
|
-
class Runner
|
|
321
|
-
def initialize(config:, verbose: false)
|
|
322
|
-
@config = config
|
|
323
|
-
@verbose = verbose
|
|
324
|
-
@evilution = EvilutionCollector.new(config.project_root)
|
|
325
|
-
@reference = ReferenceCollector.new(config.project_root, config.reference_cmd)
|
|
326
|
-
end
|
|
327
|
-
|
|
328
|
-
def call
|
|
329
|
-
FileUtils.mkdir_p(@config.output_dir)
|
|
330
|
-
reporter = Reporter.new(@config.output_dir)
|
|
331
|
-
file_results = process_files(reporter)
|
|
332
|
-
reporter.write_summary(file_results)
|
|
333
|
-
print_summary(file_results)
|
|
334
|
-
0
|
|
335
|
-
rescue ToolError => e
|
|
336
|
-
warn "Tool error: #{e.message}"
|
|
337
|
-
1
|
|
338
|
-
end
|
|
339
|
-
|
|
340
|
-
private
|
|
341
|
-
|
|
342
|
-
def process_files(reporter)
|
|
343
|
-
@config.files.map do |entry|
|
|
344
|
-
path = entry[:path]
|
|
345
|
-
reference_target = entry[:reference_target]
|
|
346
|
-
|
|
347
|
-
warn " Comparing #{path}..." if @verbose
|
|
348
|
-
compare_file(path, reference_target, reporter)
|
|
349
|
-
end
|
|
350
|
-
end
|
|
351
|
-
|
|
352
|
-
def compare_file(path, _reference_target, reporter)
|
|
353
|
-
full_path = File.join(@config.project_root, path)
|
|
354
|
-
ev_set = MutationSet.from_json(@evilution.collect(path))
|
|
355
|
-
ref_set = MutationSet.from_json(@reference.collect(full_path))
|
|
356
|
-
comparison = Comparison.new(evilution: ev_set, reference: ref_set)
|
|
357
|
-
catalog = Catalog.new(comparison.extra_in_reference)
|
|
358
|
-
|
|
359
|
-
reporter.write_file_report(path, comparison, catalog)
|
|
360
|
-
build_file_result(path, comparison, catalog)
|
|
361
|
-
end
|
|
362
|
-
|
|
363
|
-
def build_file_result(path, comparison, catalog)
|
|
364
|
-
{
|
|
365
|
-
file: path,
|
|
366
|
-
evilution_count: comparison.evilution_set.size,
|
|
367
|
-
reference_count: comparison.reference_set.size,
|
|
368
|
-
density_ratio: comparison.density_ratio.round(2),
|
|
369
|
-
extra_count: comparison.extra_in_reference.size,
|
|
370
|
-
operator_summary: catalog.summary
|
|
371
|
-
}
|
|
372
|
-
end
|
|
373
|
-
|
|
374
|
-
def print_summary(file_results)
|
|
375
|
-
totals = compute_totals(file_results)
|
|
376
|
-
|
|
377
|
-
puts "Comparison complete. Results in #{@config.output_dir}/"
|
|
378
|
-
puts " Files: #{file_results.size}"
|
|
379
|
-
puts " Evilution: #{totals[:ev]} mutations"
|
|
380
|
-
puts " Reference: #{totals[:ref]} mutations"
|
|
381
|
-
puts " Ratio: #{totals[:ratio]}x"
|
|
382
|
-
puts " Extra in reference: #{totals[:extra]}"
|
|
383
|
-
end
|
|
384
|
-
|
|
385
|
-
def compute_totals(file_results)
|
|
386
|
-
ev = file_results.sum { |r| r[:evilution_count] }
|
|
387
|
-
ref = file_results.sum { |r| r[:reference_count] }
|
|
388
|
-
extra = file_results.sum { |r| r[:extra_count] }
|
|
389
|
-
{ ev: ev, ref: ref, extra: extra,
|
|
390
|
-
ratio: ev.positive? ? (ref.to_f / ev).round(2) : 0.0 }
|
|
391
|
-
end
|
|
392
|
-
end
|
|
393
|
-
end
|
|
394
|
-
|
|
395
|
-
if __FILE__ == $PROGRAM_NAME
|
|
396
|
-
verbose = false
|
|
397
|
-
parser = OptionParser.new do |opts|
|
|
398
|
-
opts.banner = "Usage: #{$PROGRAM_NAME} CONFIG_FILE [--verbose]"
|
|
399
|
-
opts.on("-v", "--verbose", "Show progress") { verbose = true }
|
|
400
|
-
opts.on("-h", "--help", "Show help") do
|
|
401
|
-
puts opts
|
|
402
|
-
exit 0
|
|
403
|
-
end
|
|
404
|
-
end
|
|
405
|
-
parser.parse!
|
|
406
|
-
|
|
407
|
-
config_path = ARGV.first
|
|
408
|
-
unless config_path
|
|
409
|
-
warn parser.banner
|
|
410
|
-
exit 2
|
|
411
|
-
end
|
|
412
|
-
|
|
413
|
-
begin
|
|
414
|
-
config = CompareMutations::Config.new(config_path)
|
|
415
|
-
runner = CompareMutations::Runner.new(config: config, verbose: verbose)
|
|
416
|
-
exit runner.call
|
|
417
|
-
rescue CompareMutations::ConfigError => e
|
|
418
|
-
warn "Config error: #{e.message}"
|
|
419
|
-
exit 2
|
|
420
|
-
end
|
|
421
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
# Head-to-head mutation comparison configuration
|
|
2
|
-
#
|
|
3
|
-
# Each file entry specifies a source file and the corresponding reference tool
|
|
4
|
-
# subject pattern. The reference_cmd must produce JSON output with the same
|
|
5
|
-
# schema as `evilution util mutation --format json`:
|
|
6
|
-
# [{"operator": "...", "line": N, "diff": "..."}, ...]
|
|
7
|
-
#
|
|
8
|
-
# To use: set project_root and reference_cmd for your environment.
|
|
9
|
-
#
|
|
10
|
-
# Example:
|
|
11
|
-
#
|
|
12
|
-
# project_root: /path/to/rails/app
|
|
13
|
-
# reference_cmd: ["ruby", "scripts/mutant_json_adapter", "--project-root", "/path/to/rails/app"]
|
|
14
|
-
# output_dir: comparison_results
|
|
15
|
-
# files:
|
|
16
|
-
# - path: app/models/user.rb
|
|
17
|
-
# reference_target: "User"
|
|
18
|
-
# - path: app/services/payment_processor.rb
|
|
19
|
-
# reference_target: "PaymentProcessor"
|
|
20
|
-
|
|
21
|
-
project_root: null
|
|
22
|
-
reference_cmd: []
|
|
23
|
-
output_dir: comparison_results
|
|
24
|
-
files: []
|