evilution 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +30 -0
- data/CHANGELOG.md +49 -0
- data/README.md +285 -45
- data/docs/architecture.md +110 -12
- data/docs/ast_pattern_syntax.md +35 -7
- data/docs/combined_runs_spike.md +407 -0
- data/docs/integrations.md +1 -1
- data/docs/isolation.md +63 -2
- data/docs/migration-from-mutant.md +16 -11
- data/docs/mutant_schemata_spike.md +387 -0
- data/docs/operator_yield_study.md +576 -0
- data/docs/probe_first_spike.md +264 -0
- data/lib/evilution/ast/ancestor_lookup.rb +36 -0
- data/lib/evilution/ast/callback_declaration.rb +36 -3
- data/lib/evilution/ast/class_body.rb +62 -16
- data/lib/evilution/ast/def_region.rb +130 -0
- data/lib/evilution/ast/local_declaration.rb +37 -0
- data/lib/evilution/ast/parent_index.rb +50 -0
- data/lib/evilution/ast/parse_cache.rb +27 -0
- data/lib/evilution/ast/parser.rb +8 -3
- data/lib/evilution/ast/pattern/filter.rb +53 -12
- data/lib/evilution/ast/skip_filter.rb +83 -0
- data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
- data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
- data/lib/evilution/ast/skip_sites/logging.rb +93 -0
- data/lib/evilution/ast/skip_sites/message.rb +44 -0
- data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
- data/lib/evilution/ast/skip_sites/presets.rb +31 -0
- data/lib/evilution/ast/skip_sites/purity.rb +69 -0
- data/lib/evilution/ast/skip_sites/requires.rb +40 -0
- data/lib/evilution/ast/skip_sites.rb +9 -0
- data/lib/evilution/ast/source_surgeon.rb +20 -2
- data/lib/evilution/ast/syntax_check.rb +10 -8
- data/lib/evilution/baseline/report.rb +11 -2
- data/lib/evilution/baseline.rb +31 -6
- data/lib/evilution/cache/keys.rb +140 -0
- data/lib/evilution/cache/run_fingerprint.rb +51 -0
- data/lib/evilution/cache/skeleton.rb +57 -0
- data/lib/evilution/cache/spec_digest.rb +39 -0
- data/lib/evilution/cache.rb +143 -50
- data/lib/evilution/class_tree.rb +38 -0
- data/lib/evilution/cli/command.rb +12 -4
- data/lib/evilution/cli/commands/run.rb +12 -3
- data/lib/evilution/cli/commands/subjects.rb +5 -2
- data/lib/evilution/cli/parser/options_builder.rb +67 -3
- data/lib/evilution/cli/printers/environment.rb +34 -4
- data/lib/evilution/cli/printers/session_detail.rb +20 -3
- data/lib/evilution/compare/categorizer.rb +2 -1
- data/lib/evilution/compare/normalizer.rb +3 -2
- data/lib/evilution/config/sources.rb +3 -0
- data/lib/evilution/config/validators/base.rb +9 -0
- data/lib/evilution/config/validators/since.rb +22 -0
- data/lib/evilution/config/validators/skip_sites.rb +24 -0
- data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
- data/lib/evilution/config/validators/timeout_factor.rb +10 -0
- data/lib/evilution/config/validators/timeout_floor.rb +10 -0
- data/lib/evilution/config.rb +122 -6
- data/lib/evilution/coverage/load_tracker.rb +50 -0
- data/lib/evilution/coverage/map.rb +42 -5
- data/lib/evilution/coverage/map_builder.rb +123 -22
- data/lib/evilution/coverage/map_store.rb +15 -5
- data/lib/evilution/coverage/recorder.rb +42 -3
- data/lib/evilution/coverage_example_filter.rb +35 -0
- data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
- data/lib/evilution/example_filter.rb +20 -0
- data/lib/evilution/git/changed_lines.rb +103 -0
- data/lib/evilution/git/since_scope.rb +77 -0
- data/lib/evilution/git/unified_diff.rb +68 -0
- data/lib/evilution/infra_crash.rb +26 -0
- data/lib/evilution/integration/base.rb +97 -6
- data/lib/evilution/integration/crash_detector.rb +46 -6
- data/lib/evilution/integration/known_failures.rb +4 -0
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
- data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
- data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
- data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
- data/lib/evilution/integration/loading/concern_registry.rb +87 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
- data/lib/evilution/integration/loading/created_modules.rb +56 -0
- data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
- data/lib/evilution/integration/loading/neutralization.rb +41 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
- data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
- data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
- data/lib/evilution/integration/minitest.rb +22 -2
- data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
- data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
- data/lib/evilution/integration/rspec/example_ids.rb +23 -0
- data/lib/evilution/integration/rspec/example_order.rb +86 -0
- data/lib/evilution/integration/rspec/killer_first.rb +72 -0
- data/lib/evilution/integration/rspec/result_builder.rb +7 -0
- data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
- data/lib/evilution/integration/rspec/state_guard.rb +2 -2
- data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
- data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
- data/lib/evilution/integration/rspec.rb +98 -47
- data/lib/evilution/integration/stop_policy.rb +49 -0
- data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
- data/lib/evilution/integration/test_unit.rb +28 -2
- data/lib/evilution/isolation/fork.rb +58 -13
- data/lib/evilution/isolation/in_process.rb +12 -1
- data/lib/evilution/line_set.rb +72 -0
- data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
- data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
- data/lib/evilution/mcp/info_tool.rb +24 -17
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
- data/lib/evilution/mcp/mutate_tool.rb +32 -7
- data/lib/evilution/mcp/session_tool.rb +1 -2
- data/lib/evilution/mcp.rb +4 -0
- data/lib/evilution/mutation/line_diff.rb +111 -0
- data/lib/evilution/mutation.rb +17 -13
- data/lib/evilution/mutator/base.rb +121 -98
- data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
- data/lib/evilution/mutator/line_range_filter.rb +27 -0
- data/lib/evilution/mutator/load_sources.rb +88 -0
- data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
- data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
- data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
- data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
- data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
- data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
- data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
- data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
- data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
- data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
- data/lib/evilution/mutator/registry.rb +11 -5
- data/lib/evilution/mutator/restore_source_cache.rb +41 -0
- data/lib/evilution/parallel/pool.rb +11 -4
- data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
- data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
- data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
- data/lib/evilution/parallel/work_queue/worker.rb +10 -3
- data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
- data/lib/evilution/parallel/work_queue.rb +69 -12
- data/lib/evilution/parallel_db_warning.rb +39 -12
- data/lib/evilution/process_supervisor.rb +22 -2
- data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
- data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
- data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
- data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +4 -4
- data/lib/evilution/reporter/cli.rb +9 -2
- data/lib/evilution/reporter/html/assets/style.css +8 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
- data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
- data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
- data/lib/evilution/reporter/json/subjects.rb +1 -0
- data/lib/evilution/reporter/json/timings.rb +42 -0
- data/lib/evilution/reporter/json.rb +13 -50
- data/lib/evilution/result/coverage_gap.rb +5 -0
- data/lib/evilution/result/mutation_result.rb +55 -3
- data/lib/evilution/result/mutation_timings.rb +54 -0
- data/lib/evilution/result/run_timings.rb +20 -0
- data/lib/evilution/result/subject_score.rb +8 -1
- data/lib/evilution/result/subject_scorer.rb +5 -3
- data/lib/evilution/result/summary.rb +49 -12
- data/lib/evilution/runner/baseline_runner.rb +47 -5
- data/lib/evilution/runner/diagnostics.rb +0 -13
- data/lib/evilution/runner/isolation_resolver.rb +16 -4
- data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
- data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
- data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
- data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
- data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
- data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
- data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
- data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
- data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
- data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
- data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
- data/lib/evilution/runner/mutation_executor.rb +75 -10
- data/lib/evilution/runner/mutation_planner.rb +29 -21
- data/lib/evilution/runner/phase_clock.rb +22 -0
- data/lib/evilution/runner/report_publisher.rb +1 -1
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +87 -26
- data/lib/evilution/session/diff.rb +9 -2
- data/lib/evilution/session/store.rb +17 -2
- data/lib/evilution/source_ast_cache.rb +10 -0
- data/lib/evilution/spec_ast_cache.rb +15 -0
- data/lib/evilution/spec_resolver/constant_references.rb +66 -0
- data/lib/evilution/spec_resolver/project_files.rb +50 -0
- data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
- data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
- data/lib/evilution/spec_resolver.rb +76 -65
- data/lib/evilution/spec_selector.rb +17 -13
- data/lib/evilution/subject.rb +7 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution/worker_database/copies.rb +73 -0
- data/lib/evilution/worker_database/schema.rb +39 -0
- data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
- data/lib/evilution/worker_database.rb +214 -0
- data/schema/evilution.config.schema.json +74 -2
- metadata +84 -13
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
- data/scripts/benchmark_density +0 -262
- data/scripts/benchmark_density.yml +0 -19
- data/scripts/canary_manifest.yml +0 -47
- data/scripts/compare_mutations +0 -421
- data/scripts/compare_mutations.yml +0 -24
- data/scripts/compare_targeting +0 -279
- data/scripts/compare_targeting.example.yml +0 -24
- data/scripts/mutant_json_adapter +0 -227
data/scripts/compare_targeting
DELETED
|
@@ -1,279 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
# Targeting-mode comparison harness.
|
|
5
|
-
#
|
|
6
|
-
# For each repo in a manifest, runs evilution over the SAME mutation set under
|
|
7
|
-
# three example-targeting modes -- full_file (baseline), lexical (current), and
|
|
8
|
-
# coverage (new) -- joins the per-mutation results, and emits a per-repo table:
|
|
9
|
-
# score_full / score_lexical / score_coverage,
|
|
10
|
-
# lost_kills (mutations full_file KILLED but coverage did NOT -> the gate),
|
|
11
|
-
# wall_ratio_lexical / wall_ratio_coverage (vs the full_file baseline).
|
|
12
|
-
#
|
|
13
|
-
# The accuracy gate is lost_kills == 0: coverage targeting must never lose a kill
|
|
14
|
-
# the full-file run would catch. Speed is reported, not gated.
|
|
15
|
-
#
|
|
16
|
-
# Usage:
|
|
17
|
-
# scripts/compare_targeting MANIFEST.yml [--out report.md]
|
|
18
|
-
#
|
|
19
|
-
# MANIFEST.yml:
|
|
20
|
-
# repos:
|
|
21
|
-
# - name: acme/foo
|
|
22
|
-
# dir: /checkouts/foo # already bundled
|
|
23
|
-
# args: ["lib/foo.rb", "--jobs", "4"]
|
|
24
|
-
#
|
|
25
|
-
# The actual canary execution (checkout + bundle over the EV-rxob manifest) is
|
|
26
|
-
# the validation run, EV-7ydn; this harness is the reusable comparison engine.
|
|
27
|
-
|
|
28
|
-
require "json"
|
|
29
|
-
require "yaml"
|
|
30
|
-
require "open3"
|
|
31
|
-
require "optparse"
|
|
32
|
-
require "digest"
|
|
33
|
-
|
|
34
|
-
module CompareTargeting
|
|
35
|
-
MODES = %w[full_file lexical coverage].freeze
|
|
36
|
-
KEY_FIELDS = %w[file line operator].freeze
|
|
37
|
-
|
|
38
|
-
class ConfigError < StandardError; end
|
|
39
|
-
|
|
40
|
-
module_function
|
|
41
|
-
|
|
42
|
-
# Stable per-mutation identity across modes: only TARGETING differs between
|
|
43
|
-
# runs, so the same mutation has the same file/line/operator (+ diff to
|
|
44
|
-
# separate distinct mutations sharing a line+operator).
|
|
45
|
-
def key_for(detail)
|
|
46
|
-
base = KEY_FIELDS.map { |field| detail[field] }.join(":")
|
|
47
|
-
digest = detail["diff"].to_s
|
|
48
|
-
digest.empty? ? base : "#{base}##{Digest::SHA256.hexdigest(digest)[0, 8]}"
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
# The label shown in lost-kill output: file:line:operator (no diff hash).
|
|
52
|
-
def label_for(detail)
|
|
53
|
-
KEY_FIELDS.map { |field| detail[field] }.join(":")
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
# One mode's run, indexed by mutation key.
|
|
57
|
-
class ModeResult
|
|
58
|
-
CATEGORIES = %w[killed survived neutral equivalent unresolved unparseable timed_out errors].freeze
|
|
59
|
-
|
|
60
|
-
def self.from_json(data)
|
|
61
|
-
by_key = {}
|
|
62
|
-
CATEGORIES.each do |category|
|
|
63
|
-
Array(data[category]).each do |detail|
|
|
64
|
-
key = CompareTargeting.key_for(detail)
|
|
65
|
-
by_key[key] ||= {
|
|
66
|
-
status: detail["status"],
|
|
67
|
-
duration: (detail["duration"] || 0.0).to_f,
|
|
68
|
-
label: CompareTargeting.label_for(detail)
|
|
69
|
-
}
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
new(by_key)
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
def initialize(by_key)
|
|
76
|
-
@by_key = by_key
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
def keys
|
|
80
|
-
@by_key.keys
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def status(key)
|
|
84
|
-
@by_key.dig(key, :status)
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
def label(key)
|
|
88
|
-
@by_key.dig(key, :label)
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
def killed_count
|
|
92
|
-
@by_key.count { |_, value| value[:status] == "killed" }
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
# Measurable = the run actually observed a verdict: killed or survived.
|
|
96
|
-
# unresolved/equivalent/errors/unparseable are excluded from the score.
|
|
97
|
-
def measurable_count
|
|
98
|
-
@by_key.count { |_, value| %w[killed survived].include?(value[:status]) }
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
def total_duration
|
|
102
|
-
@by_key.values.sum { |value| value[:duration] }
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
# Joins the three modes for one repo and derives the comparison metrics.
|
|
107
|
-
class Comparison
|
|
108
|
-
def initialize(full_file:, lexical:, coverage:)
|
|
109
|
-
@modes = { "full_file" => full_file, "lexical" => lexical, "coverage" => coverage }
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
def score(mode)
|
|
113
|
-
result = @modes.fetch(mode)
|
|
114
|
-
return 0.0 if result.measurable_count.zero?
|
|
115
|
-
|
|
116
|
-
result.killed_count.to_f / result.measurable_count
|
|
117
|
-
end
|
|
118
|
-
|
|
119
|
-
def wall_ratio(mode)
|
|
120
|
-
baseline = @modes.fetch("full_file").total_duration
|
|
121
|
-
return 0.0 if baseline.zero?
|
|
122
|
-
|
|
123
|
-
@modes.fetch(mode).total_duration / baseline
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
# Mutation labels that full_file KILLED but `mode` failed to kill -- the
|
|
127
|
-
# lost kills that must be zero before coverage can become the default.
|
|
128
|
-
def lost_kills(mode = "coverage")
|
|
129
|
-
full = @modes.fetch("full_file")
|
|
130
|
-
other = @modes.fetch(mode)
|
|
131
|
-
full
|
|
132
|
-
.keys
|
|
133
|
-
.select { |key| full.status(key) == "killed" && other.status(key) != "killed" }
|
|
134
|
-
.map { |key| full.label(key) }
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
def to_row(repo)
|
|
138
|
-
{
|
|
139
|
-
repo: repo,
|
|
140
|
-
score_full: score("full_file"),
|
|
141
|
-
score_lexical: score("lexical"),
|
|
142
|
-
score_coverage: score("coverage"),
|
|
143
|
-
lost_kills: lost_kills("coverage").size,
|
|
144
|
-
wall_ratio_lexical: wall_ratio("lexical"),
|
|
145
|
-
wall_ratio_coverage: wall_ratio("coverage")
|
|
146
|
-
}
|
|
147
|
-
end
|
|
148
|
-
end
|
|
149
|
-
|
|
150
|
-
# Renders the per-repo rows as a markdown table with a PASS/FAIL gate line.
|
|
151
|
-
class TableReporter
|
|
152
|
-
COLUMNS = %w[repo score_full score_lexical score_coverage lost_kills
|
|
153
|
-
wall_ratio_lexical wall_ratio_coverage].freeze
|
|
154
|
-
|
|
155
|
-
def initialize(rows)
|
|
156
|
-
@rows = rows
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
def to_markdown
|
|
160
|
-
(table_lines + ["", gate_line]).join("\n")
|
|
161
|
-
end
|
|
162
|
-
|
|
163
|
-
private
|
|
164
|
-
|
|
165
|
-
def table_lines
|
|
166
|
-
[header_row, separator_row, *@rows.map { |row| data_row(row) }]
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
def header_row
|
|
170
|
-
"| #{COLUMNS.join(" | ")} |"
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
def separator_row
|
|
174
|
-
"| #{COLUMNS.map { "---" }.join(" | ")} |"
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
def data_row(row)
|
|
178
|
-
"| #{COLUMNS.map { |col| format_cell(row[col.to_sym]) }.join(" | ")} |"
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def gate_line
|
|
182
|
-
total_lost = @rows.sum { |row| row[:lost_kills] }
|
|
183
|
-
gate = total_lost.zero? ? "PASS" : "FAIL"
|
|
184
|
-
"GATE (total lost_kills == 0): #{gate} (#{total_lost} lost kills across #{@rows.size} repos)"
|
|
185
|
-
end
|
|
186
|
-
|
|
187
|
-
def format_cell(value)
|
|
188
|
-
value.is_a?(Float) ? format("%.3f", value) : value.to_s
|
|
189
|
-
end
|
|
190
|
-
end
|
|
191
|
-
|
|
192
|
-
# Runs evilution for one (repo, mode) and parses the JSON into a ModeResult.
|
|
193
|
-
# command_runner is injected so the pure pipeline is testable without a real
|
|
194
|
-
# mutation run.
|
|
195
|
-
class ModeRunner
|
|
196
|
-
# Run the inner `bundle exec evilution` in the TARGET repo's bundler context.
|
|
197
|
-
# If this harness is itself launched under `bundle exec`, Bundler exports
|
|
198
|
-
# BUNDLE_GEMFILE/RUBYOPT into the child, which would make the inner bundle
|
|
199
|
-
# resolve against evilution's Gemfile instead of the target repo's. Strip
|
|
200
|
-
# that inherited bundler env first.
|
|
201
|
-
DEFAULT_RUNNER = lambda do |cmd, dir|
|
|
202
|
-
stdout, stderr, status =
|
|
203
|
-
if defined?(Bundler)
|
|
204
|
-
Bundler.with_unbundled_env { Open3.capture3(*cmd, chdir: dir) }
|
|
205
|
-
else
|
|
206
|
-
Open3.capture3(*cmd, chdir: dir)
|
|
207
|
-
end
|
|
208
|
-
raise ConfigError, "evilution failed in #{dir}: #{stderr}" unless status.success?
|
|
209
|
-
|
|
210
|
-
stdout
|
|
211
|
-
end
|
|
212
|
-
|
|
213
|
-
def initialize(command_runner: DEFAULT_RUNNER)
|
|
214
|
-
@command_runner = command_runner
|
|
215
|
-
end
|
|
216
|
-
|
|
217
|
-
def run(repo_dir:, evilution_args:, mode:)
|
|
218
|
-
cmd = ["bundle", "exec", "evilution", *evilution_args,
|
|
219
|
-
"--example-targeting", mode, "--format", "json"]
|
|
220
|
-
json = @command_runner.call(cmd, repo_dir)
|
|
221
|
-
ModeResult.from_json(JSON.parse(json))
|
|
222
|
-
end
|
|
223
|
-
end
|
|
224
|
-
|
|
225
|
-
# Drives the manifest: every repo through every mode, then the table.
|
|
226
|
-
class Harness
|
|
227
|
-
def initialize(mode_runner: ModeRunner.new)
|
|
228
|
-
@mode_runner = mode_runner
|
|
229
|
-
end
|
|
230
|
-
|
|
231
|
-
def call(repos)
|
|
232
|
-
rows = repos.map { |repo| compare_repo(repo) }
|
|
233
|
-
TableReporter.new(rows).to_markdown
|
|
234
|
-
end
|
|
235
|
-
|
|
236
|
-
def compare_repo(repo)
|
|
237
|
-
results = MODES.to_h do |mode|
|
|
238
|
-
[mode, @mode_runner.run(repo_dir: repo.fetch(:dir), evilution_args: repo.fetch(:args), mode: mode)]
|
|
239
|
-
end
|
|
240
|
-
Comparison.new(**results.transform_keys(&:to_sym)).to_row(repo.fetch(:name))
|
|
241
|
-
end
|
|
242
|
-
end
|
|
243
|
-
|
|
244
|
-
def load_manifest(path)
|
|
245
|
-
data = YAML.safe_load_file(path, symbolize_names: true)
|
|
246
|
-
raise ConfigError, "manifest must list repos:" unless data.is_a?(Hash) && data[:repos].is_a?(Array)
|
|
247
|
-
|
|
248
|
-
data[:repos]
|
|
249
|
-
rescue Errno::ENOENT
|
|
250
|
-
raise ConfigError, "manifest not found: #{path}"
|
|
251
|
-
rescue Psych::SyntaxError => e
|
|
252
|
-
raise ConfigError, "manifest is not valid YAML: #{e.message}"
|
|
253
|
-
end
|
|
254
|
-
end
|
|
255
|
-
|
|
256
|
-
if __FILE__ == $PROGRAM_NAME
|
|
257
|
-
out_path = nil
|
|
258
|
-
parser = OptionParser.new do |opts|
|
|
259
|
-
opts.banner = "Usage: scripts/compare_targeting MANIFEST.yml [--out report.md]"
|
|
260
|
-
opts.on("--out PATH", "Write the markdown report to PATH (default: stdout)") { |p| out_path = p }
|
|
261
|
-
end
|
|
262
|
-
parser.parse!
|
|
263
|
-
|
|
264
|
-
manifest_path = ARGV.first
|
|
265
|
-
unless manifest_path
|
|
266
|
-
warn parser.banner
|
|
267
|
-
exit 2
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
begin
|
|
271
|
-
repos = CompareTargeting.load_manifest(manifest_path)
|
|
272
|
-
report = CompareTargeting::Harness.new.call(repos)
|
|
273
|
-
out_path ? File.write(out_path, report) : puts(report)
|
|
274
|
-
exit 0
|
|
275
|
-
rescue CompareTargeting::ConfigError => e
|
|
276
|
-
warn "Error: #{e.message}"
|
|
277
|
-
exit 2
|
|
278
|
-
end
|
|
279
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
# Example manifest for scripts/compare_targeting (EV-51d4).
|
|
2
|
-
#
|
|
3
|
-
# Each repo is run through evilution three times -- once per example-targeting
|
|
4
|
-
# mode (full_file, lexical, coverage) -- over the SAME mutation set, and the
|
|
5
|
-
# harness emits a per-repo table with the lost_kills gate and wall-time ratios.
|
|
6
|
-
#
|
|
7
|
-
# Repos must already be checked out and `bundle install`ed; the harness only
|
|
8
|
-
# runs evilution inside them. The real EV-rxob canary list is wired up in the
|
|
9
|
-
# validation run (EV-7ydn).
|
|
10
|
-
#
|
|
11
|
-
# scripts/compare_targeting scripts/compare_targeting.example.yml --out report.md
|
|
12
|
-
|
|
13
|
-
repos:
|
|
14
|
-
- name: thoughtbot/factory_bot
|
|
15
|
-
dir: /tmp/ev-canaries/factory_bot
|
|
16
|
-
args: ["lib", "--jobs", "4"]
|
|
17
|
-
|
|
18
|
-
- name: bblimke/webmock
|
|
19
|
-
dir: /tmp/ev-canaries/webmock
|
|
20
|
-
args: ["lib", "--jobs", "4"]
|
|
21
|
-
|
|
22
|
-
- name: rack/rack
|
|
23
|
-
dir: /tmp/ev-canaries/rack
|
|
24
|
-
args: ["lib", "--jobs", "4"]
|
data/scripts/mutant_json_adapter
DELETED
|
@@ -1,227 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
# Mutant JSON Adapter
|
|
5
|
-
#
|
|
6
|
-
# Extracts method bodies from a Ruby file using Prism, feeds each to
|
|
7
|
-
# `mutant util mutation -e`, parses the diff output, and produces JSON
|
|
8
|
-
# in the same schema as `evilution util mutation --format json`:
|
|
9
|
-
#
|
|
10
|
-
# [{"operator": "...", "subject": "...", "file": "...", "line": N, "diff": "..."}, ...]
|
|
11
|
-
#
|
|
12
|
-
# This works around mutant's inability to parse Ruby 4.0 files by processing
|
|
13
|
-
# individual method snippets through its eval mode.
|
|
14
|
-
#
|
|
15
|
-
# Usage:
|
|
16
|
-
# scripts/mutant_json_adapter FILE_PATH [--project-root DIR] [--verbose]
|
|
17
|
-
|
|
18
|
-
require "json"
|
|
19
|
-
require "open3"
|
|
20
|
-
require "optparse"
|
|
21
|
-
require "prism"
|
|
22
|
-
|
|
23
|
-
module MutantJsonAdapter
|
|
24
|
-
class ParseError < StandardError; end
|
|
25
|
-
|
|
26
|
-
class MethodExtractor
|
|
27
|
-
MethodInfo = Struct.new(:name, :line_number, :source)
|
|
28
|
-
|
|
29
|
-
def initialize(file_path)
|
|
30
|
-
@file_path = file_path
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def extract
|
|
34
|
-
source = File.read(@file_path)
|
|
35
|
-
binary = source.b
|
|
36
|
-
encoding = source.encoding
|
|
37
|
-
result = Prism.parse(source)
|
|
38
|
-
|
|
39
|
-
raise ParseError, "failed to parse #{@file_path}" if result.failure?
|
|
40
|
-
|
|
41
|
-
visitor = MethodVisitor.new(binary, encoding)
|
|
42
|
-
visitor.visit(result.value)
|
|
43
|
-
visitor.methods
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
class MethodVisitor < Prism::Visitor
|
|
47
|
-
attr_reader :methods
|
|
48
|
-
|
|
49
|
-
def initialize(source_binary, encoding)
|
|
50
|
-
super()
|
|
51
|
-
@source_binary = source_binary
|
|
52
|
-
@encoding = encoding
|
|
53
|
-
@methods = []
|
|
54
|
-
@context = []
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def visit_module_node(node)
|
|
58
|
-
@context.push(node.constant_path&.full_name || node.name.to_s)
|
|
59
|
-
super
|
|
60
|
-
@context.pop
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
def visit_class_node(node)
|
|
64
|
-
@context.push(node.constant_path&.full_name || node.name.to_s)
|
|
65
|
-
super
|
|
66
|
-
@context.pop
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
def visit_def_node(node)
|
|
70
|
-
loc = node.location
|
|
71
|
-
text = @source_binary[loc.start_offset...loc.end_offset].force_encoding(@encoding)
|
|
72
|
-
separator = node.receiver ? "." : "#"
|
|
73
|
-
name = "#{@context.join("::")}#{separator}#{node.name}"
|
|
74
|
-
|
|
75
|
-
@methods << MethodInfo.new(name: name, line_number: loc.start_line, source: text)
|
|
76
|
-
super
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
class MutantRunner
|
|
82
|
-
def initialize(project_root: nil)
|
|
83
|
-
@project_root = project_root
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
def mutations_for(source)
|
|
87
|
-
cmd = ["bundle", "exec", "mutant", "util", "mutation", "-e", source]
|
|
88
|
-
opts = @project_root ? { chdir: @project_root } : {}
|
|
89
|
-
stdout, _stderr, _status = Open3.capture3(*cmd, **opts)
|
|
90
|
-
stdout
|
|
91
|
-
end
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
class OutputParser
|
|
95
|
-
def parse(raw_output, subject:, file_path:)
|
|
96
|
-
return [] if raw_output.strip.empty?
|
|
97
|
-
|
|
98
|
-
mutations = []
|
|
99
|
-
current_diff_lines = []
|
|
100
|
-
|
|
101
|
-
raw_output.each_line do |line|
|
|
102
|
-
if line.start_with?("@@ ")
|
|
103
|
-
flush_mutation(mutations, current_diff_lines, subject, file_path) if current_diff_lines.any?
|
|
104
|
-
current_diff_lines = [line]
|
|
105
|
-
elsif line.start_with?("-", "+", " ") && current_diff_lines.any?
|
|
106
|
-
current_diff_lines << line
|
|
107
|
-
end
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
flush_mutation(mutations, current_diff_lines, subject, file_path) if current_diff_lines.any?
|
|
111
|
-
mutations
|
|
112
|
-
end
|
|
113
|
-
|
|
114
|
-
private
|
|
115
|
-
|
|
116
|
-
def flush_mutation(mutations, diff_lines, subject, file_path)
|
|
117
|
-
diff_text = diff_lines.join
|
|
118
|
-
operator = infer_operator(diff_lines)
|
|
119
|
-
|
|
120
|
-
mutations << {
|
|
121
|
-
"operator" => operator,
|
|
122
|
-
"subject" => subject.name,
|
|
123
|
-
"file" => file_path,
|
|
124
|
-
"line" => subject.line_number,
|
|
125
|
-
"diff" => diff_text
|
|
126
|
-
}
|
|
127
|
-
diff_lines.clear
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
def infer_operator(diff_lines)
|
|
131
|
-
removed = extract_diff_side(diff_lines, "-", "---")
|
|
132
|
-
added = extract_diff_side(diff_lines, "+", "+++")
|
|
133
|
-
|
|
134
|
-
categorize_mutation(removed, added)
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
def extract_diff_side(diff_lines, prefix, header)
|
|
138
|
-
diff_lines.select { |l| l.start_with?(prefix) && !l.start_with?(header) }
|
|
139
|
-
.map { |l| l[1..].strip }
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
def categorize_mutation(removed, added)
|
|
143
|
-
return "replacement" if removed.empty?
|
|
144
|
-
|
|
145
|
-
body_lines = added.reject { |l| l == "end" || l.start_with?("def ") }
|
|
146
|
-
return "removal" if body_lines.empty?
|
|
147
|
-
|
|
148
|
-
classify_added(body_lines.first)
|
|
149
|
-
end
|
|
150
|
-
|
|
151
|
-
def classify_added(added_body)
|
|
152
|
-
case added_body
|
|
153
|
-
when "raise" then "raise_insertion"
|
|
154
|
-
when "super" then "super_replacement"
|
|
155
|
-
when "nil" then "nil_replacement"
|
|
156
|
-
when "true", "false" then "boolean_swap"
|
|
157
|
-
when /\.eql\?/ then "equality_change"
|
|
158
|
-
when /\.equal\?/ then "identity_change"
|
|
159
|
-
when /-?\d+\b/ then "literal_boundary"
|
|
160
|
-
else "mutation"
|
|
161
|
-
end
|
|
162
|
-
end
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
class Runner
|
|
166
|
-
def initialize(file_path:, project_root: nil, verbose: false)
|
|
167
|
-
@file_path = File.expand_path(file_path)
|
|
168
|
-
@project_root = project_root
|
|
169
|
-
@verbose = verbose
|
|
170
|
-
end
|
|
171
|
-
|
|
172
|
-
def call
|
|
173
|
-
subjects = MethodExtractor.new(@file_path).extract
|
|
174
|
-
|
|
175
|
-
if subjects.empty?
|
|
176
|
-
warn "No methods found in #{@file_path}" if @verbose
|
|
177
|
-
puts "[]"
|
|
178
|
-
return 0
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
all_mutations = process_subjects(subjects)
|
|
182
|
-
puts JSON.pretty_generate(all_mutations)
|
|
183
|
-
0
|
|
184
|
-
rescue ParseError => e
|
|
185
|
-
warn e.message
|
|
186
|
-
2
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
private
|
|
190
|
-
|
|
191
|
-
def process_subjects(subjects)
|
|
192
|
-
mutant_runner = MutantRunner.new(project_root: @project_root)
|
|
193
|
-
parser = OutputParser.new
|
|
194
|
-
|
|
195
|
-
subjects.flat_map do |subject|
|
|
196
|
-
warn " Processing #{subject.name}..." if @verbose
|
|
197
|
-
raw = mutant_runner.mutations_for(subject.source)
|
|
198
|
-
parser.parse(raw, subject: subject, file_path: @file_path)
|
|
199
|
-
end
|
|
200
|
-
end
|
|
201
|
-
end
|
|
202
|
-
end
|
|
203
|
-
|
|
204
|
-
if __FILE__ == $PROGRAM_NAME
|
|
205
|
-
verbose = false
|
|
206
|
-
project_root = nil
|
|
207
|
-
|
|
208
|
-
opt_parser = OptionParser.new do |opts|
|
|
209
|
-
opts.banner = "Usage: #{$PROGRAM_NAME} FILE_PATH [--project-root DIR] [--verbose]"
|
|
210
|
-
opts.on("-v", "--verbose", "Show progress") { verbose = true }
|
|
211
|
-
opts.on("-p", "--project-root DIR", "Project root for bundle exec") { |d| project_root = d }
|
|
212
|
-
opts.on("-h", "--help", "Show help") do
|
|
213
|
-
puts opts
|
|
214
|
-
exit 0
|
|
215
|
-
end
|
|
216
|
-
end
|
|
217
|
-
opt_parser.parse!
|
|
218
|
-
|
|
219
|
-
file_path = ARGV.first
|
|
220
|
-
unless file_path
|
|
221
|
-
warn opt_parser.banner
|
|
222
|
-
exit 2
|
|
223
|
-
end
|
|
224
|
-
|
|
225
|
-
runner = MutantJsonAdapter::Runner.new(file_path: file_path, project_root: project_root, verbose: verbose)
|
|
226
|
-
exit runner.call
|
|
227
|
-
end
|