mutineer 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +680 -0
- data/README.md +210 -499
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/child_coverage.rb +53 -0
- data/lib/mutineer/cli.rb +293 -42
- data/lib/mutineer/concern_blocks.rb +243 -0
- data/lib/mutineer/config.rb +154 -8
- data/lib/mutineer/coverage_map.rb +265 -27
- data/lib/mutineer/daemon_backend.rb +244 -75
- data/lib/mutineer/daemon_client.rb +73 -10
- data/lib/mutineer/daemon_server.rb +226 -42
- data/lib/mutineer/external_backend.rb +159 -15
- data/lib/mutineer/init.rb +124 -0
- data/lib/mutineer/isolation.rb +78 -24
- data/lib/mutineer/job_plan.rb +234 -44
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +707 -0
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutator_registry.rb +6 -2
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +36 -7
- data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +84 -0
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +302 -14
- data/lib/mutineer/rails_run_env.rb +73 -0
- data/lib/mutineer/rails_worker_db.rb +1596 -57
- data/lib/mutineer/redaction.rb +199 -0
- data/lib/mutineer/reporter.rb +173 -31
- data/lib/mutineer/result.rb +52 -15
- data/lib/mutineer/runner.rb +148 -28
- data/lib/mutineer/statement_lines.rb +24 -11
- data/lib/mutineer/subject.rb +30 -4
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners/rspec.rb +15 -3
- data/lib/mutineer/test_runners.rb +5 -2
- data/lib/mutineer/triage.rb +111 -0
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/lib/mutineer.rb +4 -0
- data/skills/mutineer/SKILL.md +84 -0
- metadata +13 -1
data/lib/mutineer/result.rb
CHANGED
|
@@ -4,21 +4,35 @@ module Mutineer
|
|
|
4
4
|
# Immutable outcome of running one mutant. Ten distinct states:
|
|
5
5
|
# killed - a test failed/errored, so the mutation was caught.
|
|
6
6
|
# survived - every test passed, so the mutation went undetected.
|
|
7
|
-
# error - the child crashed (unhandled exception)
|
|
7
|
+
# error - the child crashed (unhandled exception), or a Minitest
|
|
8
|
+
# run's first failure was a database connection or lock
|
|
9
|
+
# error (#296): exit status 2.
|
|
8
10
|
# timeout - the parent SIGKILLed a child that overran its wall clock.
|
|
9
11
|
# skipped - the mutated source failed to re-parse (invalid); no fork.
|
|
10
12
|
# no_coverage - no test exercises the mutated line; not run, not scored.
|
|
13
|
+
# Also a mutant on a later line of a multi-line statement,
|
|
14
|
+
# in code that runs only sometimes, that the statement's
|
|
15
|
+
# tests ran but did not kill (#254): they may not reach it.
|
|
16
|
+
# Also a mutant on a later line of a statement whose first
|
|
17
|
+
# or last line holds other code (a call inside a one-line
|
|
18
|
+
# block), that the tests of that line did not kill (#289).
|
|
11
19
|
# uncapturable - the line's would-be covering test errored during capture,
|
|
12
20
|
# so coverage was lost. Excluded from the denominator exactly
|
|
13
21
|
# like no_coverage, but reported separately: it signals a
|
|
14
22
|
# broken harness (a test that failed to run), not a genuine
|
|
15
23
|
# coverage gap.
|
|
16
|
-
# unplaceable -
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
24
|
+
# unplaceable - the strategy cannot load the mutant where the tests call
|
|
25
|
+
# it, so it is not run. Under `--strategy redefine` the
|
|
26
|
+
# method belongs to a class or module that cannot be named
|
|
27
|
+
# statically (for example each class that includes a
|
|
28
|
+
# concern), and every method in a concern's `included` or
|
|
29
|
+
# `prepended` block is one (#256). Under `--strategy reload`
|
|
30
|
+
# only some methods in such a block are: those it cannot
|
|
31
|
+
# load into each class that has them (#297; the cases are
|
|
32
|
+
# listed in docs/json-schema.md).
|
|
33
|
+
# Excluded from the denominator and, unlike uncapturable,
|
|
34
|
+
# from the no-verdict gate: nothing is broken.
|
|
35
|
+
# `--test-command` runs these mutants.
|
|
22
36
|
# ran_at_load - the mutated line ran while the app booted or its class
|
|
23
37
|
# loaded. That run happened before the mutant was applied,
|
|
24
38
|
# so a test can check a value the original code computed.
|
|
@@ -34,10 +48,10 @@ module Mutineer
|
|
|
34
48
|
#
|
|
35
49
|
# `error` and `skipped` are deliberately distinct: skipped is a pre-fork
|
|
36
50
|
# validity failure (counted separately by the reporter), error is a runtime
|
|
37
|
-
# crash. Never conflate them via `details` string parsing. `
|
|
38
|
-
# `
|
|
39
|
-
# pre-fork or a reclassified
|
|
40
|
-
# denominator.
|
|
51
|
+
# crash. Never conflate them via `details` string parsing. `uncapturable` and
|
|
52
|
+
# `unplaceable` are pre-fork results, and `no_coverage` and `ran_at_load` are
|
|
53
|
+
# pre-fork or a reclassified result (JobPlan.selection_verdict,
|
|
54
|
+
# JobPlan.load_verdict): all excluded from the score denominator.
|
|
41
55
|
#
|
|
42
56
|
# `subject`, `mutation`, and `id` are nil when the Result is built by
|
|
43
57
|
# Isolation/Runner (which only know the outcome); the orchestrator attaches
|
|
@@ -53,8 +67,9 @@ module Mutineer
|
|
|
53
67
|
# from a fork back to the process that forked it, so both ends always run the
|
|
54
68
|
# same code. Nothing persists a marshaled Result (reports, baselines and the
|
|
55
69
|
# coverage cache are JSON), so a new field needs no Marshal compatibility.
|
|
56
|
-
Result = Data.define(:status, :details, :subject, :mutation, :id, :kills) do
|
|
57
|
-
# Every field but `status` defaults to nil.
|
|
70
|
+
Result = Data.define(:status, :details, :subject, :mutation, :id, :kills, :reason) do
|
|
71
|
+
# Every field but `status` defaults to nil. `reason` is the text a user
|
|
72
|
+
# gave for an ignored mutant, from an inline marker or an `ignore:` mapping.
|
|
58
73
|
#
|
|
59
74
|
# @param status [Symbol] the outcome.
|
|
60
75
|
# @param details [String, nil] error or skip details.
|
|
@@ -62,7 +77,8 @@ module Mutineer
|
|
|
62
77
|
# @param mutation [Mutineer::Mutation, nil] the mutation.
|
|
63
78
|
# @param id [String, nil] the stable mutant id.
|
|
64
79
|
# @param kills [Mutineer::Kills, nil] the matrix row of a `--matrix` run.
|
|
65
|
-
|
|
80
|
+
# @param reason [String, nil] why an ignored mutant was suppressed.
|
|
81
|
+
def initialize(status:, details: nil, subject: nil, mutation: nil, id: nil, kills: nil, reason: nil)
|
|
66
82
|
super
|
|
67
83
|
end
|
|
68
84
|
|
|
@@ -153,7 +169,8 @@ module Mutineer
|
|
|
153
169
|
# cleanly (see Isolation.finish). It is false when the child timed out,
|
|
154
170
|
# errored or exited the process early, when the recorder never armed, when
|
|
155
171
|
# its lines arrived out of order, when Minitest returned after an Interrupt
|
|
156
|
-
# with tests unseen,
|
|
172
|
+
# with tests unseen, when a test had a database connection or lock failure,
|
|
173
|
+
# or when the row does not agree with the verdict.
|
|
157
174
|
Kills = Data.define(:killed_by, :ran, :complete)
|
|
158
175
|
|
|
159
176
|
# Aggregates a flat list of Results into counts, the mutation score, and the
|
|
@@ -194,6 +211,11 @@ module Mutineer
|
|
|
194
211
|
def timeout_count = count(:timeout)
|
|
195
212
|
# @return [Integer] ignored count.
|
|
196
213
|
def ignored_count = count(:ignored)
|
|
214
|
+
# Mutants that exist but did not run: unplaceable plus skipped invalid.
|
|
215
|
+
# Neither is in the score; the reporter warns when they pass a share of the run.
|
|
216
|
+
#
|
|
217
|
+
# @return [Integer] unplaceable plus skipped-invalid count.
|
|
218
|
+
def unrun_count = unplaceable_count + skipped_invalid_count
|
|
197
219
|
|
|
198
220
|
# Every generated, classified mutation. NOT the score denominator.
|
|
199
221
|
#
|
|
@@ -214,6 +236,21 @@ module Mutineer
|
|
|
214
236
|
(killed_count.to_f / covered_count * 100).round(1)
|
|
215
237
|
end
|
|
216
238
|
|
|
239
|
+
# Checks the unrounded score against a threshold. `mutation_score` is
|
|
240
|
+
# rounded for display, so 1999 of 2000 killed reads 100.0; a gate that
|
|
241
|
+
# compared it would pass `--threshold 100` with a survivor. The comparison
|
|
242
|
+
# is exact: integer counts against the threshold as the decimal it prints
|
|
243
|
+
# as, so a float product such as `64.4 * 250` cannot tip a run at the
|
|
244
|
+
# boundary. `to_s.to_r` keeps that decimal; `rationalize` could round a
|
|
245
|
+
# long threshold such as 66.66666666666667 down to 200/3.
|
|
246
|
+
#
|
|
247
|
+
# @param threshold [Numeric] threshold percentage.
|
|
248
|
+
# @return [Boolean] true when killed / covered * 100 >= threshold; false
|
|
249
|
+
# when nothing was covered.
|
|
250
|
+
def meets_threshold?(threshold)
|
|
251
|
+
covered_count.positive? && killed_count * 100 >= threshold.to_s.to_r * covered_count
|
|
252
|
+
end
|
|
253
|
+
|
|
217
254
|
# Returns the surviving mutants.
|
|
218
255
|
#
|
|
219
256
|
# @return [Array<Mutineer::Result>] surviving results.
|
data/lib/mutineer/runner.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "pathname"
|
|
4
|
+
require "rbconfig"
|
|
4
5
|
require_relative "parser"
|
|
5
6
|
require_relative "project"
|
|
6
7
|
require_relative "result"
|
|
@@ -16,8 +17,28 @@ require_relative "file_swap"
|
|
|
16
17
|
require_relative "external_backend"
|
|
17
18
|
require_relative "daemon_backend"
|
|
18
19
|
require_relative "job_plan"
|
|
20
|
+
require_relative "rails_run_env"
|
|
21
|
+
require_relative "redaction"
|
|
19
22
|
|
|
20
23
|
module Mutineer
|
|
24
|
+
# Raised when the tool process cannot load a project file: the boot file, a
|
|
25
|
+
# standalone source, or a `--require` path. The message is redacted and the
|
|
26
|
+
# original exception is not kept as `cause`, so neither can show the
|
|
27
|
+
# DATABASE_URL password. The CLI maps it to a runtime error (exit 1).
|
|
28
|
+
class ProjectLoadError < StandardError
|
|
29
|
+
# @return [Array<String>] the redacted `Class: message` of each exception
|
|
30
|
+
# in the original `cause` chain, outermost first.
|
|
31
|
+
attr_reader :causes
|
|
32
|
+
|
|
33
|
+
# @param message [String] the redacted message.
|
|
34
|
+
# @param causes [Array<String>] see {#causes}.
|
|
35
|
+
# @return [ProjectLoadError]
|
|
36
|
+
def initialize(message, causes: [])
|
|
37
|
+
super(message)
|
|
38
|
+
@causes = causes
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
21
42
|
# Orchestrates one mutation end-to-end: apply it textually, validate the
|
|
22
43
|
# result, select its covering test files from the coverage map, then run only
|
|
23
44
|
# those against the mutated source in an isolated child process (strategy
|
|
@@ -31,7 +52,8 @@ module Mutineer
|
|
|
31
52
|
# Full orchestration: resolve operators, discover subjects, build the
|
|
32
53
|
# coverage map, run every mutation, and aggregate. Returns
|
|
33
54
|
# [AggregateResult, source_map, extras], where extras is the hash
|
|
34
|
-
# {JobPlan.collect_jobs} returns (`:legacy_ignore_matches`, `:id_map
|
|
55
|
+
# {JobPlan.collect_jobs} returns (`:legacy_ignore_matches`, `:id_map`,
|
|
56
|
+
# `:dropped_id_map`, `:declined`), plus `:unscoped_mutants` under `--since`.
|
|
35
57
|
# The CLI then reports + applies the exit code; the integration test asserts
|
|
36
58
|
# directly on the AggregateResult.
|
|
37
59
|
#
|
|
@@ -43,6 +65,10 @@ module Mutineer
|
|
|
43
65
|
# @return [Array(Mutineer::AggregateResult, Hash<String, String>, Hash)] aggregate,
|
|
44
66
|
# source map, and run extras.
|
|
45
67
|
def self.execute(config)
|
|
68
|
+
# Rails reads PARALLEL_WORKERS when a test helper calls parallelize.
|
|
69
|
+
# Set it before any boot or child spawn, then put the caller's value back.
|
|
70
|
+
previous_parallel_workers = ENV["PARALLEL_WORKERS"]
|
|
71
|
+
RailsRunEnv.pin_process!
|
|
46
72
|
operator_classes = MutatorRegistry.resolve(config.operators || MutatorRegistry::DEFAULT_NAMES)
|
|
47
73
|
|
|
48
74
|
# External backend: run the suite as a subprocess in the app's own runtime.
|
|
@@ -53,7 +79,32 @@ module Mutineer
|
|
|
53
79
|
# Daemon backend: boot the app ONCE in a persistent subprocess under the
|
|
54
80
|
# app's bundle and fork per mutant. Tool-side we only discover jobs + build
|
|
55
81
|
# payloads (Prism), so branch before any in-process boot.
|
|
56
|
-
|
|
82
|
+
if config.daemon
|
|
83
|
+
# The daemon loads these files itself and would report a missing one as
|
|
84
|
+
# a boot failure (exit 1). Check here so it stays a usage error (exit 2),
|
|
85
|
+
# as it is in process.
|
|
86
|
+
[config.boot, *config.require_paths].compact.each { |f| project_file!(config.project_root, f) }
|
|
87
|
+
return DaemonBackend.execute(config, operator_classes)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Both strategies (whole-file reload and redefine, #279) write
|
|
91
|
+
# .mutineer_mutant*.rb into each source dir (so require_relative and
|
|
92
|
+
# __dir__ resolve). Redefine falls back to the system temp dir for a
|
|
93
|
+
# read-only source dir, which this sweep does not reach; the OS cleans
|
|
94
|
+
# it, as in 1.x. A SIGKILL'd child skips the tempfile's
|
|
95
|
+
# ensure-unlink, orphaning it. `ensure` is unreliable vs SIGKILL, so the
|
|
96
|
+
# PARENT sweeps each source dir before and after the run. Orphans are
|
|
97
|
+
# impossible after a normal run. The first sweep runs before the boot: a
|
|
98
|
+
# file an older version left without the dot would stop Zeitwerk (#253).
|
|
99
|
+
dirs = JobPlan.source_dirs(config)
|
|
100
|
+
JobPlan.sweep_orphans(dirs)
|
|
101
|
+
if config.strategy == "redefine"
|
|
102
|
+
dirs.reject { |d| File.writable?(d) }.each do |d|
|
|
103
|
+
warn "[mutineer] #{d} is not writable: redefine loads its mutants from the system temp dir, " \
|
|
104
|
+
"so a method there that uses __dir__ or require_relative can be falsely killed. " \
|
|
105
|
+
"Make the directory writable to avoid this (#339)."
|
|
106
|
+
end
|
|
107
|
+
end
|
|
57
108
|
|
|
58
109
|
# Boot mode: require the boot file ONCE so the app env (e.g. Rails) is booted
|
|
59
110
|
# in the parent and inherited by every fork. Do NOT manually require the
|
|
@@ -78,11 +129,11 @@ module Mutineer
|
|
|
78
129
|
when :idle then Coverage.start(lines: true, methods: true)
|
|
79
130
|
when :suspended then Coverage.resume
|
|
80
131
|
end
|
|
81
|
-
|
|
132
|
+
require_project_file(config.project_root, config.boot)
|
|
82
133
|
else
|
|
83
|
-
config.sources.each { |f|
|
|
134
|
+
config.sources.each { |f| require_project_file(config.project_root, f) }
|
|
84
135
|
end
|
|
85
|
-
config.require_paths.each { |f|
|
|
136
|
+
config.require_paths.each { |f| require_project_file(config.project_root, f) }
|
|
86
137
|
preload_owners(config) if config.boot && config.strategy == "redefine"
|
|
87
138
|
|
|
88
139
|
if config.boot
|
|
@@ -102,6 +153,7 @@ module Mutineer
|
|
|
102
153
|
boot_path: File.expand_path(config.boot, config.project_root),
|
|
103
154
|
require_paths: config.require_paths, # loaded above; here only for the cache digest
|
|
104
155
|
verbose: config.verbose,
|
|
156
|
+
coverage_from: config.coverage_from && File.expand_path(config.coverage_from, config.project_root),
|
|
105
157
|
capture_timeout: config.capture_timeout || CoverageMap::DEFAULT_CAPTURE_TIMEOUT
|
|
106
158
|
).build_via_fork(after_fork: (config.rails ? -> { reconnect_active_record } : nil))
|
|
107
159
|
# Nothing reads Coverage once the map is built (cache hit or not), so
|
|
@@ -118,24 +170,16 @@ module Mutineer
|
|
|
118
170
|
source_paths: config.sources, test_paths: config.tests,
|
|
119
171
|
cache_dir: File.expand_path(config.cache_dir, config.project_root), project_root: config.project_root,
|
|
120
172
|
load_paths: config.load_paths + rel_roots, framework: config.framework,
|
|
121
|
-
require_paths: config.require_paths,
|
|
173
|
+
require_paths: config.require_paths, verbose: config.verbose,
|
|
122
174
|
capture_timeout: config.capture_timeout || CoverageMap::DEFAULT_CAPTURE_TIMEOUT
|
|
123
175
|
).build_or_load
|
|
124
176
|
end
|
|
125
|
-
JobPlan.abort_if_unclean!(coverage_map)
|
|
177
|
+
JobPlan.abort_if_unclean!(coverage_map, framework: config.framework)
|
|
126
178
|
|
|
127
179
|
# Collect every (subject, mutation) up front so the pool can fan them out.
|
|
128
180
|
jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
|
|
129
181
|
|
|
130
|
-
jobs = JobPlan.
|
|
131
|
-
|
|
132
|
-
# Whole-file reload writes mutineer_mutant*.rb into each source dir (so
|
|
133
|
-
# require_relative resolves). A SIGKILL'd child skips the tempfile's
|
|
134
|
-
# ensure-unlink, orphaning it. `ensure` is unreliable vs SIGKILL, so the
|
|
135
|
-
# PARENT sweeps each source dir before and after the run. Orphans are
|
|
136
|
-
# impossible after a normal run.
|
|
137
|
-
dirs = JobPlan.source_dirs(config)
|
|
138
|
-
JobPlan.sweep_orphans(dirs)
|
|
182
|
+
jobs, ignored_results, extras[:unscoped_mutants] = JobPlan.scope_since(jobs, ignored_results, source_map, config)
|
|
139
183
|
|
|
140
184
|
strategy = config.strategy
|
|
141
185
|
# Fail-fast must be serial: a parallel stop_when fires on the first survivor
|
|
@@ -165,6 +209,9 @@ module Mutineer
|
|
|
165
209
|
end
|
|
166
210
|
|
|
167
211
|
[AggregateResult.new(results + ignored_results), source_map, extras]
|
|
212
|
+
ensure
|
|
213
|
+
# A caller that set PARALLEL_WORKERS keeps that value after the run.
|
|
214
|
+
RailsRunEnv.restore_process!(previous_parallel_workers)
|
|
168
215
|
end
|
|
169
216
|
|
|
170
217
|
# A `--matrix` error that carries no {Kills} row (the worker crashed, or its
|
|
@@ -206,7 +253,7 @@ module Mutineer
|
|
|
206
253
|
FileSwap.restore_orphans(dirs)
|
|
207
254
|
|
|
208
255
|
jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
|
|
209
|
-
jobs = JobPlan.
|
|
256
|
+
jobs, ignored_results, extras[:unscoped_mutants] = JobPlan.scope_since(jobs, ignored_results, source_map, config)
|
|
210
257
|
|
|
211
258
|
# Nothing to mutate: return before the smoke check, which runs the whole
|
|
212
259
|
# --test set to calibrate a timeout no mutant would use (#76).
|
|
@@ -217,15 +264,23 @@ module Mutineer
|
|
|
217
264
|
# 3x the clean run, floor 30s, ceiling 300s: a heuristic. The floor covers
|
|
218
265
|
# a fast suite; the ceiling bounds a hung mutant (infinite loop) so a
|
|
219
266
|
# handful cannot stall a serial run for ~45min on a slow suite.
|
|
220
|
-
smoke_elapsed = ExternalBackend.smoke_check!(config.test_command, abs_tests)
|
|
267
|
+
smoke_elapsed, check_summary = ExternalBackend.smoke_check!(config.test_command, abs_tests)
|
|
221
268
|
timeout = [[smoke_elapsed * 3, 30].max, 300].min.ceil
|
|
269
|
+
# #292: the summary check needs a summary line to look for. Without one
|
|
270
|
+
# on the clean run, keep the exit-code rule and say what that costs.
|
|
271
|
+
unless check_summary
|
|
272
|
+
warn "[mutineer] the clean run of --test-command printed no Minitest or RSpec summary line " \
|
|
273
|
+
"(such as \"3 runs, 3 assertions, 0 failures, 0 errors\" or \"3 examples, 0 failures\"), " \
|
|
274
|
+
"so the crash check is off: an exit 1 counts as a kill even when the suite crashed " \
|
|
275
|
+
"before its tests ran. Use the default Minitest or RSpec progress output to turn it on."
|
|
276
|
+
end
|
|
222
277
|
|
|
223
278
|
results = []
|
|
224
279
|
progress = Progress.new(jobs.size)
|
|
225
280
|
begin
|
|
226
281
|
jobs.each do |subject, mutation, id|
|
|
227
282
|
r = run_external(subject, mutation, config.test_command, abs_tests,
|
|
228
|
-
timeout: timeout, verbose: config.verbose)
|
|
283
|
+
timeout: timeout, verbose: config.verbose, check_summary: check_summary)
|
|
229
284
|
results << r.with(subject: subject, mutation: mutation, id: id)
|
|
230
285
|
progress.tick
|
|
231
286
|
break if config.fail_fast && r.survived? # stop at the first survivor
|
|
@@ -240,18 +295,19 @@ module Mutineer
|
|
|
240
295
|
|
|
241
296
|
# Runs one mutant through the external backend: apply the whole-file mutation
|
|
242
297
|
# on disk, run the command, restore. An invalid (non-reparsing) mutant would
|
|
243
|
-
# fail to load
|
|
244
|
-
# cheap) and never write the file, preserving the `skipped` verdict the
|
|
298
|
+
# fail to load (scored `error`, or a false `killed` without the summary
|
|
299
|
+
# check), so skip it tool-side (Prism, already cheap) and never write the file, preserving the `skipped` verdict the
|
|
245
300
|
# in-process path gives at the pre-fork check.
|
|
246
301
|
#
|
|
302
|
+
# @param check_summary [Boolean] passed to {ExternalBackend.run}.
|
|
247
303
|
# @return [Mutineer::Result] verdict for this mutant.
|
|
248
|
-
def self.run_external(subject, mutation, command, abs_tests, timeout:, verbose:)
|
|
249
|
-
source = File.read(subject.file)
|
|
304
|
+
def self.run_external(subject, mutation, command, abs_tests, timeout:, verbose:, check_summary:)
|
|
305
|
+
source = File.read(subject.file, encoding: "UTF-8")
|
|
250
306
|
mutated = mutation.apply(source)
|
|
251
307
|
return Result.skipped if Parser.parse_string(mutated).errors.any?
|
|
252
308
|
|
|
253
309
|
FileSwap.with(subject.file, mutated) do
|
|
254
|
-
ExternalBackend.run(command, abs_tests, timeout: timeout, verbose: verbose)
|
|
310
|
+
ExternalBackend.run(command, abs_tests, timeout: timeout, verbose: verbose, check_summary: check_summary)
|
|
255
311
|
end
|
|
256
312
|
end
|
|
257
313
|
|
|
@@ -294,7 +350,8 @@ module Mutineer
|
|
|
294
350
|
# @param mutation [Mutineer::Mutation] mutation to run.
|
|
295
351
|
# @param source_file [String] source file path.
|
|
296
352
|
# @param coverage_map [Mutineer::CoverageMap, nil] coverage map.
|
|
297
|
-
# @param subject [Mutineer::Subject, nil] subject for surgical strategy
|
|
353
|
+
# @param subject [Mutineer::Subject, nil] subject for surgical strategy;
|
|
354
|
+
# redefine also loads each of its enclosing subjects.
|
|
298
355
|
# @param strategy [String] mutation strategy.
|
|
299
356
|
# @param timeout [Integer] child timeout in seconds.
|
|
300
357
|
# @param rails [Boolean] whether Rails reconnect handling is enabled.
|
|
@@ -304,7 +361,7 @@ module Mutineer
|
|
|
304
361
|
# @return [Mutineer::Result] mutant result.
|
|
305
362
|
def self.run(mutation, source_file:, coverage_map: nil, subject: nil, strategy: "reload",
|
|
306
363
|
timeout: Isolation::DEFAULT_TIMEOUT, rails: false, framework: "minitest", matrix: false)
|
|
307
|
-
source = File.read(source_file)
|
|
364
|
+
source = File.read(source_file, encoding: "UTF-8")
|
|
308
365
|
mutated = mutation.apply(source)
|
|
309
366
|
|
|
310
367
|
# Validity rule: a mutant that does not re-parse is skipped before forking.
|
|
@@ -316,7 +373,8 @@ module Mutineer
|
|
|
316
373
|
# narrow identically (score parity).
|
|
317
374
|
kind, payload = JobPlan.coverage_selection(source_file, mutation, subject, source, coverage_map)
|
|
318
375
|
return payload if kind == :verdict
|
|
319
|
-
|
|
376
|
+
# A :maybe mutant that cannot be placed stays no_coverage, as before #254.
|
|
377
|
+
return kind == :maybe ? Result.no_coverage : Result.unplaceable if subject&.unplaceable?(strategy)
|
|
320
378
|
|
|
321
379
|
abs_tests = payload
|
|
322
380
|
|
|
@@ -325,7 +383,16 @@ module Mutineer
|
|
|
325
383
|
# across processes corrupts it. Drop it so AR reconnects per child.
|
|
326
384
|
reconnect_active_record if rails
|
|
327
385
|
if strategy == "redefine"
|
|
386
|
+
# Load each enclosing def too: its body holds the original nested def,
|
|
387
|
+
# which a call to it would define again over the mutant (#290).
|
|
328
388
|
Isolation.apply_surgical(mutation, subject, source)
|
|
389
|
+
s = subject
|
|
390
|
+
Isolation.apply_surgical(mutation, s, source) while (s = s.enclosing_subject)
|
|
391
|
+
elsif subject&.reload_hook_call
|
|
392
|
+
# #297: a def in a concern's hook block reaches each class that has it.
|
|
393
|
+
code, hook_def = JobPlan.hook_mutant(mutated, mutation, subject, source)
|
|
394
|
+
Isolation.apply_whole_file(code, source_file)
|
|
395
|
+
ConcernBlocks.place_hook_def(source_file: source_file, **hook_def)
|
|
329
396
|
else
|
|
330
397
|
Isolation.apply_whole_file(mutated, source_file)
|
|
331
398
|
end
|
|
@@ -338,7 +405,8 @@ module Mutineer
|
|
|
338
405
|
end
|
|
339
406
|
end
|
|
340
407
|
result = relative_kills(result, coverage_map.project_root)
|
|
341
|
-
JobPlan.load_verdict(result, source_file, mutation, subject, source, coverage_map)
|
|
408
|
+
result = JobPlan.load_verdict(result, source_file, mutation, subject, source, coverage_map)
|
|
409
|
+
JobPlan.selection_verdict(kind, result)
|
|
342
410
|
end
|
|
343
411
|
|
|
344
412
|
# Rewrites the test files of a result's {Kills} row relative to the project
|
|
@@ -403,6 +471,58 @@ module Mutineer
|
|
|
403
471
|
end
|
|
404
472
|
private_class_method :reconnect_active_record
|
|
405
473
|
|
|
474
|
+
# The absolute path of a project file `require` can load: as given, or
|
|
475
|
+
# with `.rb` or the native-extension suffix.
|
|
476
|
+
#
|
|
477
|
+
# @api private
|
|
478
|
+
# @param root [String] the project root.
|
|
479
|
+
# @param path [String] the path as the user gave it.
|
|
480
|
+
# @raise [Errno::ENOENT] when the requested file does not exist.
|
|
481
|
+
# @raise [ArgumentError] when Ruby cannot expand the path.
|
|
482
|
+
# @return [String]
|
|
483
|
+
def self.project_file!(root, path)
|
|
484
|
+
file = File.expand_path(path, root)
|
|
485
|
+
suffixes = ["", ".rb", ".#{RbConfig::CONFIG.fetch('DLEXT')}"]
|
|
486
|
+
raise Errno::ENOENT, path unless suffixes.any? { |suffix| File.file?("#{file}#{suffix}") }
|
|
487
|
+
|
|
488
|
+
file
|
|
489
|
+
end
|
|
490
|
+
private_class_method :project_file!
|
|
491
|
+
|
|
492
|
+
# Require one project file in the tool process. A failure becomes a
|
|
493
|
+
# {ProjectLoadError} whose message, and each cause in its list, is
|
|
494
|
+
# redacted. The original backtrace is kept; the cause is not, because
|
|
495
|
+
# Ruby would print its raw message.
|
|
496
|
+
#
|
|
497
|
+
# @api private
|
|
498
|
+
# @param root [String] the project root.
|
|
499
|
+
# @param path [String] the path as the user gave it.
|
|
500
|
+
# @raise [Errno::ENOENT] when the requested file does not exist.
|
|
501
|
+
# @raise [ProjectLoadError] when the file raises while it loads.
|
|
502
|
+
# @return [void]
|
|
503
|
+
def self.require_project_file(root, path)
|
|
504
|
+
# Outside the rescue: a path Ruby cannot expand (`~nosuchuser/boot.rb`)
|
|
505
|
+
# stays a usage error (exit 2), not a load failure.
|
|
506
|
+
file = project_file!(root, path)
|
|
507
|
+
|
|
508
|
+
begin
|
|
509
|
+
require file
|
|
510
|
+
rescue StandardError, ScriptError => e
|
|
511
|
+
# Every part is redacted on its own, so the message build cannot raise
|
|
512
|
+
# on mixed encodings and leave the raw error as the new one's cause.
|
|
513
|
+
causes = []
|
|
514
|
+
cause = e.cause
|
|
515
|
+
while cause
|
|
516
|
+
causes << Redaction.describe(cause)
|
|
517
|
+
cause = cause.cause
|
|
518
|
+
end
|
|
519
|
+
error = ProjectLoadError.new("cannot load #{Redaction.text(path)}: #{Redaction.describe(e)}", causes: causes)
|
|
520
|
+
error.set_backtrace(e.backtrace)
|
|
521
|
+
raise error, cause: nil
|
|
522
|
+
end
|
|
523
|
+
end
|
|
524
|
+
private_class_method :require_project_file
|
|
525
|
+
|
|
406
526
|
# Pure, injectable predicate: true when a transactional-fixture transaction is
|
|
407
527
|
# already open on the connection. Keys off open_transactions so it is correct
|
|
408
528
|
# whenever the transaction exists, regardless of when it opened. Any probe
|
|
@@ -12,12 +12,21 @@ module Mutineer
|
|
|
12
12
|
# @param def_node [Prism::DefNode] the method that holds the position.
|
|
13
13
|
# @param source [String] the source text of the file.
|
|
14
14
|
# @param offset [Integer] byte offset of the position.
|
|
15
|
+
# @param loose [Boolean] return the lines also when their tests may not
|
|
16
|
+
# have run the position: when it is in code that runs only sometimes
|
|
17
|
+
# when the statement runs (the right side of `a && b`, a branch, the body
|
|
18
|
+
# of `x if c`, #254), and with a first or last line that holds other code
|
|
19
|
+
# (the opener of a one-line block, #289). The `def` line is never
|
|
20
|
+
# given, because it counts when the file loads. So every test that ran
|
|
21
|
+
# the position is among the tests of these lines, unless the statement
|
|
22
|
+
# starts on the `def` line.
|
|
15
23
|
# @return [Array<Integer>] the lines of the statement, with the body of each
|
|
16
24
|
# heredoc in it. Empty when no statement in the body holds the position.
|
|
17
|
-
#
|
|
18
|
-
# can count that line without the statement: the `def` line
|
|
19
|
-
# the file loads, and `x if done; y(` counts when `y` does
|
|
20
|
-
|
|
25
|
+
# Unless `loose`, a first or last line that holds other code is left out,
|
|
26
|
+
# because Ruby can count that line without the statement: the `def` line
|
|
27
|
+
# counts when the file loads, and `x if done; y(` counts when `y` does
|
|
28
|
+
# not run.
|
|
29
|
+
def self.for(def_node, source, offset, loose: false)
|
|
21
30
|
return [] unless def_node.body
|
|
22
31
|
|
|
23
32
|
path = path_to(def_node.body, offset) || []
|
|
@@ -25,16 +34,20 @@ module Mutineer
|
|
|
25
34
|
return [] unless index
|
|
26
35
|
|
|
27
36
|
statement = path[index]
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
37
|
+
unless loose
|
|
38
|
+
# Code that runs only sometimes inside the statement: the tests that ran
|
|
39
|
+
# the statement need not have run it.
|
|
40
|
+
return [] if path[index..].each_cons(2).any? { |parent, child| sometimes?(parent, child) }
|
|
41
|
+
|
|
42
|
+
# Ruby counts the line of `x while c` or `x if c` when it checks `c`,
|
|
43
|
+
# also when `x` never runs. The path is the modifier, its body, statement.
|
|
44
|
+
return [] if index >= 2 && modifier?(path[index - 2])
|
|
45
|
+
end
|
|
35
46
|
|
|
36
47
|
location = statement.location
|
|
37
48
|
lines = (location.start_line..last_line(statement)).to_a
|
|
49
|
+
return lines - [def_node.location.start_line] if loose
|
|
50
|
+
|
|
38
51
|
lines.delete(location.start_line) unless before(source, location).strip.empty?
|
|
39
52
|
after = after(source, location).strip
|
|
40
53
|
lines.delete(location.end_line) unless after.empty? || after.start_with?("#")
|
data/lib/mutineer/subject.rb
CHANGED
|
@@ -8,11 +8,24 @@ module Mutineer
|
|
|
8
8
|
# (`module ::X` inside `Outer` owns into `X`); `lexical` is the class/module
|
|
9
9
|
# chain as written (`["Outer", "::X"]`), which the redefine strategy needs to
|
|
10
10
|
# rebuild the same Module.nesting as a whole-file reload (#145). `block_owner`
|
|
11
|
-
# is the constant a `Data.define`-style block is assigned to, as written
|
|
12
|
-
# `
|
|
13
|
-
#
|
|
11
|
+
# is the constant a `Data.define`-style block is assigned to, as written, or
|
|
12
|
+
# the `ClassMethods` of a concern's `class_methods` block (#252);
|
|
13
|
+
# `owner_unknown` is true when the class that owns the method cannot be named
|
|
14
|
+
# (an unassigned builder block, or a concern block whose class or module no
|
|
15
|
+
# constant names, such as an `included` block that runs in each includer),
|
|
16
|
+
# so the redefine strategy cannot load a mutant onto it. `hook_block` is true
|
|
17
|
+
# for a def anywhere inside a concern's `included` or `prepended` block, which
|
|
18
|
+
# ActiveSupport runs only when a class first includes the concern, so a
|
|
19
|
+
# reload of the file does not reach a class that already has it (#256).
|
|
20
|
+
# `reload_hook_call` is the `included` or `prepended` call node of such a
|
|
21
|
+
# block when reload can still place the def: the mutant file stores the
|
|
22
|
+
# mutated def just before that call, and the child loads it into each class
|
|
23
|
+
# that has the method (#297). It is nil when reload cannot place the def, even
|
|
24
|
+
# when `hook_block` is true (see docs/json-schema.md for the cases).
|
|
25
|
+
# `enclosing_subject` is the subject of the `def` whose body holds this one,
|
|
26
|
+
# or nil, so the redefine strategy can load each enclosing method too (#290).
|
|
14
27
|
Subject = Struct.new(:file, :namespace, :name, :singleton, :def_node, :lexical, :block_owner, :owner_unknown,
|
|
15
|
-
keyword_init: true) do
|
|
28
|
+
:hook_block, :reload_hook_call, :enclosing_subject, keyword_init: true) do
|
|
16
29
|
# Returns the fully-qualified subject name.
|
|
17
30
|
#
|
|
18
31
|
# @return [String] namespaced method name like `Billing::Invoice#total`.
|
|
@@ -28,6 +41,19 @@ module Mutineer
|
|
|
28
41
|
lexical || namespace
|
|
29
42
|
end
|
|
30
43
|
|
|
44
|
+
# True when the strategy cannot load a mutant of this method where the tests
|
|
45
|
+
# call it, so the mutant is not run. Redefine needs a named owner. Reload
|
|
46
|
+
# loads the whole file again, which redefines everything except what a
|
|
47
|
+
# concern's `included` or `prepended` block defines: that block does not run
|
|
48
|
+
# again in a class that already includes the concern. Reload places such a
|
|
49
|
+
# def only when discovery found its {#reload_hook_call} (#297).
|
|
50
|
+
#
|
|
51
|
+
# @param strategy [String] `"reload"` or `"redefine"`.
|
|
52
|
+
# @return [Boolean]
|
|
53
|
+
def unplaceable?(strategy)
|
|
54
|
+
strategy == "redefine" ? owner_unknown == true : hook_block == true && reload_hook_call.nil?
|
|
55
|
+
end
|
|
56
|
+
|
|
31
57
|
# Returns the body location for the subject, if any.
|
|
32
58
|
#
|
|
33
59
|
# @return [Prism::Location, nil] body location or nil for empty methods.
|
|
@@ -13,7 +13,8 @@ module Mutineer
|
|
|
13
13
|
# first failing test.
|
|
14
14
|
# @param record_to [IO, nil] a `--matrix` run: send each test's outcome to
|
|
15
15
|
# this channel and run every test.
|
|
16
|
-
# @return [Integer] 0 on success, 1 on failure
|
|
16
|
+
# @return [Integer] 0 on success, 1 on failure, 2 when the first failure
|
|
17
|
+
# was an Active Record connection or lock error.
|
|
17
18
|
def self.run(test_files, stop_at_first_failure: false, record_to: nil)
|
|
18
19
|
MinitestIntegration.run(test_files, stop_at_first_failure: stop_at_first_failure, record_to: record_to)
|
|
19
20
|
end
|
|
@@ -12,7 +12,8 @@ module Mutineer
|
|
|
12
12
|
# Child-process-only RSpec runner.
|
|
13
13
|
#
|
|
14
14
|
# Mirrors MinitestIntegration's contract: run the given spec files and
|
|
15
|
-
# return 0 (all passed) or
|
|
15
|
+
# return 0 (all passed), 1 (an example failed) or 2 (only an error
|
|
16
|
+
# outside examples, such as a spec file that raised while loading).
|
|
16
17
|
module RSpec
|
|
17
18
|
# Runs the given RSpec files.
|
|
18
19
|
#
|
|
@@ -21,7 +22,7 @@ module Mutineer
|
|
|
21
22
|
# first failing example (RSpec `--fail-fast`).
|
|
22
23
|
# @param record_to [IO, nil] a `--matrix` run: send each example's outcome
|
|
23
24
|
# to this channel (see {KillFormatter}) and run every example.
|
|
24
|
-
# @return [Integer] 0 on success,
|
|
25
|
+
# @return [Integer] 0 on success, else {.failure_code}.
|
|
25
26
|
# @raise [ArgumentError] when both options are given: a matrix run never stops.
|
|
26
27
|
def self.run(spec_files, stop_at_first_failure: false, record_to: nil)
|
|
27
28
|
if stop_at_first_failure && record_to
|
|
@@ -56,11 +57,22 @@ module Mutineer
|
|
|
56
57
|
# early, so `end` needs a reported outcome for every planned example.
|
|
57
58
|
KillChannel.write_end(record_to) if record_to && KillFormatter.last&.saw_every_example?
|
|
58
59
|
|
|
59
|
-
status.zero? ? 0 :
|
|
60
|
+
status.zero? ? 0 : failure_code
|
|
60
61
|
ensure
|
|
61
62
|
FileOrder.rank = nil
|
|
62
63
|
end
|
|
63
64
|
|
|
65
|
+
# The exit status of a failed run: 2 when no example failed and RSpec
|
|
66
|
+
# recorded an error outside examples, else 1.
|
|
67
|
+
#
|
|
68
|
+
# @api private
|
|
69
|
+
# @return [Integer] 1 or 2.
|
|
70
|
+
def self.failure_code
|
|
71
|
+
world = ::RSpec.world
|
|
72
|
+
outside_examples = world.respond_to?(:non_example_failure) && world.non_example_failure
|
|
73
|
+
outside_examples && ::RSpec.configuration.reporter.failed_examples.empty? ? 2 : 1
|
|
74
|
+
end
|
|
75
|
+
|
|
64
76
|
# Runs the top-level example groups in the order of `spec_files` (#203),
|
|
65
77
|
# after RSpec's own ordering, which then decides the order within a file.
|
|
66
78
|
#
|
|
@@ -7,8 +7,11 @@ module Mutineer
|
|
|
7
7
|
# Picks the test-framework runner.
|
|
8
8
|
#
|
|
9
9
|
# Each runner responds to
|
|
10
|
-
# `.run(files, stop_at_first_failure: false, record_to: nil) -> 0/1`
|
|
11
|
-
# (0 = all passed, 1 = any failure
|
|
10
|
+
# `.run(files, stop_at_first_failure: false, record_to: nil) -> 0/1/2`
|
|
11
|
+
# (0 = all passed, 1 = any failure, 2 = no verdict: an error outside tests,
|
|
12
|
+
# which the RSpec runner reports, or a Minitest run whose first failure was
|
|
13
|
+
# an Active Record connection or lock error, #296) and is called only inside
|
|
14
|
+
# a forked child.
|
|
12
15
|
# Only the mutant path passes `stop_at_first_failure: true`; a `--matrix`
|
|
13
16
|
# mutant run passes `record_to:` instead (a {KillChannel} pipe) and runs every
|
|
14
17
|
# test. The fork boundary silences stdout (see ChildStdout); the runners do not.
|