mutineer 1.8.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +156 -0
- data/README.md +1 -1
- data/lib/mutineer/cli.rb +17 -2
- data/lib/mutineer/concern_blocks.rb +103 -13
- data/lib/mutineer/daemon_backend.rb +1 -1
- data/lib/mutineer/daemon_client.rb +55 -20
- data/lib/mutineer/daemon_server.rb +10 -9
- data/lib/mutineer/external_backend.rb +46 -5
- data/lib/mutineer/init.rb +1 -1
- data/lib/mutineer/isolation.rb +9 -5
- data/lib/mutineer/job_plan.rb +6 -4
- data/lib/mutineer/mutation.rb +4 -3
- data/lib/mutineer/parser.rb +39 -0
- data/lib/mutineer/rails_worker_db.rb +10 -0
- data/lib/mutineer/redaction.rb +45 -17
- data/lib/mutineer/reload_guards.rb +564 -0
- data/lib/mutineer/reporter.rb +2 -1
- data/lib/mutineer/runner.rb +20 -5
- data/lib/mutineer/triage.rb +1 -1
- data/lib/mutineer/version.rb +1 -1
- metadata +2 -1
|
@@ -61,6 +61,17 @@ module Mutineer
|
|
|
61
61
|
RSPEC_OUTSIDE_ERROR = /errors? occurred outside of examples/
|
|
62
62
|
# ANSI colour codes, removed before matching so a coloured summary still counts.
|
|
63
63
|
ANSI_CODE = /\e\[[\d;]*m/
|
|
64
|
+
# The seed line Minitest prints before every run. Captures the seed.
|
|
65
|
+
MINITEST_SEED = /^Run options:.*--seed (\d+)/
|
|
66
|
+
# The seed line RSpec prints when it shuffles the tests. Captures the seed.
|
|
67
|
+
RSPEC_SEED = /^Randomized with seed (\d+)/
|
|
68
|
+
# A Minitest seed given on the command or in TESTOPTS (`SEED=N`, `--seed N`
|
|
69
|
+
# `-s N` or `-sN`). Minitest prints its seed line for a fixed seed too, so
|
|
70
|
+
# the line alone cannot tell.
|
|
71
|
+
MINITEST_SEED_GIVEN = /\bSEED=|--seed\b|(?:\A|\s)-s(?![A-Za-z_])/
|
|
72
|
+
# An RSpec seed given on the command or in SPEC_OPTS (`--seed N` or
|
|
73
|
+
# `--order rand:N`). RSpec ignores SEED.
|
|
74
|
+
RSPEC_SEED_GIVEN = /--seed\b|\brand(?:om)?:\d/
|
|
64
75
|
|
|
65
76
|
# PATH entries that pin a concrete Ruby under a version manager (ahead of
|
|
66
77
|
# shims). Optional trailing slash; normal bins are left alone.
|
|
@@ -286,20 +297,50 @@ module Mutineer
|
|
|
286
297
|
output.b.gsub(ANSI_CODE, "")
|
|
287
298
|
end
|
|
288
299
|
|
|
300
|
+
# The runner and seed of a random test order the output shows, or nil.
|
|
301
|
+
# Nil also when Mutineer sees a fixed seed: for Minitest, SEED, TESTOPTS
|
|
302
|
+
# or the command sets one; for RSpec, SPEC_OPTS or the command does. RSpec
|
|
303
|
+
# `--order defined` prints no seed line. Each runner is checked on its
|
|
304
|
+
# own, so a seeded Minitest suite does not hide an unseeded RSpec suite
|
|
305
|
+
# that the same command runs. Ceiling: a seed set in a wrapper script or
|
|
306
|
+
# `.rspec`, or Minitest's `test_order`, is not seen, so the warning still
|
|
307
|
+
# prints; a seed-like argument meant for a wrapper hides it (#338, limits
|
|
308
|
+
# tracked in #368).
|
|
309
|
+
#
|
|
310
|
+
# @api private
|
|
311
|
+
# @param command [String] the --test-command template.
|
|
312
|
+
# @param text [String] output from {summary_text}.
|
|
313
|
+
# @return [Array(Symbol, String), nil] `[:minitest | :rspec, seed]`.
|
|
314
|
+
def self.random_order(command, text)
|
|
315
|
+
minitest = text[MINITEST_SEED, 1]
|
|
316
|
+
if minitest && ENV["SEED"].to_s.empty? &&
|
|
317
|
+
[command, ENV["TESTOPTS"]].none? { |s| s.to_s.match?(MINITEST_SEED_GIVEN) }
|
|
318
|
+
return [:minitest, minitest]
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
rspec = text[RSPEC_SEED, 1]
|
|
322
|
+
[:rspec, rspec] if rspec && [command, ENV["SPEC_OPTS"]].none? { |s| s.to_s.match?(RSPEC_SEED_GIVEN) }
|
|
323
|
+
end
|
|
324
|
+
|
|
289
325
|
# Pre-flight: run the command once against the UNMUTATED tree. Green (exit 0)
|
|
290
|
-
# returns the elapsed seconds (used to calibrate the per-mutant timeout)
|
|
326
|
+
# returns the elapsed seconds (used to calibrate the per-mutant timeout),
|
|
291
327
|
# whether the output has a Minitest or RSpec summary line (used to turn on
|
|
292
|
-
# the summary check in {run})
|
|
293
|
-
#
|
|
328
|
+
# the summary check in {run}), and the runner and seed of a random test
|
|
329
|
+
# order (see {random_order}); anything else raises SmokeCheckError so the run
|
|
330
|
+
# aborts before scoring.
|
|
294
331
|
#
|
|
295
332
|
# @param command [String] the --test-command template.
|
|
296
333
|
# @param files [Array<String>] test file paths.
|
|
297
334
|
# @param timeout [Numeric] ceiling for the calibration run.
|
|
298
|
-
# @return [Array(Float, Boolean
|
|
335
|
+
# @return [Array(Float, Boolean, Array(Symbol, String), nil)]
|
|
336
|
+
# `[elapsed_seconds, summary_seen, random_order]`.
|
|
299
337
|
# @raise [Mutineer::SmokeCheckError] when the clean suite is not green.
|
|
300
338
|
def self.smoke_check!(command, files, timeout: SMOKE_TIMEOUT)
|
|
301
339
|
kind, code, output, elapsed = spawn_capture(command, files, timeout)
|
|
302
|
-
|
|
340
|
+
if kind == :exited && code&.zero?
|
|
341
|
+
text = summary_text(output)
|
|
342
|
+
return [elapsed, summary_line?(text), random_order(command, text)]
|
|
343
|
+
end
|
|
303
344
|
|
|
304
345
|
reason =
|
|
305
346
|
if kind == :timeout then "did not finish within #{timeout}s"
|
data/lib/mutineer/init.rb
CHANGED
data/lib/mutineer/isolation.rb
CHANGED
|
@@ -193,15 +193,19 @@ module Mutineer
|
|
|
193
193
|
# A concern's stored `included` and `prepended` blocks from the source file
|
|
194
194
|
# are forgotten around the load, so ActiveSupport accepts the blocks the
|
|
195
195
|
# temp file stores (#256). For a def in such a block, the caller passes the
|
|
196
|
-
# text from {JobPlan.hook_mutant} and
|
|
197
|
-
#
|
|
196
|
+
# text and `hook_def` from {JobPlan.hook_mutant}, and the load places the
|
|
197
|
+
# mutated def (#297, #348).
|
|
198
|
+
#
|
|
199
|
+
# {ConcernBlocks.load_mutant} turns Ruby's warnings off for the load, so
|
|
200
|
+
# the constants it assigns again do not warn (#364).
|
|
198
201
|
#
|
|
199
202
|
# @api private
|
|
200
203
|
# @param mutated [String] mutated source text.
|
|
201
204
|
# @param source_file [String] original source file path.
|
|
205
|
+
# @param hook_def [Hash{Symbol => Object}, nil] see {ConcernBlocks.load_mutant}.
|
|
202
206
|
# @return [Object] whatever `load` returns.
|
|
203
|
-
def self.apply_whole_file(mutated, source_file)
|
|
204
|
-
write_beside(mutated, source_file) { |path| ConcernBlocks.
|
|
207
|
+
def self.apply_whole_file(mutated, source_file, hook_def: nil)
|
|
208
|
+
write_beside(mutated, source_file) { |path| ConcernBlocks.load_mutant(source_file, path, hook_def: hook_def) }
|
|
205
209
|
end
|
|
206
210
|
|
|
207
211
|
# Writes `code` to a hidden `.mutineer_mutant` file beside `source_file`,
|
|
@@ -270,7 +274,7 @@ module Mutineer
|
|
|
270
274
|
# running the ORIGINAL method (C2 false-survived). Raise -> the fork
|
|
271
275
|
# block aborts before any test runs -> Result.error, never a bogus
|
|
272
276
|
# `survived`.
|
|
273
|
-
raise "surgical snippet failed to reparse" if Parser.
|
|
277
|
+
raise "surgical snippet failed to reparse" if Parser.invalid?(wrapped)
|
|
274
278
|
|
|
275
279
|
# Preserve original visibility — class/module bodies define methods
|
|
276
280
|
# public, but 7a's `load` would re-apply the file's private/protected
|
data/lib/mutineer/job_plan.rb
CHANGED
|
@@ -183,9 +183,10 @@ module Mutineer
|
|
|
183
183
|
return if files.empty?
|
|
184
184
|
|
|
185
185
|
seed_hint = TestRunners.for(framework) == TestRunners::Minitest ? minitest_seed_hint : ""
|
|
186
|
+
# The hint opens with "the unmutated suite is not green", so the files
|
|
187
|
+
# follow it rather than repeat that sentence (#360).
|
|
186
188
|
raise SmokeCheckError,
|
|
187
|
-
"
|
|
188
|
-
"#{ExternalBackend.generic_env_hint}.#{seed_hint}"
|
|
189
|
+
"#{ExternalBackend.generic_env_hint}. Failed: #{files.join(', ')}.#{seed_hint}"
|
|
189
190
|
end
|
|
190
191
|
|
|
191
192
|
# Names the configured seed, and accounts for helper overrides.
|
|
@@ -205,8 +206,9 @@ module Mutineer
|
|
|
205
206
|
# block (#297): the mutated file with one statement added just before the
|
|
206
207
|
# hook call, `::Mutineer::ConcernBlocks.store_hook_def do <mutated def> end`.
|
|
207
208
|
# The statement sits in the same body as the hook call, so the stored def has
|
|
208
|
-
# the block's constant scope.
|
|
209
|
-
#
|
|
209
|
+
# the block's constant scope. The child passes the hook-def values to
|
|
210
|
+
# {ConcernBlocks.load_mutant}, which places the def when the load reaches
|
|
211
|
+
# that statement (#348). Shared by {Runner.run} and
|
|
210
212
|
# {DaemonBackend.job_result}, so both build the same text. The hook call starts
|
|
211
213
|
# before the def, so the mutation does not move its offset. A bad insert is
|
|
212
214
|
# not checked here: the child's `load` raises `SyntaxError` before any test
|
data/lib/mutineer/mutation.rb
CHANGED
|
@@ -16,12 +16,13 @@ module Mutineer
|
|
|
16
16
|
source.byteslice(0, start_offset) + replacement + source.byteslice(end_offset..)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
# Checks whether the
|
|
19
|
+
# Checks whether the running Ruby can still load the mutated source (see
|
|
20
|
+
# {Parser.invalid?}).
|
|
20
21
|
#
|
|
21
22
|
# @param source [String] original source text.
|
|
22
|
-
# @return [Boolean] true when the mutated source re-parses
|
|
23
|
+
# @return [Boolean] true when the mutated source re-parses.
|
|
23
24
|
def valid?(source)
|
|
24
|
-
Parser.
|
|
25
|
+
!Parser.invalid?(apply(source))
|
|
25
26
|
end
|
|
26
27
|
end
|
|
27
28
|
end
|
data/lib/mutineer/parser.rb
CHANGED
|
@@ -33,6 +33,45 @@ module Mutineer
|
|
|
33
33
|
Prism.parse(source)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
+
# True when Ruby cannot load `source`: the loaded Prism reports a parse
|
|
37
|
+
# error and the running Ruby cannot compile the text either. Prism alone is
|
|
38
|
+
# not enough. Prism 2 reports a bare-constant operator write inside a
|
|
39
|
+
# method (`T += 1`) as a dynamic constant assignment, while the Prism 1
|
|
40
|
+
# that Ruby 3.4 and 4.0 load files with accepts it (#392). Compiling does
|
|
41
|
+
# not run the code.
|
|
42
|
+
#
|
|
43
|
+
# @param source [String] source text.
|
|
44
|
+
# @return [Boolean]
|
|
45
|
+
def self.invalid?(source)
|
|
46
|
+
return false if parse_string(source).errors.empty? # the usual case costs one parse
|
|
47
|
+
|
|
48
|
+
!compiles?(source)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Serializes {.compiles?}: it swaps the process-wide `$VERBOSE`, and
|
|
52
|
+
# `--daemon` checks mutants on several threads.
|
|
53
|
+
COMPILE_LOCK = Mutex.new
|
|
54
|
+
|
|
55
|
+
# Whether the running Ruby compiles `source`, without running it. Mutineer
|
|
56
|
+
# forks, so it runs on CRuby, which has RubyVM. Warnings are off for the
|
|
57
|
+
# compile: a mutant can trigger one (`if a = 1`).
|
|
58
|
+
#
|
|
59
|
+
# @param source [String] source text.
|
|
60
|
+
# @return [Boolean]
|
|
61
|
+
def self.compiles?(source)
|
|
62
|
+
COMPILE_LOCK.synchronize do
|
|
63
|
+
verbose = $VERBOSE
|
|
64
|
+
$VERBOSE = nil
|
|
65
|
+
RubyVM::InstructionSequence.compile(source)
|
|
66
|
+
true
|
|
67
|
+
rescue SyntaxError
|
|
68
|
+
false
|
|
69
|
+
ensure
|
|
70
|
+
$VERBOSE = verbose
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
private_class_method :compiles?
|
|
74
|
+
|
|
36
75
|
# The comments of source text, without building its syntax tree: a few
|
|
37
76
|
# times cheaper than {.parse_string} for a caller that needs only these.
|
|
38
77
|
#
|
|
@@ -488,6 +488,16 @@ module Mutineer
|
|
|
488
488
|
end
|
|
489
489
|
end
|
|
490
490
|
|
|
491
|
+
# Turns off Rails' pending-migration check in this mutant fork (#385).
|
|
492
|
+
# `rails/test_help` runs it when the tests load, and it queries every
|
|
493
|
+
# database, shared ones too, so {touched_shared_databases} would flag every
|
|
494
|
+
# mutant. The clean check still runs it, before any mutant.
|
|
495
|
+
#
|
|
496
|
+
# @return [void]
|
|
497
|
+
def self.skip_schema_check!
|
|
498
|
+
ActiveRecord.maintain_test_schema = false if ActiveRecord.respond_to?(:maintain_test_schema=)
|
|
499
|
+
end
|
|
500
|
+
|
|
491
501
|
# Names from `shared` whose database this fork used. Run in a fork after
|
|
492
502
|
# its tests: a name here means a test used a database that parallel
|
|
493
503
|
# workers share, so the verdict cannot be trusted. A pool counts when it
|
data/lib/mutineer/redaction.rb
CHANGED
|
@@ -5,18 +5,32 @@ require "uri"
|
|
|
5
5
|
module Mutineer
|
|
6
6
|
# Hides database credentials in text bound for a terminal: an error reply,
|
|
7
7
|
# a log line, a warning. Rails parses DATABASE_URL itself, so an error about
|
|
8
|
-
# a malformed one can quote the whole URL. Outside a URL a password
|
|
9
|
-
#
|
|
10
|
-
# letters inside other words. A password that exactly equals the
|
|
11
|
-
# left visible outside a URL, so `role "postgres"` stays readable.
|
|
8
|
+
# a malformed one can quote the whole URL. Outside a URL a password form
|
|
9
|
+
# made only of word characters is hidden only as a whole token, so it cannot
|
|
10
|
+
# blank letters inside other words (#341). A password that exactly equals the
|
|
11
|
+
# username is left visible outside a URL, so `role "postgres"` stays readable.
|
|
12
12
|
#
|
|
13
13
|
# Stdlib only: the app-side daemon loads this file without the rest of
|
|
14
14
|
# mutineer.
|
|
15
15
|
module Redaction
|
|
16
|
-
# The password in a `scheme://user:password@host` URL.
|
|
17
|
-
# `@` before an unescaped quote or line end
|
|
18
|
-
#
|
|
19
|
-
|
|
16
|
+
# The password in a `scheme://user:password@host` URL. It ends at the last
|
|
17
|
+
# `@` before an unescaped quote or line end that starts a host: a run with
|
|
18
|
+
# no `@`, then a `/`, `:`, `?` or `#`, and no other `@` before the next
|
|
19
|
+
# space. So in `u:p@ss@h/db and p@ss` only `p@ss` is hidden (#341). With no
|
|
20
|
+
# such `@`, it ends at the last `@`, as before. A malformed password with
|
|
21
|
+
# an `@`, a `:`, a space, or a quote `inspect` escaped (`\"`) is hidden
|
|
22
|
+
# too, so when the two goals conflict, hiding wins: in
|
|
23
|
+
# `u:pw@h/x and y@h2/db` the text up to `h2` is hidden. One case still
|
|
24
|
+
# shows part of a password: one that holds an `@`, a host, a `/`, `:`,
|
|
25
|
+
# `?` or `#`, and a space (`a@b/c d`), in a URL whose own host has no
|
|
26
|
+
# port, path or query, or has an `@` in them (`?n=ci@corp`).
|
|
27
|
+
URL_PASSWORD = %r{
|
|
28
|
+
([a-z][a-z0-9+.\-]*://[^\s:/@"'\\]*):
|
|
29
|
+
(?:
|
|
30
|
+
(?:\\.|[^"'\r\n\\])*@(?=[^@\s"'\\/:?\#]+[/:?\#][^@\s]*(?:\s|\z))
|
|
31
|
+
| (?:\\.|[^"'\r\n\\])*@
|
|
32
|
+
)
|
|
33
|
+
}xi
|
|
20
34
|
# What replaces a hidden password.
|
|
21
35
|
REDACTED = "[REDACTED]"
|
|
22
36
|
# What {text} returns when it fails: never the input, which may hold the
|
|
@@ -24,18 +38,27 @@ module Mutineer
|
|
|
24
38
|
FAILED = "[message hidden: redaction failed]"
|
|
25
39
|
# An ASCII word character, the edge a whole-token match must not touch.
|
|
26
40
|
WORD_CHAR = "[A-Za-z0-9_]"
|
|
27
|
-
# The longest password form hidden only as a whole token
|
|
28
|
-
#
|
|
29
|
-
#
|
|
41
|
+
# The longest password form hidden only as a whole token whatever its
|
|
42
|
+
# characters. A longer form with a non-word character is hidden wherever
|
|
43
|
+
# it appears, because its edges are no reliable token boundary.
|
|
30
44
|
SHORT_FORM = 3
|
|
45
|
+
# An escape or prefix that glues a word character to the left of a
|
|
46
|
+
# password form in a message: `%40`, `\x41`, `\u0041`, `\u{41}`, `\101`,
|
|
47
|
+
# `\n`, an ANSI code such as the colour Rails logs put on a span (raw,
|
|
48
|
+
# escaped as `\e`, `\033`, `\x1b` or `\u001b`, or with its ESC byte
|
|
49
|
+
# stripped), or a `-p` flag after a space, quote, bracket, comma or `=`
|
|
50
|
+
# (`mysql -ps3cret`, `["-ps3cret"]`). A form right after one is still a
|
|
51
|
+
# whole token.
|
|
52
|
+
ESCAPE = /%\h{2}|\\x\h{2}|\\u(?:\h{4}|\{\h+\})|\\[0-7]{1,3}|(?:\e|\\(?:e|033|x1[bB]|u001[bB]))?\[[0-9;]*[A-Za-z]|\\[a-z]|(?<![^\s"'(\[,=])-p/.source
|
|
31
53
|
|
|
32
54
|
module_function
|
|
33
55
|
|
|
34
56
|
# Hide the password in any URL, and the DATABASE_URL password (from its
|
|
35
57
|
# `user:password@` pair and from its `password=` query parameter) wherever
|
|
36
58
|
# it appears: whole, percent-decoded, `inspect` escaped, or cut short after
|
|
37
|
-
# its `scheme://user:` prefix. Outside a URL a form of
|
|
38
|
-
# characters or fewer is hidden only as a whole token
|
|
59
|
+
# its `scheme://user:` prefix. Outside a URL a form of word characters, or
|
|
60
|
+
# of {SHORT_FORM} characters or fewer, is hidden only as a whole token; a
|
|
61
|
+
# longer form with another character is hidden anywhere. Never raises: text in
|
|
39
62
|
# any encoding comes back as UTF-8, and an internal error returns {FAILED}.
|
|
40
63
|
#
|
|
41
64
|
# @param text [#to_s]
|
|
@@ -68,7 +91,8 @@ module Mutineer
|
|
|
68
91
|
raws.concat(query_passwords(url))
|
|
69
92
|
secrets = raws.compact.uniq.flat_map { |raw| password_forms(raw) }.uniq.sort_by { |form| -form.length }
|
|
70
93
|
secrets.reduce(out) do |acc, secret|
|
|
71
|
-
secret.length
|
|
94
|
+
whole_token = secret.length <= SHORT_FORM || secret.match?(/\A#{WORD_CHAR}+\z/)
|
|
95
|
+
whole_token ? redact_word(acc, secret) : acc.gsub(secret, REDACTED)
|
|
72
96
|
end
|
|
73
97
|
rescue StandardError
|
|
74
98
|
FAILED
|
|
@@ -93,14 +117,18 @@ module Mutineer
|
|
|
93
117
|
end
|
|
94
118
|
|
|
95
119
|
# Replace `form` in `text` only where it is a whole token: no ASCII word
|
|
96
|
-
# character touches either edge, including a punctuation edge.
|
|
120
|
+
# character touches either edge, including a punctuation edge. A form
|
|
121
|
+
# right after an {ESCAPE} also counts as a whole token, and the escape is
|
|
122
|
+
# kept. An escape may not start where a copy of the form starts, so it
|
|
123
|
+
# cannot swallow that copy (`%41%41` with password `%41`).
|
|
97
124
|
#
|
|
98
125
|
# @param text [String]
|
|
99
126
|
# @param form [String] one form of the password.
|
|
100
127
|
# @return [String]
|
|
101
128
|
def redact_word(text, form)
|
|
102
|
-
|
|
103
|
-
|
|
129
|
+
quoted = Regexp.escape(form)
|
|
130
|
+
pattern = "((?!#{quoted})(?:#{ESCAPE})|(?<!#{WORD_CHAR}))#{quoted}(?!#{WORD_CHAR})"
|
|
131
|
+
text.gsub(Regexp.new(pattern)) { "#{Regexp.last_match(1)}#{REDACTED}" }
|
|
104
132
|
end
|
|
105
133
|
|
|
106
134
|
# An exception as `Class: message`, redacted. Each part is redacted on its
|