mutineer 1.8.0 → 1.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -61,6 +61,17 @@ module Mutineer
61
61
  RSPEC_OUTSIDE_ERROR = /errors? occurred outside of examples/
62
62
  # ANSI colour codes, removed before matching so a coloured summary still counts.
63
63
  ANSI_CODE = /\e\[[\d;]*m/
64
+ # The seed line Minitest prints before every run. Captures the seed.
65
+ MINITEST_SEED = /^Run options:.*--seed (\d+)/
66
+ # The seed line RSpec prints when it shuffles the tests. Captures the seed.
67
+ RSPEC_SEED = /^Randomized with seed (\d+)/
68
+ # A Minitest seed given on the command or in TESTOPTS (`SEED=N`, `--seed N`
69
+ # `-s N` or `-sN`). Minitest prints its seed line for a fixed seed too, so
70
+ # the line alone cannot tell.
71
+ MINITEST_SEED_GIVEN = /\bSEED=|--seed\b|(?:\A|\s)-s(?![A-Za-z_])/
72
+ # An RSpec seed given on the command or in SPEC_OPTS (`--seed N` or
73
+ # `--order rand:N`). RSpec ignores SEED.
74
+ RSPEC_SEED_GIVEN = /--seed\b|\brand(?:om)?:\d/
64
75
 
65
76
  # PATH entries that pin a concrete Ruby under a version manager (ahead of
66
77
  # shims). Optional trailing slash; normal bins are left alone.
@@ -286,20 +297,50 @@ module Mutineer
286
297
  output.b.gsub(ANSI_CODE, "")
287
298
  end
288
299
 
300
+ # The runner and seed of a random test order the output shows, or nil.
301
+ # Nil also when Mutineer sees a fixed seed: for Minitest, SEED, TESTOPTS
302
+ # or the command sets one; for RSpec, SPEC_OPTS or the command does. RSpec
303
+ # `--order defined` prints no seed line. Each runner is checked on its
304
+ # own, so a seeded Minitest suite does not hide an unseeded RSpec suite
305
+ # that the same command runs. Ceiling: a seed set in a wrapper script or
306
+ # `.rspec`, or Minitest's `test_order`, is not seen, so the warning still
307
+ # prints; a seed-like argument meant for a wrapper hides it (#338, limits
308
+ # tracked in #368).
309
+ #
310
+ # @api private
311
+ # @param command [String] the --test-command template.
312
+ # @param text [String] output from {summary_text}.
313
+ # @return [Array(Symbol, String), nil] `[:minitest | :rspec, seed]`.
314
+ def self.random_order(command, text)
315
+ minitest = text[MINITEST_SEED, 1]
316
+ if minitest && ENV["SEED"].to_s.empty? &&
317
+ [command, ENV["TESTOPTS"]].none? { |s| s.to_s.match?(MINITEST_SEED_GIVEN) }
318
+ return [:minitest, minitest]
319
+ end
320
+
321
+ rspec = text[RSPEC_SEED, 1]
322
+ [:rspec, rspec] if rspec && [command, ENV["SPEC_OPTS"]].none? { |s| s.to_s.match?(RSPEC_SEED_GIVEN) }
323
+ end
324
+
289
325
  # Pre-flight: run the command once against the UNMUTATED tree. Green (exit 0)
290
- # returns the elapsed seconds (used to calibrate the per-mutant timeout) and
326
+ # returns the elapsed seconds (used to calibrate the per-mutant timeout),
291
327
  # whether the output has a Minitest or RSpec summary line (used to turn on
292
- # the summary check in {run}); anything else raises SmokeCheckError so the
293
- # run aborts before scoring.
328
+ # the summary check in {run}), and the runner and seed of a random test
329
+ # order (see {random_order}); anything else raises SmokeCheckError so the run
330
+ # aborts before scoring.
294
331
  #
295
332
  # @param command [String] the --test-command template.
296
333
  # @param files [Array<String>] test file paths.
297
334
  # @param timeout [Numeric] ceiling for the calibration run.
298
- # @return [Array(Float, Boolean)] `[elapsed_seconds, summary_seen]`.
335
+ # @return [Array(Float, Boolean, Array(Symbol, String), nil)]
336
+ # `[elapsed_seconds, summary_seen, random_order]`.
299
337
  # @raise [Mutineer::SmokeCheckError] when the clean suite is not green.
300
338
  def self.smoke_check!(command, files, timeout: SMOKE_TIMEOUT)
301
339
  kind, code, output, elapsed = spawn_capture(command, files, timeout)
302
- return [elapsed, summary_line?(summary_text(output))] if kind == :exited && code&.zero?
340
+ if kind == :exited && code&.zero?
341
+ text = summary_text(output)
342
+ return [elapsed, summary_line?(text), random_order(command, text)]
343
+ end
303
344
 
304
345
  reason =
305
346
  if kind == :timeout then "did not finish within #{timeout}s"
data/lib/mutineer/init.rb CHANGED
@@ -31,7 +31,7 @@ module Mutineer
31
31
  end
32
32
  begin
33
33
  parser.parse!(argv)
34
- rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
34
+ rescue OptionParser::ParseError => e
35
35
  err.puts "mutineer: #{e.message}"
36
36
  return 2
37
37
  end
@@ -193,15 +193,19 @@ module Mutineer
193
193
  # A concern's stored `included` and `prepended` blocks from the source file
194
194
  # are forgotten around the load, so ActiveSupport accepts the blocks the
195
195
  # temp file stores (#256). For a def in such a block, the caller passes the
196
- # text from {JobPlan.hook_mutant} and then calls
197
- # {ConcernBlocks.place_hook_def} (#297).
196
+ # text and `hook_def` from {JobPlan.hook_mutant}, and the load places the
197
+ # mutated def (#297, #348).
198
+ #
199
+ # {ConcernBlocks.load_mutant} turns Ruby's warnings off for the load, so
200
+ # the constants it assigns again do not warn (#364).
198
201
  #
199
202
  # @api private
200
203
  # @param mutated [String] mutated source text.
201
204
  # @param source_file [String] original source file path.
205
+ # @param hook_def [Hash{Symbol => Object}, nil] see {ConcernBlocks.load_mutant}.
202
206
  # @return [Object] whatever `load` returns.
203
- def self.apply_whole_file(mutated, source_file)
204
- write_beside(mutated, source_file) { |path| ConcernBlocks.replace(source_file) { load path } }
207
+ def self.apply_whole_file(mutated, source_file, hook_def: nil)
208
+ write_beside(mutated, source_file) { |path| ConcernBlocks.load_mutant(source_file, path, hook_def: hook_def) }
205
209
  end
206
210
 
207
211
  # Writes `code` to a hidden `.mutineer_mutant` file beside `source_file`,
@@ -270,7 +274,7 @@ module Mutineer
270
274
  # running the ORIGINAL method (C2 false-survived). Raise -> the fork
271
275
  # block aborts before any test runs -> Result.error, never a bogus
272
276
  # `survived`.
273
- raise "surgical snippet failed to reparse" if Parser.parse_string(wrapped).errors.any?
277
+ raise "surgical snippet failed to reparse" if Parser.invalid?(wrapped)
274
278
 
275
279
  # Preserve original visibility — class/module bodies define methods
276
280
  # public, but 7a's `load` would re-apply the file's private/protected
@@ -183,9 +183,10 @@ module Mutineer
183
183
  return if files.empty?
184
184
 
185
185
  seed_hint = TestRunners.for(framework) == TestRunners::Minitest ? minitest_seed_hint : ""
186
+ # The hint opens with "the unmutated suite is not green", so the files
187
+ # follow it rather than repeat that sentence (#360).
186
188
  raise SmokeCheckError,
187
- "the unmutated suite is not green (#{files.join(', ')}) — " \
188
- "#{ExternalBackend.generic_env_hint}.#{seed_hint}"
189
+ "#{ExternalBackend.generic_env_hint}. Failed: #{files.join(', ')}.#{seed_hint}"
189
190
  end
190
191
 
191
192
  # Names the configured seed, and accounts for helper overrides.
@@ -205,8 +206,9 @@ module Mutineer
205
206
  # block (#297): the mutated file with one statement added just before the
206
207
  # hook call, `::Mutineer::ConcernBlocks.store_hook_def do <mutated def> end`.
207
208
  # The statement sits in the same body as the hook call, so the stored def has
208
- # the block's constant scope. After the load, the child passes the hook-def
209
- # values to {ConcernBlocks.place_hook_def}. Shared by {Runner.run} and
209
+ # the block's constant scope. The child passes the hook-def values to
210
+ # {ConcernBlocks.load_mutant}, which places the def when the load reaches
211
+ # that statement (#348). Shared by {Runner.run} and
210
212
  # {DaemonBackend.job_result}, so both build the same text. The hook call starts
211
213
  # before the def, so the mutation does not move its offset. A bad insert is
212
214
  # not checked here: the child's `load` raises `SyntaxError` before any test
@@ -16,12 +16,13 @@ module Mutineer
16
16
  source.byteslice(0, start_offset) + replacement + source.byteslice(end_offset..)
17
17
  end
18
18
 
19
- # Checks whether the mutated source still parses cleanly.
19
+ # Checks whether the running Ruby can still load the mutated source (see
20
+ # {Parser.invalid?}).
20
21
  #
21
22
  # @param source [String] original source text.
22
- # @return [Boolean] true when the mutated source re-parses without errors.
23
+ # @return [Boolean] true when the mutated source re-parses.
23
24
  def valid?(source)
24
- Parser.parse_string(apply(source)).errors.empty?
25
+ !Parser.invalid?(apply(source))
25
26
  end
26
27
  end
27
28
  end
@@ -33,6 +33,45 @@ module Mutineer
33
33
  Prism.parse(source)
34
34
  end
35
35
 
36
+ # True when Ruby cannot load `source`: the loaded Prism reports a parse
37
+ # error and the running Ruby cannot compile the text either. Prism alone is
38
+ # not enough. Prism 2 reports a bare-constant operator write inside a
39
+ # method (`T += 1`) as a dynamic constant assignment, while the Prism 1
40
+ # that Ruby 3.4 and 4.0 load files with accepts it (#392). Compiling does
41
+ # not run the code.
42
+ #
43
+ # @param source [String] source text.
44
+ # @return [Boolean]
45
+ def self.invalid?(source)
46
+ return false if parse_string(source).errors.empty? # the usual case costs one parse
47
+
48
+ !compiles?(source)
49
+ end
50
+
51
+ # Serializes {.compiles?}: it swaps the process-wide `$VERBOSE`, and
52
+ # `--daemon` checks mutants on several threads.
53
+ COMPILE_LOCK = Mutex.new
54
+
55
+ # Whether the running Ruby compiles `source`, without running it. Mutineer
56
+ # forks, so it runs on CRuby, which has RubyVM. Warnings are off for the
57
+ # compile: a mutant can trigger one (`if a = 1`).
58
+ #
59
+ # @param source [String] source text.
60
+ # @return [Boolean]
61
+ def self.compiles?(source)
62
+ COMPILE_LOCK.synchronize do
63
+ verbose = $VERBOSE
64
+ $VERBOSE = nil
65
+ RubyVM::InstructionSequence.compile(source)
66
+ true
67
+ rescue SyntaxError
68
+ false
69
+ ensure
70
+ $VERBOSE = verbose
71
+ end
72
+ end
73
+ private_class_method :compiles?
74
+
36
75
  # The comments of source text, without building its syntax tree: a few
37
76
  # times cheaper than {.parse_string} for a caller that needs only these.
38
77
  #
@@ -488,6 +488,16 @@ module Mutineer
488
488
  end
489
489
  end
490
490
 
491
+ # Turns off Rails' pending-migration check in this mutant fork (#385).
492
+ # `rails/test_help` runs it when the tests load, and it queries every
493
+ # database, shared ones too, so {touched_shared_databases} would flag every
494
+ # mutant. The clean check still runs it, before any mutant.
495
+ #
496
+ # @return [void]
497
+ def self.skip_schema_check!
498
+ ActiveRecord.maintain_test_schema = false if ActiveRecord.respond_to?(:maintain_test_schema=)
499
+ end
500
+
491
501
  # Names from `shared` whose database this fork used. Run in a fork after
492
502
  # its tests: a name here means a test used a database that parallel
493
503
  # workers share, so the verdict cannot be trusted. A pool counts when it
@@ -5,18 +5,32 @@ require "uri"
5
5
  module Mutineer
6
6
  # Hides database credentials in text bound for a terminal: an error reply,
7
7
  # a log line, a warning. Rails parses DATABASE_URL itself, so an error about
8
- # a malformed one can quote the whole URL. Outside a URL a password of 3
9
- # characters or fewer is hidden only as a whole token, so it cannot blank
10
- # letters inside other words. A password that exactly equals the username is
11
- # left visible outside a URL, so `role "postgres"` stays readable.
8
+ # a malformed one can quote the whole URL. Outside a URL a password form
9
+ # made only of word characters is hidden only as a whole token, so it cannot
10
+ # blank letters inside other words (#341). A password that exactly equals the
11
+ # username is left visible outside a URL, so `role "postgres"` stays readable.
12
12
  #
13
13
  # Stdlib only: the app-side daemon loads this file without the rest of
14
14
  # mutineer.
15
15
  module Redaction
16
- # The password in a `scheme://user:password@host` URL. Greedy up to the last
17
- # `@` before an unescaped quote or line end, so a malformed password with a
18
- # space, an unescaped `@`, or a quote `inspect` escaped (`\"`) is hidden too.
19
- URL_PASSWORD = %r{([a-z][a-z0-9+.\-]*://[^\s:/@"'\\]*):(?:\\.|[^"'\r\n\\])*@}i
16
+ # The password in a `scheme://user:password@host` URL. It ends at the last
17
+ # `@` before an unescaped quote or line end that starts a host: a run with
18
+ # no `@`, then a `/`, `:`, `?` or `#`, and no other `@` before the next
19
+ # space. So in `u:p@ss@h/db and p@ss` only `p@ss` is hidden (#341). With no
20
+ # such `@`, it ends at the last `@`, as before. A malformed password with
21
+ # an `@`, a `:`, a space, or a quote `inspect` escaped (`\"`) is hidden
22
+ # too, so when the two goals conflict, hiding wins: in
23
+ # `u:pw@h/x and y@h2/db` the text up to `h2` is hidden. One case still
24
+ # shows part of a password: one that holds an `@`, a host, a `/`, `:`,
25
+ # `?` or `#`, and a space (`a@b/c d`), in a URL whose own host has no
26
+ # port, path or query, or has an `@` in them (`?n=ci@corp`).
27
+ URL_PASSWORD = %r{
28
+ ([a-z][a-z0-9+.\-]*://[^\s:/@"'\\]*):
29
+ (?:
30
+ (?:\\.|[^"'\r\n\\])*@(?=[^@\s"'\\/:?\#]+[/:?\#][^@\s]*(?:\s|\z))
31
+ | (?:\\.|[^"'\r\n\\])*@
32
+ )
33
+ }xi
20
34
  # What replaces a hidden password.
21
35
  REDACTED = "[REDACTED]"
22
36
  # What {text} returns when it fails: never the input, which may hold the
@@ -24,18 +38,27 @@ module Mutineer
24
38
  FAILED = "[message hidden: redaction failed]"
25
39
  # An ASCII word character, the edge a whole-token match must not touch.
26
40
  WORD_CHAR = "[A-Za-z0-9_]"
27
- # The longest password form hidden only as a whole token. A longer form is
28
- # hidden wherever it appears, even glued to an escape such as `%40` or
29
- # `\x41`: failing toward hiding too much is safer for a long secret.
41
+ # The longest password form hidden only as a whole token whatever its
42
+ # characters. A longer form with a non-word character is hidden wherever
43
+ # it appears, because its edges are no reliable token boundary.
30
44
  SHORT_FORM = 3
45
+ # An escape or prefix that glues a word character to the left of a
46
+ # password form in a message: `%40`, `\x41`, `\u0041`, `\u{41}`, `\101`,
47
+ # `\n`, an ANSI code such as the colour Rails logs put on a span (raw,
48
+ # escaped as `\e`, `\033`, `\x1b` or `\u001b`, or with its ESC byte
49
+ # stripped), or a `-p` flag after a space, quote, bracket, comma or `=`
50
+ # (`mysql -ps3cret`, `["-ps3cret"]`). A form right after one is still a
51
+ # whole token.
52
+ ESCAPE = /%\h{2}|\\x\h{2}|\\u(?:\h{4}|\{\h+\})|\\[0-7]{1,3}|(?:\e|\\(?:e|033|x1[bB]|u001[bB]))?\[[0-9;]*[A-Za-z]|\\[a-z]|(?<![^\s"'(\[,=])-p/.source
31
53
 
32
54
  module_function
33
55
 
34
56
  # Hide the password in any URL, and the DATABASE_URL password (from its
35
57
  # `user:password@` pair and from its `password=` query parameter) wherever
36
58
  # it appears: whole, percent-decoded, `inspect` escaped, or cut short after
37
- # its `scheme://user:` prefix. Outside a URL a form of {SHORT_FORM}
38
- # characters or fewer is hidden only as a whole token. Never raises: text in
59
+ # its `scheme://user:` prefix. Outside a URL a form of word characters, or
60
+ # of {SHORT_FORM} characters or fewer, is hidden only as a whole token; a
61
+ # longer form with another character is hidden anywhere. Never raises: text in
39
62
  # any encoding comes back as UTF-8, and an internal error returns {FAILED}.
40
63
  #
41
64
  # @param text [#to_s]
@@ -68,7 +91,8 @@ module Mutineer
68
91
  raws.concat(query_passwords(url))
69
92
  secrets = raws.compact.uniq.flat_map { |raw| password_forms(raw) }.uniq.sort_by { |form| -form.length }
70
93
  secrets.reduce(out) do |acc, secret|
71
- secret.length > SHORT_FORM ? acc.gsub(secret, REDACTED) : redact_word(acc, secret)
94
+ whole_token = secret.length <= SHORT_FORM || secret.match?(/\A#{WORD_CHAR}+\z/)
95
+ whole_token ? redact_word(acc, secret) : acc.gsub(secret, REDACTED)
72
96
  end
73
97
  rescue StandardError
74
98
  FAILED
@@ -93,14 +117,18 @@ module Mutineer
93
117
  end
94
118
 
95
119
  # Replace `form` in `text` only where it is a whole token: no ASCII word
96
- # character touches either edge, including a punctuation edge.
120
+ # character touches either edge, including a punctuation edge. A form
121
+ # right after an {ESCAPE} also counts as a whole token, and the escape is
122
+ # kept. An escape may not start where a copy of the form starts, so it
123
+ # cannot swallow that copy (`%41%41` with password `%41`).
97
124
  #
98
125
  # @param text [String]
99
126
  # @param form [String] one form of the password.
100
127
  # @return [String]
101
128
  def redact_word(text, form)
102
- pattern = "(?<!#{WORD_CHAR})#{Regexp.escape(form)}(?!#{WORD_CHAR})"
103
- text.gsub(Regexp.new(pattern), REDACTED)
129
+ quoted = Regexp.escape(form)
130
+ pattern = "((?!#{quoted})(?:#{ESCAPE})|(?<!#{WORD_CHAR}))#{quoted}(?!#{WORD_CHAR})"
131
+ text.gsub(Regexp.new(pattern)) { "#{Regexp.last_match(1)}#{REDACTED}" }
104
132
  end
105
133
 
106
134
  # An exception as `Class: message`, redacted. Each part is redacted on its