mutineer 1.8.1 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +733 -0
  3. data/README.md +85 -50
  4. data/lib/mutineer/changed_lines.rb +2 -1
  5. data/lib/mutineer/child_cause.rb +149 -0
  6. data/lib/mutineer/child_stdout.rb +12 -1
  7. data/lib/mutineer/cli.rb +540 -59
  8. data/lib/mutineer/command_redact.rb +38 -0
  9. data/lib/mutineer/concern_blocks.rb +95 -24
  10. data/lib/mutineer/config.rb +112 -6
  11. data/lib/mutineer/coverage_map.rb +1345 -156
  12. data/lib/mutineer/daemon_backend.rb +41 -18
  13. data/lib/mutineer/daemon_client.rb +104 -21
  14. data/lib/mutineer/daemon_server.rb +85 -14
  15. data/lib/mutineer/database_urls.rb +189 -0
  16. data/lib/mutineer/external_backend.rb +93 -6
  17. data/lib/mutineer/fs_path.rb +125 -0
  18. data/lib/mutineer/isolation.rb +380 -25
  19. data/lib/mutineer/job_plan.rb +352 -41
  20. data/lib/mutineer/minitest_integration/file_order.rb +56 -0
  21. data/lib/mutineer/minitest_integration/in_test_database_error.rb +243 -19
  22. data/lib/mutineer/minitest_integration.rb +26 -58
  23. data/lib/mutineer/mutant_id.rb +4 -1
  24. data/lib/mutineer/mutation.rb +10 -4
  25. data/lib/mutineer/mutator_registry.rb +6 -3
  26. data/lib/mutineer/mutators/collection_method.rb +195 -6
  27. data/lib/mutineer/mutators/literal_mutation.rb +33 -3
  28. data/lib/mutineer/mutators/range_literal.rb +78 -29
  29. data/lib/mutineer/mutators/rescue_body.rb +126 -0
  30. data/lib/mutineer/mutators/return_nil.rb +23 -6
  31. data/lib/mutineer/output_filter.rb +451 -0
  32. data/lib/mutineer/pairing.rb +463 -31
  33. data/lib/mutineer/parser.rb +53 -0
  34. data/lib/mutineer/project_path.rb +15 -8
  35. data/lib/mutineer/rails_worker_db.rb +366 -6
  36. data/lib/mutineer/reach_probe.rb +229 -0
  37. data/lib/mutineer/redaction.rb +301 -41
  38. data/lib/mutineer/reload_guards.rb +985 -0
  39. data/lib/mutineer/reporter.rb +467 -58
  40. data/lib/mutineer/result.rb +6 -7
  41. data/lib/mutineer/run_seed.rb +16 -0
  42. data/lib/mutineer/runner.rb +194 -55
  43. data/lib/mutineer/source_bytes.rb +187 -0
  44. data/lib/mutineer/statement_lines.rb +108 -14
  45. data/lib/mutineer/test_runners/rspec.rb +118 -10
  46. data/lib/mutineer/test_runners/rspec_seed.rb +151 -0
  47. data/lib/mutineer/test_runners.rb +3 -2
  48. data/lib/mutineer/triage.rb +55 -10
  49. data/lib/mutineer/version.rb +1 -1
  50. data/lib/mutineer.rb +1 -0
  51. data/skills/mutineer/SKILL.md +12 -4
  52. metadata +13 -1
@@ -0,0 +1,189 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mutineer
4
+ # DATABASE_URL values stored in a project's dotenv files. The tool process
5
+ # does not see a URL the app loads itself during boot. Reading the files
6
+ # lets redaction hide that password (#341). Stdlib only, so the app-side
7
+ # daemon can load this file.
8
+ module DatabaseUrls
9
+ # Files dotenv-rails reads. Later files override at runtime. Every value
10
+ # is kept: hiding a password the running app did not use is safe.
11
+ FILE_NAMES = %w[
12
+ .env
13
+ .env.local
14
+ .env.development
15
+ .env.development.local
16
+ .env.test
17
+ .env.test.local
18
+ ].freeze
19
+ module_function
20
+
21
+ # DATABASE_URL values from the dotenv files under `root`. Missing files
22
+ # add nothing. An unreadable file adds nothing: a permission error must
23
+ # not stop a run. Estimate: a project dotenv file is readable by the user
24
+ # who starts the run.
25
+ #
26
+ # @param root [String, nil] project directory.
27
+ # @return [Array<String>]
28
+ def from_files(root)
29
+ return [] if root.nil? || root.empty?
30
+
31
+ FILE_NAMES.flat_map { |name| from_file(File.join(root, name)) }.uniq
32
+ end
33
+
34
+ # DATABASE_URL values in one file. Every line is read. A double-quoted
35
+ # value may continue on later lines until its closing quote. A quote that
36
+ # never closes is a plain value on its own line, as in dotenv. `${NAME}` and
37
+ # `$NAME` expand from an earlier assignment in the file, or from the
38
+ # environment when that variable is already set. A backslash before `$`
39
+ # is a literal dollar. A single-quoted value is not expanded. An invalid
40
+ # byte is replaced and does not stop the read. A directory, a missing
41
+ # path, and an unreadable file add nothing. An unknown `$NAME` is left as
42
+ # written, so that reference does not become a password.
43
+ #
44
+ # @param path [String]
45
+ # @return [Array<String>]
46
+ def from_file(path)
47
+ return [] unless File.file?(path)
48
+
49
+ vars = {}
50
+ urls = []
51
+ lines = File.open(path, "rb") { |io| io.each_line.map { |line| utf8_line(line) } }
52
+ index = 0
53
+ while index < lines.size
54
+ stop = index
55
+ chunk = lines[index]
56
+ parsed = parse_assignment(chunk)
57
+ while parsed == :open && stop + 1 < lines.size
58
+ stop += 1
59
+ chunk += lines[stop]
60
+ parsed = parse_assignment(chunk)
61
+ end
62
+ if parsed == :open
63
+ # A quote that never closes: dotenv reads that line as a plain
64
+ # value, and the lines after it as their own assignments.
65
+ stop = index
66
+ parsed = parse_assignment(lines[index], plain: true)
67
+ end
68
+ index = stop + 1
69
+ next unless parsed.is_a?(Array)
70
+
71
+ key, raw, expand_value = parsed
72
+ value = expand_value ? expand(raw, vars) : raw
73
+ vars[key] = value
74
+ urls << value if key == "DATABASE_URL" && !value.empty?
75
+ end
76
+ urls
77
+ rescue SystemCallError
78
+ []
79
+ end
80
+
81
+ # The DATABASE_URL value on one dotenv line, or nil when the line is a
82
+ # comment, another key, or an empty value.
83
+ #
84
+ # @param line [String]
85
+ # @return [String, nil]
86
+ def line_url(line)
87
+ parsed = parse_assignment(line)
88
+ return nil unless parsed.is_a?(Array) && parsed[0] == "DATABASE_URL"
89
+
90
+ parsed[1].empty? ? nil : parsed[1]
91
+ end
92
+
93
+ # One `KEY=value` assignment. The third element is false for a
94
+ # single-quoted value, which dotenv does not expand. `:open` means a
95
+ # quoted value has not reached its closing quote yet. `plain` reads a
96
+ # leading quote as part of an unquoted value.
97
+ #
98
+ # @param line [String]
99
+ # @param plain [Boolean]
100
+ # @return [Array(String, String, Boolean), Symbol, nil]
101
+ def parse_assignment(line, plain: false)
102
+ text = utf8_line(line).sub(/\A\uFEFF/, "").strip
103
+ return nil if text.empty? || text.start_with?("#")
104
+
105
+ text = text.sub(/\Aexport\s+/, "")
106
+ match = text.match(/\A([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\z/m)
107
+ return nil unless match
108
+
109
+ key = match[1]
110
+ raw = match[2]
111
+ if !plain && raw.start_with?("'")
112
+ stop = closing_quote(raw, "'")
113
+ return :open unless stop
114
+
115
+ [key, raw[1...stop], false]
116
+ elsif !plain && raw.start_with?('"')
117
+ stop = closing_quote(raw, '"')
118
+ return :open unless stop
119
+
120
+ [key, decode_double(raw[1...stop]), true]
121
+ else
122
+ [key, raw.sub(/\s+#.*\z/, "").strip, true]
123
+ end
124
+ end
125
+
126
+ # Replace `${NAME}` and `$NAME`. A backslash before `$` stays a dollar and
127
+ # is not a reference. The environment wins over an earlier assignment in
128
+ # the file. Anything else is left as written.
129
+ #
130
+ # @param value [String]
131
+ # @param vars [Hash{String => String}]
132
+ # @return [String]
133
+ def expand(value, vars)
134
+ value.gsub(/\\\$|\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)/) do
135
+ match = Regexp.last_match
136
+ next "$" if match[0] == "\\$"
137
+
138
+ name = match[1] || match[2]
139
+ if ENV.key?(name)
140
+ ENV[name].to_s
141
+ elsif vars.key?(name)
142
+ vars[name]
143
+ else
144
+ match[0]
145
+ end
146
+ end
147
+ end
148
+
149
+ # One line as UTF-8. An invalid byte becomes U+FFFD so a binary dotenv
150
+ # line cannot raise before redaction starts.
151
+ #
152
+ # @param line [String]
153
+ # @return [String]
154
+ def utf8_line(line)
155
+ text = line.to_s.dup
156
+ text.force_encoding(Encoding::UTF_8)
157
+ text.scrub
158
+ end
159
+
160
+ # Index of the closing quote. A backslash escapes the next character
161
+ # inside double quotes, so `\"` does not end the value.
162
+ #
163
+ # @param raw [String]
164
+ # @param quote [String]
165
+ # @return [Integer, nil]
166
+ def closing_quote(raw, quote)
167
+ index = 1
168
+ while index < raw.length
169
+ if quote == '"' && raw[index] == "\\"
170
+ index += 2
171
+ elsif raw[index] == quote
172
+ return index
173
+ else
174
+ index += 1
175
+ end
176
+ end
177
+ nil
178
+ end
179
+
180
+ # Double-quoted dotenv escapes this reader understands. Other backslashes
181
+ # stay as written.
182
+ #
183
+ # @param text [String]
184
+ # @return [String]
185
+ def decode_double(text)
186
+ text.gsub(/\\[nrt"\\]/, "\\n" => "\n", "\\r" => "\r", "\\t" => "\t", "\\\"" => '"', "\\\\" => "\\")
187
+ end
188
+ end
189
+ end
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "fileutils"
3
4
  require "shellwords"
4
5
  require "tempfile"
6
+ require "tmpdir"
5
7
  require_relative "result"
6
8
  require_relative "rails_run_env"
7
9
  require_relative "redaction"
@@ -61,6 +63,17 @@ module Mutineer
61
63
  RSPEC_OUTSIDE_ERROR = /errors? occurred outside of examples/
62
64
  # ANSI colour codes, removed before matching so a coloured summary still counts.
63
65
  ANSI_CODE = /\e\[[\d;]*m/
66
+ # The seed line Minitest prints before every run. Captures the seed.
67
+ MINITEST_SEED = /^Run options:.*--seed (\d+)/
68
+ # The seed line RSpec prints when it shuffles the tests. Captures the seed.
69
+ RSPEC_SEED = /^Randomized with seed (\d+)/
70
+ # A Minitest seed given on the command or in TESTOPTS (`SEED=N`, `--seed N`
71
+ # `-s N` or `-sN`). Minitest prints its seed line for a fixed seed too, so
72
+ # the line alone cannot tell.
73
+ MINITEST_SEED_GIVEN = /\bSEED=|--seed\b|(?:\A|\s)-s(?![A-Za-z_])/
74
+ # An RSpec seed given on the command or in SPEC_OPTS (`--seed N` or
75
+ # `--order rand:N`). RSpec ignores SEED.
76
+ RSPEC_SEED_GIVEN = /--seed\b|\brand(?:om)?:\d/
64
77
 
65
78
  # PATH entries that pin a concrete Ruby under a version manager (ahead of
66
79
  # shims). Optional trailing slash; normal bins are left alone.
@@ -286,20 +299,50 @@ module Mutineer
286
299
  output.b.gsub(ANSI_CODE, "")
287
300
  end
288
301
 
302
+ # The runner and seed of a random test order the output shows, or nil.
303
+ # Nil also when Mutineer sees a fixed seed: for Minitest, SEED, TESTOPTS
304
+ # or the command sets one; for RSpec, SPEC_OPTS or the command does. RSpec
305
+ # `--order defined` prints no seed line. Each runner is checked on its
306
+ # own, so a seeded Minitest suite does not hide an unseeded RSpec suite
307
+ # that the same command runs. Ceiling: a seed set in a wrapper script or
308
+ # `.rspec`, or Minitest's `test_order`, is not seen, so the warning still
309
+ # prints; a seed-like argument meant for a wrapper hides it (#338, limits
310
+ # tracked in #368).
311
+ #
312
+ # @api private
313
+ # @param command [String] the --test-command template.
314
+ # @param text [String] output from {summary_text}.
315
+ # @return [Array(Symbol, String), nil] `[:minitest | :rspec, seed]`.
316
+ def self.random_order(command, text)
317
+ minitest = text[MINITEST_SEED, 1]
318
+ if minitest && ENV["SEED"].to_s.empty? &&
319
+ [command, ENV["TESTOPTS"]].none? { |s| s.to_s.match?(MINITEST_SEED_GIVEN) }
320
+ return [:minitest, minitest]
321
+ end
322
+
323
+ rspec = text[RSPEC_SEED, 1]
324
+ [:rspec, rspec] if rspec && [command, ENV["SPEC_OPTS"]].none? { |s| s.to_s.match?(RSPEC_SEED_GIVEN) }
325
+ end
326
+
289
327
  # Pre-flight: run the command once against the UNMUTATED tree. Green (exit 0)
290
- # returns the elapsed seconds (used to calibrate the per-mutant timeout) and
328
+ # returns the elapsed seconds (used to calibrate the per-mutant timeout),
291
329
  # whether the output has a Minitest or RSpec summary line (used to turn on
292
- # the summary check in {run}); anything else raises SmokeCheckError so the
293
- # run aborts before scoring.
330
+ # the summary check in {run}), and the runner and seed of a random test
331
+ # order (see {random_order}); anything else raises SmokeCheckError so the run
332
+ # aborts before scoring.
294
333
  #
295
334
  # @param command [String] the --test-command template.
296
335
  # @param files [Array<String>] test file paths.
297
336
  # @param timeout [Numeric] ceiling for the calibration run.
298
- # @return [Array(Float, Boolean)] `[elapsed_seconds, summary_seen]`.
337
+ # @return [Array(Float, Boolean, Array(Symbol, String), nil)]
338
+ # `[elapsed_seconds, summary_seen, random_order]`.
299
339
  # @raise [Mutineer::SmokeCheckError] when the clean suite is not green.
300
340
  def self.smoke_check!(command, files, timeout: SMOKE_TIMEOUT)
301
341
  kind, code, output, elapsed = spawn_capture(command, files, timeout)
302
- return [elapsed, summary_line?(summary_text(output))] if kind == :exited && code&.zero?
342
+ if kind == :exited && code&.zero?
343
+ text = summary_text(output)
344
+ return [elapsed, summary_line?(text), random_order(command, text)]
345
+ end
303
346
 
304
347
  reason =
305
348
  if kind == :timeout then "did not finish within #{timeout}s"
@@ -345,6 +388,47 @@ module Mutineer
345
388
  parts
346
389
  end
347
390
 
391
+ # RUBYOPT for a `--test-command` child: the copier file alone. The tool's
392
+ # own RUBYOPT (`-rbundler/setup` under `bundle exec`) stays out, as
393
+ # {child_env} clears every `RUBY*` key so the suite picks its own Ruby and
394
+ # gems. Empty when the copier file cannot be named without whitespace.
395
+ #
396
+ # @api private
397
+ # @param dir [String] the directory that holds command_redact.rb.
398
+ # @return [String]
399
+ def self.command_rubyopt(dir = __dir__)
400
+ path = command_redact_file(dir)
401
+ path ? "-r#{path}" : ""
402
+ end
403
+
404
+ # Path of command_redact.rb that RUBYOPT can carry. Ruby splits RUBYOPT at
405
+ # whitespace and has no quoting or escape, so a path with a space (a home
406
+ # directory named "Jane Doe") makes every Ruby child stop with "invalid
407
+ # switch in RUBYOPT". Such a path is reached through a symlink in a
408
+ # temporary directory, removed when this process exits. `require_relative`
409
+ # in the file resolves the real path, so the rest of the copier loads.
410
+ #
411
+ # @api private
412
+ # @param dir [String] the directory that holds command_redact.rb.
413
+ # @return [String, nil] nil when the temporary directory name has whitespace too.
414
+ def self.command_redact_file(dir)
415
+ path = File.expand_path("command_redact.rb", dir)
416
+ return path unless path.match?(/\s/)
417
+
418
+ COMMAND_REDACT_LINKS[dir] ||= begin
419
+ tmp = Dir.mktmpdir("mutineer-redact")
420
+ at_exit { FileUtils.remove_entry(tmp, true) }
421
+ link = File.join(tmp, "command_redact.rb")
422
+ File.symlink(path, link)
423
+ link
424
+ end
425
+ link = COMMAND_REDACT_LINKS[dir]
426
+ link.match?(/\s/) ? nil : link
427
+ end
428
+
429
+ # Symlinks {command_redact_file} made, by source directory.
430
+ COMMAND_REDACT_LINKS = {} # rubocop:disable Style/MutableConstant
431
+
348
432
  # Spawns the command to a captured combined-output tempfile, enforces a
349
433
  # wall-clock timeout (SIGKILL past the deadline), and returns
350
434
  # [kind, exit_code, output, elapsed]. Mirrors Isolation's single-waiter loop:
@@ -368,7 +452,10 @@ module Mutineer
368
452
  # explicit [program, argv0] form guarantees the no-shell exec path even for a
369
453
  # degenerate single-element argv (Process.spawn(*argv) would route a lone
370
454
  # metachar-bearing string through /bin/sh, breaking the argv-only invariant).
371
- pid = Process.spawn(child_env, [argv.first, argv.first], *argv[1..],
455
+ # RUBYOPT loads the copier in each Ruby process this command starts, so a
456
+ # password that process assigns is hidden before the bytes are captured
457
+ # (#457). A program that is not Ruby ignores RUBYOPT.
458
+ pid = Process.spawn(child_env.merge("RUBYOPT" => command_rubyopt), [argv.first, argv.first], *argv[1..],
372
459
  out: out, err: %i[child out], pgroup: true)
373
460
  kind, code = wait_with_timeout(pid, timeout)
374
461
  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
@@ -0,0 +1,125 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mutineer
4
+ # Filesystem paths as one encoding (#346).
5
+ #
6
+ # With `LANG` and `LC_ALL` unset, Ruby's filesystem encoding is US-ASCII.
7
+ # A non-ASCII name then comes back as ASCII-8BIT, or as US-ASCII with those
8
+ # bytes marked invalid (`Coverage` result keys). The same bytes in two
9
+ # encodings compare unequal, a regexp raises, and `JSON.generate` refuses
10
+ # the string. Read every such path as UTF-8 when the bytes are UTF-8, and
11
+ # as binary otherwise, so both copies share one encoding.
12
+ module FsPath
13
+ # This file, so a coverage subprocess can require the same rule.
14
+ PATH = File.expand_path(__FILE__)
15
+
16
+ # `path` as UTF-8 when its bytes are UTF-8, otherwise as binary.
17
+ # ASCII-only text is returned as it is: ASCII encodings already compare
18
+ # equal. `nil` stays `nil`.
19
+ #
20
+ # @param path [String, nil]
21
+ # @return [String, nil]
22
+ def self.read(path)
23
+ return path if path.nil?
24
+
25
+ text = String.try_convert(path) || path.to_s
26
+ return text if stable?(text)
27
+
28
+ utf8 = text.dup.force_encoding(Encoding::UTF_8)
29
+ return utf8 if utf8.valid_encoding?
30
+
31
+ text.dup.force_encoding(Encoding::ASCII_8BIT)
32
+ end
33
+
34
+ # True when `text`'s bytes begin with `prefix`'s bytes.
35
+ # Encodings are not compared, so a UTF-8 root and a binary file name
36
+ # do not raise (#346).
37
+ #
38
+ # @param text [String]
39
+ # @param prefix [String]
40
+ # @return [Boolean]
41
+ def self.bytes_start_with?(text, prefix)
42
+ text.to_s.b.start_with?(prefix.to_s.b)
43
+ end
44
+
45
+ # `text` without a byte prefix, read again as UTF-8 or binary.
46
+ # `text` itself when the prefix is absent.
47
+ #
48
+ # @param text [String]
49
+ # @param prefix [String]
50
+ # @return [String]
51
+ def self.bytes_delete_prefix(text, prefix)
52
+ raw = text.to_s.b
53
+ pre = prefix.to_s.b
54
+ return read(text) unless raw.start_with?(pre)
55
+
56
+ read(raw.byteslice(pre.bytesize..).b)
57
+ end
58
+
59
+ # Absolute form of `path`, passed through {read}.
60
+ #
61
+ # @param path [String]
62
+ # @param base [String, nil] directory `path` is relative to.
63
+ # @return [String]
64
+ def self.expand(path, base = nil)
65
+ expanded = base.nil? ? File.expand_path(path) : File.expand_path(path, base)
66
+ read(expanded)
67
+ end
68
+
69
+ # `path` tagged the way `Dir.pwd` is tagged in this process.
70
+ #
71
+ # {read} is for comparison and JSON. `load` must use this encoding:
72
+ # Minitest records the path it was given and later calls `delete_prefix`
73
+ # with `"#{Dir.pwd}/"`. Those two encodings differ for a non-ASCII
74
+ # directory under `LC_ALL=C`, and `delete_prefix` raises (#346).
75
+ #
76
+ # @param path [String]
77
+ # @return [String]
78
+ def self.filesystem(path)
79
+ text = read(path)
80
+ tagged = text.dup.force_encoding(Dir.pwd.encoding)
81
+ return tagged if tagged.valid_encoding?
82
+
83
+ text.dup.force_encoding(Encoding::ASCII_8BIT)
84
+ end
85
+
86
+ # Minitest filters a failure backtrace with a regexp and then shortens
87
+ # the path against Dir.pwd. Under `LC_ALL=C` those lines are US-ASCII
88
+ # with the directory's bytes, so the regexp raises and a killed mutant
89
+ # is reported as an error (#346). Retag each line to {filesystem} first.
90
+ # No-op until Minitest is loaded. Safe to call more than once.
91
+ #
92
+ # @return [void]
93
+ def self.install_minitest_path_filter!
94
+ return unless defined?(::Minitest::BacktraceFilter)
95
+ return if ::Minitest::BacktraceFilter < PathFilter
96
+
97
+ ::Minitest::BacktraceFilter.prepend(PathFilter)
98
+ end
99
+
100
+ # Prepended onto `Minitest::BacktraceFilter`. See {install_minitest_path_filter!}.
101
+ module PathFilter
102
+ # @param bt [Array<String>, nil]
103
+ # @return [Array<String>]
104
+ def filter(bt)
105
+ return super if bt.nil?
106
+
107
+ super(bt.map { |line| FsPath.filesystem(line.to_s) })
108
+ end
109
+ end
110
+
111
+ # True when `text` needs no retag: valid UTF-8, or bytes that are ASCII
112
+ # in whatever encoding they carry. An invalid byte sequence is not stable.
113
+ #
114
+ # @param text [String]
115
+ # @return [Boolean]
116
+ def self.stable?(text)
117
+ return true if text.encoding == Encoding::UTF_8 && text.valid_encoding?
118
+
119
+ text.ascii_only?
120
+ rescue ArgumentError
121
+ false
122
+ end
123
+ private_class_method :stable?
124
+ end
125
+ end