mutineer 1.6.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +680 -0
  3. data/README.md +210 -499
  4. data/lib/mutineer/baseline.rb +1 -1
  5. data/lib/mutineer/changed_lines.rb +2 -1
  6. data/lib/mutineer/child_coverage.rb +53 -0
  7. data/lib/mutineer/cli.rb +293 -42
  8. data/lib/mutineer/concern_blocks.rb +243 -0
  9. data/lib/mutineer/config.rb +154 -8
  10. data/lib/mutineer/coverage_map.rb +265 -27
  11. data/lib/mutineer/daemon_backend.rb +244 -75
  12. data/lib/mutineer/daemon_client.rb +73 -10
  13. data/lib/mutineer/daemon_server.rb +226 -42
  14. data/lib/mutineer/external_backend.rb +159 -15
  15. data/lib/mutineer/init.rb +124 -0
  16. data/lib/mutineer/isolation.rb +78 -24
  17. data/lib/mutineer/job_plan.rb +234 -44
  18. data/lib/mutineer/kill_channel.rb +24 -14
  19. data/lib/mutineer/migrate.rb +707 -0
  20. data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
  21. data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
  22. data/lib/mutineer/minitest_integration.rb +44 -14
  23. data/lib/mutineer/mutator_registry.rb +6 -2
  24. data/lib/mutineer/mutators/array_literal.rb +4 -2
  25. data/lib/mutineer/mutators/base.rb +36 -7
  26. data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
  27. data/lib/mutineer/mutators/chain_link.rb +6 -6
  28. data/lib/mutineer/mutators/condition_forcing.rb +6 -3
  29. data/lib/mutineer/mutators/operand_removal.rb +4 -2
  30. data/lib/mutineer/mutators/range_literal.rb +105 -4
  31. data/lib/mutineer/mutators/return_nil.rb +1 -0
  32. data/lib/mutineer/orphan_guard.rb +84 -0
  33. data/lib/mutineer/pairing.rb +62 -1
  34. data/lib/mutineer/project.rb +302 -14
  35. data/lib/mutineer/rails_run_env.rb +73 -0
  36. data/lib/mutineer/rails_worker_db.rb +1596 -57
  37. data/lib/mutineer/redaction.rb +199 -0
  38. data/lib/mutineer/reporter.rb +173 -31
  39. data/lib/mutineer/result.rb +52 -15
  40. data/lib/mutineer/runner.rb +148 -28
  41. data/lib/mutineer/statement_lines.rb +24 -11
  42. data/lib/mutineer/subject.rb +30 -4
  43. data/lib/mutineer/test_runners/minitest.rb +2 -1
  44. data/lib/mutineer/test_runners/rspec.rb +15 -3
  45. data/lib/mutineer/test_runners.rb +5 -2
  46. data/lib/mutineer/triage.rb +111 -0
  47. data/lib/mutineer/version.rb +1 -1
  48. data/lib/mutineer/worker_pool.rb +2 -1
  49. data/lib/mutineer.rb +4 -0
  50. data/skills/mutineer/SKILL.md +84 -0
  51. metadata +13 -1
@@ -3,7 +3,10 @@
3
3
  require "json"
4
4
  require "tempfile"
5
5
  require_relative "child_stdout"
6
+ require_relative "child_coverage"
6
7
  require_relative "orphan_guard"
8
+ require_relative "concern_blocks"
9
+ require_relative "redaction"
7
10
 
8
11
  module Mutineer
9
12
  # App-side daemon (persistent worker).
@@ -25,28 +28,47 @@ module Mutineer
25
28
  #
26
29
  # Protocol (one JSON object per line, both directions):
27
30
  # boot in : {"cmd":"boot","project_root":"...","boot":"config/environment","require_paths":[...],
28
- # "load_paths":["test"],"framework":"minitest","rails":true,"schema":"db/schema.rb"}
29
- # ready out: {"ready":true,"ruby":"3.3.6"} (or {"ready":false,"error":"..."} then exit)
31
+ # "load_paths":["test"],"framework":"minitest","rails":true}
32
+ # ready out: {"ready":true,"ruby":"3.3.6","databases":{"names":["primary"],"primary":"primary"}}
33
+ # ("databases" is null with no worker-DB adapter, {"error":"...","primary":"primary"|null} when the lookup fails)
34
+ # (or {"ready":false,"error":"..."} then exit)
35
+ # provision: {"cmd":"provision","slots":N} -> {"ok":true} or {"ok":false,"error":"..."}
30
36
  # run in : {"id":N,"worker":I,"payload":{"code":"<ruby>","source_file":"app/models/order.rb"},
31
37
  # "tests":["test/models/order_test.rb"],"timeout":30}
38
+ # (a mutant of a def in a concern's hook block adds to the payload
39
+ # "hook_def":{"line":12,"name":"total","singleton":false}, #297)
32
40
  # verdict : {"id":N,"verdict":"survived"|"killed"|"error"|"timeout"}
33
41
  # quit in : {"cmd":"quit"}
34
42
  #
35
43
  # Worker isolation: when the app is Rails, each fork is routed to its own
36
- # database `<db>-<worker>` via {RailsWorkerDb} BEFORE any test loads, so
37
- # concurrent workers cannot clobber each other's transactional fixtures.
38
- # `worker` defaults to 0 (serial). SQLite this pass; Postgres provisioning is
39
- # not yet implemented.
44
+ # database via {RailsWorkerDb} BEFORE any test loads, so concurrent workers
45
+ # cannot clobber each other's transactional fixtures. `worker` defaults to 0
46
+ # (serial). SQLite copies a file per slot. Postgres and MySQL use
47
+ # `<base>-mutineer-<w>`, created once before the worker daemons start.
40
48
  #
41
49
  # Verdict mapping: child exit 0=survived (suite passed), 1=killed (suite
42
50
  # failed), 2=error (child raised AROUND the test: load, boot, or worker-DB
43
- # routing failure); parent-detected timeout. Tagging an in-test DB error (one
44
- # fired inside a test body, vs at routing time) as `error` rather than
45
- # `killed` is only observable under the concurrent gate and is not yet
46
- # implemented.
51
+ # routing failure); parent-detected timeout. Exit 2 also covers a run whose
52
+ # first failing test failed with an Active Record connection or lock error
53
+ # inside the test (#296): {MinitestIntegration::InTestDatabaseError} decides.
47
54
  module DaemonServer
48
55
  # Poll interval (seconds) for the per-fork deadline wait loop.
49
56
  POLL = 0.02
57
+ # Skips mutant files of both modes ({OrphanGuard::TEMP_BASENAME}) when
58
+ # Zeitwerk scans an autoload directory. The boot sweep keeps a file another
59
+ # process still holds. Zeitwerk skips a hidden one (#253), but one an older
60
+ # version wrote has no dot, and it would stop the next boot, because its
61
+ # name is not a constant.
62
+ module DaemonTempIgnore
63
+ # @return [void]
64
+ def setup
65
+ dirs = Array(Mutineer::DaemonServer.instance_variable_get(:@source_dirs))
66
+ dirs.each do |dir|
67
+ ignore(File.join(dir, Mutineer::OrphanGuard::TEMP_BASENAME))
68
+ end
69
+ super
70
+ end
71
+ end
50
72
 
51
73
  class << self
52
74
  # Serve the protocol on the given IO pair (defaults to stdio). Returns on quit.
@@ -62,11 +84,13 @@ module Mutineer
62
84
  output ||= @protocol
63
85
  @errio = errio
64
86
  @output = output
87
+ # The client sends UTF-8 JSON; with LANG unset the pipe would read as US-ASCII (#303).
88
+ input.set_encoding(Encoding::UTF_8)
65
89
  boot_line = input.gets
66
90
  return if boot_line.nil? # client vanished before boot
67
91
 
68
92
  boot!(JSON.parse(boot_line.strip))
69
- output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION))
93
+ output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION, "databases" => databases))
70
94
  output.flush
71
95
 
72
96
  input.each_line do |line|
@@ -84,6 +108,14 @@ module Mutineer
84
108
  end
85
109
  break if req["cmd"] == "quit"
86
110
 
111
+ # Create worker databases before coverage forks or mutant forks.
112
+ # The first daemon does this. A restarted worker daemon does not.
113
+ if req["cmd"] == "provision"
114
+ output.puts(JSON.generate(provision_slots(req["slots"])))
115
+ output.flush
116
+ next
117
+ end
118
+
87
119
  # Build the coverage map app-side and ship it to the tool, which then
88
120
  # selects covering tests per mutant. One-shot control message.
89
121
  if req["cmd"] == "coverage"
@@ -129,11 +161,17 @@ module Mutineer
129
161
  require "coverage"
130
162
  Coverage.start(lines: true, methods: true)
131
163
  end
132
- # Clear any mutant tempfile a prior SIGKILLed timeout child orphaned in a
133
- # source dir BEFORE the app boots. Zeitwerk would otherwise choke on the
134
- # tempfile's non-constant name during autoload setup.
164
+ # Clear a mutant tempfile a prior SIGKILLed child orphaned in a source
165
+ # dir BEFORE the app boots. Zeitwerk skips a hidden one, but it would
166
+ # choke on the non-constant name of one an older version wrote without
167
+ # the dot (#253). A file another
168
+ # live process still owns stays: this runs before the database lock,
169
+ # so a refused boot must not delete that run's mutant. The kept file
170
+ # is hidden from Zeitwerk instead.
135
171
  sweep_temps
172
+ install_zeitwerk_ignore! if cfg["rails"]
136
173
  require File.expand_path(cfg["boot"]) if cfg["boot"]
174
+ @zeitwerk_trace&.disable
137
175
  # The --require files, after the boot as in-process (Runner.execute), so
138
176
  # the coverage peek sees what they ran at load.
139
177
  Array(cfg["require_paths"]).each { |f| require File.expand_path(f) }
@@ -141,7 +179,7 @@ module Mutineer
141
179
  rescue Exception => e # rubocop:disable Lint/RescueException
142
180
  # Boot failed (bad boot path, app error). Tell the client and exit so it
143
181
  # can surface a clean error rather than hang on the handshake.
144
- @output.puts(JSON.generate("ready" => false, "error" => "#{e.class}: #{e.message}"))
182
+ @output.puts(JSON.generate("ready" => false, "error" => describe_error(e)))
145
183
  @output.flush
146
184
  exit!(1)
147
185
  end
@@ -149,20 +187,89 @@ module Mutineer
149
187
  # Load the per-worker DB adapter app-side (sibling gem file, by relative path
150
188
  # so it bypasses the app bundle, like this daemon itself). No-op unless the
151
189
  # app has ActiveRecord. Records the adapter + schema path so each fork can
152
- # route to its own database. SQLite-only this pass; a non-SQLite config
153
- # raises in the fork and reads as `error`, never a mis-routed verdict.
190
+ # route to its own database.
191
+ #
192
+ # `db_role` "prepare" waits for a provision command (the first daemon).
193
+ # "worker" only takes the shared lock (a worker daemon, including a
194
+ # respawn). Anything else provisions now, which is what a direct client
195
+ # that never sends the command needs. SQLite provisioning does nothing.
154
196
  def setup_worker_db(cfg)
155
197
  require_relative "rails_worker_db"
156
198
  @worker_db = RailsWorkerDb.available? ? RailsWorkerDb : nil
157
- schema = cfg["schema"] && File.expand_path(cfg["schema"])
158
- @schema_path = schema if schema && File.exist?(schema)
159
- # Each worker slot is seeded once, on first use (not every mutant fork).
199
+ # Only the booted app knows its schema file (`schema_dump:` can rename it).
200
+ @schema_path = @worker_db&.ruby_schema_path
201
+ # Only a parallel worker daemon gets these names (DaemonBackend.boot_config).
202
+ @shared_databases = Array(cfg["shared_databases"]).map(&:to_s)
203
+ # Each SQLite slot is seeded once, on first use (not every mutant fork).
204
+ # Postgres and MySQL slots are copied at provision time, so this flag is unused there.
160
205
  @slot_ready = {}
206
+ return unless @worker_db
207
+
208
+ case cfg["db_role"]
209
+ when "prepare"
210
+ nil
211
+ when "worker"
212
+ @worker_db.prepare_worker!
213
+ else
214
+ @worker_db.provision(cfg.fetch("slots", 2).to_i)
215
+ end
161
216
  rescue LoadError => e
162
- @errio.puts("[daemon] worker-DB routing unavailable: #{e.message}")
217
+ @errio.puts("[daemon] worker-DB routing unavailable: #{redact(e.message)}")
163
218
  @worker_db = nil
164
219
  end
165
220
 
221
+ # The booted test environment's database config names and the one each
222
+ # worker copies, for the tool's worker count. nil when the app has no
223
+ # worker-DB adapter. A failed lookup is reported, not hidden: the tool
224
+ # then keeps one worker. That report still names the primary when it
225
+ # resolves, so the tool still rejects a `shared_databases` that lists it.
226
+ #
227
+ # @return [Hash, nil]
228
+ def databases
229
+ return nil unless @worker_db
230
+
231
+ @worker_db.database_names
232
+ rescue StandardError => e
233
+ { "error" => describe_error(e), "primary" => primary_database_name }
234
+ end
235
+
236
+ # The primary config name alone, or nil when it does not resolve either.
237
+ #
238
+ # @return [String, nil]
239
+ def primary_database_name
240
+ @worker_db.primary_database_name
241
+ rescue StandardError
242
+ nil
243
+ end
244
+
245
+ # Raise when a test in this fork used a database that parallel workers
246
+ # share (`shared_databases`). Its verdict may rest on another worker's
247
+ # rows, so the mutant is scored `error`. A serial run sends no names.
248
+ #
249
+ # @return [void]
250
+ # @raise [RuntimeError] when a shared database holds a connection.
251
+ def check_shared_databases!
252
+ return if @worker_db.nil? || @shared_databases.nil? || @shared_databases.empty?
253
+
254
+ used = @worker_db.touched_shared_databases(@shared_databases)
255
+ return if used.empty?
256
+
257
+ raise "shared database used during a parallel run: #{used.join(", ")}. " \
258
+ "Remove it from shared_databases, or see https://github.com/davidteren/mutineer/blob/main/docs/rails-setup.md#more-than-one-database"
259
+ end
260
+
261
+ # Run {RailsWorkerDb.provision} and report success or the cause. A failure
262
+ # stays on this reply. It does not become a mutant verdict.
263
+ #
264
+ # @param slots [Integer, String]
265
+ # @return [Hash]
266
+ def provision_slots(slots)
267
+ @worker_db.provision(Integer(slots)) if @worker_db
268
+ { "ok" => true }
269
+ rescue Exception => e # rubocop:disable Lint/RescueException
270
+ { "ok" => false, "error" => describe_error(e) }
271
+ end
272
+
166
273
  # Build the coverage map app-side (Coverage was started at boot) and return
167
274
  # it as `{map, failed_test_files, load_lines, load_methods}` for the tool to select covering tests.
168
275
  # Capture forks route to worker 0's DB (isolated, serial). On any failure
@@ -185,8 +292,28 @@ module Mutineer
185
292
  "failed_clean_tests" => cmap.failed_clean_tests, "load_lines" => cmap.load_lines.to_a,
186
293
  "load_methods" => cmap.load_methods.to_a }
187
294
  rescue Exception => e # rubocop:disable Lint/RescueException
188
- @errio.puts("[daemon] coverage build failed: #{e.class}: #{e.message}")
189
- { "map" => {}, "failed_test_files" => [], "error" => "#{e.class}: #{e.message}" }
295
+ error = describe_error(e)
296
+ @errio.puts("[daemon] coverage build failed: #{error}")
297
+ { "map" => {}, "failed_test_files" => [], "error" => error }
298
+ end
299
+
300
+ # `Class: message` for an error the daemon reports, with credentials hidden.
301
+ # Rails parses DATABASE_URL itself, so its URI error can carry the password.
302
+ # {Redaction.describe} never raises, so a rescue that calls this cannot
303
+ # make the raw error the cause of a new one.
304
+ #
305
+ # @param error [Exception]
306
+ # @return [String]
307
+ def describe_error(error)
308
+ Redaction.describe(error)
309
+ end
310
+
311
+ # Hide database credentials in text bound for the client or stderr.
312
+ #
313
+ # @param text [String]
314
+ # @return [String]
315
+ def redact(text)
316
+ Redaction.text(text)
190
317
  end
191
318
 
192
319
  # Fork-safety hook for coverage capture. Each capture fork drops its copy
@@ -225,35 +352,75 @@ module Mutineer
225
352
  code =
226
353
  begin
227
354
  ChildStdout.silence
355
+ ChildCoverage.allow_start
228
356
  close_protocol
229
357
  # Route THIS fork at its own worker database before any test loads.
230
358
  # A routing failure raises here and is scored `error`, never a false verdict.
231
359
  @worker_db&.after_fork(worker, first_use ? @schema_path : nil, seed: first_use)
232
360
  apply_payload(req["payload"])
233
- run_tests(Array(req["tests"]))
361
+ # A parallel worker records each pool that runs SQL, so a test
362
+ # that closes its shared connection is still caught below.
363
+ @worker_db.watch_sql if @worker_db && @shared_databases&.any?
364
+ result = run_tests(Array(req["tests"]))
365
+ check_shared_databases!
366
+ result
234
367
  rescue Exception => e # rubocop:disable Lint/RescueException
235
- @errio.puts("[daemon-child] #{e.class}: #{e.message}")
368
+ @errio.puts("[daemon-child] #{describe_error(e)}")
236
369
  2
237
370
  end
238
371
  exit!(code)
239
372
  end
240
373
  verdict = wait_verdict(pid, timeout)
241
374
  # Mark ready only when the child finished cleanly after seeding
242
- # (killed/survived). Timeout can interrupt mid-copy; error is a routing
243
- # failure. Both leave the slot unready so the next fork seeds again.
375
+ # (killed/survived). Timeout can interrupt mid-copy; error can be a
376
+ # routing failure or a database error inside a test (#296). Both leave
377
+ # the slot unready so the next fork seeds again. A shared-database
378
+ # error (check_shared_databases!) also exits 2, so it seeds again too:
379
+ # the exit code cannot tell it from a routing failure. That costs only
380
+ # time, in a run where that check then fails every mutant.
244
381
  @slot_ready[worker] = true if first_use && %w[killed survived].include?(verdict)
245
382
  # A SIGKILLed timeout child skipped its Tempfile unlink. Sweep the orphan
246
- # so it cannot outlive the run or trip Zeitwerk on a later fork.
383
+ # so it cannot outlive the run.
247
384
  sweep_temps if verdict == "timeout"
248
385
  { "id" => req["id"], "verdict" => verdict }
249
386
  end
250
387
 
251
- # Remove orphaned mutant tempfiles from the source dirs (parent-side; the
252
- # SIGKILL path cannot run the child's ensure). Mirrors JobPlan.sweep_orphans.
388
+ # Hide kept mutant files from Zeitwerk before the app builds its loaders.
389
+ # The loader class may not exist yet. Watch for it, then stop watching.
390
+ #
391
+ # @return [void]
392
+ def install_zeitwerk_ignore!
393
+ if defined?(Zeitwerk::Loader)
394
+ prepend_daemon_temp_ignore
395
+ return
396
+ end
397
+
398
+ @zeitwerk_trace = TracePoint.new(:end) do |event|
399
+ next unless event.self.is_a?(Class) && event.self.name == "Zeitwerk::Loader"
400
+
401
+ @zeitwerk_trace.disable
402
+ prepend_daemon_temp_ignore
403
+ end
404
+ @zeitwerk_trace.enable
405
+ end
406
+
407
+ # @return [void]
408
+ def prepend_daemon_temp_ignore
409
+ return if Zeitwerk::Loader.ancestors.include?(DaemonTempIgnore)
410
+
411
+ Zeitwerk::Loader.prepend(DaemonTempIgnore)
412
+ end
413
+
414
+ # Remove orphaned mutant tempfiles of both modes from the source dirs
415
+ # (parent-side; the SIGKILL path cannot run the child's ensure). Mirrors
416
+ # JobPlan.sweep_orphans. Skips a file another process still owns
417
+ # ({OrphanGuard.mutant_file_in_use?}).
253
418
  def sweep_temps
254
419
  @source_dirs.to_a.each do |dir|
255
- Dir.glob(File.join(dir, "mutineer_daemon*.rb")).each do |f|
256
- File.unlink(f) rescue nil # rubocop:disable Style/RescueModifier
420
+ Dir.glob(File.join(dir, OrphanGuard::TEMP_BASENAME)).each do |path|
421
+ next if OrphanGuard.mutant_file_in_use?(path)
422
+
423
+ File.unlink(path) rescue nil # rubocop:disable Style/RescueModifier
257
424
  end
258
425
  end
259
426
  end
@@ -300,27 +467,44 @@ module Mutineer
300
467
  # `require_relative` in the mutated source resolves against its real
301
468
  # neighbours. Writing it to the tmpdir would LoadError on such files and
302
469
  # score a spurious `error` that diverges from the in-process path. The
303
- # Zeitwerk hazard (a stray `.rb` in an autoload dir) is handled by the
304
- # boot/timeout `sweep_temps`, not by relocating the file. Same path for
305
- # reload (whole file) and redefine (wrapped snippet).
470
+ # leading dot keeps the file away from Zeitwerk, which skips hidden files:
471
+ # loading it can be the first time Zeitwerk scans a namespace directory,
472
+ # and a parallel worker's file may sit there too (#253). `sweep_temps` only
473
+ # removes orphans and never touches a live file. Same path for reload
474
+ # (whole file) and redefine (wrapped snippet). The backend sends the whole
475
+ # file, so the concern blocks it stores replace the forgotten ones (#256).
476
+ # With a `"hook_def"`, the loaded file stored a mutated hook-block def, and
477
+ # {ConcernBlocks.place_hook_def} loads it into each class that has the
478
+ # method; it raises when none was stored, so the mutant scores `error` (#297).
306
479
  def apply_payload(payload)
307
480
  dir = File.dirname(File.expand_path(payload.fetch("source_file")))
308
- Tempfile.create(["mutineer_daemon", ".rb"], dir) do |f|
481
+ Tempfile.create([".mutineer_daemon", ".rb"], dir) do |f|
482
+ # Held until this block ends, including SIGKILL (the kernel drops it).
483
+ # Another run's sweep sees the lock and does not delete the file.
484
+ f.flock(File::LOCK_EX) rescue nil # rubocop:disable Style/RescueModifier
309
485
  f.write(payload.fetch("code"))
310
486
  f.flush
311
- load f.path
487
+ ConcernBlocks.replace(payload.fetch("source_file")) { load f.path }
312
488
  end
489
+ hook = payload["hook_def"] or return
490
+
491
+ ConcernBlocks.place_hook_def(source_file: payload.fetch("source_file"), line: hook.fetch("line"),
492
+ name: hook.fetch("name").to_sym, singleton: hook.fetch("singleton"))
313
493
  end
314
494
 
315
495
  # Load the covering test files and run them; 0 = all passed (survived),
316
- # 1 = a failure/error (killed). Minitest only; rspec is not yet on this path.
496
+ # 1 = a failure/error (killed), 2 = the first failure was an in-test
497
+ # database error (error, #296). Minitest only; rspec is not yet on this path.
498
+ # {MinitestIntegration.run} is the method the daemon's clean check uses,
499
+ # so each mutant fork gets the same seed and class list (#282). It loads
500
+ # only stdlib, so the daemon still needs no Prism. The fork requires
501
+ # nothing else first: the test files' `require "test_helper"` loads
502
+ # rails/test_help after the class list is reset, as in the clean check.
317
503
  def run_tests(tests)
318
504
  raise "unsupported framework #{@framework.inspect}" unless @framework == "minitest"
319
505
 
320
- require "minitest"
321
- require "rails/test_help" if defined?(Rails)
322
- tests.each { |t| load File.expand_path(t) }
323
- Minitest.run([]) ? 0 : 1
506
+ require_relative "minitest_integration"
507
+ MinitestIntegration.run(tests.map { |t| File.expand_path(t) })
324
508
  end
325
509
  end
326
510
  end
@@ -3,6 +3,8 @@
3
3
  require "shellwords"
4
4
  require "tempfile"
5
5
  require_relative "result"
6
+ require_relative "rails_run_env"
7
+ require_relative "redaction"
6
8
 
7
9
  module Mutineer
8
10
  # Raised when the smoke check (the unmutated suite) is not green, so the run
@@ -17,14 +19,18 @@ module Mutineer
17
19
  # This is deliberately NOT a `TestRunners` framework adapter: those return an
18
20
  # Integer 0/1 from inside a fork and are dispatched by framework name. This is a
19
21
  # whole backend — it spawns a process, enforces a wall-clock timeout, and maps
20
- # the exit status to a Result. The mapping is the same direction as in-process
21
- # (suite passes => survived, suite fails => killed) but coarser: it cannot tell an
22
- # infrastructure error from a genuine kill, so the smoke check (below) guards the
23
- # persistent case and the score is disclosed as an upper bound.
22
+ # the exit status to a Result. The mapping follows Isolation.decode: exit 0 is
23
+ # survived, exit 1 is killed, and any other exit is error (#292). When the clean
24
+ # smoke run printed a Minitest or RSpec summary line, exit 1 is a kill only when
25
+ # the output also shows a failing test, so a suite that crashed before its tests
26
+ # ran is not a kill. The smoke check (below) guards a persistently broken
27
+ # environment, and the score is disclosed as an upper bound.
24
28
  #
25
29
  # Child environment: bundler/gem env injected by Mutineer's Ruby is stripped, and
26
30
  # version-manager concrete version bins (e.g. `~/.rbenv/versions/3.4.9/bin`) are
27
31
  # removed from PATH so shims / `.ruby-version` can select the app's Ruby (#32).
32
+ # PARALLEL_WORKERS and DISABLE_SPRING are set to 1 ({RailsRunEnv}), and so is
33
+ # DISABLE_BOOTSNAP_COMPILE_CACHE ({child_env}).
28
34
  module ExternalBackend
29
35
  # Generous ceiling for the one-off smoke/calibration run (a cold app boot plus
30
36
  # the full suite). The per-mutant timeout is derived from how long this took.
@@ -32,6 +38,29 @@ module Mutineer
32
38
  # Poll interval for the deadline wait loop. Independent of Isolation's loop —
33
39
  # this backend waits on an external process TREE, not an in-process fork.
34
40
  POLL = 0.02
41
+ # Exit codes a shell or wrapper uses when the command could not run at all
42
+ # (126: not executable, 127: not found). Scored `error`, never `killed`.
43
+ CANNOT_RUN_CODES = [126, 127].freeze
44
+ # The exit code Minitest, `bin/rails test` and RSpec use for a failing test.
45
+ # Any other non-zero code is scored `error`, the same rule as Isolation.decode.
46
+ FAILURE_CODE = 1
47
+
48
+ # The end-of-run summary line of Minitest 5 and 6 ("N runs") and of
49
+ # minitest-reporters ("N tests"). Captures the failure and error counts.
50
+ MINITEST_SUMMARY = /\d+ (?:runs|tests), \d+ assertions, (\d+) failures, (\d+) errors/
51
+ # The end-of-run summary line of RSpec ("N examples, N failures").
52
+ # Captures the failure count.
53
+ RSPEC_SUMMARY = /\d+ examples?, (\d+) failures?/
54
+ # The line Minitest prints once every test file has loaded, before the first
55
+ # test runs. Exit 1 after it with no summary line means code under test
56
+ # called `exit 1` or `abort`, which the in-process backend also scores killed.
57
+ MINITEST_RUNNING = "# Running:"
58
+ # The line SimpleCov prints when its minimum_coverage gate fails the run.
59
+ SIMPLECOV_GATE = /SimpleCov failed with exit \d+ due to a coverage related error/
60
+ # RSpec's note that a file raised outside any example, such as at load.
61
+ RSPEC_OUTSIDE_ERROR = /errors? occurred outside of examples/
62
+ # ANSI colour codes, removed before matching so a coloured summary still counts.
63
+ ANSI_CODE = /\e\[[\d;]*m/
35
64
 
36
65
  # PATH entries that pin a concrete Ruby under a version manager (ahead of
37
66
  # shims). Optional trailing slash; normal bins are left alone.
@@ -67,7 +96,15 @@ module Mutineer
67
96
  # onto the parent env: set a key to +nil+ to unset it. Scrubs Mutineer/bundler
68
97
  # injection and version-manager PATH pins so shims / `.ruby-version` can win.
69
98
  # Keeps other vars (e.g. `RAILS_ENV`) so setting them on the Mutineer command
70
- # still reaches the suite.
99
+ # still reaches the suite. Also sets +PARALLEL_WORKERS+, +DISABLE_SPRING+ and
100
+ # +DISABLE_BOOTSNAP_COMPILE_CACHE+ to 1. A wrapper can change them again.
101
+ # Bootsnap checks only that its variable is set, so only unsetting it turns
102
+ # the cache back on. The Bootsnap pin is here, not in
103
+ # {RailsRunEnv.pin_child!}: this backend rewrites each source file in place,
104
+ # and Bootsnap keys its compile cache on path, size and whole-second mtime,
105
+ # so a same-size mutant written in the same second can load the previous
106
+ # mutant's cached code (#285). The daemon loads each mutant from a new file
107
+ # and keeps the cache.
71
108
  #
72
109
  # @api private
73
110
  # @return [Hash{String => String, nil}] env for Process.spawn.
@@ -90,6 +127,8 @@ module Mutineer
90
127
  kept
91
128
  end
92
129
  env["PATH"] = path.join(File::PATH_SEPARATOR)
130
+ RailsRunEnv.pin_child!(env, spring: true)
131
+ env["DISABLE_BOOTSNAP_COMPILE_CACHE"] = "1"
93
132
  env
94
133
  end
95
134
 
@@ -125,42 +164,142 @@ module Mutineer
125
164
  # @param files [Array<String>] test file paths.
126
165
  # @param timeout [Numeric] per-mutant wall-clock timeout in seconds.
127
166
  # @param verbose [Boolean] print the child's captured output on a non-pass.
167
+ # @param check_summary [Boolean] score exit 1 as killed only when the output
168
+ # has a summary line that reports a failing test (see {failure_reported?}).
169
+ # The runner turns this on when the clean smoke run printed a summary line.
128
170
  # @return [Mutineer::Result]
129
- def self.run(command, files, timeout:, verbose: false)
171
+ def self.run(command, files, timeout:, verbose: false, check_summary: false)
130
172
  kind, code, output, = spawn_capture(command, files, timeout)
131
173
  case kind
132
174
  when :timeout
133
175
  # A timeout is the one non-pass we flag by default — a normal kill is also
134
176
  # a non-zero exit, so notifying on every non-zero would spam every kill.
135
177
  warn "[mutineer] test-command exceeded #{timeout}s and was killed (scored timeout)."
136
- warn output if verbose && !output.empty?
178
+ warn Redaction.text(output) if verbose && !output.empty?
137
179
  Result.timeout
138
180
  else # :exited
139
181
  return Result.survived if code&.zero?
140
182
  # Signal death (nil exitstatus) is infrastructure, not a suite assertion
141
183
  # failure — match Isolation/daemon (error), do not inflate kill rate.
142
184
  if code.nil?
143
- warn output if verbose && !output.empty?
185
+ warn Redaction.text(output) if verbose && !output.empty?
144
186
  return Result.error("test-command terminated by signal")
145
187
  end
146
188
 
147
- warn output if verbose && !output.empty?
189
+ warn Redaction.text(output) if verbose && !output.empty?
190
+ # 126 (cannot execute) and 127 (command not found) come from a wrapper
191
+ # such as `bundle exec` that could not start the suite: no test ran.
192
+ return Result.error("test-command could not start (exit #{code})") if CANNOT_RUN_CODES.include?(code)
193
+
194
+ text = summary_text(output)
195
+ # SimpleCov's minimum_coverage gate replaced the exit code. Under
196
+ # Minitest it runs in Minitest.after_run even after a failing test, so a
197
+ # failing summary plus its message is still a kill. A passing summary
198
+ # plus its message means no test caught the mutant: in process, the
199
+ # fork exits before that hook, and the mutant survives. Keyed on
200
+ # SimpleCov's own message, so code under test that calls `exit 2`
201
+ # still scores error. Ceiling: with `SimpleCov.print_error_status =
202
+ # false` the gate prints nothing, and its exit scores error too.
203
+ if text.match?(SIMPLECOV_GATE)
204
+ return Result.killed if failure_reported?(text)
205
+ return Result.survived if tests_passed?(text)
206
+ end
207
+
208
+ unless code == FAILURE_CODE
209
+ return Result.error("test-command exited #{code} (only exit 1 counts as a kill; " \
210
+ "check RSpec --failure-exit-code, a coverage gate, or your wrapper)")
211
+ end
212
+
213
+ if check_summary && !failure_reported?(text) && !exited_during_tests?(text)
214
+ return Result.error("test-command exited 1 but its output shows no failing test " \
215
+ "(the suite or the mutant crashed before tests ran; --verbose shows the output)")
216
+ end
217
+
148
218
  Result.killed
149
219
  end
150
220
  end
151
221
 
222
+ # True when the text has a Minitest or RSpec summary line.
223
+ #
224
+ # @api private
225
+ # @param text [String] output from {summary_text}.
226
+ # @return [Boolean]
227
+ def self.summary_line?(text)
228
+ text.match?(MINITEST_SUMMARY) || text.match?(RSPEC_SUMMARY)
229
+ end
230
+
231
+ # True when a summary line reports a failing test: a Minitest failure or
232
+ # error, or an RSpec failure. An RSpec error outside examples alone (a spec
233
+ # file that raised while loading) is not one, as in TestRunners::RSpec.
234
+ # Any matching line counts, so a wrapper that runs two suites still kills.
235
+ # A test that prints a fake summary line to stdout fools this check.
236
+ #
237
+ # @api private
238
+ # @param text [String] output from {summary_text}.
239
+ # @return [Boolean]
240
+ def self.failure_reported?(text)
241
+ text.scan(MINITEST_SUMMARY).any? { |failures, errors| (failures.to_i + errors.to_i).positive? } ||
242
+ text.scan(RSPEC_SUMMARY).any? { |(failures)| failures.to_i.positive? }
243
+ end
244
+
245
+ # True when Minitest started its tests and the process then ended with no
246
+ # summary line after that start: code under test called `exit 1` or
247
+ # `abort` mid-run. Only the last start counts, so a wrapper whose earlier
248
+ # suite printed a summary still kills. A load crash ends before Minitest
249
+ # prints {MINITEST_RUNNING}, so it stays false.
250
+ # Only the default Minitest reporter prints that line: under RSpec or
251
+ # minitest-reporters, such an exit scores error, not killed. Ceiling: an
252
+ # exception that escapes Minitest.run itself (a reporter plugin bug) also
253
+ # reads as a kill here, where the in-process backend scores error.
254
+ #
255
+ # @api private
256
+ # @param text [String] output from {summary_text}.
257
+ # @return [Boolean]
258
+ def self.exited_during_tests?(text)
259
+ running_at = text.rindex(MINITEST_RUNNING)
260
+ return false unless running_at
261
+
262
+ summary_at = [text.rindex(MINITEST_SUMMARY), text.rindex(RSPEC_SUMMARY)].compact.max
263
+ summary_at.nil? || summary_at < running_at
264
+ end
265
+
266
+ # True when a recognized summary line is present and none shows a failing
267
+ # test or an RSpec error outside examples. With no recognized summary line
268
+ # (a JUnit or custom reporter), it is false, so a SimpleCov gate exit there
269
+ # scores error rather than a survivor the output cannot prove.
270
+ #
271
+ # @api private
272
+ # @param text [String] output from {summary_text}.
273
+ # @return [Boolean]
274
+ def self.tests_passed?(text)
275
+ summary_line?(text) && !failure_reported?(text) && !text.match?(RSPEC_OUTSIDE_ERROR)
276
+ end
277
+
278
+ # The output as binary with ANSI colour codes removed. Binary, so bytes that
279
+ # are invalid in the default encoding (an unset locale) cannot raise.
280
+ # Computed once per run and passed to the predicates above.
281
+ #
282
+ # @api private
283
+ # @param output [String] captured child stdout+stderr.
284
+ # @return [String]
285
+ def self.summary_text(output)
286
+ output.b.gsub(ANSI_CODE, "")
287
+ end
288
+
152
289
  # Pre-flight: run the command once against the UNMUTATED tree. Green (exit 0)
153
- # returns the elapsed seconds (used to calibrate the per-mutant timeout);
154
- # anything else raises SmokeCheckError so the run aborts before scoring.
290
+ # returns the elapsed seconds (used to calibrate the per-mutant timeout) and
291
+ # whether the output has a Minitest or RSpec summary line (used to turn on
292
+ # the summary check in {run}); anything else raises SmokeCheckError so the
293
+ # run aborts before scoring.
155
294
  #
156
295
  # @param command [String] the --test-command template.
157
296
  # @param files [Array<String>] test file paths.
158
297
  # @param timeout [Numeric] ceiling for the calibration run.
159
- # @return [Float] elapsed seconds of the clean run.
298
+ # @return [Array(Float, Boolean)] `[elapsed_seconds, summary_seen]`.
160
299
  # @raise [Mutineer::SmokeCheckError] when the clean suite is not green.
161
300
  def self.smoke_check!(command, files, timeout: SMOKE_TIMEOUT)
162
301
  kind, code, output, elapsed = spawn_capture(command, files, timeout)
163
- return elapsed if kind == :exited && code&.zero?
302
+ return [elapsed, summary_line?(summary_text(output))] if kind == :exited && code&.zero?
164
303
 
165
304
  reason =
166
305
  if kind == :timeout then "did not finish within #{timeout}s"
@@ -202,7 +341,7 @@ module Mutineer
202
341
  "ASDF_RUBY_VERSION from the child env; the suite still picked the wrong " \
203
342
  "Ruby. Use a wrapper that re-selects the app Ruby and ensure " \
204
343
  ".ruby-version / .tool-versions is present " \
205
- "(see README: Apps on Ruby < 3.4 → Under a version manager)"
344
+ "(see https://github.com/davidteren/mutineer/blob/main/docs/rails-setup.md#apps-on-ruby--34, section Under a version manager)"
206
345
  parts
207
346
  end
208
347
 
@@ -234,7 +373,12 @@ module Mutineer
234
373
  kind, code = wait_with_timeout(pid, timeout)
235
374
  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
236
375
  out.rewind
237
- [kind, code, out.read, elapsed]
376
+ # Test output is arbitrary bytes. Read it as UTF-8 whatever the locale (#303).
377
+ # Each invalid byte becomes `\xNN`, as in Reporter#printable, so the
378
+ # smoke-failure checks can match on the text and nothing is lost.
379
+ output = out.read.force_encoding(Encoding::UTF_8)
380
+ .scrub { |bytes| bytes.unpack("C*").map { |b| format("\\x%02X", b) }.join }
381
+ [kind, code, output, elapsed]
238
382
  ensure
239
383
  if out
240
384
  out.close