mutineer 1.6.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +680 -0
  3. data/README.md +210 -499
  4. data/lib/mutineer/baseline.rb +1 -1
  5. data/lib/mutineer/changed_lines.rb +2 -1
  6. data/lib/mutineer/child_coverage.rb +53 -0
  7. data/lib/mutineer/cli.rb +293 -42
  8. data/lib/mutineer/concern_blocks.rb +243 -0
  9. data/lib/mutineer/config.rb +154 -8
  10. data/lib/mutineer/coverage_map.rb +265 -27
  11. data/lib/mutineer/daemon_backend.rb +244 -75
  12. data/lib/mutineer/daemon_client.rb +73 -10
  13. data/lib/mutineer/daemon_server.rb +226 -42
  14. data/lib/mutineer/external_backend.rb +159 -15
  15. data/lib/mutineer/init.rb +124 -0
  16. data/lib/mutineer/isolation.rb +78 -24
  17. data/lib/mutineer/job_plan.rb +234 -44
  18. data/lib/mutineer/kill_channel.rb +24 -14
  19. data/lib/mutineer/migrate.rb +707 -0
  20. data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
  21. data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
  22. data/lib/mutineer/minitest_integration.rb +44 -14
  23. data/lib/mutineer/mutator_registry.rb +6 -2
  24. data/lib/mutineer/mutators/array_literal.rb +4 -2
  25. data/lib/mutineer/mutators/base.rb +36 -7
  26. data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
  27. data/lib/mutineer/mutators/chain_link.rb +6 -6
  28. data/lib/mutineer/mutators/condition_forcing.rb +6 -3
  29. data/lib/mutineer/mutators/operand_removal.rb +4 -2
  30. data/lib/mutineer/mutators/range_literal.rb +105 -4
  31. data/lib/mutineer/mutators/return_nil.rb +1 -0
  32. data/lib/mutineer/orphan_guard.rb +84 -0
  33. data/lib/mutineer/pairing.rb +62 -1
  34. data/lib/mutineer/project.rb +302 -14
  35. data/lib/mutineer/rails_run_env.rb +73 -0
  36. data/lib/mutineer/rails_worker_db.rb +1596 -57
  37. data/lib/mutineer/redaction.rb +199 -0
  38. data/lib/mutineer/reporter.rb +173 -31
  39. data/lib/mutineer/result.rb +52 -15
  40. data/lib/mutineer/runner.rb +148 -28
  41. data/lib/mutineer/statement_lines.rb +24 -11
  42. data/lib/mutineer/subject.rb +30 -4
  43. data/lib/mutineer/test_runners/minitest.rb +2 -1
  44. data/lib/mutineer/test_runners/rspec.rb +15 -3
  45. data/lib/mutineer/test_runners.rb +5 -2
  46. data/lib/mutineer/triage.rb +111 -0
  47. data/lib/mutineer/version.rb +1 -1
  48. data/lib/mutineer/worker_pool.rb +2 -1
  49. data/lib/mutineer.rb +4 -0
  50. data/skills/mutineer/SKILL.md +84 -0
  51. metadata +13 -1
@@ -6,6 +6,7 @@ require_relative "coverage_map"
6
6
  require_relative "daemon_client"
7
7
  require_relative "progress"
8
8
  require_relative "job_plan"
9
+ require_relative "orphan_guard"
9
10
 
10
11
  module Mutineer
11
12
  # Daemon execution backend. Boots the app ONCE in a persistent subprocess under
@@ -31,8 +32,16 @@ module Mutineer
31
32
  DEFAULT_TIMEOUT = 60
32
33
 
33
34
  # The daemon's per-mutant tempfile, written into the source dir so
34
- # require_relative resolves. Kept in step with DaemonServer#sweep_temps.
35
- DAEMON_TEMP_GLOB = "mutineer_daemon*.rb"
35
+ # require_relative resolves. The leading dot hides it from Zeitwerk (#253);
36
+ # older versions wrote it without one. Daemon files only, unlike the boot
37
+ # sweeps ({OrphanGuard::TEMP_BASENAME}): this sweep runs after the boot and
38
+ # waits up to {SWEEP_GRACE} while any file it matches is still owned, so a
39
+ # parallel in-process run's live files would add that wait to every daemon run.
40
+ DAEMON_TEMP_GLOB = "{.,}mutineer_daemon#{"[0-9]" * 8}-*.rb"
41
+
42
+ # Seconds the end-of-run sweep waits for an orphaned child to die. Its
43
+ # watchdog notices a dead daemon within {OrphanGuard::POLL}.
44
+ SWEEP_GRACE = (OrphanGuard::POLL * 2) + 0.5
36
45
 
37
46
  # Full daemon run: collect jobs, build the coverage map once, then execute
38
47
  # serially or across N worker daemons. Fail-fast forces serial so the survivor
@@ -44,25 +53,14 @@ module Mutineer
44
53
  # source map, and the {JobPlan.collect_jobs} extras.
45
54
  def self.execute(config, operator_classes)
46
55
  jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
47
- jobs = JobPlan.filter_since(jobs, source_map, config) if config.since
56
+ jobs, ignored_results, extras[:unscoped_mutants] = JobPlan.scope_since(jobs, ignored_results, source_map, config)
48
57
  abs_tests = config.tests.map { |t| File.expand_path(t, config.project_root) }
49
58
 
50
59
  # Nothing to mutate (`--since` matched no changed line, or every mutant is
51
60
  # suppressed). Return before booting anything: the coverage daemon and the
52
61
  # worker daemons below each boot the whole app, and README documents
53
62
  # `--since origin/<base>` for PR CI, where a docs-only PR is routine.
54
- if jobs.empty?
55
- # The daemon sweeps orphaned temps at boot and nothing boots here, so sweep
56
- # tool-side. A file a hard-killed run left in app/models breaks the app's own
57
- # Zeitwerk boot, not just Mutineer's next run.
58
- JobPlan.sweep_orphans(JobPlan.source_dirs(config), DAEMON_TEMP_GLOB)
59
- return [AggregateResult.new(ignored_results), source_map, extras]
60
- end
61
-
62
- # Build the coverage map once (app-side). nil when the build fails: runners
63
- # fall back to the full --test set (and emit a stderr warning) rather than
64
- # mis-scoring everything as no_coverage.
65
- coverage_map = build_coverage_map(config, abs_tests)
63
+ return [AggregateResult.new(ignored_results), source_map, extras] if jobs.empty?
66
64
 
67
65
  # Worker count = resolved --jobs, capped at the job count (no idle daemons).
68
66
  # >1 → N concurrent daemon handles, each on its OWN worker DB (N-handles, the
@@ -71,18 +69,60 @@ module Mutineer
71
69
  # input index, so the verdict set would diverge from serial (a different,
72
70
  # non-deterministic survivor set/score). The "identical to --jobs 1" guarantee
73
71
  # below only holds when fail-fast cannot race.
72
+ # A second database config is only known after the app boots, so the first
73
+ # daemon can still lower this count before any worker starts.
74
74
  worker_count = [config.jobs || 1, 1].max
75
75
  worker_count = 1 if config.fail_fast
76
76
  worker_count = [worker_count, jobs.size].min
77
77
 
78
- results =
79
- if worker_count > 1
80
- run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map)
81
- else
82
- run_serial(jobs, config, abs_tests, coverage_map, source_map)
83
- end
78
+ # One booted process copies worker databases, then builds the coverage map.
79
+ # A cached map still provisions: the copy has to happen while no worker is
80
+ # connected to the template database. That process keeps its database lock
81
+ # until the worker daemons have taken theirs.
82
+ prepare_client = nil
83
+ begin
84
+ coverage_map, worker_count, prepare_client = prepare_run(config, abs_tests, worker_count)
85
+
86
+ results =
87
+ if worker_count > 1
88
+ run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client)
89
+ else
90
+ run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client)
91
+ end
92
+ ensure
93
+ # run_serial and run_parallel quit this client once workers hold locks.
94
+ # Quit again here when they never reached that point. A second quit is a no-op.
95
+ prepare_client&.quit
96
+ end
84
97
 
85
98
  [AggregateResult.new(results + ignored_results), source_map, extras]
99
+ ensure
100
+ sweep_daemon_temps(config)
101
+ end
102
+
103
+ # Removes daemon mutant files left in the source dirs, on every exit path.
104
+ # The daemon sweeps at boot, but an empty run boots nothing, and a daemon
105
+ # that dies mid-mutant orphans a child that lives on until its watchdog
106
+ # fires (#250). The respawned daemon sweeps before that, and a run that
107
+ # ends or aborts right after the crash does too. So a file whose owner is
108
+ # still alive gets {SWEEP_GRACE} to go. A clean run leaves no file and pays
109
+ # nothing. A file a concurrent run still owns stays in place, but looks the
110
+ # same as a dying child's, so this run then waits the full grace.
111
+ #
112
+ # @param config [Mutineer::Config] the run config.
113
+ # @return [void]
114
+ def self.sweep_daemon_temps(config)
115
+ dirs = JobPlan.source_dirs(config)
116
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + SWEEP_GRACE
117
+ loop do
118
+ still_owned = JobPlan.sweep_orphans(dirs, DAEMON_TEMP_GLOB)
119
+ break if still_owned.empty?
120
+ break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
121
+
122
+ sleep 0.1
123
+ end
124
+ rescue SystemCallError
125
+ nil # never replace the run's own result or exception with a cleanup error
86
126
  end
87
127
 
88
128
  # Build the coverage map via a short-lived daemon (boots the app once, captures
@@ -101,11 +141,109 @@ module Mutineer
101
141
  def self.build_coverage_map(config, abs_tests)
102
142
  client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
103
143
  app_root: config.project_root).start
104
- data = begin
105
- client.coverage
144
+ begin
145
+ coverage_map_from_client(client, config)
106
146
  ensure
107
147
  client.quit
108
148
  end
149
+ rescue DaemonBootTimeout
150
+ raise
151
+ rescue DaemonBootError => e
152
+ warn_coverage_fallback("#{e.class}: #{e.message}")
153
+ nil
154
+ end
155
+
156
+ # Boot the first daemon, copy worker databases, then read the coverage map
157
+ # from that same process. A boot failure still falls back to the full test
158
+ # set. A provisioning failure does not: it stops the run.
159
+ #
160
+ # @param config [Mutineer::Config]
161
+ # @param abs_tests [Array<String>]
162
+ # @param worker_count [Integer] slots requested before the database count is known.
163
+ # @return [Array] the coverage map (or nil), the worker count to start, and
164
+ # the prepare daemon. The caller quits that daemon after worker daemons
165
+ # hold their locks. It is nil when this process could not boot one.
166
+ def self.prepare_run(config, abs_tests, worker_count)
167
+ client = nil
168
+ begin
169
+ client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
170
+ app_root: config.project_root).start
171
+ rescue DaemonBootTimeout
172
+ raise
173
+ rescue DaemonBootError => e
174
+ warn_coverage_fallback("#{e.class}: #{e.message}")
175
+ # No daemon listed a Rails app's databases, so no second worker may
176
+ # share them. A --boot app without --rails has no database routing.
177
+ unknown = { "error" => "the first daemon did not boot" }
178
+ return [nil, config.rails ? worker_count_for(unknown, [], worker_count) : worker_count, nil]
179
+ end
180
+
181
+ begin
182
+ # Raises ConfigError when shared_databases lists the primary database.
183
+ worker_count = worker_count_for(client.databases, config.shared_databases, worker_count)
184
+ # Raises DaemonBootError. Callers must not score mutants after this.
185
+ client.provision(worker_count)
186
+ [coverage_map_from_client(client, config), worker_count, client]
187
+ rescue Exception # rubocop:disable Lint/RescueException
188
+ # An interrupt must also quit it, or the daemon keeps its database lock.
189
+ client.quit
190
+ raise
191
+ end
192
+ end
193
+
194
+ # The worker count once the booted app's databases are known. Each worker
195
+ # copies only the primary database, so a second database forces one worker
196
+ # unless `shared_databases` lists it. Listing the primary is a config error.
197
+ # A failed lookup also forces one worker: a database nobody counted must
198
+ # never be shared by accident.
199
+ #
200
+ # @param databases [Hash, nil] the daemon's report: "names" and "primary",
201
+ # or "error" (with "primary" when it resolves). nil when the app has no
202
+ # worker-DB adapter.
203
+ # @param shared [Array<String>] the `shared_databases` config names.
204
+ # @param worker_count [Integer] the workers requested.
205
+ # @return [Integer] the workers to start.
206
+ # @raise [Mutineer::ConfigError] when `shared` lists the primary database.
207
+ def self.worker_count_for(databases, shared, worker_count)
208
+ return worker_count if databases.nil?
209
+
210
+ primary = databases["primary"]
211
+ if shared.include?(primary)
212
+ raise ConfigError, "shared_databases lists #{primary}, the database each worker copies. " \
213
+ "List only the other databases, the ones parallel workers may share."
214
+ end
215
+
216
+ if databases["error"]
217
+ if worker_count > 1
218
+ warn "[mutineer] could not list this app's databases (#{databases["error"]}); Mutineer runs one worker."
219
+ end
220
+ return 1
221
+ end
222
+
223
+ names = Array(databases["names"])
224
+ unknown = shared - names
225
+ if unknown.any?
226
+ warn "[mutineer] shared_databases names no test database: #{unknown.join(", ")}. " \
227
+ "This app has #{names.join(", ")}."
228
+ end
229
+
230
+ unshared = names - shared
231
+ return worker_count unless unshared.size > 1 && worker_count > 1
232
+
233
+ outside = shared.empty? ? "" : " outside shared_databases"
234
+ warn "[mutineer] this app has #{unshared.size} databases#{outside}; Mutineer runs one worker. " \
235
+ "Parallel runs support one database. To run in parallel, list each one " \
236
+ "your tests never use under shared_databases: #{(unshared - [primary]).join(", ")}."
237
+ 1
238
+ end
239
+
240
+ # Turn one daemon's coverage reply into a map. Does not start or stop the daemon.
241
+ #
242
+ # @param client [Mutineer::DaemonClient]
243
+ # @param config [Mutineer::Config]
244
+ # @return [Mutineer::CoverageMap, nil]
245
+ def self.coverage_map_from_client(client, config)
246
+ data = client.coverage
109
247
  # A red unmutated suite must abort, even when the shipped map is empty.
110
248
  # Falling back to the full --test set would treat those failures as kills.
111
249
  if data.is_a?(Hash) && Array(data["failed_clean_tests"]).any?
@@ -114,7 +252,7 @@ module Mutineer
114
252
  failed_test_files: data["failed_test_files"] || [],
115
253
  project_root: config.project_root,
116
254
  failed_clean_tests: data["failed_clean_tests"]
117
- ))
255
+ ), framework: config.framework)
118
256
  end
119
257
 
120
258
  unless data && !(data["map"] || {}).empty?
@@ -155,59 +293,70 @@ module Mutineer
155
293
  # (stop at the first survivor).
156
294
  #
157
295
  # @api private
296
+ # @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
297
+ # the databases. Quit once this worker has booted and taken its lock.
158
298
  # @return [Array<Mutineer::Result>] results in input order.
159
- def self.run_serial(jobs, config, abs_tests, coverage_map, source_map)
160
- client = DaemonClient.new(boot: boot_config(config, abs_tests),
161
- app_root: config.project_root).start
162
- results = []
163
- progress = Progress.new(jobs.size)
299
+ def self.run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client = nil)
300
+ client = nil
164
301
  begin
302
+ client = DaemonClient.new(boot: boot_config(config, abs_tests),
303
+ app_root: config.project_root).start
304
+ prepare_client&.quit
305
+ prepare_client = nil
306
+ results = []
307
+ progress = Progress.new(jobs.size)
165
308
  jobs.each_with_index do |job, i|
166
309
  r = job_result(job, i, client, 0, config, coverage_map, abs_tests, source_map)
167
310
  results << r
168
311
  progress.tick
169
312
  break if config.fail_fast && r.survived?
170
313
  end
314
+ results
171
315
  ensure
172
- client.quit
316
+ prepare_client&.quit
317
+ client&.quit
173
318
  end
174
- results
175
319
  end
176
320
 
177
321
  # Parallel path: N daemon handles, each pinned to its own worker slot (own DB).
178
322
  # A shared queue of job indices feeds N tool-side threads; results are placed by
179
323
  # input index so the verdict set matches serial. Callers must not pass fail_fast
180
324
  # here ({execute} forces serial for fail-fast). Per-mutant crashes are classified
181
- # in {job_result}, shared with the serial path; a {DaemonBootError} ends the run
182
- # here rather than scoring the remainder against a daemon that has given up.
325
+ # in {job_result}, shared with the serial path. Any error that escapes a worker
326
+ # (a {DaemonBootError}, or a defect) clears the queue, and the run ends once
327
+ # every worker has joined, rather than scoring the remainder first.
183
328
  #
184
329
  # @api private
330
+ # @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
331
+ # the databases. Quit once every worker has booted and taken its lock.
185
332
  # @return [Array<Mutineer::Result>] one result per input job, in input order.
186
- def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map)
333
+ def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client = nil)
187
334
  results = Array.new(jobs.size)
188
335
  progress = Progress.new(jobs.size)
189
336
  queue = Queue.new
337
+ errors = Queue.new
190
338
  jobs.each_index { |i| queue << i }
191
339
 
192
- # Built one at a time so a refused spawn part-way (EMFILE under a high --jobs)
193
- # can still quit the daemons already up. Array.new would lose every reference.
340
+ # Both lists grow one entry at a time, so a refused spawn part-way (EMFILE
341
+ # under a high --jobs) or a Ctrl-C leaves the ensure below every daemon
342
+ # already up. Each client joins its list before it starts, so one
343
+ # interrupted mid-boot is quit too. The prepare daemon stays up through
344
+ # this loop so its lock still covers the databases until each worker
345
+ # holds one.
194
346
  clients = []
347
+ threads = []
195
348
  begin
196
349
  worker_count.times do
197
- clients << DaemonClient.new(boot: boot_config(config, abs_tests),
198
- app_root: config.project_root).start
350
+ clients << DaemonClient.new(boot: boot_config(config, abs_tests, parallel: true),
351
+ app_root: config.project_root)
352
+ clients.last.start
199
353
  end
200
- rescue StandardError
201
- clients.each(&:quit)
202
- raise
354
+ ensure
355
+ prepare_client&.quit
203
356
  end
204
357
 
205
- clients.each_with_index.map do |client, worker|
206
- Thread.new do
207
- # The abort below is re-raised by join and reported once there; without
208
- # this Ruby also dumps the thread's backtrace, which the serial path never
209
- # does. Same fault, same output, whatever --jobs is set to.
210
- Thread.current.report_on_exception = false
358
+ clients.each_with_index do |client, worker|
359
+ threads << Thread.new do
211
360
  loop do
212
361
  i = begin
213
362
  queue.pop(true)
@@ -217,20 +366,35 @@ module Mutineer
217
366
  results[i] = job_result(jobs[i], i, client, worker, config, coverage_map, abs_tests, source_map)
218
367
  progress.tick
219
368
  end
220
- rescue DaemonBootError
221
- # The daemon gave up for good. Stop feeding the other workers rather
222
- # than letting them score the rest of the run against a dead client;
223
- # Thread#join re-raises this and ends the run.
369
+ rescue Exception => e # rubocop:disable Lint/RescueException
370
+ # The daemon gave up for good, or a defect escaped. Stop feeding the
371
+ # other workers, so they stop after their current mutant, and hand the
372
+ # error to the main thread. The thread itself ends cleanly.
224
373
  queue.clear
225
- raise
374
+ errors << e
226
375
  ensure
227
376
  client.quit
228
377
  end
229
- end.each(&:join)
378
+ end
379
+ # Join every worker before raising, so all daemons have quit and the
380
+ # end-of-run sweep sees each child gone.
381
+ threads.each(&:join)
382
+ raise errors.pop unless errors.empty?
230
383
 
231
384
  # Every job was popped by some worker and every pop assigns, so no slot can
232
- # be nil here: an escaping exception aborts the run via join instead.
385
+ # be nil here: a worker error is raised above instead.
233
386
  results
387
+ ensure
388
+ # An exception from outside (Ctrl-C, a caller's Timeout) skips the joins
389
+ # above. Kill every worker first, so their daemons quit at the same time,
390
+ # then wait for them. A second Ctrl-C during that wait still exits.
391
+ queue&.clear
392
+ threads&.each(&:kill)
393
+ threads&.each(&:join)
394
+ # Quit every client too. One may have no thread yet (startup failed or
395
+ # was interrupted), and a thread killed before its block began never runs
396
+ # its own ensure. A second quit of a client is a no-op.
397
+ clients&.each(&:quit)
234
398
  end
235
399
 
236
400
  # Build the payload for one job, run it on the given daemon/worker, and attach
@@ -265,14 +429,24 @@ module Mutineer
265
429
  Result.skipped
266
430
  elsif sel && sel[0] == :verdict
267
431
  sel[1]
432
+ elsif subject.unplaceable?("reload") # the daemon always reloads the whole file
433
+ # A :maybe mutant that cannot be placed stays no_coverage, as in-process.
434
+ sel && sel[0] == :maybe ? Result.no_coverage : Result.unplaceable
268
435
  else
436
+ payload = { "code" => mutated, "source_file" => File.expand_path(subject.file, config.project_root) }
437
+ if subject.reload_hook_call
438
+ # #297: the daemon child places the hook-block def, as in-process (Runner.run).
439
+ payload["code"], hook_def = JobPlan.hook_mutant(mutated, mutation, subject, source)
440
+ payload["hook_def"] = hook_def.to_h { |key, value| [key.to_s, key == :name ? value.to_s : value] }
441
+ end
269
442
  verdict = client.request(
270
443
  id: req_id, worker: worker, timeout: config.daemon_timeout || DEFAULT_TIMEOUT,
271
- payload: { "code" => mutated, "source_file" => File.expand_path(subject.file, config.project_root) },
272
- tests: sel ? sel[1] : abs_tests
444
+ payload: payload, tests: sel ? sel[1] : abs_tests
273
445
  )
274
446
  # A survivor whose line ran at load, as in-process (Runner.run).
275
- JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
447
+ loaded = JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
448
+ # Tests that may not have run the mutant, as in-process (#254).
449
+ JobPlan.selection_verdict(sel && sel[0], loaded)
276
450
  end
277
451
  r.with(subject: subject, mutation: mutation, id: id)
278
452
  end
@@ -285,8 +459,12 @@ module Mutineer
285
459
  # @param config [Mutineer::Config] the run config.
286
460
  # @param abs_tests [Array<String>] absolute --test paths.
287
461
  # @param coverage [Boolean] whether this daemon builds the coverage map.
462
+ # @param db_role [String, nil] "prepare" (copy databases on command),
463
+ # "worker" (lock only), or nil to pick from `coverage`.
464
+ # @param parallel [Boolean] whether this daemon is one of several workers.
465
+ # Only then does it check `shared_databases` after each mutant.
288
466
  # @return [Hash] the boot config shipped to the daemon.
289
- def self.boot_config(config, abs_tests, coverage: false)
467
+ def self.boot_config(config, abs_tests, coverage: false, db_role: nil, parallel: false)
290
468
  {
291
469
  project_root: config.project_root,
292
470
  boot: File.expand_path(config.boot || "config/environment", config.project_root),
@@ -298,14 +476,17 @@ module Mutineer
298
476
  source_dirs: JobPlan.source_dirs(config), # so the daemon can sweep orphan mutant temps
299
477
  framework: config.framework,
300
478
  rails: config.rails,
301
- # Schema for per-worker DB isolation. Sent when present; the daemon loads
302
- # it over a worker's copy of the test DB only when that copy is out of date.
303
- schema: schema_path(config),
479
+ # Databases the workers share. A parallel worker fails a mutant whose
480
+ # tests used one. One worker shares nothing, so it gets no names.
481
+ shared_databases: parallel ? config.shared_databases : [],
304
482
  # Coverage narrowing. Only the short-lived map-building daemon starts
305
483
  # Coverage (before boot); worker daemons boot with it OFF (no wasted
306
484
  # instrumentation/memory across every mutant fork). `sources`/`tests` are the
307
485
  # map-build inputs.
308
486
  coverage: coverage,
487
+ # "prepare" copies worker databases when asked. "worker" only locks.
488
+ # A direct client that omits this provisions at boot instead.
489
+ db_role: db_role || (coverage ? "prepare" : "worker"),
309
490
  # The map-building daemon's capture limit (--capture-timeout); nil = default.
310
491
  capture_timeout: config.capture_timeout,
311
492
  sources: config.sources.map { |s| File.expand_path(s, config.project_root) },
@@ -313,19 +494,6 @@ module Mutineer
313
494
  }
314
495
  end
315
496
 
316
- # Absolute path to the app's `db/schema.rb` if it exists, else nil. Each worker
317
- # DB starts as a copy of the test DB; the daemon loads this file over the copy
318
- # when the copy's schema differs. `structure.sql` apps get nil and keep the
319
- # copy as it is.
320
- #
321
- # @param config [Mutineer::Config] the run config.
322
- # @api private
323
- # @return [String, nil] absolute schema path or nil.
324
- def self.schema_path(config)
325
- path = File.expand_path("db/schema.rb", config.project_root)
326
- File.exist?(path) ? path : nil
327
- end
328
-
329
497
  # Map a daemon verdict string to a Result. The daemon reports the four
330
498
  # run-time states it can decide; pre-fork states (skipped/no_coverage/…) are
331
499
  # resolved tool-side before a request is ever sent.
@@ -346,6 +514,7 @@ module Mutineer
346
514
  # tests drive directly: {boot_config} from the zero-dep suite and
347
515
  # {build_coverage_map} from the daemon suite. Everything else is daemon-pipeline
348
516
  # internals with no caller outside this file.
349
- private_class_method :run_serial, :run_parallel, :job_result, :schema_path, :result_for
517
+ private_class_method :sweep_daemon_temps, :run_serial, :run_parallel, :job_result, :result_for,
518
+ :prepare_run, :coverage_map_from_client, :worker_count_for
350
519
  end
351
520
  end
@@ -4,6 +4,8 @@ require "json"
4
4
  require "io/wait"
5
5
  require "open3"
6
6
  require_relative "external_backend"
7
+ require_relative "rails_run_env"
8
+ require_relative "redaction"
7
9
 
8
10
  module Mutineer
9
11
  # Raised when the daemon cannot be booted or is gone for good: a bad boot path, an
@@ -42,6 +44,9 @@ module Mutineer
42
44
  # Seconds the daemon gets to boot the app and answer the handshake. A boot
43
45
  # that never returns ends the run instead of hanging it (#101).
44
46
  BOOT_TIMEOUT = 600
47
+ # Seconds close_io waits for the stderr drain to reach EOF before it stops
48
+ # the thread, so a last line without a newline is not lost.
49
+ DRAIN_GRACE = 1
45
50
  # Bundler's marker for a variable that was unset before it activated.
46
51
  BUNDLER_UNSET = "BUNDLER_ENVIRONMENT_PRESERVER_INTENTIONALLY_NIL"
47
52
 
@@ -71,6 +76,31 @@ module Mutineer
71
76
  self
72
77
  end
73
78
 
79
+ # The booted test environment's databases, as the daemon reported them:
80
+ # "names" and "primary", or "error" when the lookup failed. nil when the
81
+ # app has no worker-DB adapter.
82
+ #
83
+ # @return [Hash, nil]
84
+ attr_reader :databases
85
+
86
+ # Ask the daemon to create `slots` worker databases. Raises when the copy
87
+ # fails or another run holds the database, so the caller stops before any
88
+ # mutant is scored.
89
+ #
90
+ # @param slots [Integer] worker slots to create.
91
+ # @return [void]
92
+ # @raise [Mutineer::DaemonBootError]
93
+ def provision(slots)
94
+ raise DaemonBootError, "daemon is not running" if @stdin.nil?
95
+
96
+ send_line("cmd" => "provision", "slots" => slots)
97
+ reply = read_line(BOOT_TIMEOUT)
98
+ return if reply && reply["ok"]
99
+
100
+ detail = reply && reply["error"] ? reply["error"] : "provision failed"
101
+ raise DaemonBootError, detail
102
+ end
103
+
74
104
  # Run one mutant: ship the payload + covering tests, return the verdict string.
75
105
  # On a daemon crash (EOF/dead pipe) respawn (bounded) and return `"error"` for
76
106
  # this mutant. Never a wrong verdict, never a wedged run.
@@ -150,6 +180,8 @@ module Mutineer
150
180
  env["BUNDLE_GEMFILE"] = @gemfile
151
181
  env["RBENV_VERSION"] = @ruby_version if @ruby_version
152
182
  env["RAILS_ENV"] = "test" if rails_boot? && !env.key?("RAILS_ENV")
183
+ # The daemon loads tests after boot. Rails reads this when parallelize runs.
184
+ RailsRunEnv.pin_child!(env)
153
185
  env
154
186
  end
155
187
 
@@ -317,15 +349,14 @@ module Mutineer
317
349
  "-r", DAEMON_PATH, "-e", "Mutineer::DaemonServer.run",
318
350
  unsetenv_others: true, chdir: @app_root
319
351
  )
320
- # Drain daemon stderr to the tool's stderr so child/boot errors are visible.
321
- # Tracked (not fire-and-forget) so close_io can reclaim it on quit/respawn;
322
- # the rescue swallows the benign EBADF/IOError raised when close_io closes
323
- # the pipe out from under an in-flight copy_stream.
324
- @drain = Thread.new do # rubocop:disable ThreadSafety/NewThread
325
- IO.copy_stream(@stderr, @errio)
326
- rescue IOError, Errno::EBADF
327
- nil
328
- end
352
+ # Replies are UTF-8 JSON, and the app's stderr is redacted as text: with
353
+ # LANG unset both pipes would read as US-ASCII (#303).
354
+ @stdout.set_encoding(Encoding::UTF_8)
355
+ @stderr.set_encoding(Encoding::UTF_8)
356
+ # Drain daemon stderr to the tool's stderr so child/boot errors are visible,
357
+ # redacted line by line (#288: the app's output goes through here too).
358
+ # Tracked (not fire-and-forget) so close_io can reclaim it on quit/respawn.
359
+ @drain = Thread.new { drain_stderr(@stderr) } # rubocop:disable ThreadSafety/NewThread
329
360
 
330
361
  send_line(@boot)
331
362
  read_line(BOOT_TIMEOUT)
@@ -344,6 +375,7 @@ module Mutineer
344
375
  close_io
345
376
  raise (timed_out ? DaemonBootTimeout : DaemonBootError), "daemon failed to boot under the app bundle: #{detail}"
346
377
  end
378
+ @databases = ready["databases"]
347
379
  end
348
380
 
349
381
  # Respawn after a crash, up to MAX_RESTARTS, then hard-fail loudly.
@@ -383,6 +415,33 @@ module Mutineer
383
415
  nil
384
416
  end
385
417
 
418
+ # Copy each line of the daemon's stderr to the tool's stderr, redacted.
419
+ # A line is the unit the daemon and the app write, so a password is never
420
+ # split across two redaction calls. The line break is kept outside the
421
+ # redaction, so a line redaction hides as a whole still ends its line.
422
+ # After a write fails (EPIPE from a tool stderr that went away), the drain
423
+ # keeps reading and drops the lines, so the daemon never blocks on a full
424
+ # pipe. The outer rescue swallows the benign EBADF/IOError raised when
425
+ # close_io closes the pipe under an in-flight read, so the thread never
426
+ # ends with an error that close_io's join would re-raise.
427
+ #
428
+ # @param io [IO] the daemon's stderr pipe.
429
+ # @return [nil]
430
+ def drain_stderr(io)
431
+ writable = true
432
+ io.each_line do |line|
433
+ next unless writable
434
+
435
+ body = line.chomp("\n").scrub # an invalid byte must not stop the drain
436
+ @errio.write(Redaction.text(body), body.length < line.length ? "\n" : "")
437
+ rescue StandardError
438
+ writable = false
439
+ end
440
+ nil
441
+ rescue StandardError
442
+ nil
443
+ end
444
+
386
445
  # Close the IPC pipes, stop the stderr-drain thread, and reap the daemon so a
387
446
  # respawn or quit leaves no leaked fd, thread, or zombie.
388
447
  #
@@ -395,7 +454,11 @@ module Mutineer
395
454
  rescue Errno::ESRCH
396
455
  nil # it exited between the check and the kill
397
456
  end
398
- @drain&.kill # stop the drain BEFORE closing its fd (avoids a copy_stream EBADF)
457
+ # The drain holds a partial last line until EOF. Give it a moment to reach
458
+ # EOF now the daemon is dead, so a boot error with no final newline still
459
+ # prints; then stop it BEFORE closing its fd (avoids an EBADF mid-read).
460
+ @drain&.join(DRAIN_GRACE)
461
+ @drain&.kill
399
462
  [@stdin, @stdout, @stderr].each { |io| io&.close rescue nil } # rubocop:disable Style/RescueModifier
400
463
  @wait_thr&.join # reap the exited daemon so respawn/quit leaves no zombie
401
464
  @stdin = @stdout = @stderr = @drain = @wait_thr = nil