mutineer 1.5.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,10 +9,15 @@ require "set"
9
9
  require_relative "minitest_integration"
10
10
  require_relative "test_runners"
11
11
  require_relative "child_stdout"
12
+ require_relative "orphan_guard"
13
+ require_relative "redaction"
12
14
  require_relative "project_path"
13
15
  require_relative "pairing"
14
16
 
15
17
  module Mutineer
18
+ # A `--coverage-from` capture that cannot be read, or does not fit this run.
19
+ class CoverageFromError < StandardError; end
20
+
16
21
  # Maps `(source_file, line) -> [test_files]` so each mutant runs only against
17
22
  # the tests that actually exercise its line. Built once, then queried per
18
23
  # mutant via #tests_for. Persisted to .mutineer/coverage.json with a
@@ -28,8 +33,36 @@ module Mutineer
28
33
  # the parent. Stdout stays free for test output, which goes to File::NULL.
29
34
  RESULT_FD = 3
30
35
 
36
+ # Seconds a capture waits for its result after the child exits in time.
37
+ # The result is already in the pipe then; the wait only lets the reader
38
+ # thread run, even when the capture deadline has passed (#129).
39
+ RESULT_GRACE = 1
40
+
31
41
  attr_reader :project_root, :failed_test_files, :failed_clean_tests, :phase_a_ran, :map
32
42
 
43
+ # The source lines that ran while the app booted or the sources and the
44
+ # `--require` files loaded, before any test ran, as a Set of "file:line"
45
+ # keys like {#map}'s (#187, #217).
46
+ # A mutant on such a line can change a value computed before the mutant was
47
+ # applied, so its verdict is not trusted (`ran_at_load`).
48
+ #
49
+ # @return [Set<String>]
50
+ attr_reader :load_lines
51
+
52
+ # The source methods that were called while the app booted or the sources
53
+ # and the `--require` files loaded, as a Set of "file:line:column" keys of
54
+ # their `def` (#209). Line coverage cannot tell this for a one-line or
55
+ # endless method: its `def` line counts when the method is defined.
56
+ #
57
+ # @return [Set<String>]
58
+ attr_reader :load_methods
59
+
60
+ # Seconds each test file took to capture, keyed by project-relative path
61
+ # (#203). {#order_tests} runs the cheaper files first.
62
+ #
63
+ # @return [Hash{String => Float}]
64
+ attr_reader :timings
65
+
33
66
  # Build a QUERY-ONLY map from data captured elsewhere (the daemon builds the
34
67
  # map app-side and ships `map` + `failed_test_files` over IPC; the tool
35
68
  # reconstructs it here for per-mutant selection). Skips the capture machinery
@@ -40,10 +73,18 @@ module Mutineer
40
73
  # @param failed_test_files [Array<String>] test files whose capture failed.
41
74
  # @param project_root [String] project root (for path relativization).
42
75
  # @param failed_clean_tests [Array<String>] test files whose unmutated run failed.
76
+ # @param load_lines [Array<String>, Set<String>] "file:line" keys that ran at load ({#load_lines}).
77
+ # @param load_methods [Array<String>, Set<String>] "file:line:column" keys of methods called at load
78
+ # ({#load_methods}).
79
+ # @param timings [Hash{String => Float}] capture seconds per test file ({#timings}).
43
80
  # @return [Mutineer::CoverageMap] a query-only map.
44
- def self.from_data(map:, failed_test_files:, project_root:, failed_clean_tests: [])
81
+ def self.from_data(map:, failed_test_files:, project_root:, failed_clean_tests: [], load_lines: [],
82
+ load_methods: [], timings: {})
45
83
  instance = allocate
46
84
  instance.instance_variable_set(:@map, map || {})
85
+ instance.instance_variable_set(:@timings, timings || {})
86
+ instance.instance_variable_set(:@load_lines, Set.new(load_lines || []))
87
+ instance.instance_variable_set(:@load_methods, Set.new(load_methods || []))
47
88
  instance.instance_variable_set(:@failed_test_files, failed_test_files || [])
48
89
  instance.instance_variable_set(:@failed_clean_tests, failed_clean_tests || [])
49
90
  instance.instance_variable_set(:@project_root, project_root)
@@ -53,10 +94,12 @@ module Mutineer
53
94
  def initialize(source_paths:, test_paths:, cache_dir: ".mutineer",
54
95
  load_paths: ["lib"], project_root: Dir.pwd,
55
96
  capture_timeout: DEFAULT_CAPTURE_TIMEOUT, boot_path: nil,
56
- framework: "minitest", verbose: false)
97
+ framework: "minitest", verbose: false, require_paths: [], coverage_from: nil)
57
98
  @source_paths = Array(source_paths)
99
+ @require_paths = Array(require_paths)
58
100
  @test_paths = Array(test_paths)
59
101
  @cache_dir = cache_dir
102
+ @coverage_from = coverage_from
60
103
  @load_paths = Array(load_paths)
61
104
  @project_root = project_root
62
105
  @capture_timeout = capture_timeout
@@ -66,6 +109,9 @@ module Mutineer
66
109
  @map = {}
67
110
  @failed_test_files = []
68
111
  @failed_clean_tests = []
112
+ @load_lines = Set.new
113
+ @load_methods = Set.new
114
+ @timings = {}
69
115
  @loaded_dependencies = {}
70
116
  @phase_a_ran = false
71
117
  end
@@ -83,8 +129,15 @@ module Mutineer
83
129
  # the booted parent instead. Inverts into the same map #tests_for reads, and
84
130
  # reuses the digest cache (the digest mixes in the boot file so a boot cache
85
131
  # never collides with a standalone one).
132
+ #
133
+ # The lines that ran during boot are read here on every run, cache or not:
134
+ # the boot runs every time, and no test is credited with those lines.
86
135
  def build_via_fork(after_fork: nil)
87
136
  warn_external_sources
137
+ # Matched by real path in #record_load: a Coverage key is the path as
138
+ # required, which can differ from the configured one (/var, /private/var).
139
+ booted = Coverage.running? ? Coverage.peek_result : {}
140
+ record_load(booted.transform_values { |v| v.is_a?(Hash) ? load_entry(v[:lines], v[:methods]) : v })
88
141
  cached_or(after_fork: after_fork) { run_phase_a_via_fork(after_fork: after_fork) }
89
142
  end
90
143
 
@@ -95,6 +148,54 @@ module Mutineer
95
148
  @map["#{relativize(file)}:#{line}"] || []
96
149
  end
97
150
 
151
+ # The covering `tests` of a mutant in `file`, in the order a mutant run
152
+ # loads them (#203): the files {Pairing.infer_tests} pairs with `file`
153
+ # first, then the cheapest by {#timings}. A file with no timing comes
154
+ # after the timed ones, and the path breaks a tie, so the same cache gives
155
+ # the same order on every run. Costs compare in doubling buckets
156
+ # (`log2(seconds - fastest + 1)`), so files of close cost keep their path
157
+ # order when a rebuild measures them again. Taking off the fastest
158
+ # captured file's time removes the startup cost every standalone capture
159
+ # pays; a failed capture is timed again on each run, so it does not count.
160
+ # A run that stops at the first failure then reaches a fast killing test
161
+ # before a slow file uses up the timeout.
162
+ #
163
+ # @param file [String] the mutated source file path.
164
+ # @param tests [Array<String>] project-relative test paths from {#tests_for}.
165
+ # @return [Array<String>] the same paths, reordered.
166
+ def order_tests(file, tests)
167
+ rel = relativize(absolute(file))
168
+ @paired ||= {}
169
+ paired = @paired[rel] ||= Pairing.infer_tests(rel, project_root: @project_root,
170
+ prefer: @framework || "minitest")
171
+ base = @timings.except(*@failed_test_files).values.min || 0
172
+ tests.sort_by do |t|
173
+ cost = @timings[t]
174
+ [paired.include?(t) ? 0 : 1, cost ? Math.log2(cost - base + 1).floor : Float::INFINITY, t]
175
+ end
176
+ end
177
+
178
+ # True when `file:line` ran while the app booted or the sources loaded
179
+ # (see {#load_lines}).
180
+ #
181
+ # @param file [String] source file path.
182
+ # @param line [Integer] 1-based line.
183
+ # @return [Boolean]
184
+ def ran_at_load?(file, line)
185
+ @load_lines.include?("#{relativize(file)}:#{line}")
186
+ end
187
+
188
+ # True when the method whose `def` starts at `line` and `column` of `file`
189
+ # was called while the app booted or the sources loaded (see {#load_methods}).
190
+ #
191
+ # @param file [String] source file path.
192
+ # @param line [Integer] 1-based line of the `def`.
193
+ # @param column [Integer] 0-based byte column of the `def`.
194
+ # @return [Boolean]
195
+ def method_ran_at_load?(file, line, column)
196
+ @load_methods.include?("#{relativize(file)}:#{line}:#{column}")
197
+ end
198
+
98
199
  # Is this source file's empty coverage the result of an *errored* capture
99
200
  # rather than a genuine coverage gap? True iff some capture failed this run
100
201
  # AND this file got zero coverage from any successful capture AND a failed
@@ -271,9 +372,26 @@ module Mutineer
271
372
  # @return [Mutineer::CoverageMap] self.
272
373
  def cached_or(after_fork: nil)
273
374
  @digest = compute_digest
375
+ return reuse_capture(after_fork) if @coverage_from
376
+
274
377
  cached = read_cache
275
- if cached && cached["digest"] == @digest && dependencies_match?(cached)
378
+ # A cache from before load methods (#209) or test timings (#203) were
379
+ # saved rebuilds once. A cache that does not record this process's
380
+ # PARALLEL_WORKERS value rebuilds too. Rails reads that value inside
381
+ # parallelize, so a map from another worker count selects the wrong
382
+ # tests. The daemon writes this same file. Boot mode reads its load
383
+ # lines and methods from the live boot instead (#build_via_fork), but
384
+ # its map from before #209 lacks the `def` lines of the methods each
385
+ # test called.
386
+ if cached && cached["digest"] == @digest && dependencies_match?(cached) &&
387
+ cached.key?("load_methods") && cached.key?("timings") &&
388
+ parallel_workers_match?(cached)
276
389
  @map = cached["map"] || {}
390
+ @timings = cached["timings"] || {}
391
+ unless @boot_path
392
+ @load_lines = Set.new(cached["load_lines"])
393
+ @load_methods = Set.new(cached["load_methods"])
394
+ end
277
395
  @failed_test_files = cached["failed_test_files"] || []
278
396
  @failed_clean_tests = []
279
397
  @loaded_dependencies = cached["dependencies"] || {}
@@ -291,18 +409,152 @@ module Mutineer
291
409
  self
292
410
  end
293
411
 
412
+ # Takes the map from the `--coverage-from` capture instead of capturing,
413
+ # keeping only this run's sources, then runs the tests clean. A mutant runs
414
+ # only the tests that cover its line, often one file alone, so each test
415
+ # that covers one of this run's sources runs alone, as on a cache hit: a
416
+ # test that fails alone here would otherwise kill every mutant it runs. The
417
+ # other tests never run alone in this run, so only the combined check
418
+ # covers them.
419
+ #
420
+ # @api private
421
+ # @param after_fork [Proc, nil] boot-mode fork hook.
422
+ # @return [Mutineer::CoverageMap] self.
423
+ # @raise [Mutineer::CoverageFromError] when the capture does not fit.
424
+ def reuse_capture(after_fork)
425
+ cached = accepted_capture
426
+ sources = source_fingerprints.keys.to_set
427
+ @map = cached["map"].select { |key, _| sources.include?(key_source(key)) }
428
+ @timings = cached["timings"] || {}
429
+ @failed_test_files = []
430
+ @failed_clean_tests = []
431
+ @loaded_dependencies = cached["dependencies"]
432
+ covering = @map.values.flatten.to_set
433
+ verify_cached_clean(after_fork: after_fork,
434
+ test_paths: @test_paths.select { |t| covering.include?(relativize(t)) })
435
+ verify_combined_clean(after_fork: after_fork)
436
+ save
437
+ self
438
+ end
439
+
440
+ # The parsed `--coverage-from` capture, once it fits this run.
441
+ #
442
+ # @api private
443
+ # @return [Hash] the parsed coverage.json.
444
+ # @raise [Mutineer::CoverageFromError] when it cannot be read or does not fit.
445
+ def accepted_capture
446
+ if File.expand_path(@coverage_from) == File.expand_path(cache_path)
447
+ raise CoverageFromError, "--coverage-from #{@coverage_from} is this run's own cache; give the run another --cache-dir"
448
+ end
449
+
450
+ cached = JSON.parse(File.read(@coverage_from))
451
+ reason = capture_mismatch(cached)
452
+ raise CoverageFromError, "--coverage-from #{@coverage_from} cannot be reused: #{reason}" if reason
453
+
454
+ cached
455
+ rescue SystemCallError, JSON::ParserError => e
456
+ raise CoverageFromError, "--coverage-from #{@coverage_from} cannot be read: #{e.message}"
457
+ end
458
+
459
+ # Why a capture does not fit this run, or nil when it does.
460
+ #
461
+ # @api private
462
+ # @param cached [Object] the parsed coverage.json.
463
+ # @return [String, nil]
464
+ def capture_mismatch(cached)
465
+ return "it is not a coverage cache" unless cached.is_a?(Hash) && cached["map"].is_a?(Hash)
466
+ unless cached["sources"] && cached["owners"] && cached["inputs"]
467
+ return "it has no source fingerprints; capture it again with this version and --rails/--boot"
468
+ end
469
+ return "its entries are not the shape a coverage cache saves; capture it again" unless well_formed_capture?(cached)
470
+ return "its tests, --require files, boot file, load paths or framework differ" unless cached["inputs"] == compute_digest(inputs_only: true)
471
+
472
+ fingerprints = source_fingerprints
473
+ missing = fingerprints.keys - cached["sources"].keys
474
+ return "it was not captured over #{missing.join(', ')}" if missing.any?
475
+
476
+ changed = fingerprints.reject { |rel, fingerprint| cached["sources"][rel] == fingerprint }.keys
477
+ return "#{changed.join(', ')} changed since the capture" if changed.any?
478
+
479
+ owners = ownership_paths - cached["owners"]
480
+ return "#{owners.join(', ')} appeared since the capture and can own a split test" if owners.any?
481
+
482
+ # The capture can list owners of sources outside this run, so compare
483
+ # with the disk, not with this run's owners.
484
+ gone = cached["owners"].reject { |rel| File.file?(absolute(rel)) }
485
+ return "#{gone.join(', ')} disappeared since the capture and could own a split test" if gone.any?
486
+
487
+ return "a file its tests loaded changed since the capture" unless dependencies_match?(cached)
488
+ unless parallel_workers_match?(cached)
489
+ return "it was captured with PARALLEL_WORKERS=#{cached['parallel_workers'].inspect}, " \
490
+ "this run has #{ENV.fetch('PARALLEL_WORKERS', nil).inspect}"
491
+ end
492
+
493
+ failed = Array(cached["failed_test_files"])
494
+ return "capture failed for #{failed.join(', ')}" if failed.any?
495
+
496
+ moved = load_mismatch(cached, fingerprints.keys.to_set)
497
+ return nil if moved.empty?
498
+
499
+ "lines of #{moved.join(', ')} ran at load in only one of the capture and this run: loading the capture's " \
500
+ "other sources ran them. Under --strategy redefine, eager-load in both (config.eager_load = true)"
501
+ end
502
+
503
+ # True when every entry the reuse reads has the type {#save} writes, so a
504
+ # hand-edited or foreign file is rejected with a reason, not a backtrace.
505
+ #
506
+ # @api private
507
+ # @param cached [Hash] the parsed coverage.json.
508
+ # @return [Boolean]
509
+ def well_formed_capture?(cached)
510
+ strings = ->(value) { value.is_a?(Array) && value.all?(String) }
511
+ cached["inputs"].is_a?(String) && strings.(cached["owners"]) &&
512
+ cached["sources"].is_a?(Hash) && cached["sources"].all? { |k, v| k.is_a?(String) && v.is_a?(String) } &&
513
+ cached["map"].all? { |k, v| k.is_a?(String) && strings.(v) } &&
514
+ %w[load_lines load_methods].all? { |key| cached[key].nil? || strings.(cached[key]) } &&
515
+ (cached["timings"].nil? || cached["timings"].is_a?(Hash))
516
+ end
517
+
518
+ # This run's sources whose load lines or load methods differ in the capture.
519
+ #
520
+ # @api private
521
+ # @param cached [Hash] the parsed coverage.json.
522
+ # @param sources [Set<String>] project-relative source paths.
523
+ # @return [Array<String>] the sources that differ, sorted.
524
+ def load_mismatch(cached, sources)
525
+ owned = ->(keys) { Array(keys).select { |key| sources.include?(key_source(key)) }.to_set }
526
+ moved = (owned.(cached["load_lines"]) ^ owned.(@load_lines)) + (owned.(cached["load_methods"]) ^ owned.(@load_methods))
527
+ moved.map { |key| key_source(key) }.uniq.sort
528
+ end
529
+
530
+ # Fingerprint of each source, by project-relative path.
531
+ #
532
+ # @return [Hash{String => String}]
533
+ def source_fingerprints
534
+ @source_paths.to_h { |p| [relativize(absolute(p)), file_fingerprint(absolute(p))] }
535
+ end
536
+
537
+ # The source path of a "file:line" or "file:line:column" key.
538
+ #
539
+ # @param key [String] a map, load line or load method key.
540
+ # @return [String]
541
+ def key_source(key) = key.sub(/(?::\d+)+\z/, "")
542
+
294
543
  # Runs standalone coverage capture.
295
544
  #
296
545
  # @api private
297
546
  def run_phase_a
298
547
  @phase_a_ran = true
299
548
  @map = {}
549
+ @load_lines = Set.new
550
+ @load_methods = Set.new
551
+ @timings = {}
300
552
  @failed_test_files = []
301
553
  @failed_clean_tests = []
302
554
  @loaded_dependencies = {}
303
555
 
304
556
  @test_paths.each do |test_path|
305
- payload = capture(test_path)
557
+ payload = timed(test_path) { capture(test_path) }
306
558
  next unless payload
307
559
 
308
560
  accept_capture_payload(test_path, payload)
@@ -317,51 +569,83 @@ module Mutineer
317
569
  def run_phase_a_via_fork(after_fork:)
318
570
  @phase_a_ran = true
319
571
  @map = {}
572
+ @timings = {}
320
573
  @failed_test_files = []
321
574
  @failed_clean_tests = []
322
575
  @loaded_dependencies = {}
323
576
  abs_sources = abs_source_paths
324
577
 
325
578
  @test_paths.each do |test_path|
326
- # Tri-state payload: Hash = capture result, String = error diagnostic from
327
- # the child, nil = pipe gone / empty. The String diagnostic is what
328
- # becomes an :uncapturable status.
329
- case (payload = fork_capture(absolute(test_path), abs_sources, after_fork))
330
- when Hash then accept_capture_payload(test_path, payload)
331
- when String
332
- fail_test(test_path, @verbose ? "fork capture failed: #{payload}" :
333
- "fork capture produced no result (re-run with --verbose for the error)")
334
- else fail_test(test_path, "fork capture produced no result")
335
- end
579
+ accept_fork_payload(test_path, timed(test_path) { fork_capture(absolute(test_path), abs_sources, after_fork) })
580
+ end
581
+ end
582
+
583
+ # Runs the block, a capture of `test_path`, and records its wall-clock
584
+ # seconds in {#timings}. The parent times the whole capture, so the
585
+ # numbers compare files, not exact test cost.
586
+ #
587
+ # @api private
588
+ # @param test_path [String] test file path.
589
+ # @yieldreturn [Object] the capture result.
590
+ # @return [Object] the block's value.
591
+ def timed(test_path)
592
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
593
+ value = yield
594
+ @timings[relativize(test_path)] = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round(3)
595
+ value
596
+ end
597
+
598
+ # Records a {#fork_capture} result. Hash = capture result, String = error
599
+ # diagnostic from the child, :timeout = the capture deadline passed, nil =
600
+ # pipe gone / empty. The String diagnostic is what becomes an
601
+ # :uncapturable status.
602
+ #
603
+ # @api private
604
+ # @param test_path [String] test file path.
605
+ # @param payload [Hash, String, Symbol, nil] what {#fork_capture} returned.
606
+ # @return [void]
607
+ def accept_fork_payload(test_path, payload)
608
+ case payload
609
+ when Hash then accept_capture_payload(test_path, payload)
610
+ when :timeout then fail_test(test_path, "timed out after #{@capture_timeout}s")
611
+ when String
612
+ fail_test(test_path, @verbose ? "fork capture failed: #{payload}" :
613
+ "fork capture produced no result (re-run with --verbose for the error)")
614
+ else fail_test(test_path, "fork capture produced no result")
336
615
  end
337
616
  end
338
617
 
339
618
  # Fork the booted parent, run one test under the inherited Coverage, and
340
- # return its per-source counts hash (or nil on failure). Reuses the same
619
+ # return its per-source counts hash, a String diagnostic on failure, or
620
+ # :timeout after `@capture_timeout` (see {#await_child}). Reuses the same
341
621
  # fork + Marshal-over-pipe + hard-exit! discipline as WorkerPool/Isolation.
342
622
  def fork_capture(abs_test, abs_sources, after_fork)
623
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @capture_timeout
343
624
  rd, wr = IO.pipe
344
625
  # Marshal output is binary: an un-binmoded pipe can raise
345
626
  # Encoding::UndefinedConversionError on write, which the child's rescue then
346
627
  # swallows, losing the real error and yielding a bare "no result".
347
628
  rd.binmode
348
629
  wr.binmode
630
+ parent = Process.pid
349
631
  pid = fork do
350
632
  rd.close
633
+ Process.setpgid(0, 0) rescue nil # rubocop:disable Style/RescueModifier
634
+ OrphanGuard.start(parent)
351
635
  payload =
352
636
  begin
353
637
  ChildStdout.silence
354
638
  # Fork-safety hook: the in-process path reconnects AR; the daemon
355
- # routes to its worker DB. Nil (non-Rails) = no-op. Injected so this
356
- # file needs neither Runner (Prism) nor Rails.
639
+ # drops its protocol channel and routes to its worker DB. Nil =
640
+ # no-op. Injected so this file needs neither Runner (Prism) nor Rails.
357
641
  after_fork&.call
358
642
  Coverage.result(clear: true, stop: false) # discard pre-test delta
359
643
  passed = TestRunners.for(@framework).run([abs_test]).zero?
360
- # lines:true yields {file => {lines: [...]}}; reduce to the counts
644
+ # {file => {lines: [...], methods: {...}}}; reduce to the counts
361
645
  # array record() expects, keeping only our source files.
362
646
  coverage = Coverage.result(stop: false)
363
647
  .select { |f, _| abs_sources.include?(f) }
364
- .transform_values { |v| v.is_a?(Hash) ? v[:lines] : v }
648
+ .transform_values { |v| v.is_a?(Hash) ? lines_with_called_defs(v) : v }
365
649
  { "passed" => passed, "coverage" => coverage,
366
650
  "loaded_files" => capture_loaded_files }
367
651
  rescue Exception => e # rubocop:disable Lint/RescueException
@@ -379,18 +663,22 @@ module Mutineer
379
663
  end
380
664
  end
381
665
  wr.close
382
- data = rd.read
383
- rd.close
384
- _, status = Process.waitpid2(pid)
666
+ # Marshal.load reads exactly one object, not until EOF: a process that the
667
+ # test leaves running (even outside the group) can hold the pipe open.
668
+ status, payload = await_child(pid, deadline) { read_marshal(rd) }
669
+ return :timeout unless status
670
+
385
671
  # An empty pipe means the child died before writing (e.g. a hard crash,
386
672
  # OOM, or a signal from the test's own subprocess handling). Report HOW it
387
673
  # died (exit status / signal) as a diagnostic string so --verbose has
388
674
  # something actionable instead of a silent "no result".
389
- return "child wrote no result (#{describe_status(status)})" if data.empty?
675
+ return "child wrote no result (#{describe_status(status)})" if payload.nil?
390
676
 
391
- Marshal.load(data)
677
+ payload
392
678
  rescue StandardError => e
393
679
  "parent could not read capture result: #{e.class}: #{e.message}"
680
+ ensure
681
+ [rd, wr].compact.each { |io| io.close unless io.closed? }
394
682
  end
395
683
 
396
684
  # Human description of a child Process::Status for capture diagnostics.
@@ -433,7 +721,8 @@ module Mutineer
433
721
  def fail_test(test_path, reason)
434
722
  rel = relativize(test_path)
435
723
  @failed_test_files << rel
436
- warn "[mutineer] coverage skipped for #{rel}: #{reason}"
724
+ # The reason can quote an app error, and so a credentialed DATABASE_URL.
725
+ warn "[mutineer] coverage skipped for #{rel}: #{Redaction.text(reason)}"
437
726
  nil
438
727
  end
439
728
 
@@ -462,6 +751,7 @@ module Mutineer
462
751
  end
463
752
 
464
753
  record_loaded(payload["loaded_files"])
754
+ record_load(payload["load_coverage"])
465
755
 
466
756
  unless payload["passed"]
467
757
  @failed_clean_tests << relativize(test_path)
@@ -477,9 +767,10 @@ module Mutineer
477
767
  #
478
768
  # @api private
479
769
  # @param after_fork [Proc, nil] boot-mode fork hook.
770
+ # @param test_paths [Array<String>] the tests to run, each alone.
480
771
  # @return [void]
481
- def verify_cached_clean(after_fork: nil)
482
- @test_paths.each do |test_path|
772
+ def verify_cached_clean(after_fork: nil, test_paths: @test_paths)
773
+ test_paths.each do |test_path|
483
774
  rel = relativize(test_path)
484
775
  next if @failed_test_files.include?(rel)
485
776
 
@@ -507,16 +798,9 @@ module Mutineer
507
798
  pending.each do |rel|
508
799
  test_path = @test_paths.find { |t| relativize(t) == rel } || rel
509
800
  if @boot_path
510
- payload = fork_capture(absolute(test_path), abs_sources, after_fork)
511
- case payload
512
- when Hash then accept_capture_payload(test_path, payload)
513
- when String
514
- fail_test(test_path, @verbose ? "fork capture failed: #{payload}" :
515
- "fork capture produced no result (re-run with --verbose for the error)")
516
- else fail_test(test_path, "fork capture produced no result")
517
- end
801
+ accept_fork_payload(test_path, timed(test_path) { fork_capture(absolute(test_path), abs_sources, after_fork) })
518
802
  else
519
- payload = capture(test_path)
803
+ payload = timed(test_path) { capture(test_path) }
520
804
  accept_capture_payload(test_path, payload) if payload
521
805
  end
522
806
  end
@@ -556,8 +840,8 @@ module Mutineer
556
840
  # or the result. With `result: true`, the child writes its result as one
557
841
  # line to fd {RESULT_FD}, a pipe that only the script uses. The child's
558
842
  # stderr is the parent's stderr, so warnings from the script reach the
559
- # user. A wall clock of `@capture_timeout` bounds the whole call, so a hung
560
- # test cannot wedge the run.
843
+ # user. A wall clock of `@capture_timeout` bounds the whole call (see
844
+ # {#await_child}), so a hung test cannot wedge the run.
561
845
  #
562
846
  # The parent reads one line, not until EOF: a process that a test leaves
563
847
  # running can inherit fd {RESULT_FD} (a `fork` without `exec` keeps it
@@ -574,27 +858,72 @@ module Mutineer
574
858
  deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @capture_timeout
575
859
  script_rd, script_wr = IO.pipe
576
860
  result_rd, result_wr = IO.pipe if result
577
- options = { in: script_rd, out: File::NULL }
861
+ options = { in: script_rd, out: File::NULL, pgroup: true }
578
862
  options[RESULT_FD] = result_wr if result
579
863
  pid = Process.spawn(RbConfig.ruby, "-", **options)
580
- waiter = Process.detach(pid)
581
864
  script_rd.close
582
865
  result_wr&.close
583
- reader = Thread.new { result_rd.gets.to_s } if result
584
- script_wr.write(script)
585
- script_wr.close
586
- unless waiter.join(remaining(deadline))
587
- Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
588
- waiter.join
589
- reader&.kill
590
- return [nil, ""]
866
+ begin
867
+ script_wr.write(script)
868
+ rescue Errno::EPIPE
869
+ # The child exited before reading its script; await_child still reaps it.
591
870
  end
592
- [waiter.value, reader&.join(remaining(deadline))&.value.to_s]
871
+ script_wr.close
872
+ status, out = await_child(pid, deadline) { read_result_line(result_rd) if result }
873
+ [status, out.to_s]
593
874
  ensure
594
- reader&.kill
595
875
  [script_rd, script_wr, result_rd, result_wr].compact.each { |io| io.close unless io.closed? }
596
876
  end
597
877
 
878
+ # Reads the one result line a {#spawn_script} child writes to fd
879
+ # {RESULT_FD}.
880
+ #
881
+ # @api private
882
+ # @param io [IO] the parent end of the result pipe.
883
+ # @return [String] the line, or `""` at end of file.
884
+ def read_result_line(io) = io.gets.to_s
885
+
886
+ # Reads the one Marshal object a forked capture child writes, without
887
+ # waiting for end of file.
888
+ #
889
+ # @api private
890
+ # @param io [IO] the parent end of the binary result pipe.
891
+ # @return [Object, nil] the object, or nil when the child wrote none.
892
+ def read_marshal(io)
893
+ Marshal.load(io)
894
+ rescue EOFError
895
+ nil
896
+ end
897
+
898
+ # Waits for capture child `pid`, which leads its own process group, while
899
+ # a thread runs the block to read the child's result. One `deadline`
900
+ # bounds the run, the read, and the reap. Past it, the whole group gets
901
+ # SIGKILL, so a process the test started does not outlive a hung capture.
902
+ # Process.detach owns reaping. The same deadline and group kill as
903
+ # DaemonServer#wait_verdict, plus a reader, because the child reports
904
+ # through a pipe.
905
+ #
906
+ # A child that exits before the deadline has written its result, so the
907
+ # reader gets the time left, and at least {RESULT_GRACE} seconds, to finish.
908
+ #
909
+ # @api private
910
+ # @param pid [Integer] child pid, also its process group id.
911
+ # @param deadline [Float] a CLOCK_MONOTONIC time.
912
+ # @yieldreturn [Object] what the child wrote.
913
+ # @return [Array(Process::Status, Object)] the exit status and the block's
914
+ # value (nil when the reader did not finish); `[nil, nil]` after a timeout.
915
+ def await_child(pid, deadline, &read)
916
+ waiter = Process.detach(pid)
917
+ reader = Thread.new(&read)
918
+ reader.report_on_exception = false
919
+ return [nil, nil] unless waiter.join(remaining(deadline))
920
+
921
+ [waiter.value, reader.join([remaining(deadline), RESULT_GRACE].max)&.value]
922
+ ensure
923
+ kill_group(pid, waiter) if waiter&.alive?
924
+ reader&.kill
925
+ end
926
+
598
927
  # Seconds left before `deadline`, never negative.
599
928
  #
600
929
  # @api private
@@ -623,12 +952,15 @@ module Mutineer
623
952
  # @param after_fork [Proc, nil] boot-mode fork hook.
624
953
  # @return [Boolean]
625
954
  def fork_clean_pass?(abs_tests, after_fork)
955
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @capture_timeout
626
956
  rd, wr = IO.pipe
627
957
  rd.binmode
628
958
  wr.binmode
959
+ parent = Process.pid
629
960
  pid = fork do
630
961
  rd.close
631
962
  Process.setpgid(0, 0) rescue nil # rubocop:disable Style/RescueModifier
963
+ OrphanGuard.start(parent)
632
964
  begin
633
965
  ChildStdout.silence
634
966
  after_fork&.call
@@ -642,34 +974,32 @@ module Mutineer
642
974
  end
643
975
  end
644
976
  wr.close
645
- readable, = IO.select([rd], nil, nil, @capture_timeout)
646
- unless readable
647
- kill_fork_clean(pid)
648
- rd.close
649
- return false
650
- end
651
- data = rd.read
652
- rd.close
653
- Process.waitpid2(pid)
654
- return false if data.empty?
655
-
656
- Marshal.load(data)
977
+ _, passed = await_child(pid, deadline) { read_marshal(rd) }
978
+ passed == true
657
979
  rescue StandardError
658
980
  false
981
+ ensure
982
+ [rd, wr].compact.each { |io| io.close unless io.closed? }
659
983
  end
660
984
 
661
- # SIGKILLs a hung clean-check child (and its group) then reaps it.
985
+ # SIGKILLs capture child `pid` and its process group, then waits for
986
+ # `waiter` to reap the child. Falls back to the pid alone when the group
987
+ # does not exist yet.
662
988
  #
663
989
  # @api private
664
- # @param pid [Integer] child pid.
990
+ # @param pid [Integer] child pid, also its process group id.
991
+ # @param waiter [Thread] the Process.detach thread for `pid`.
665
992
  # @return [void]
666
- def kill_fork_clean(pid)
993
+ def kill_group(pid, waiter)
667
994
  begin
668
995
  Process.kill(:KILL, -pid)
669
996
  rescue Errno::ESRCH, Errno::EPERM
670
- Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
997
+ # No group yet: the child has not run setpgid. Never signal a reaped pid.
998
+ if waiter.alive?
999
+ Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
1000
+ end
671
1001
  end
672
- Process.waitpid2(pid) rescue nil # rubocop:disable Style/RescueModifier
1002
+ waiter.join
673
1003
  end
674
1004
 
675
1005
  # Builds a pass/fail-only subprocess script (no coverage instrumentation).
@@ -681,8 +1011,9 @@ module Mutineer
681
1011
  @framework == "rspec" ? rspec_clean_check_script(test_paths) : minitest_clean_check_script(test_paths)
682
1012
  end
683
1013
 
684
- # Minitest clean-suite check. Preloads configured sources like capture and
685
- # standalone {Runner.execute}, so tests that rely on that preload stay green.
1014
+ # Minitest clean-suite check. Preloads configured sources and `--require`
1015
+ # files like capture and standalone {Runner.execute}, so tests that rely on
1016
+ # that preload stay green.
686
1017
  #
687
1018
  # @api private
688
1019
  # @param test_paths [Array<String>] test file paths.
@@ -697,7 +1028,7 @@ module Mutineer
697
1028
  Minitest.define_singleton_method(:plugin_mutineer_report_init) { |options| reporter << Minitest::SummaryReporter.new(_report, options) }
698
1029
  Minitest.extensions << "mutineer_report"
699
1030
  $LOAD_PATH.unshift(*#{abs_load_paths.inspect})
700
- #{abs_source_paths.inspect}.each { |f| require f }
1031
+ #{abs_preload_paths.inspect}.each { |f| require f }
701
1032
  #{loads}
702
1033
  _passed = Minitest.run([])
703
1034
  $stderr.write(_report.string) unless _passed
@@ -705,7 +1036,8 @@ module Mutineer
705
1036
  RUBY
706
1037
  end
707
1038
 
708
- # RSpec clean-suite check. Preloads configured sources like capture.
1039
+ # RSpec clean-suite check. Preloads configured sources and `--require`
1040
+ # files like capture.
709
1041
  #
710
1042
  # @api private
711
1043
  # @param test_paths [Array<String>] spec file paths.
@@ -721,7 +1053,7 @@ module Mutineer
721
1053
  end
722
1054
  RSpec::Core::Runner.disable_autorun!
723
1055
  $LOAD_PATH.unshift(*#{abs_load_paths.inspect})
724
- #{abs_source_paths.inspect}.each { |f| require f }
1056
+ #{abs_preload_paths.inspect}.each { |f| require f }
725
1057
  _sink = StringIO.new
726
1058
  status = RSpec::Core::Runner.run(["--no-color", #{specs}], _sink, _sink)
727
1059
  $stderr.write(_sink.string) unless status.zero?
@@ -754,13 +1086,16 @@ module Mutineer
754
1086
  _report = StringIO.new
755
1087
  Minitest.define_singleton_method(:plugin_mutineer_report_init) { |options| reporter << Minitest::SummaryReporter.new(_report, options) }
756
1088
  Minitest.extensions << "mutineer_report"
757
- Coverage.start(lines: true)
1089
+ Coverage.start(lines: true, methods: true)
758
1090
  $LOAD_PATH.unshift(*#{abs_load_paths.inspect})
759
- #{abs_source_paths.inspect}.each { |f| require f }
1091
+ #{abs_preload_paths.inspect}.each { |f| require f }
1092
+ _load = #{load_coverage_expression}
760
1093
  load #{absolute(test_path).inspect}
761
1094
  _passed = Minitest.run([])
762
1095
  $stderr.write(_report.string) unless _passed
763
- _result.puts JSON.generate("passed" => _passed == true, "coverage" => Coverage.result,
1096
+ _coverage = Coverage.result.transform_values { |v| v.is_a?(Hash) ? v[:lines] : v }
1097
+ _result.puts JSON.generate("passed" => _passed == true, "coverage" => _coverage,
1098
+ "load_coverage" => _load,
764
1099
  "loaded_files" => #{loaded_files_expression})
765
1100
  _result.close
766
1101
  RUBY
@@ -785,13 +1120,16 @@ module Mutineer
785
1120
  exit 3
786
1121
  end
787
1122
  RSpec::Core::Runner.disable_autorun!
788
- Coverage.start(lines: true)
1123
+ Coverage.start(lines: true, methods: true)
789
1124
  $LOAD_PATH.unshift(*#{abs_load_paths.inspect})
790
- #{abs_source_paths.inspect}.each { |f| require f }
1125
+ #{abs_preload_paths.inspect}.each { |f| require f }
1126
+ _load = #{load_coverage_expression}
791
1127
  _sink = StringIO.new
792
1128
  _status = RSpec::Core::Runner.run(["--no-color", #{absolute(test_path).inspect}], _sink, _sink)
793
1129
  $stderr.write(_sink.string) unless _status.zero?
794
- _result.puts JSON.generate("passed" => _status.zero?, "coverage" => Coverage.result,
1130
+ _coverage = Coverage.result.transform_values { |v| v.is_a?(Hash) ? v[:lines] : v }
1131
+ _result.puts JSON.generate("passed" => _status.zero?, "coverage" => _coverage,
1132
+ "load_coverage" => _load,
795
1133
  "loaded_files" => #{loaded_files_expression})
796
1134
  _result.close
797
1135
  RUBY
@@ -802,16 +1140,95 @@ module Mutineer
802
1140
  # path outside the project (stdlib/gem files).
803
1141
  def record(coverage, test_path)
804
1142
  rel_test = relativize(test_path)
1143
+ each_covered_key(coverage) { |key| (@map[key] ||= []) << rel_test }
1144
+ end
1145
+
1146
+ # Adds the configured source lines that ran at load to {#load_lines}, and
1147
+ # the methods called at load to {#load_methods}. Lines of other files
1148
+ # (tests, helpers, gems) are dropped.
1149
+ #
1150
+ # @api private
1151
+ # @param coverage [Hash, nil] counts per absolute file, as {#record} reads
1152
+ # them; a Hash entry can also hold "methods", as {#load_entry} builds it.
1153
+ # @return [void]
1154
+ def record_load(coverage)
1155
+ return unless coverage.is_a?(Hash)
1156
+
1157
+ @source_rels ||= @source_paths.to_set { |p| relativize(absolute(p)) }
1158
+ ours = coverage.select { |abs_file, _| @source_rels.include?(relativize(abs_file)) }
1159
+ each_covered_key(ours) { |key| @load_lines << key }
1160
+ ours.each do |abs_file, data|
1161
+ next unless data.is_a?(Hash) && data["methods"].is_a?(Array)
1162
+
1163
+ data["methods"].each { |line, column| @load_methods << "#{relativize(abs_file)}:#{line}:#{column}" }
1164
+ end
1165
+ end
1166
+
1167
+ # One file's line counts, with the `def` line of each method the test
1168
+ # called counted too (#209). A forked capture does not see the `def` lines
1169
+ # run, since the boot defined the methods before the test. Ruby counts no
1170
+ # line when an endless method runs, so without this its mutants would be
1171
+ # `no_coverage` even when a test calls it. A line where more than one
1172
+ # method starts is not counted: the map keys tests by line, so the test
1173
+ # would also be credited with the methods it did not call.
1174
+ #
1175
+ # @api private
1176
+ # @param data [Hash] `Coverage` result for one file, with `:lines` and `:methods`.
1177
+ # @return [Array, nil] line counts.
1178
+ def lines_with_called_defs(data)
1179
+ lines = data[:lines]
1180
+ return lines unless lines && data[:methods]
1181
+
1182
+ lines = lines.dup
1183
+ starts = data[:methods].keys.map { |key| key[2] }.tally
1184
+ data[:methods].each do |key, count|
1185
+ line = key[2]
1186
+ lines[line - 1] = [lines[line - 1].to_i, count].max if count.positive? && starts[line] == 1
1187
+ end
1188
+ lines
1189
+ end
1190
+
1191
+ # One file's load-time coverage in the shape {#record_load} reads: the line
1192
+ # counts, and the `[line, column]` of each method called at least once.
1193
+ # {#load_coverage_expression} builds the same shape in a capture child.
1194
+ #
1195
+ # @api private
1196
+ # @param lines [Array, nil] line counts.
1197
+ # @param methods [Hash, nil] `Coverage` method counts, keyed by
1198
+ # `[owner, name, start_line, start_column, end_line, end_column]`.
1199
+ # @return [Hash{String => Array}]
1200
+ def load_entry(lines, methods)
1201
+ { "lines" => lines, "methods" => (methods || {}).filter_map { |key, count| key[2, 2] if count.positive? } }
1202
+ end
1203
+
1204
+ # Ruby source of the capture child's load-time coverage: `Coverage.peek_result`
1205
+ # for the configured sources only, matched by real path (a Coverage key is
1206
+ # the path as required), so the payload does not carry every loaded gem.
1207
+ # Each entry has the shape {#load_entry} builds.
1208
+ #
1209
+ # @api private
1210
+ # @return [String] expression to embed in a capture subprocess script.
1211
+ def load_coverage_expression
1212
+ "Coverage.peek_result.select { |f, _| #{abs_source_paths.inspect}.include?((File.realpath(f) rescue f)) }" \
1213
+ ".transform_values { |v| { \"lines\" => v[:lines], " \
1214
+ "\"methods\" => v[:methods].filter_map { |k, n| k[2, 2] if n.positive? } } }"
1215
+ end
1216
+
1217
+ # Yields a "file:line" key for every project line with a non-zero count.
1218
+ #
1219
+ # @api private
1220
+ # @param coverage [Hash] counts per absolute file: an Array, or a Hash with "lines".
1221
+ # @yieldparam key [String] the "file:line" key.
1222
+ # @return [void]
1223
+ def each_covered_key(coverage)
805
1224
  coverage.each do |abs_file, data|
806
1225
  rel = relativize(abs_file)
807
1226
  next if rel.start_with?("/") # outside project_root: not our source
808
1227
 
809
- counts = data.is_a?(Array) ? data : data["lines"]
810
- counts.each_with_index do |count, idx|
811
- next unless count&.positive?
1228
+ counts = data.is_a?(Hash) ? data["lines"] : data
1229
+ next unless counts.is_a?(Array) # a Coverage mode without line counts
812
1230
 
813
- (@map["#{rel}:#{idx + 1}"] ||= []) << rel_test
814
- end
1231
+ counts.each_with_index { |count, idx| yield "#{rel}:#{idx + 1}" if count&.positive? }
815
1232
  end
816
1233
  end
817
1234
 
@@ -906,16 +1323,32 @@ module Mutineer
906
1323
  end
907
1324
  end
908
1325
 
1326
+ # True when `cached` records the same +PARALLEL_WORKERS+ value this process
1327
+ # has now. Rails reads that variable inside +parallelize+ and can run one
1328
+ # test file in several workers. A missing key is a map saved before this
1329
+ # value was stored. In-process runs and the daemon write this same file.
1330
+ #
1331
+ # @api private
1332
+ # @param cached [Hash] parsed coverage.json.
1333
+ # @return [Boolean]
1334
+ def parallel_workers_match?(cached)
1335
+ cached.key?("parallel_workers") && cached["parallel_workers"] == ENV.fetch("PARALLEL_WORKERS", nil)
1336
+ end
1337
+
909
1338
  # Digest each file's ROLE + relative path + content length + content, plus
910
1339
  # the load_paths. Without role/path/length delimiters the digest collides
911
1340
  # (("ab","c") == ("a","bc")) and is blind to source/test role swaps, silently
912
1341
  # accepting a stale cached map.
913
- def compute_digest
1342
+ #
1343
+ # @param inputs_only [Boolean] leave out the sources and their owners.
1344
+ def compute_digest(inputs_only: false)
914
1345
  d = Digest::SHA256.new
915
- digest_group(d, "source", @source_paths)
1346
+ digest_group(d, "source", @source_paths) unless inputs_only
916
1347
  digest_group(d, "test", @test_paths)
917
- digest_group(d, "boot", [boot_digest_path]) if @boot_path
918
- ownership_paths.each do |rel|
1348
+ # One group per file: digest_group sorts, and the load order matters.
1349
+ @require_paths.each { |p| digest_group(d, "require", [digest_path(p)]) }
1350
+ digest_group(d, "boot", [digest_path(@boot_path)]) if @boot_path
1351
+ (inputs_only ? [] : ownership_paths).each do |rel|
919
1352
  d.update("owner\0")
920
1353
  d.update(rel)
921
1354
  d.update("\0")
@@ -961,10 +1394,13 @@ module Mutineer
961
1394
  paths.uniq.sort
962
1395
  end
963
1396
 
964
- # boot_path is a require-style path (e.g. "config/environment", no extension);
965
- # resolve it to the real file for reading, appending ".rb" when needed.
966
- def boot_digest_path
967
- File.exist?(absolute(@boot_path)) ? @boot_path : "#{@boot_path}.rb"
1397
+ # boot_path and require_paths are require-style paths (e.g. "config/environment",
1398
+ # no extension); resolve one to the file `require` loads: the path itself, then
1399
+ # ".rb", then a native extension. A folder of the same name (lib/my_gem/ next to
1400
+ # lib/my_gem.rb) is not the file.
1401
+ def digest_path(path)
1402
+ ["", ".rb", ".#{RbConfig::CONFIG['DLEXT']}"].map { |ext| "#{path}#{ext}" }
1403
+ .find { |p| File.file?(absolute(p)) } || "#{path}.rb"
968
1404
  end
969
1405
 
970
1406
  # Groups a digest with its role and paths.
@@ -1026,7 +1462,13 @@ module Mutineer
1026
1462
 
1027
1463
  FileUtils.mkdir_p(@cache_dir)
1028
1464
  data = { "digest" => @digest, "failed_test_files" => @failed_test_files,
1029
- "dependencies" => @loaded_dependencies, "map" => @map }
1465
+ "dependencies" => @loaded_dependencies, "map" => @map,
1466
+ "load_lines" => @load_lines.to_a.sort, "load_methods" => @load_methods.to_a.sort,
1467
+ "timings" => @timings, "parallel_workers" => ENV.fetch("PARALLEL_WORKERS", nil) }
1468
+ if @boot_path
1469
+ data.merge!("inputs" => compute_digest(inputs_only: true), "sources" => source_fingerprints,
1470
+ "owners" => ownership_paths)
1471
+ end
1030
1472
  tmp = "#{cache_path}.tmp"
1031
1473
  File.write(tmp, JSON.generate(data))
1032
1474
  File.rename(tmp, cache_path) # atomic swap
@@ -1046,6 +1488,13 @@ module Mutineer
1046
1488
  # @return [Array<String>] absolute source paths.
1047
1489
  def abs_source_paths = @source_paths.map { |p| absolute(p) }
1048
1490
 
1491
+ # The sources, then the `--require` files, in the order standalone
1492
+ # {Runner.execute} requires them before it forks the mutants (#217).
1493
+ #
1494
+ # @api private
1495
+ # @return [Array<String>] absolute paths.
1496
+ def abs_preload_paths = abs_source_paths + @require_paths.map { |p| absolute(p) }
1497
+
1049
1498
  # Returns absolute load paths.
1050
1499
  #
1051
1500
  # @return [Array<String>] absolute load paths.