saboteur 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1024 -0
  3. data/README.md +199 -952
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/workflow.rb +2 -1
  6. data/lib/saboteur/baseline_health.rb +40 -6
  7. data/lib/saboteur/cli/exit_status.rb +80 -4
  8. data/lib/saboteur/cli/help_options.rb +17 -0
  9. data/lib/saboteur/cli/history_command.rb +135 -0
  10. data/lib/saboteur/cli/presets.rb +40 -0
  11. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  12. data/lib/saboteur/cli/run_reporting.rb +74 -5
  13. data/lib/saboteur/cli/selection_options.rb +31 -0
  14. data/lib/saboteur/cli/shard_options.rb +17 -0
  15. data/lib/saboteur/cli/source_set.rb +53 -0
  16. data/lib/saboteur/cli/survivors_command.rb +157 -0
  17. data/lib/saboteur/cli.rb +136 -98
  18. data/lib/saboteur/config.rb +163 -15
  19. data/lib/saboteur/config_file.rb +8 -0
  20. data/lib/saboteur/constant_reads.rb +182 -0
  21. data/lib/saboteur/constant_subject.rb +70 -0
  22. data/lib/saboteur/constant_subject_finder.rb +281 -0
  23. data/lib/saboteur/coverage_reads.rb +29 -0
  24. data/lib/saboteur/crash_report.rb +154 -0
  25. data/lib/saboteur/custom_map.rb +3 -1
  26. data/lib/saboteur/dirty_tree_warning.rb +128 -0
  27. data/lib/saboteur/doctor.rb +49 -1
  28. data/lib/saboteur/engine/baseline_guard.rb +136 -14
  29. data/lib/saboteur/engine/boot.rb +16 -1
  30. data/lib/saboteur/engine/canary.rb +41 -10
  31. data/lib/saboteur/engine/candidate_selection.rb +28 -1
  32. data/lib/saboteur/engine/candidates.rb +43 -3
  33. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  34. data/lib/saboteur/engine/coverage_store.rb +79 -15
  35. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  36. data/lib/saboteur/engine/diagnostics.rb +41 -2
  37. data/lib/saboteur/engine/diff_scope.rb +24 -5
  38. data/lib/saboteur/engine/distribution.rb +134 -8
  39. data/lib/saboteur/engine/early_stop_recording.rb +47 -0
  40. data/lib/saboteur/engine/gated_run.rb +35 -0
  41. data/lib/saboteur/engine/load_coverage.rb +45 -0
  42. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  43. data/lib/saboteur/engine/pipelines.rb +107 -0
  44. data/lib/saboteur/engine/result_cache.rb +67 -9
  45. data/lib/saboteur/engine/run_budget.rb +90 -0
  46. data/lib/saboteur/engine/run_fingerprint.rb +4 -0
  47. data/lib/saboteur/engine/sandbox_policy.rb +56 -11
  48. data/lib/saboteur/engine/scheduler.rb +222 -20
  49. data/lib/saboteur/engine/selection_announcement.rb +54 -0
  50. data/lib/saboteur/engine/shard.rb +69 -0
  51. data/lib/saboteur/engine/subject_selection.rb +373 -19
  52. data/lib/saboteur/engine/suite_files.rb +3 -1
  53. data/lib/saboteur/engine/test_guess.rb +438 -25
  54. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  55. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  56. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  57. data/lib/saboteur/engine/verdict_check.rb +54 -18
  58. data/lib/saboteur/engine.rb +202 -158
  59. data/lib/saboteur/execution/external_runner.rb +471 -0
  60. data/lib/saboteur/execution/marker_sweep.rb +79 -0
  61. data/lib/saboteur/execution/peak_memory.rb +28 -0
  62. data/lib/saboteur/execution/process_table.rb +12 -0
  63. data/lib/saboteur/execution/process_tree.rb +65 -5
  64. data/lib/saboteur/execution/protocol.rb +2 -1
  65. data/lib/saboteur/execution/reevaluator.rb +68 -12
  66. data/lib/saboteur/execution/runner.rb +225 -24
  67. data/lib/saboteur/execution/sandbox.rb +21 -3
  68. data/lib/saboteur/execution/seatbelt.rb +226 -0
  69. data/lib/saboteur/execution/supervisor.rb +45 -13
  70. data/lib/saboteur/execution/worker_pool.rb +110 -8
  71. data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
  72. data/lib/saboteur/gates.rb +43 -2
  73. data/lib/saboteur/help.rb +108 -13
  74. data/lib/saboteur/ignore_list.rb +40 -7
  75. data/lib/saboteur/install.rb +4 -3
  76. data/lib/saboteur/integration/base.rb +7 -2
  77. data/lib/saboteur/integration/rspec.rb +175 -9
  78. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  79. data/lib/saboteur/mutation_source/builtin.rb +44 -14
  80. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  81. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  82. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  83. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  84. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  85. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  86. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  87. data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
  88. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  89. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  90. data/lib/saboteur/mutation_source/method_swap.rb +193 -0
  91. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  92. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  93. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  94. data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
  95. data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
  96. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  97. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  98. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  99. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  100. data/lib/saboteur/preview.rb +122 -13
  101. data/lib/saboteur/refinement_activations.rb +60 -0
  102. data/lib/saboteur/reporter/agent.rb +9 -1
  103. data/lib/saboteur/reporter/agent_commands.rb +47 -15
  104. data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
  105. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  106. data/lib/saboteur/reporter/findings.rb +5 -2
  107. data/lib/saboteur/reporter/history_policy.rb +11 -0
  108. data/lib/saboteur/reporter/history_reader.rb +47 -0
  109. data/lib/saboteur/reporter/html.rb +26 -4
  110. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  111. data/lib/saboteur/reporter/next_steps.rb +14 -4
  112. data/lib/saboteur/reporter/score_history.rb +5 -1
  113. data/lib/saboteur/reporter/stryker.rb +40 -7
  114. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  115. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  116. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  117. data/lib/saboteur/reporter/terminal.rb +74 -17
  118. data/lib/saboteur/result.rb +38 -7
  119. data/lib/saboteur/selection/coverage_map.rb +107 -11
  120. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  121. data/lib/saboteur/selection/external_test_map.rb +27 -0
  122. data/lib/saboteur/selection/git_diff.rb +105 -13
  123. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  124. data/lib/saboteur/selection/recorder.rb +22 -4
  125. data/lib/saboteur/selection/source_scope.rb +22 -0
  126. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  127. data/lib/saboteur/subject.rb +53 -7
  128. data/lib/saboteur/subject_finder.rb +173 -29
  129. data/lib/saboteur/text.rb +42 -0
  130. data/lib/saboteur/version.rb +1 -1
  131. data/lib/saboteur/working_tree.rb +38 -15
  132. data/lib/saboteur.rb +29 -1
  133. metadata +50 -2
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require_relative "process_table"
5
+ require_relative "marker_sweep"
5
6
 
6
7
  module Saboteur
7
8
  module Execution
@@ -12,14 +13,19 @@ module Saboteur
12
13
  # setsid (a nested saboteur run does exactly that) and leave the group.
13
14
  # Parent links survive that, so the tree is rebuilt from the process table
14
15
  # on every poll, and every pid ever seen in it is remembered, so a
15
- # descendant whose parent exits (re-parented to init) is still known.
16
+ # descendant whose parent exits (re-parented to init) is still known -
17
+ # but only if some poll SAW the parent link while the intermediate lived.
18
+ # A descendant that leaves the group and loses that parent between two
19
+ # polls is never seen (#365); the marker sweep in kill_all is the net
20
+ # for exactly that case.
16
21
  #
17
22
  # Identity is pid plus start time, so a recycled pid is never signalled.
18
23
  class ProcessTree
19
24
  attr_reader :root
20
25
 
21
- def initialize(root)
26
+ def initialize(root, marker: nil)
22
27
  @root = root
28
+ @marker = marker
23
29
  @known = {} # pid => start time
24
30
  end
25
31
 
@@ -34,6 +40,27 @@ module Saboteur
34
40
  end
35
41
  end
36
42
 
43
+ # kill_all minus an excluded root, with the pre-kill member
44
+ # descriptions: [killed pids, notes]. The notes are taken first -
45
+ # after the kill the command lines are gone (#486).
46
+ def kill_all_except(root)
47
+ notes = describe_except(root)
48
+ [kill_all - [root], notes]
49
+ end
50
+
51
+ # One "pid (command)" note per live member other than an excluded
52
+ # root, for the leftover-children refusal (#486). Best effort: a
53
+ # process that vanished mid-look lists its pid alone. Call before the
54
+ # kill - after it, the command lines are gone.
55
+ def describe_except(root, table = ProcessTable.snapshot)
56
+ live_members(table).filter_map do |entry|
57
+ next if entry.pid == root
58
+
59
+ command = ProcessTable.command(entry.pid)
60
+ command ? "#{entry.pid} (#{command})" : entry.pid.to_s
61
+ end
62
+ end
63
+
37
64
  # Count and resident memory of the live members. rss_baseline_kb is the
38
65
  # supervisor's own RSS: forked children share its pages, so only growth
39
66
  # beyond it is counted per process.
@@ -44,15 +71,20 @@ module Saboteur
44
71
  end
45
72
 
46
73
  # Stops every member (a stopped process cannot fork), rescans until no new
47
- # member appears, then kills them all. Returns the pids it killed.
74
+ # member appears, then kills them all. Each round also sweeps for
75
+ # never-seen escapees carrying the run's marker (#365). Returns the
76
+ # pids it killed.
48
77
  def kill_all(rounds: 20)
49
78
  killed = []
50
79
  rounds.times do
51
80
  live = stop_until_stable
52
- break if live.empty?
81
+ escaped = marked_escapees
82
+ break if live.empty? && escaped.empty?
53
83
 
54
84
  live.each { |entry| signal("KILL", entry.pid) }
55
- killed.concat(live.map(&:pid))
85
+ signalled = escaped.select { |entry| escapee_alive?(entry) }
86
+ signalled.each { |entry| signal("KILL", entry.pid) }
87
+ killed.concat(live.map(&:pid), signalled.map(&:pid))
56
88
  sleep 0.01
57
89
  end
58
90
  killed.uniq
@@ -60,6 +92,34 @@ module Saboteur
60
92
 
61
93
  private
62
94
 
95
+ # Live, non-zombie pids carrying the run's marker that the tree never
96
+ # learned: the setsid-and-orphan escapees (#365). Members match the
97
+ # marker too, but the same round kills them anyway.
98
+ def marked_escapees
99
+ return [] unless @marker && MarkerSweep.available?
100
+
101
+ table = ProcessTable.snapshot
102
+ root_started = @known[root]
103
+ MarkerSweep.marked_pids(@marker, Supervisor::TREE_ENV).filter_map do |pid|
104
+ entry = table[pid]
105
+ next unless entry && !entry.zombie && !@known.key?(pid)
106
+
107
+ # A process started before the run's root carries the marker only
108
+ # through an exec chain the run did not spawn: never ours (#389).
109
+ next if root_started && entry.started < root_started
110
+
111
+ entry
112
+ end
113
+ end
114
+
115
+ # A pid is not an identity: the escapee can exit between the sweep's
116
+ # snapshot and the KILL, and the kernel can hand the pid to someone
117
+ # else. Signal only the process the sweep actually saw (#389).
118
+ def escapee_alive?(entry)
119
+ fresh = ProcessTable.snapshot[entry.pid]
120
+ fresh && !fresh.zombie && fresh.started == entry.started
121
+ end
122
+
63
123
  def grow(table)
64
124
  @known[root] = table[root].started if @known.empty? && table[root]
65
125
  loop do
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require "json"
5
+ require_relative "../text"
5
6
 
6
7
  module Saboteur
7
8
  module Execution
@@ -31,7 +32,7 @@ module Saboteur
31
32
  # Events in order; lines that are not this protocol's are ignored.
32
33
  def parse(payload)
33
34
  payload.each_line.filter_map do |line|
34
- message = JSON.parse(line)
35
+ message = JSON.parse(Text.utf8_strict(line))
35
36
  message if message.is_a?(Hash) && message["v"] == VERSION
36
37
  rescue JSON::ParserError
37
38
  nil
@@ -54,9 +54,54 @@ module Saboteur
54
54
  nil
55
55
  end
56
56
 
57
+ # The method object the subject's def currently resolves to,
58
+ # normalized through a known def-time wrapper. The runner compares
59
+ # what the mutant's evaluation installed with what is in place after
60
+ # the tests: a suite that re-loads the file mid-run (a before(:each)
61
+ # `load "lib/..."`, rolify-style) quietly replaces the mutant, and a
62
+ # "survivor" then only proves the tests never ran it. nil when this
63
+ # cannot be told (a missing or non-Ruby method): the caller keeps the
64
+ # verdict.
65
+ def installed_definition(subject)
66
+ owner = resolve_owner(subject)
67
+ target = subject.singleton ? owner.singleton_class : owner
68
+ method = target.instance_method(subject.name)
69
+ Wrapper.original_def(method) || method
70
+ rescue StandardError
71
+ nil
72
+ end
73
+
74
+ # The definition now in place for the subject versus the one the
75
+ # mutant's evaluation installed: a "file:line" site when the tests
76
+ # provably replaced it - a before(:each) re-loading the subject's
77
+ # own file (rolify-style `load "lib/..."`, #597) puts the original
78
+ # back before any test calls it, so a "survivor" proves nothing.
79
+ # Only a positively identified overwrite counts: the replacement
80
+ # comes from the subject's own file, and no prepend or
81
+ # allow_any_instance_of-style wrapper whose super chain still
82
+ # reaches the mutant (those tests did run it). Anything
83
+ # undecidable returns nil and the verdict stands.
84
+ def overwritten_site(subject, installed)
85
+ now = installed_definition(subject)
86
+ return nil if now.nil? || now == installed
87
+
88
+ method = now
89
+ method = method.super_method until method.nil? || method == installed
90
+ return nil if method # the mutant is still reachable through super
91
+
92
+ file, line = now.source_location
93
+ return nil unless file && File.expand_path(file) == File.expand_path(subject.file)
94
+
95
+ "#{file}:#{line}"
96
+ rescue StandardError
97
+ nil
98
+ end
99
+
57
100
  # True when the subject's id matches a whole_file entry (exact or a
58
101
  # File.fnmatch glob).
59
102
  def whole_file?(subject)
103
+ return true if subject.whole_file_only?
104
+
60
105
  @whole_file.any? { |pattern| File.fnmatch?(pattern, subject.id) }
61
106
  end
62
107
 
@@ -87,12 +132,15 @@ module Saboteur
87
132
 
88
133
  # Resolves the subject's owner the way Ruby would resolve the nesting in
89
134
  # the file. const_get also triggers autoload (Zeitwerk) before any reopening.
135
+ # A `class << self` scope resolves the scopes after it (a factory
136
+ # constant assigned there binds on the singleton class, #352); when the
137
+ # subject's own scope is the singleton class, the owner is still the
138
+ # attached class.
90
139
  def resolve_owner(subject)
91
- scopes = [Object]
92
- subject.nesting.reject(&:sclass?).each do |scope|
93
- scopes << resolve_path(scope.path, scopes)
140
+ scopes = subject.nesting.each_with_object([Object]) do |scope, resolved|
141
+ resolved << (scope.sclass? ? resolved.last.singleton_class : resolve_path(scope.path, resolved))
94
142
  end
95
- owner = scopes.last
143
+ owner = subject.sclass ? scopes[-2] : scopes.last
96
144
  raise Error, "#{subject.owner} is not a class or module" unless owner.is_a?(Module)
97
145
 
98
146
  owner
@@ -139,28 +187,36 @@ module Saboteur
139
187
 
140
188
  # The file as it is on disk, with the method's bytes replaced by text,
141
189
  # evaluated at top level under the file's own name and first line.
142
- # Redefinition warnings are expected and silenced.
143
190
  def eval_whole_file(subject, text)
144
191
  source = subject.file_source.dup.force_encoding(Encoding::UTF_8)
145
192
  spliced = source.byteslice(0, subject.start_byte) + text + source.byteslice(subject.end_byte..)
146
- verbose = $VERBOSE
147
- $VERBOSE = nil
148
- TOPLEVEL_BINDING.eval(spliced, subject.file, 1)
149
- ensure
150
- $VERBOSE = verbose
193
+ silencing_warnings { TOPLEVEL_BINDING.eval(spliced, subject.file, 1) }
151
194
  end
152
195
 
153
196
  def eval_in_owner(owner, subject, text, extra_lines)
154
197
  code, line = with_magic_comment(subject, text, extra_lines)
155
198
  eval_target = subject.sclass ? owner.singleton_class : owner
156
- eval_target.class_eval(code, subject.file, line)
199
+ silencing_warnings { eval_target.class_eval(code, subject.file, line) }
157
200
  end
158
201
 
159
202
  def eval_nested(subject, text, extra_lines)
160
203
  headers = subject.nesting.map(&:header)
161
204
  wrapped = "#{headers.join('; ')}; #{text}; #{(['end'] * headers.size).join('; ')}"
162
205
  code, line = with_magic_comment(subject, wrapped, extra_lines)
163
- TOPLEVEL_BINDING.eval(code, subject.file, line)
206
+ silencing_warnings { TOPLEVEL_BINDING.eval(code, subject.file, line) }
207
+ end
208
+
209
+ # Re-evaluation redefines the method by design, and a "method
210
+ # redefined" warning is only ever emitted under $VERBOSE. Suites that
211
+ # raise on warnings (--warnings plus a raising Warning.warn or
212
+ # Warning.process handler, dry-rb/hanami-style) must not see it:
213
+ # before the guard every mutant errored as RuntimeError (#603).
214
+ def silencing_warnings
215
+ verbose = $VERBOSE
216
+ $VERBOSE = nil
217
+ yield
218
+ ensure
219
+ $VERBOSE = verbose
164
220
  end
165
221
 
166
222
  # The first line of code sits extra_lines above the def, so the def
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "json"
5
5
  require_relative "stale_captures"
6
+ require_relative "../crash_report"
6
7
 
7
8
  module Saboteur
8
9
  module Execution
@@ -39,17 +40,42 @@ module Saboteur
39
40
  end
40
41
  end
41
42
 
43
+ # The child log (or its whole directory) vanished mid-job (#234).
44
+ # Carries the digest of the mutant whose run just ended when the job
45
+ # had one, so a parallel parent can prefer this report over a
46
+ # bystander's bare "unknown mutant" report (#285).
47
+ class LogVanished < Error
48
+ attr_reader :culprit_digest
49
+
50
+ def initialize(message, culprit_digest: nil)
51
+ @culprit_digest = culprit_digest
52
+ super(message)
53
+ end
54
+ end
55
+
42
56
  LOG_TAIL_BYTES = 2000
57
+ CRASH_LOG_BYTES = 2_000
58
+ CRASH_LOG_LINES = 8
59
+ # File mtimes follow the filesystem's clock, which can trail the
60
+ # process clock; a log written during this run can stamp a little
61
+ # before its start, so the freshness check allows that skew.
62
+ CRASH_LOG_CLOCK_SLACK = 2
63
+ # Freshness matching can attach another run's log (clock slack, two
64
+ # nested runs in one child), so the label must not claim exact
65
+ # attribution (#296).
66
+ CRASH_LOG_LABEL = "recent saboteur crash log (freshness-matched, may be another run's):"
43
67
 
44
68
  # log_dir: where children write their output; a child's log is kept
45
- # only when it ends in an error.
69
+ # only when it ends in an error. crash_log: the crash log of a saboteur
70
+ # run nested inside a test; by default it sits next to the logs dir,
71
+ # where the engine's cache dir keeps it.
46
72
  # seed: every child seeds Ruby's random numbers with it before its
47
73
  # tests, so a test that uses rand sees the same values in every run.
48
74
  # sandbox: when given, a callable run first in every child (Landlock
49
75
  # makes the project tree read-only there, #152). It raises rather than
50
76
  # run half-restricted, which the child reports as an error.
51
77
  def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1,
52
- sandbox: nil)
78
+ sandbox: nil, canary: nil, crash_log: nil)
53
79
  @integration = integration
54
80
  @reevaluator = reevaluator
55
81
  @hooks = hooks
@@ -57,6 +83,8 @@ module Saboteur
57
83
  @supervisor = supervisor
58
84
  @seed = seed
59
85
  @sandbox = sandbox
86
+ @canary = canary
87
+ @crash_log = crash_log || File.join(File.dirname(log_dir), CrashReport::LOG_NAME)
60
88
  end
61
89
 
62
90
  # text: method source to evaluate in the child (the unchanged source for
@@ -69,23 +97,33 @@ module Saboteur
69
97
  @sequence = @sequence.to_i + 1
70
98
  job = Job.new(subject:, text:, test_ids:, timeout:, mutation:, eval_prefix:,
71
99
  log_path: File.join(@log_dir, "child-#{Process.pid}-#{@sequence}.log"))
72
- result = @supervisor.run(timeout: timeout) { |writer| child(writer, job) }
100
+ log_file = open_log(job)
101
+ result = @supervisor.run(timeout: timeout) { |writer| child(writer, job, log_file) }
73
102
  outcome = outcome_for(result, job)
74
- File.delete(job.log_path) if outcome.status != :error && File.exist?(job.log_path)
103
+ check_tree_still_there(job)
104
+ check_log_still_there(job)
105
+ File.delete(job.log_path) if outcome.status != :error
75
106
  outcome
107
+ ensure
108
+ log_file&.close
76
109
  end
77
110
 
78
111
  private
79
112
 
80
113
  # --- child ----------------------------------------------------------
81
114
 
82
- def child(writer, job)
115
+ def child(writer, job, log_file)
83
116
  # Redirect before restricting: the sweep closes inherited stdio
84
117
  # redirected to a regular file outside the grants (#178), and a
85
118
  # reopened-afterwards $stdout would start from a closed stream.
86
- # The log lives under the granted cache dir, so it survives.
87
- redirect_output(job.log_path)
88
- @sandbox&.call([writer.fileno])
119
+ # The parent opened the log before the fork, so the inherited
120
+ # descriptor survives on the keep list - the cache dir itself
121
+ # stays read-only inside the sandbox (#234). stdio was just
122
+ # reopened onto that log, so 1 and 2 point at a regular file
123
+ # outside the grants and the sweep would close them; keep them
124
+ # explicitly - keeping only the log's own fd is not enough (#283).
125
+ redirect_output(log_file)
126
+ @sandbox&.call([writer.fileno, log_file.fileno, $stdout.fileno, $stderr.fileno])
89
127
  Process.setproctitle(title(job))
90
128
  @hooks.run(:after_fork)
91
129
  # Before the evaluation: a whole-file subject runs the file's own
@@ -106,6 +144,7 @@ module Saboteur
106
144
  @reevaluator.call(job.subject, job.text, prefix: job.eval_prefix)
107
145
  replaced = @reevaluator.replaced_definition(job.subject)
108
146
  Protocol.emit(writer, "replaced", site: replaced) if replaced
147
+ @installed_definition = job.mutation ? @reevaluator.installed_definition(job.subject) : nil
109
148
  end
110
149
 
111
150
  # Capture detection is best-effort: a subject shape it cannot scan
@@ -118,18 +157,103 @@ module Saboteur
118
157
 
119
158
  def emit_result(writer, job, has_captures)
120
159
  method = replacement_method(job) if has_captures && job.mutation
121
- called = method ? false : nil
122
- trace = TracePoint.new(:call) { called = true } if method
123
- result = if trace
124
- trace.enable(target: method) { @hooks.around(job.mutation) { run_tests(job.test_ids) } }
125
- else
126
- @hooks.around(job.mutation) { run_tests(job.test_ids) }
127
- end
160
+ flags = { replacement_called: method ? false : nil, mutant_called: false }
161
+ target = method || (job.mutation ? @installed_definition : nil)
162
+ trace = call_probe(target, flags)
163
+ started_at = Time.now
164
+ suite = proc { @hooks.around(job.mutation) { run_tests(job.test_ids) } }
165
+ result = trace ? trace.enable(target: target, &suite) : suite.call
128
166
  if result.nil?
129
167
  Protocol.emit(writer, "error", message: "around_mutant hook did not run the tests")
130
168
  else
131
- Protocol.emit(writer, "result", **result, replacement_called: called)
169
+ called = flags[:replacement_called]
170
+ Protocol.emit(writer, "result", **with_crash_detail(result, started_at), replacement_called: called)
171
+ # A reload after the mutant already ran (an after-hook restore)
172
+ # is no reason to doubt the verdict; only a never-invoked mutant
173
+ # can have been hidden by the overwrite.
174
+ emit_overwritten(writer, job, flags[:mutant_called])
175
+ end
176
+ end
177
+
178
+ # A reload after the mutant already ran (an after-hook restore) is
179
+ # no reason to doubt the verdict; only a never-invoked mutant can
180
+ # have been hidden by the overwrite.
181
+ def emit_overwritten(writer, job, mutant_called)
182
+ overwritten = job.mutation && @installed_definition &&
183
+ @reevaluator.overwritten_site(job.subject, @installed_definition)
184
+ Protocol.emit(writer, "overwritten", site: overwritten) if overwritten && !mutant_called
185
+ end
186
+
187
+ # One call probe on the installed definition feeds both verdicts:
188
+ # replacement_called for stale captures, and overwrite detection for
189
+ # a mutant the suite replaced before any test ran it (#597).
190
+ def call_probe(target, flags)
191
+ target && TracePoint.new(:call) do
192
+ flags[:mutant_called] = true
193
+ flags[:replacement_called] = true unless flags[:replacement_called].nil?
194
+ end
195
+ end
196
+
197
+ # A test can run saboteur itself (the dogfood's cli_spec): the nested
198
+ # CLI reports its own crashes as exit 3 and logs them to
199
+ # internal-error.log. A log written during this run explains the
200
+ # failure better than the bare expectation diff, so its first lines
201
+ # join each failure detail; an older log is another run's (#296).
202
+ def with_crash_detail(result, started_at)
203
+ excerpt = crash_log_excerpt(started_at)
204
+ details = result[:failure_details]
205
+ return result unless excerpt && details && !details.empty?
206
+
207
+ result.merge(failure_details: details.transform_values { |d| [d, CRASH_LOG_LABEL, excerpt].join("\n") })
208
+ end
209
+
210
+ # The nested CLI writes its log to the cache dir, or to the pid-keyed
211
+ # temp fallback when the sandbox holds the cache dir read-only (#296).
212
+ def crash_log_excerpt(started_at)
213
+ excerpt = tmp_log_candidates.filter_map { |path| fresh_log_excerpt(path, started_at) }
214
+ .max_by { |mtime, _text| mtime }
215
+ excerpt&.last
216
+ rescue StandardError
217
+ nil
218
+ end
219
+
220
+ # The log paths are predictable and live in shared dirs, so read the
221
+ # log the way CrashReport writes it: through a no-follow, non-blocking
222
+ # descriptor,
223
+ # and validate on that same fd - a regular file, owned by this uid,
224
+ # with no group/other permission bits, fresh by the fd's own mtime.
225
+ # One open, no check-then-reopen race; a symlink, a foreign, wide or
226
+ # non-regular file, or any IO failure is no excerpt, never an
227
+ # exception (#296).
228
+ def fresh_log_excerpt(path, started_at)
229
+ # No safe open, no excerpt: without O_NOFOLLOW a symlink redirects
230
+ # the read, without O_NONBLOCK a no-writer FIFO blocks it (#296).
231
+ return unless CrashReport.safe_log_io?
232
+
233
+ flags = File::RDONLY | CrashReport.nofollow_flag | CrashReport.nonblock_flag
234
+ File.open(path, flags) do |file|
235
+ stat = file.stat
236
+ next unless stat.file? && stat.uid == Process.uid && stat.mode.nobits?(0o077)
237
+ next unless stat.mtime >= started_at - CRASH_LOG_CLOCK_SLACK
238
+
239
+ text = file.read(CRASH_LOG_BYTES)
240
+ # Scrub before the excerpt travels as JSON: the bounded read can
241
+ # end mid-character and the log can hold locale-invalid bytes, and
242
+ # a JSON::GeneratorError here would turn a kill into a child error
243
+ # (#296).
244
+ [stat.mtime, text ? CrashReport.scrub(text).lines.first(CRASH_LOG_LINES).join : nil]
132
245
  end
246
+ rescue SystemCallError, IOError
247
+ nil
248
+ end
249
+
250
+ # The excerpt is diagnostic enrichment: when CrashReport itself is the
251
+ # mutated file, tmp_log_path can raise or return garbage - keep the
252
+ # primary candidate and never let the lookup change a verdict (#296).
253
+ def tmp_log_candidates
254
+ [@crash_log, CrashReport.tmp_log_path].compact.uniq.grep(String)
255
+ rescue StandardError
256
+ [@crash_log].compact
133
257
  end
134
258
 
135
259
  # Targeted tracing distinguishes the new Ruby body from an old Method
@@ -151,19 +275,69 @@ module Saboteur
151
275
  def run_tests(test_ids)
152
276
  result = @integration.run(test_ids)
153
277
  { passed: result.passed, failed: result.failed_ids, executed: result.executed_ids, runtime: result.runtime,
154
- failure_classes: result.failure_classes, failure_messages: result.failure_messages }
278
+ failure_classes: result.failure_classes, failure_messages: result.failure_messages,
279
+ failure_details: result.failure_details }
155
280
  rescue SystemExit => e
156
281
  # A test called `exit`. The tests noticed something; count it as a failure.
157
282
  { passed: false, failed: [], executed: [], runtime: 0.0, exited: e.status }
158
283
  end
159
284
 
160
- def redirect_output(log_path)
161
- $stdout.reopen(log_path, "w")
285
+ def redirect_output(log_file)
286
+ $stdout.reopen(log_file)
162
287
  $stderr.reopen($stdout)
163
288
  $stdout.sync = true
164
289
  $stderr.sync = true
165
290
  end
166
291
 
292
+ # Names the just-run mutant when there is one (a verdict-check rerun
293
+ # has none): the most likely culprit, worth naming for the report.
294
+ def culprit(job)
295
+ culprit_digest(job) || "an unknown mutant"
296
+ end
297
+
298
+ # The 16-hex digest of the just-run mutant, or nil when the job ran
299
+ # none (a neutral or verdict-check rerun): a parallel parent prefers
300
+ # the vanished-log report that carries one (#285).
301
+ def culprit_digest(job)
302
+ return unless job.mutation && job.subject.respond_to?(:start_byte)
303
+
304
+ job.mutation.digest(subject_start: job.subject.start_byte,
305
+ original_source: job.subject.file_source)
306
+ end
307
+
308
+ # Opened in the parent, not the sandboxed child: the cache dir is
309
+ # not writable inside the sandbox (#234), and a log directory that
310
+ # is already gone is a tool problem to report, not a mutant result.
311
+ def open_log(job)
312
+ File.open(job.log_path, "w")
313
+ rescue SystemCallError => e
314
+ raise Error, "cannot open the child log #{job.log_path} (#{e.class}: #{e.message})"
315
+ end
316
+
317
+ # A mutant's tests can delete saboteur's own files when the sandbox
318
+ # is off or degraded (#234): a vanished log makes the verdict
319
+ # worthless, so fail as a tool error instead of trusting the run.
320
+ # The mutant that just ran is the likely - not proven - cause: a
321
+ # leftover process from an earlier mutant could have done it.
322
+ # The per-mutant probe (#235): names the mutant whose run just wiped
323
+ # tracked files, instead of blaming the subject's last mutant when the
324
+ # loss surfaces at subject end. Runs before the log check, so the
325
+ # hazard report still wins when a mutant took both (#283).
326
+ def check_tree_still_there(job)
327
+ # culprit_digest, not the display fallback: the digest doubles as
328
+ # trip metadata, and "an unknown mutant" must not rank as one (#285).
329
+ @canary&.probe!(subject: job.subject, digest: culprit_digest(job))
330
+ end
331
+
332
+ def check_log_still_there(job)
333
+ return if File.exist?(job.log_path)
334
+
335
+ cause = "a mutant's tests may have hit saboteur's files"
336
+ raise LogVanished.new("the child log #{job.log_path} vanished mid-run - the mutant that just ran, " \
337
+ "#{culprit(job)}, is the likely cause: #{cause}",
338
+ culprit_digest: culprit_digest(job))
339
+ end
340
+
167
341
  # --- parent ---------------------------------------------------------
168
342
 
169
343
  def outcome_for(result, job)
@@ -197,7 +371,7 @@ module Saboteur
197
371
  end
198
372
  end
199
373
 
200
- def classify_result(result, runtime, test_ids)
374
+ def classify_result(result, runtime, test_ids, verbose_failure: false)
201
375
  failed = Array(result["failed"])
202
376
  executed = Array(result["executed"])
203
377
  return Outcome.new(status: :killed, runtime: runtime, message: "test called exit") if result.key?("exited")
@@ -206,9 +380,7 @@ module Saboteur
206
380
  unexpected = failed - test_ids
207
381
  return mismatch(runtime, "failed tests not requested: #{unexpected.first(3).join(', ')}") if unexpected.any?
208
382
 
209
- return Outcome.new(status: :killed, runtime: runtime, failed_ids: failed, executed_ids: executed,
210
- message: result.fetch("failure_messages", {})[failed.first],
211
- kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
383
+ return killed_outcome(result, runtime, failed, executed, verbose_failure)
212
384
  end
213
385
  unless result["passed"]
214
386
  return Outcome.new(status: :killed, runtime: runtime, executed_ids: executed,
@@ -222,14 +394,30 @@ module Saboteur
222
394
  Outcome.new(status: :survived, runtime: runtime, executed_ids: executed)
223
395
  end
224
396
 
397
+ def killed_outcome(result, runtime, failed, executed, verbose_failure)
398
+ Outcome.new(status: :killed, runtime: runtime, failed_ids: failed, executed_ids: executed,
399
+ message: failure_message(result, failed.first, verbose_failure),
400
+ kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
401
+ end
402
+
403
+ # A neutral check (the verdict re-run, no mutation) needs the full
404
+ # failure to explain why the killer fails unchanged (#296).
405
+ def failure_message(result, test_id, verbose_failure)
406
+ message = result.fetch("failure_messages", {})[test_id]
407
+ verbose_failure ? result.fetch("failure_details", {})[test_id] || message : message
408
+ end
409
+
225
410
  def classify_events(result, events, runtime, job)
226
- outcome = classify_result(result, runtime, job.test_ids)
411
+ outcome = classify_result(result, runtime, job.test_ids, verbose_failure: job.mutation.nil?)
227
412
  replaced = events.find { |event| event["event"] == "replaced" }
228
413
  return replaced_by_later_definition(outcome, replaced) if never_ran?(outcome, replaced, job)
229
414
 
230
415
  captures = events.find { |event| event["event"] == "captures" }
231
416
  return unreachable(outcome, captures) if unobservable?(outcome, captures, job, result["replacement_called"])
232
417
 
418
+ overwritten = events.find { |event| event["event"] == "overwritten" }
419
+ return overwritten_by_later_load(outcome, overwritten) if replaced_while_running?(outcome, overwritten, job)
420
+
233
421
  outcome
234
422
  end
235
423
 
@@ -252,6 +440,19 @@ module Saboteur
252
440
  captures && job.mutation && outcome.status == :survived && replacement_called == false
253
441
  end
254
442
 
443
+ # A surviving mutant the tests themselves overwrote: a before(:each)
444
+ # re-loading the file replaced it before any test ran (rolify's suite
445
+ # shape, #597), so the verdict says nothing about the tests.
446
+ def replaced_while_running?(outcome, overwritten, job)
447
+ overwritten && job.mutation && outcome.status == :survived
448
+ end
449
+
450
+ def overwritten_by_later_load(outcome, overwritten)
451
+ Outcome.new(status: :unreachable, runtime: outcome.runtime,
452
+ message: "the tests (or their hooks) replaced the definition while they ran " \
453
+ "(#{overwritten['site']}), so this mutant never ran")
454
+ end
455
+
255
456
  # A survivor that no test could have observed: a Method object
256
457
  # captured at load still calls the original. Not a verdict on the
257
458
  # tests, so it is its own status, out of the score.
@@ -57,8 +57,15 @@ module Saboteur
57
57
  return @abi_version if defined?(@abi_version)
58
58
  return @abi_version = 0 unless linux?
59
59
 
60
- version = LibC.syscall_fn.call(SYS_CREATE_RULESET, 0, 0, CREATE_RULESET_VERSION, 0)
61
- errno = last_errno
60
+ # Build the errno reader BEFORE the probe: its lazy dlopen/dlsym run
61
+ # libc calls that can set errno, so building it between the probe and
62
+ # the read can replace the real answer with a stale 0.
63
+ LibC.errno_location_fn
64
+ version, errno = probe_once
65
+ # A failed syscall always sets errno, so (failure, errno 0) is never
66
+ # the kernel's real answer - the read was clobbered between the call
67
+ # and here. Probe once more, now with every Fiddle function warm.
68
+ version, errno = probe_once if !version.positive? && errno.zero?
62
69
  @abi_version = probe_abi(version, errno)
63
70
  rescue Fiddle::DLError
64
71
  # No usable libc (an unusual musl/ldso layout): the sandbox cannot
@@ -70,6 +77,12 @@ module Saboteur
70
77
  # no Landlock to give: run unsandboxed, noted once. Anything else
71
78
  # (EPERM under a seccomp policy, ...) means the kernel is there but
72
79
  # refused - fail closed rather than run without the promised sandbox.
80
+ # One create-ruleset version probe with its errno read paired to it.
81
+ def probe_once
82
+ version = LibC.syscall_fn.call(SYS_CREATE_RULESET, 0, 0, CREATE_RULESET_VERSION, 0)
83
+ [version, last_errno]
84
+ end
85
+
73
86
  def probe_abi(version, errno)
74
87
  return version if version.positive?
75
88
  return 0 if UNSUPPORTED_ERRNOS.include?(errno)
@@ -160,7 +173,12 @@ module Saboteur
160
173
  return unless stat.file?
161
174
 
162
175
  granted = target.delete_suffix(" (deleted)")
163
- return if writable.any? { |dir| granted == dir || granted.start_with?("#{dir}/") }
176
+ # readlink targets arrive tagged with default_external, so under
177
+ # LANG=C a non-ASCII target compared against a UTF-8 grant pattern
178
+ # raises Encoding::CompatibilityError before any test runs (CI #272).
179
+ # Compare bytes.
180
+ target_bytes = granted.b
181
+ return if writable.any? { |dir| target_bytes == dir.b || target_bytes.start_with?(dir.b + "/") } # rubocop:disable Style/StringConcatenation -- interpolation with a BINARY part raises on high bytes
164
182
 
165
183
  flags = LibC.fcntl_fn.call(descriptor, F_GETFL, 0)
166
184
  return close_fd(descriptor) if flags.negative?