saboteur 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +215 -0
  3. data/README.md +44 -6
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +116 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/presets.rb +41 -1
  20. data/lib/saboteur/cli/preview_command.rb +26 -0
  21. data/lib/saboteur/cli/run_budget_options.rb +6 -1
  22. data/lib/saboteur/cli/run_command.rb +33 -0
  23. data/lib/saboteur/cli/run_output.rb +38 -0
  24. data/lib/saboteur/cli/run_reporting.rb +6 -5
  25. data/lib/saboteur/cli/run_validation.rb +54 -0
  26. data/lib/saboteur/cli/source_set.rb +0 -18
  27. data/lib/saboteur/cli.rb +29 -305
  28. data/lib/saboteur/config.rb +76 -7
  29. data/lib/saboteur/config_file.rb +12 -8
  30. data/lib/saboteur/constant_subject_finder.rb +3 -0
  31. data/lib/saboteur/custom_map.rb +24 -5
  32. data/lib/saboteur/deep_source.rb +49 -0
  33. data/lib/saboteur/dirty_tree_warning.rb +6 -4
  34. data/lib/saboteur/doctor.rb +10 -0
  35. data/lib/saboteur/engine/baseline_guard.rb +118 -15
  36. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  37. data/lib/saboteur/engine/boot.rb +11 -3
  38. data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
  39. data/lib/saboteur/engine/candidate_selection.rb +16 -5
  40. data/lib/saboteur/engine/candidates.rb +4 -1
  41. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  42. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  43. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  44. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  45. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  46. data/lib/saboteur/engine/coverage_store.rb +296 -21
  47. data/lib/saboteur/engine/diagnostics.rb +24 -7
  48. data/lib/saboteur/engine/distribution.rb +2 -2
  49. data/lib/saboteur/engine/gated_run.rb +4 -1
  50. data/lib/saboteur/engine/load_coverage.rb +72 -8
  51. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  52. data/lib/saboteur/engine/partial_merge.rb +40 -0
  53. data/lib/saboteur/engine/pipelines.rb +11 -3
  54. data/lib/saboteur/engine/reference_specs.rb +71 -0
  55. data/lib/saboteur/engine/result_cache.rb +68 -5
  56. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  57. data/lib/saboteur/engine/run_budget.rb +10 -7
  58. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  59. data/lib/saboteur/engine/sandbox_policy.rb +26 -5
  60. data/lib/saboteur/engine/scheduler.rb +33 -25
  61. data/lib/saboteur/engine/selection_announcement.rb +9 -3
  62. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  63. data/lib/saboteur/engine/shadow_run.rb +38 -0
  64. data/lib/saboteur/engine/subject_selection.rb +144 -64
  65. data/lib/saboteur/engine/suite_partition.rb +22 -0
  66. data/lib/saboteur/engine/test_guess.rb +186 -32
  67. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  68. data/lib/saboteur/engine.rb +44 -11
  69. data/lib/saboteur/environment_hints.rb +55 -0
  70. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  71. data/lib/saboteur/execution/external_runner.rb +123 -1
  72. data/lib/saboteur/execution/fiddle_loader.rb +34 -0
  73. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  74. data/lib/saboteur/execution/marker_sweep.rb +2 -1
  75. data/lib/saboteur/execution/process_table.rb +7 -3
  76. data/lib/saboteur/execution/reevaluator.rb +7 -0
  77. data/lib/saboteur/execution/sandbox.rb +39 -20
  78. data/lib/saboteur/execution/seatbelt.rb +10 -3
  79. data/lib/saboteur/execution/supervisor.rb +42 -8
  80. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  81. data/lib/saboteur/execution/test_command_shape.rb +149 -0
  82. data/lib/saboteur/execution/worker_pool.rb +48 -4
  83. data/lib/saboteur/frameworks/rails.rb +2 -5
  84. data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
  85. data/lib/saboteur/gates.rb +121 -28
  86. data/lib/saboteur/install.rb +6 -1
  87. data/lib/saboteur/integration/base.rb +19 -2
  88. data/lib/saboteur/integration/rspec.rb +145 -18
  89. data/lib/saboteur/line_hits.rb +81 -0
  90. data/lib/saboteur/method_calls.rb +91 -0
  91. data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
  92. data/lib/saboteur/mutation_source/builtin.rb +33 -21
  93. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  94. data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
  95. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  96. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  97. data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
  98. data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
  99. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  100. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  101. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  102. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  103. data/lib/saboteur/preview.rb +13 -0
  104. data/lib/saboteur/refinement_activations.rb +22 -10
  105. data/lib/saboteur/reporter/agent.rb +2 -2
  106. data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
  107. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  108. data/lib/saboteur/reporter/review_comments.rb +151 -0
  109. data/lib/saboteur/reporter/safe_out.rb +32 -0
  110. data/lib/saboteur/reporter/stryker.rb +8 -6
  111. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  112. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  113. data/lib/saboteur/reporter/terminal.rb +45 -5
  114. data/lib/saboteur/result.rb +14 -4
  115. data/lib/saboteur/selection/coverage_map.rb +150 -9
  116. data/lib/saboteur/selection/external_test_map.rb +4 -0
  117. data/lib/saboteur/selection/git_diff.rb +24 -2
  118. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  119. data/lib/saboteur/selection/recorder.rb +52 -22
  120. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  121. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  122. data/lib/saboteur/subject.rb +16 -0
  123. data/lib/saboteur/subject_finder.rb +91 -6
  124. data/lib/saboteur/text.rb +31 -0
  125. data/lib/saboteur/version.rb +1 -1
  126. data/lib/saboteur/working_tree.rb +7 -2
  127. data/lib/saboteur.rb +9 -0
  128. metadata +48 -16
@@ -0,0 +1,115 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # Which spec files a partial coverage rebuild would have to rerun, or
7
+ # :full when the whole map must be rebuilt (#46, step 2). Pure: it reads
8
+ # two inputs snapshots and a coverage map and touches nothing else.
9
+ #
10
+ # An allowlist, not a blocklist. Coverage never records a test reading a
11
+ # fixture, a YAML file, an ENV value or a template, so no changed file
12
+ # can be proven harmless by absence. Only two kinds of change may take
13
+ # the partial path: a spec file of the suite, and an included source
14
+ # file with recorded coverage evidence that did not run while the app
15
+ # loaded. Every other change, and every added or removed file, is :full,
16
+ # as is a change of root, settings, runtime or snapshot format.
17
+ #
18
+ # The stale files come back in the order the map's full run used, so a
19
+ # rerun keeps the relative order that run had.
20
+ class PartialInvalidation
21
+ FULL = :full
22
+ # Placeholder, to be fitted from measurements; not configuration.
23
+ THRESHOLD = 0.5
24
+ SAME = %w[format root runtime settings].freeze
25
+
26
+ # included: relative paths of the included source files.
27
+ # spec_files: relative paths of the suite's spec files.
28
+ # status_file: whether RSpec picks examples from its generated status
29
+ # file (--only-failures, --next-failure), which no snapshot covers.
30
+ # RSpec ids name their file: "./spec/a_spec.rb[1:2]" is spec/a_spec.rb.
31
+ def self.spec_of(id)
32
+ id.split("[").first.delete_prefix("./")
33
+ end
34
+
35
+ def initialize(map:, root:, included:, spec_files:, status_file: false)
36
+ @status_file = status_file
37
+ @map = map
38
+ @root = Selection::CoverageMap.canonical(root)
39
+ @included = included
40
+ @spec_files = spec_files
41
+ end
42
+
43
+ def stale_specs(base, current)
44
+ return FULL if @status_file
45
+ return FULL unless SAME.all? { |key| base[key] == current[key] }
46
+
47
+ changed = changed_files([base, current].map { |snapshot| snapshot["files"] })
48
+ return FULL unless changed
49
+
50
+ stale = changed.each_with_object([]) do |file, specs|
51
+ found = specs_for(file)
52
+ return FULL unless found
53
+
54
+ specs.concat(found)
55
+ end
56
+ ordered(stale.uniq).then { |specs| heavy?(specs) ? FULL : specs }
57
+ end
58
+
59
+ # The part of the recorded spec time these files hold, 0 when none was recorded.
60
+ def share(specs)
61
+ total = weights.values.sum
62
+ total.zero? ? 0.0 : specs.sum { |file| weights.fetch(file, 0) }.fdiv(total)
63
+ end
64
+
65
+ private
66
+
67
+ # nil when a file was added, removed or unreadable on either side.
68
+ def changed_files(snapshots)
69
+ before, after = snapshots
70
+ return nil unless before.keys.sort == after.keys.sort
71
+ return nil if (before.values + after.values).include?(false)
72
+
73
+ before.keys.reject { |file| before[file] == after[file] }
74
+ end
75
+
76
+ def specs_for(file)
77
+ return [file] if @spec_files.include?(file)
78
+ return nil unless @included.include?(file) && evidence?(File.join(@root, file))
79
+
80
+ touching(File.join(@root, file)).then { |specs| specs if specs.all? { |spec| @spec_files.include?(spec) } }
81
+ end
82
+
83
+ def evidence?(path)
84
+ [@map.lines, @map.methods].any? { |hits| (hits[path] || {}).values.any?(&:any?) } &&
85
+ [@map.load_lines, @map.load_methods].none? { |hits| hits[path]&.any? } &&
86
+ !@map.preloaded?(path)
87
+ end
88
+
89
+ def touching(path)
90
+ ids = [@map.lines, @map.methods].flat_map { |hits| (hits[path] || {}).values.flatten }
91
+ ids.map { |id| spec_of(id) }.uniq
92
+ end
93
+
94
+ def spec_of(id)
95
+ self.class.spec_of(id)
96
+ end
97
+
98
+ def ordered(stale)
99
+ run_order = @map.tests.map { |id| spec_of(id) }.uniq
100
+ run_order.select { |file| stale.include?(file) } + (stale - run_order).sort
101
+ end
102
+
103
+ # Stale share of the recorded time, by spec file, never above THRESHOLD.
104
+ def heavy?(specs)
105
+ total = weights.values.sum
106
+ total.zero? || share(specs) > THRESHOLD
107
+ end
108
+
109
+ def weights
110
+ @map.tests.group_by { |id| spec_of(id) }
111
+ .transform_values { |ids| ids.sum { |id| @map.durations.fetch(id, 0) } }
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,40 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # The map a partial rebuild produces (#46, step 3): the base map with all
7
+ # evidence of the rerun spec files removed, plus the fresh map the child
8
+ # recorded over just those files. Whole spec files are replaced, never
9
+ # single examples, since a file's example ids can shift when it changes.
10
+ # Load-time evidence, loaded files and preloaded files carry over from
11
+ # both: every run boots the same app.
12
+ module PartialMerge
13
+ module_function
14
+
15
+ def call(base, fresh, spec_files)
16
+ specs = spec_files.to_set
17
+ dropped = base.tests.select { |id| specs.include?(PartialInvalidation.spec_of(id)) }.to_set
18
+ trimmed = Selection::CoverageMap.new(
19
+ lines: without(base.lines, dropped), methods: without(base.methods, dropped),
20
+ branches: without(base.branches, dropped), tests: base.tests.reject { |id| dropped.include?(id) },
21
+ durations: base.durations.except(*dropped),
22
+ load_lines: base.load_lines, load_methods: base.load_methods,
23
+ loaded_files: base.loaded_files, preloaded: base.preloaded.to_a, build_seconds: base.build_seconds
24
+ )
25
+ Selection::CoverageMap.combine([trimmed, fresh])
26
+ end
27
+
28
+ # {file => {line-or-span => ids}} without the dropped ids. An entry left
29
+ # with no test is removed, and so is a file left with no entry: a file
30
+ # with no test behind it must read as no evidence, not as covered.
31
+ def without(hits, dropped)
32
+ hits.each_with_object({}) do |(file, by_line), acc|
33
+ kept = by_line.transform_values { |ids| ids.reject { |id| dropped.include?(id) } }
34
+ kept.reject! { |_, ids| ids.empty? }
35
+ acc[file] = kept unless kept.empty?
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "../execution/test_command_shape"
5
+
4
6
  module Saboteur
5
7
  class Engine
6
8
  # The two run pipelines: in-process (boot, coverage, workers) and the
@@ -22,7 +24,7 @@ module Saboteur
22
24
  mutate(subjects, files, integration, scope, ignore_list,
23
25
  load_lines:, load_methods:)
24
26
  end
25
- @diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
27
+ @diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
26
28
  build_report(results, subjects, scope, started)
27
29
  ensure
28
30
  # Cleanup runs however the call ends - a partway boot failure, no
@@ -43,10 +45,16 @@ module Saboteur
43
45
  def call_external(ignore_list, started)
44
46
  @diagnostics.inform("external test command: #{config['test_command']} " \
45
47
  "(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
48
+ if Execution::TestCommandShape.multi_file_ruby?(config["test_command"])
49
+ @diagnostics.warn("use the framework's multi-file form (bin/rails test f1 f2, rspec f1 f2) or a " \
50
+ "shell loop (for f in ...; do ruby \"$f\" || exit 1; done)\n " \
51
+ "the test command hands several test files to one ruby: ruby executes only " \
52
+ "the first, so the rest never run and their mutants silently survive (#611)")
53
+ end
46
54
  files = config.included_files
47
55
  subjects, scope = select_subjects(files)
48
56
  results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
49
- @diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
57
+ @diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
50
58
  build_report(results, subjects, scope, started)
51
59
  end
52
60
 
@@ -78,7 +86,7 @@ module Saboteur
78
86
  end
79
87
  run_candidates(runnable_subjects, runnable_lists, scheduler, nil, ignore_list)
80
88
  ensure
81
- runner.close
89
+ runner&.close
82
90
  end
83
91
 
84
92
  # Warn loudly when the command cannot see mutants at all: an
@@ -0,0 +1,71 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # The specs that read a constant a source file defines, or require it: the
7
+ # cold-start choice for a file with no spec by name. The constant match is
8
+ # the lexical matching #538 uses for changed specs; the require match
9
+ # covers a file that defines only top-level methods. A file no spec reads or requires, or one read by more than
10
+ # CAP specs (a hub: the choice is not selective), gets no entry.
11
+ class ReferenceSpecs
12
+ CAP = 10
13
+ REQUIRE = /^\s*(require_relative|require)[ (]\s*["']([^"']+)["']/
14
+
15
+ # constants: file => the constants it defines, as TestGuess shapes them
16
+ # for ConstantReads.
17
+ def initialize(root, constants)
18
+ @root = root
19
+ @constants = constants
20
+ end
21
+
22
+ # chosen: file => sorted root-relative spec paths; crowded: file => how
23
+ # many specs read it, for the files over the cap (no choice was made).
24
+ Found = Data.define(:chosen, :crowded)
25
+
26
+ # The announcement for the files that got specs by reference.
27
+ def self.note(referenced)
28
+ named = referenced.map { |file, specs| "#{file} (#{specs.join(', ')})" }
29
+ "inferred specs: selected by constant reference or require, not by name: #{named.join(', ')}; " \
30
+ "a spec that reaches the file another way is not found"
31
+ end
32
+
33
+ def self.crowd_note(count) = "#{count} specs reference or require it, over the cap of #{CAP}"
34
+
35
+ # files: root-relative source files; spec_paths: root-relative spec
36
+ # files of the suite.
37
+ def call(files, spec_paths)
38
+ absolute = spec_paths.map { |path| File.join(@root, path) }
39
+ absolute.select! { |path| File.file?(path) && !File.symlink?(path) }
40
+ found = files.to_h { |file| [file, readers_of(file, absolute)] }.reject { |_, readers| readers.empty? }
41
+ crowded, chosen = found.partition { |_, readers| readers.size > CAP }.map(&:to_h)
42
+ Found.new(chosen: chosen, crowded: crowded.transform_values(&:size))
43
+ end
44
+
45
+ private
46
+
47
+ def readers_of(file, absolute_specs)
48
+ read = @constants.call(file).flat_map { |constant| ConstantReads.across(absolute_specs, constant).keys }
49
+ (read + requirers_of(file, absolute_specs)).uniq.sort.map { |path| path.delete_prefix("#{@root}/") }
50
+ end
51
+
52
+ # Specs that require the file: by path from the spec (require_relative)
53
+ # or by load path from the file's source root (require "a/b" for
54
+ # lib/a/b.rb). A bare basename is not a load path, so it matches nothing.
55
+ def requirers_of(file, absolute_specs)
56
+ stem = File.join(@root, file).delete_suffix(".rb")
57
+ loadable = file.delete_suffix(".rb").sub(%r{\A[^/]+/}, "")
58
+ absolute_specs.select { |spec| requires?(spec, stem, loadable) }
59
+ end
60
+
61
+ def requires?(spec, stem, loadable)
62
+ File.read(spec, encoding: "UTF-8").scrub.scan(REQUIRE).any? do |kind, target|
63
+ target = target.delete_suffix(".rb")
64
+ kind == "require_relative" ? File.expand_path(target, File.dirname(spec)) == stem : target == loadable
65
+ end
66
+ rescue SystemCallError
67
+ false
68
+ end
69
+ end
70
+ end
71
+ end
@@ -4,6 +4,7 @@
4
4
  require "digest"
5
5
  require "fileutils"
6
6
  require "json"
7
+ require "securerandom"
7
8
  require_relative "../text"
8
9
 
9
10
  module Saboteur
@@ -25,6 +26,7 @@ module Saboteur
25
26
  # stale.
26
27
  class ResultCache
27
28
  FORMAT = 1
29
+ MAX_BYTES = 64 * 1024 * 1024
28
30
  STORED = %i[killed survived timeout].freeze
29
31
  LOCKFILES = %w[Gemfile.lock gems.locked].freeze
30
32
  # Dependency declarations themselves, so the no-lockfile library
@@ -34,9 +36,15 @@ module Saboteur
34
36
 
35
37
  SOURCE_EXCLUDES = %w[.git vendor node_modules].freeze
36
38
 
39
+ # Seconds between checkpoint saves while verdicts arrive, so a run
40
+ # killed mid-phase (OOM, CI timeout, SIGKILL) keeps what it finished
41
+ # (#471).
42
+ SAVE_INTERVAL = 30
43
+
37
44
  # suite_digest: SuiteFiles#digest, so an edit to a spec helper or
38
45
  # support file invalidates every verdict.
39
- def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [])
46
+ def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [],
47
+ save_interval: SAVE_INTERVAL, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
40
48
  @path = path
41
49
  @root = root
42
50
  @inputs = inputs
@@ -47,6 +55,9 @@ module Saboteur
47
55
  @fallback_files = fallback_files.map { |file| File.expand_path(file, root) }.sort
48
56
  @entries = load
49
57
  @file_digests = {}
58
+ @save_interval = save_interval
59
+ @clock = clock
60
+ @last_saved = clock.call
50
61
  end
51
62
 
52
63
  # The stored verdict as a hash (status, reason, killed_by, kill_cause,
@@ -67,15 +78,39 @@ module Saboteur
67
78
  "reason" => result.reason, "killed_by" => result.killed_by, "kill_cause" => result.kill_cause,
68
79
  "runtime" => result.runtime
69
80
  }
81
+ checkpoint if @clock.call - @last_saved >= @save_interval
70
82
  end
71
83
 
84
+ # Written to a temp file and renamed, so a kill mid-write leaves the
85
+ # previous complete file, never a truncated one. A failed write or
86
+ # rename removes the temp file and raises.
72
87
  def save
73
88
  FileUtils.mkdir_p(File.dirname(@path))
74
- File.write(@path, JSON.generate(format: FORMAT, entries: @entries))
89
+ temp = "#{@path}.#{SecureRandom.hex(8)}.tmp"
90
+ begin
91
+ # "wx": a planted file or symlink at the temp name is never followed.
92
+ File.open(temp, "wx", 0o600) { |out| out.write(JSON.generate(format: FORMAT, entries: @entries)) }
93
+ File.rename(temp, @path)
94
+ rescue StandardError
95
+ FileUtils.rm_f(temp)
96
+ raise
97
+ end
98
+ @last_saved = @clock.call
75
99
  end
76
100
 
77
101
  private
78
102
 
103
+ # A mid-run save is a safety net, so it is best effort: a full disk or
104
+ # an unwritable directory must not abort a run that is still producing
105
+ # verdicts. The wait restarts either way, so a broken disk is tried
106
+ # once per interval, not once per verdict. The final save is not
107
+ # swallowed (Engine calls #save).
108
+ def checkpoint
109
+ save
110
+ rescue SystemCallError
111
+ @last_saved = @clock.call
112
+ end
113
+
79
114
  # The current project source files, relative and sorted. A file added
80
115
  # after the map was built changes the set and invalidates every stored
81
116
  # verdict, whether or not any watched file references it yet.
@@ -86,13 +121,41 @@ module Saboteur
86
121
  .join("\0")
87
122
  end
88
123
 
124
+ # The file is untrusted input (a restored cache, a --resume directory):
125
+ # read only as a regular file, never a symlink or FIFO, under a size
126
+ # cap, and kept only when entries is a Hash of Hashes.
89
127
  def load
90
- data = JSON.parse(Text.utf8_strict(File.read(@path)))
91
- data["format"] == FORMAT ? data["entries"] : {}
92
- rescue SystemCallError, JSON::ParserError
128
+ return {} unless File.lstat(@path).file?
129
+
130
+ text = slurp
131
+ return {} unless text
132
+
133
+ data = JSON.parse(Text.utf8_strict(text))
134
+ entries = data["entries"] if data.is_a?(Hash) && data["format"] == FORMAT
135
+ well_formed?(entries) ? entries : {}
136
+ rescue SystemCallError, JSON::ParserError, ArgumentError, EncodingError
93
137
  {}
94
138
  end
95
139
 
140
+ def slurp
141
+ File.open(@path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
142
+ text = io.read(MAX_BYTES)
143
+ text if io.eof?
144
+ end
145
+ end
146
+
147
+ def well_formed?(entries)
148
+ entries.is_a?(Hash) && entries.values.all? { |entry| entry.is_a?(Hash) && typed?(entry) }
149
+ end
150
+
151
+ # The fields the reports read, typed so a crafted entry cannot crash
152
+ # them.
153
+ def typed?(entry)
154
+ STORED.map(&:to_s).include?(entry["status"]) && (entry["runtime"].nil? || entry["runtime"].is_a?(Numeric)) &&
155
+ %w[reason kill_cause].all? { |key| entry[key].nil? || entry[key].is_a?(String) } &&
156
+ entry["killed_by"].is_a?(Array) && entry["killed_by"].all?(String)
157
+ end
158
+
96
159
  def key(subject, mutation)
97
160
  "#{subject.id} #{mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)}"
98
161
  end
@@ -0,0 +1,110 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "fileutils"
5
+ require "json"
6
+ require "securerandom"
7
+
8
+ module Saboteur
9
+ class Engine
10
+ # Identity file for a --resume directory (#471). The verdicts live in
11
+ # ResultCache; this only records what the run was computed from, so a
12
+ # later run with other inputs fails loudly instead of reusing the
13
+ # directory under a different meaning.
14
+ #
15
+ # The file is untrusted input: it is read only when it is a regular
16
+ # file (never a FIFO or a symlink), under a size cap, and must hold
17
+ # exactly the three expected keys with bounded String values.
18
+ class ResumeManifest
19
+ FORMAT = 1
20
+ FILE = "manifest.json"
21
+ KEYS = %w[format identity version].freeze
22
+ MAX_BYTES = 4096
23
+ MAX_FIELD = 512
24
+
25
+ # Raised when the directory belongs to another run or cannot be read.
26
+ class Mismatch < StandardError; end
27
+
28
+ # identity: a String that changes whenever the run's inputs change
29
+ # (the RunFingerprint digest joined with the settings that pick the
30
+ # mutants), at most MAX_FIELD characters.
31
+ def initialize(directory:, identity:)
32
+ @directory = directory
33
+ @identity = identity
34
+ end
35
+
36
+ def path
37
+ File.join(@directory, FILE)
38
+ end
39
+
40
+ # Creates the manifest in a fresh directory, accepts a matching one,
41
+ # and raises Mismatch for anything else.
42
+ def verify!
43
+ File.lstat(path)
44
+ compare(read)
45
+ rescue Errno::ENOENT
46
+ write
47
+ end
48
+
49
+ private
50
+
51
+ def compare(stored)
52
+ return if stored["identity"] == @identity && stored["version"] == VERSION
53
+
54
+ raise Mismatch, "#{path}: the run inputs changed since this directory was created; " \
55
+ "delete it or choose a new --resume path"
56
+ end
57
+
58
+ def read
59
+ raise Mismatch, "#{path}: unreadable manifest" unless File.lstat(path).file?
60
+
61
+ validate(parse(slurp))
62
+ rescue SystemCallError
63
+ raise Mismatch, "#{path}: unreadable manifest"
64
+ end
65
+
66
+ def slurp
67
+ File.open(path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
68
+ text = io.read(MAX_BYTES) || +""
69
+ raise Mismatch, "#{path}: manifest too large" unless io.eof?
70
+
71
+ text
72
+ end
73
+ end
74
+
75
+ def parse(text)
76
+ raise Mismatch, "#{path}: unreadable manifest" unless text.force_encoding(Encoding::UTF_8).valid_encoding?
77
+
78
+ JSON.parse(text)
79
+ rescue JSON::ParserError
80
+ raise Mismatch, "#{path}: unreadable manifest"
81
+ end
82
+
83
+ def validate(data)
84
+ raise Mismatch, "#{path}: unsupported manifest format" unless
85
+ data.is_a?(Hash) && data.keys.sort == KEYS && data["format"] == FORMAT && bounded?(data)
86
+
87
+ data
88
+ end
89
+
90
+ def bounded?(data)
91
+ %w[identity version].all? { |key| data[key].is_a?(String) && data[key].length <= MAX_FIELD }
92
+ end
93
+
94
+ # Written to a private temp file and linked into place: link fails
95
+ # when another run created the manifest first, and then that run's
96
+ # manifest is the one compared.
97
+ def write
98
+ FileUtils.mkdir_p(@directory)
99
+ temp = "#{path}.#{Process.pid}.#{SecureRandom.hex(4)}.tmp"
100
+ File.write(temp, JSON.generate(format: FORMAT, version: VERSION, identity: @identity))
101
+ File.link(temp, path)
102
+ nil
103
+ rescue Errno::EEXIST
104
+ compare(read)
105
+ ensure
106
+ FileUtils.rm_f(temp) if temp
107
+ end
108
+ end
109
+ end
110
+ end
@@ -35,8 +35,11 @@ module Saboteur
35
35
 
36
36
  # Absolute monotonic deadline shared with the schedulers (forked
37
37
  # workers inherit it): a subject's remaining mutants stop once it
38
- # passes, so the overshoot after a spent budget stays under one
39
- # mutant deadline.
38
+ # passes. New mutation work never overshoots more than the fork in
39
+ # flight; a subject the deadline cut mid-list still confirms the
40
+ # kills it already measured (#588), and those checks and retries
41
+ # fork uncapped - the overshoot is bounded by the subject's
42
+ # distinct killing sequences times their full timeout.
40
43
  def budget_deadline
41
44
  config["max_duration"] && (@run_started + config["max_duration"])
42
45
  end
@@ -61,11 +64,11 @@ module Saboteur
61
64
 
62
65
  needed = per_pass - remaining
63
66
  raise BudgetPhaseTimeout,
64
- "the --max-duration budget cannot cover the coverage map build: baseline passes took about " \
65
- "#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about one " \
66
- "more full pass and keeps nothing when it is cut off. Re-run with --max-duration about " \
67
- "#{needed.round}s higher, fewer --baseline-passes, or once without --max-duration so the " \
68
- "coverage cache is built for later --incremental runs"
67
+ "re-run with --max-duration about #{needed.round}s higher, fewer --baseline-passes, or once " \
68
+ "without --max-duration so the coverage cache is built for later --incremental runs: " \
69
+ "the budget cannot cover the coverage map build - baseline passes took about " \
70
+ "#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about " \
71
+ "one more full pass and keeps nothing when it is cut off"
69
72
  end
70
73
 
71
74
  # Boot and baselines can spend the budget by themselves: stop before
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "digest"
5
5
  require "json"
6
+ require_relative "../config"
6
7
  require_relative "../text"
7
8
  require_relative "data_inputs"
8
9
 
@@ -13,7 +14,7 @@ module Saboteur
13
14
  class RunFingerprint
14
15
  FORMAT_VERSION = 1
15
16
  MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
16
- LOCATION_SETTINGS = %w[cache_dir json_report html_report score_history].freeze
17
+ LOCATION_SETTINGS = Config::OUTPUT_KEYS
17
18
  ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
18
19
  EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
19
20
  Entry = Data.define(:path, :kind, :state, :sha256, :required)
@@ -39,8 +39,7 @@ module Saboteur
39
39
  # same line twice (D's review, #178).
40
40
  unless @sandbox_note_given
41
41
  @sandbox_note_given = true
42
- @diagnostics.warn("sandbox is on but #{self.class.sandbox_gap(abi)}; children run with the hazard guard " \
43
- "and canary only")
42
+ @diagnostics.warn(degraded_note(abi))
44
43
  end
45
44
  nil
46
45
  end
@@ -55,11 +54,30 @@ module Saboteur
55
54
  end
56
55
 
57
56
  def self.sandbox_gap(abi)
58
- return "this kernel lacks Landlock (needs Linux 6.2+)" if abi.zero?
57
+ if abi.zero?
58
+ # Landlock is Linux-only: on macOS there is no kernel to upgrade,
59
+ # so name the platform instead of a Linux version (#667).
60
+ return "macOS has no Landlock" if Execution::Seatbelt.darwin?
61
+
62
+ return "this kernel lacks Landlock (needs Linux 6.2+)"
63
+ end
59
64
 
60
65
  "this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
61
66
  end
62
67
 
68
+ # The degraded-run note: on macOS the gap is permanent, so point at
69
+ # the Seatbelt opt-in; on Linux the kernel wording stands alone
70
+ # (#667). Severity stays a warning - degrading confinement is never
71
+ # routine.
72
+ def degraded_note(abi)
73
+ gap = self.class.sandbox_gap(abi)
74
+ tail = "children run with the hazard guard and canary only"
75
+ return "sandbox is on but #{gap}; #{tail}" unless Execution::Seatbelt.darwin?
76
+
77
+ "sandbox is on but #{gap}; #{tail} - opt into the macOS sandbox with c.sandbox = \"seatbelt\" " \
78
+ "(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
79
+ end
80
+
63
81
  def restrict_hook
64
82
  writable = sandbox_writable_dirs
65
83
  ->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
@@ -126,7 +144,8 @@ module Saboteur
126
144
  def check_sandbox_write_set(all_dirs, user_dirs)
127
145
  holder = all_dirs.find { |dir| path_contains?(dir, @root) }
128
146
  if holder
129
- message = "sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
147
+ message = "narrow the sandbox grant #{holder} so it no longer contains the project root\n " \
148
+ "sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
130
149
  raise Error, %(#{message} - refusing under c.sandbox = "strict") if @config["sandbox"] == "strict"
131
150
 
132
151
  @diagnostics.warn(message)
@@ -135,7 +154,9 @@ module Saboteur
135
154
  (all_dirs - [dir]).each do |other|
136
155
  next unless path_contains?(dir, other)
137
156
 
138
- @diagnostics.warn("sandbox grant #{dir} contains the granted dir #{other}; " \
157
+ safe_other = Text.display_safe(other)
158
+ @diagnostics.warn("drop the redundant sandbox grant #{safe_other}\n " \
159
+ "sandbox grant #{Text.display_safe(dir)} contains the granted dir #{safe_other}; " \
139
160
  "the narrower grant is redundant")
140
161
  end
141
162
  end