saboteur 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +10 -0
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +3 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/preview_command.rb +26 -0
  20. data/lib/saboteur/cli/run_budget_options.rb +3 -0
  21. data/lib/saboteur/cli/run_command.rb +33 -0
  22. data/lib/saboteur/cli/run_output.rb +38 -0
  23. data/lib/saboteur/cli/run_reporting.rb +6 -5
  24. data/lib/saboteur/cli/run_validation.rb +54 -0
  25. data/lib/saboteur/cli.rb +29 -307
  26. data/lib/saboteur/config.rb +72 -6
  27. data/lib/saboteur/constant_subject_finder.rb +3 -0
  28. data/lib/saboteur/custom_map.rb +20 -12
  29. data/lib/saboteur/deep_source.rb +49 -0
  30. data/lib/saboteur/dirty_tree_warning.rb +1 -1
  31. data/lib/saboteur/doctor.rb +6 -0
  32. data/lib/saboteur/engine/baseline_guard.rb +50 -13
  33. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  34. data/lib/saboteur/engine/boot.rb +4 -0
  35. data/lib/saboteur/engine/candidate_selection.rb +15 -5
  36. data/lib/saboteur/engine/candidates.rb +4 -1
  37. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  38. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  39. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  40. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  41. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  42. data/lib/saboteur/engine/coverage_store.rb +275 -22
  43. data/lib/saboteur/engine/diagnostics.rb +23 -6
  44. data/lib/saboteur/engine/distribution.rb +2 -2
  45. data/lib/saboteur/engine/gated_run.rb +1 -0
  46. data/lib/saboteur/engine/load_coverage.rb +54 -8
  47. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  48. data/lib/saboteur/engine/partial_merge.rb +40 -0
  49. data/lib/saboteur/engine/pipelines.rb +7 -7
  50. data/lib/saboteur/engine/reference_specs.rb +71 -0
  51. data/lib/saboteur/engine/result_cache.rb +68 -5
  52. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  53. data/lib/saboteur/engine/run_budget.rb +5 -5
  54. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  55. data/lib/saboteur/engine/sandbox_policy.rb +5 -2
  56. data/lib/saboteur/engine/scheduler.rb +1 -1
  57. data/lib/saboteur/engine/selection_announcement.rb +1 -0
  58. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  59. data/lib/saboteur/engine/shadow_run.rb +38 -0
  60. data/lib/saboteur/engine/subject_selection.rb +33 -37
  61. data/lib/saboteur/engine/suite_partition.rb +22 -0
  62. data/lib/saboteur/engine/test_guess.rb +82 -18
  63. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  64. data/lib/saboteur/engine.rb +41 -9
  65. data/lib/saboteur/environment_hints.rb +55 -0
  66. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  67. data/lib/saboteur/execution/external_runner.rb +105 -1
  68. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  69. data/lib/saboteur/execution/sandbox.rb +30 -17
  70. data/lib/saboteur/execution/supervisor.rb +40 -7
  71. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  72. data/lib/saboteur/execution/worker_pool.rb +48 -4
  73. data/lib/saboteur/frameworks/rails.rb +2 -5
  74. data/lib/saboteur/gates.rb +121 -28
  75. data/lib/saboteur/install.rb +6 -1
  76. data/lib/saboteur/integration/base.rb +19 -2
  77. data/lib/saboteur/integration/rspec.rb +46 -7
  78. data/lib/saboteur/line_hits.rb +81 -0
  79. data/lib/saboteur/method_calls.rb +91 -0
  80. data/lib/saboteur/mutation_source/builtin.rb +32 -22
  81. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  82. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  83. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  84. data/lib/saboteur/mutation_source/operator_examples.rb +5 -0
  85. data/lib/saboteur/mutation_source/operator_registry.rb +5 -0
  86. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  87. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  88. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  89. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  90. data/lib/saboteur/preview.rb +9 -1
  91. data/lib/saboteur/refinement_activations.rb +22 -10
  92. data/lib/saboteur/reporter/agent.rb +2 -2
  93. data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
  94. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  95. data/lib/saboteur/reporter/review_comments.rb +151 -0
  96. data/lib/saboteur/reporter/safe_out.rb +32 -0
  97. data/lib/saboteur/reporter/stryker.rb +8 -6
  98. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  99. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  100. data/lib/saboteur/reporter/terminal.rb +29 -4
  101. data/lib/saboteur/result.rb +14 -4
  102. data/lib/saboteur/selection/coverage_map.rb +136 -10
  103. data/lib/saboteur/selection/external_test_map.rb +4 -0
  104. data/lib/saboteur/selection/git_diff.rb +24 -2
  105. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  106. data/lib/saboteur/selection/recorder.rb +52 -22
  107. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  108. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  109. data/lib/saboteur/subject.rb +16 -0
  110. data/lib/saboteur/subject_finder.rb +91 -6
  111. data/lib/saboteur/text.rb +31 -0
  112. data/lib/saboteur/version.rb +1 -1
  113. data/lib/saboteur/working_tree.rb +7 -2
  114. data/lib/saboteur.rb +8 -0
  115. metadata +44 -2
@@ -0,0 +1,40 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # The map a partial rebuild produces (#46, step 3): the base map with all
7
+ # evidence of the rerun spec files removed, plus the fresh map the child
8
+ # recorded over just those files. Whole spec files are replaced, never
9
+ # single examples, since a file's example ids can shift when it changes.
10
+ # Load-time evidence, loaded files and preloaded files carry over from
11
+ # both: every run boots the same app.
12
+ module PartialMerge
13
+ module_function
14
+
15
+ def call(base, fresh, spec_files)
16
+ specs = spec_files.to_set
17
+ dropped = base.tests.select { |id| specs.include?(PartialInvalidation.spec_of(id)) }.to_set
18
+ trimmed = Selection::CoverageMap.new(
19
+ lines: without(base.lines, dropped), methods: without(base.methods, dropped),
20
+ branches: without(base.branches, dropped), tests: base.tests.reject { |id| dropped.include?(id) },
21
+ durations: base.durations.except(*dropped),
22
+ load_lines: base.load_lines, load_methods: base.load_methods,
23
+ loaded_files: base.loaded_files, preloaded: base.preloaded.to_a, build_seconds: base.build_seconds
24
+ )
25
+ Selection::CoverageMap.combine([trimmed, fresh])
26
+ end
27
+
28
+ # {file => {line-or-span => ids}} without the dropped ids. An entry left
29
+ # with no test is removed, and so is a file left with no entry: a file
30
+ # with no test behind it must read as no evidence, not as covered.
31
+ def without(hits, dropped)
32
+ hits.each_with_object({}) do |(file, by_line), acc|
33
+ kept = by_line.transform_values { |ids| ids.reject { |id| dropped.include?(id) } }
34
+ kept.reject! { |_, ids| ids.empty? }
35
+ acc[file] = kept unless kept.empty?
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -24,7 +24,7 @@ module Saboteur
24
24
  mutate(subjects, files, integration, scope, ignore_list,
25
25
  load_lines:, load_methods:)
26
26
  end
27
- @diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
27
+ @diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
28
28
  build_report(results, subjects, scope, started)
29
29
  ensure
30
30
  # Cleanup runs however the call ends - a partway boot failure, no
@@ -46,15 +46,15 @@ module Saboteur
46
46
  @diagnostics.inform("external test command: #{config['test_command']} " \
47
47
  "(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
48
48
  if Execution::TestCommandShape.multi_file_ruby?(config["test_command"])
49
- @diagnostics.warn("the test command hands several test files to one ruby: ruby executes only " \
50
- "the first, so the rest never run and their mutants silently survive (#611) - " \
51
- "use the framework's multi-file form (bin/rails test f1 f2, rspec f1 f2) or a " \
52
- "shell loop (for f in ...; do ruby \"$f\" || exit 1; done)")
49
+ @diagnostics.warn("use the framework's multi-file form (bin/rails test f1 f2, rspec f1 f2) or a " \
50
+ "shell loop (for f in ...; do ruby \"$f\" || exit 1; done)\n " \
51
+ "the test command hands several test files to one ruby: ruby executes only " \
52
+ "the first, so the rest never run and their mutants silently survive (#611)")
53
53
  end
54
54
  files = config.included_files
55
55
  subjects, scope = select_subjects(files)
56
56
  results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
57
- @diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
57
+ @diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
58
58
  build_report(results, subjects, scope, started)
59
59
  end
60
60
 
@@ -86,7 +86,7 @@ module Saboteur
86
86
  end
87
87
  run_candidates(runnable_subjects, runnable_lists, scheduler, nil, ignore_list)
88
88
  ensure
89
- runner.close
89
+ runner&.close
90
90
  end
91
91
 
92
92
  # Warn loudly when the command cannot see mutants at all: an
@@ -0,0 +1,71 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # The specs that read a constant a source file defines, or require it: the
7
+ # cold-start choice for a file with no spec by name. The constant match is
8
+ # the lexical matching #538 uses for changed specs; the require match
9
+ # covers a file that defines only top-level methods. A file no spec reads or requires, or one read by more than
10
+ # CAP specs (a hub: the choice is not selective), gets no entry.
11
+ class ReferenceSpecs
12
+ CAP = 10
13
+ REQUIRE = /^\s*(require_relative|require)[ (]\s*["']([^"']+)["']/
14
+
15
+ # constants: file => the constants it defines, as TestGuess shapes them
16
+ # for ConstantReads.
17
+ def initialize(root, constants)
18
+ @root = root
19
+ @constants = constants
20
+ end
21
+
22
+ # chosen: file => sorted root-relative spec paths; crowded: file => how
23
+ # many specs read it, for the files over the cap (no choice was made).
24
+ Found = Data.define(:chosen, :crowded)
25
+
26
+ # The announcement for the files that got specs by reference.
27
+ def self.note(referenced)
28
+ named = referenced.map { |file, specs| "#{file} (#{specs.join(', ')})" }
29
+ "inferred specs: selected by constant reference or require, not by name: #{named.join(', ')}; " \
30
+ "a spec that reaches the file another way is not found"
31
+ end
32
+
33
+ def self.crowd_note(count) = "#{count} specs reference or require it, over the cap of #{CAP}"
34
+
35
+ # files: root-relative source files; spec_paths: root-relative spec
36
+ # files of the suite.
37
+ def call(files, spec_paths)
38
+ absolute = spec_paths.map { |path| File.join(@root, path) }
39
+ absolute.select! { |path| File.file?(path) && !File.symlink?(path) }
40
+ found = files.to_h { |file| [file, readers_of(file, absolute)] }.reject { |_, readers| readers.empty? }
41
+ crowded, chosen = found.partition { |_, readers| readers.size > CAP }.map(&:to_h)
42
+ Found.new(chosen: chosen, crowded: crowded.transform_values(&:size))
43
+ end
44
+
45
+ private
46
+
47
+ def readers_of(file, absolute_specs)
48
+ read = @constants.call(file).flat_map { |constant| ConstantReads.across(absolute_specs, constant).keys }
49
+ (read + requirers_of(file, absolute_specs)).uniq.sort.map { |path| path.delete_prefix("#{@root}/") }
50
+ end
51
+
52
+ # Specs that require the file: by path from the spec (require_relative)
53
+ # or by load path from the file's source root (require "a/b" for
54
+ # lib/a/b.rb). A bare basename is not a load path, so it matches nothing.
55
+ def requirers_of(file, absolute_specs)
56
+ stem = File.join(@root, file).delete_suffix(".rb")
57
+ loadable = file.delete_suffix(".rb").sub(%r{\A[^/]+/}, "")
58
+ absolute_specs.select { |spec| requires?(spec, stem, loadable) }
59
+ end
60
+
61
+ def requires?(spec, stem, loadable)
62
+ File.read(spec, encoding: "UTF-8").scrub.scan(REQUIRE).any? do |kind, target|
63
+ target = target.delete_suffix(".rb")
64
+ kind == "require_relative" ? File.expand_path(target, File.dirname(spec)) == stem : target == loadable
65
+ end
66
+ rescue SystemCallError
67
+ false
68
+ end
69
+ end
70
+ end
71
+ end
@@ -4,6 +4,7 @@
4
4
  require "digest"
5
5
  require "fileutils"
6
6
  require "json"
7
+ require "securerandom"
7
8
  require_relative "../text"
8
9
 
9
10
  module Saboteur
@@ -25,6 +26,7 @@ module Saboteur
25
26
  # stale.
26
27
  class ResultCache
27
28
  FORMAT = 1
29
+ MAX_BYTES = 64 * 1024 * 1024
28
30
  STORED = %i[killed survived timeout].freeze
29
31
  LOCKFILES = %w[Gemfile.lock gems.locked].freeze
30
32
  # Dependency declarations themselves, so the no-lockfile library
@@ -34,9 +36,15 @@ module Saboteur
34
36
 
35
37
  SOURCE_EXCLUDES = %w[.git vendor node_modules].freeze
36
38
 
39
+ # Seconds between checkpoint saves while verdicts arrive, so a run
40
+ # killed mid-phase (OOM, CI timeout, SIGKILL) keeps what it finished
41
+ # (#471).
42
+ SAVE_INTERVAL = 30
43
+
37
44
  # suite_digest: SuiteFiles#digest, so an edit to a spec helper or
38
45
  # support file invalidates every verdict.
39
- def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [])
46
+ def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [],
47
+ save_interval: SAVE_INTERVAL, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
40
48
  @path = path
41
49
  @root = root
42
50
  @inputs = inputs
@@ -47,6 +55,9 @@ module Saboteur
47
55
  @fallback_files = fallback_files.map { |file| File.expand_path(file, root) }.sort
48
56
  @entries = load
49
57
  @file_digests = {}
58
+ @save_interval = save_interval
59
+ @clock = clock
60
+ @last_saved = clock.call
50
61
  end
51
62
 
52
63
  # The stored verdict as a hash (status, reason, killed_by, kill_cause,
@@ -67,15 +78,39 @@ module Saboteur
67
78
  "reason" => result.reason, "killed_by" => result.killed_by, "kill_cause" => result.kill_cause,
68
79
  "runtime" => result.runtime
69
80
  }
81
+ checkpoint if @clock.call - @last_saved >= @save_interval
70
82
  end
71
83
 
84
+ # Written to a temp file and renamed, so a kill mid-write leaves the
85
+ # previous complete file, never a truncated one. A failed write or
86
+ # rename removes the temp file and raises.
72
87
  def save
73
88
  FileUtils.mkdir_p(File.dirname(@path))
74
- File.write(@path, JSON.generate(format: FORMAT, entries: @entries))
89
+ temp = "#{@path}.#{SecureRandom.hex(8)}.tmp"
90
+ begin
91
+ # "wx": a planted file or symlink at the temp name is never followed.
92
+ File.open(temp, "wx", 0o600) { |out| out.write(JSON.generate(format: FORMAT, entries: @entries)) }
93
+ File.rename(temp, @path)
94
+ rescue StandardError
95
+ FileUtils.rm_f(temp)
96
+ raise
97
+ end
98
+ @last_saved = @clock.call
75
99
  end
76
100
 
77
101
  private
78
102
 
103
+ # A mid-run save is a safety net, so it is best effort: a full disk or
104
+ # an unwritable directory must not abort a run that is still producing
105
+ # verdicts. The wait restarts either way, so a broken disk is tried
106
+ # once per interval, not once per verdict. The final save is not
107
+ # swallowed (Engine calls #save).
108
+ def checkpoint
109
+ save
110
+ rescue SystemCallError
111
+ @last_saved = @clock.call
112
+ end
113
+
79
114
  # The current project source files, relative and sorted. A file added
80
115
  # after the map was built changes the set and invalidates every stored
81
116
  # verdict, whether or not any watched file references it yet.
@@ -86,13 +121,41 @@ module Saboteur
86
121
  .join("\0")
87
122
  end
88
123
 
124
+ # The file is untrusted input (a restored cache, a --resume directory):
125
+ # read only as a regular file, never a symlink or FIFO, under a size
126
+ # cap, and kept only when entries is a Hash of Hashes.
89
127
  def load
90
- data = JSON.parse(Text.utf8_strict(File.read(@path)))
91
- data["format"] == FORMAT ? data["entries"] : {}
92
- rescue SystemCallError, JSON::ParserError
128
+ return {} unless File.lstat(@path).file?
129
+
130
+ text = slurp
131
+ return {} unless text
132
+
133
+ data = JSON.parse(Text.utf8_strict(text))
134
+ entries = data["entries"] if data.is_a?(Hash) && data["format"] == FORMAT
135
+ well_formed?(entries) ? entries : {}
136
+ rescue SystemCallError, JSON::ParserError, ArgumentError, EncodingError
93
137
  {}
94
138
  end
95
139
 
140
+ def slurp
141
+ File.open(@path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
142
+ text = io.read(MAX_BYTES)
143
+ text if io.eof?
144
+ end
145
+ end
146
+
147
+ def well_formed?(entries)
148
+ entries.is_a?(Hash) && entries.values.all? { |entry| entry.is_a?(Hash) && typed?(entry) }
149
+ end
150
+
151
+ # The fields the reports read, typed so a crafted entry cannot crash
152
+ # them.
153
+ def typed?(entry)
154
+ STORED.map(&:to_s).include?(entry["status"]) && (entry["runtime"].nil? || entry["runtime"].is_a?(Numeric)) &&
155
+ %w[reason kill_cause].all? { |key| entry[key].nil? || entry[key].is_a?(String) } &&
156
+ entry["killed_by"].is_a?(Array) && entry["killed_by"].all?(String)
157
+ end
158
+
96
159
  def key(subject, mutation)
97
160
  "#{subject.id} #{mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)}"
98
161
  end
@@ -0,0 +1,110 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "fileutils"
5
+ require "json"
6
+ require "securerandom"
7
+
8
+ module Saboteur
9
+ class Engine
10
+ # Identity file for a --resume directory (#471). The verdicts live in
11
+ # ResultCache; this only records what the run was computed from, so a
12
+ # later run with other inputs fails loudly instead of reusing the
13
+ # directory under a different meaning.
14
+ #
15
+ # The file is untrusted input: it is read only when it is a regular
16
+ # file (never a FIFO or a symlink), under a size cap, and must hold
17
+ # exactly the three expected keys with bounded String values.
18
+ class ResumeManifest
19
+ FORMAT = 1
20
+ FILE = "manifest.json"
21
+ KEYS = %w[format identity version].freeze
22
+ MAX_BYTES = 4096
23
+ MAX_FIELD = 512
24
+
25
+ # Raised when the directory belongs to another run or cannot be read.
26
+ class Mismatch < StandardError; end
27
+
28
+ # identity: a String that changes whenever the run's inputs change
29
+ # (the RunFingerprint digest joined with the settings that pick the
30
+ # mutants), at most MAX_FIELD characters.
31
+ def initialize(directory:, identity:)
32
+ @directory = directory
33
+ @identity = identity
34
+ end
35
+
36
+ def path
37
+ File.join(@directory, FILE)
38
+ end
39
+
40
+ # Creates the manifest in a fresh directory, accepts a matching one,
41
+ # and raises Mismatch for anything else.
42
+ def verify!
43
+ File.lstat(path)
44
+ compare(read)
45
+ rescue Errno::ENOENT
46
+ write
47
+ end
48
+
49
+ private
50
+
51
+ def compare(stored)
52
+ return if stored["identity"] == @identity && stored["version"] == VERSION
53
+
54
+ raise Mismatch, "#{path}: the run inputs changed since this directory was created; " \
55
+ "delete it or choose a new --resume path"
56
+ end
57
+
58
+ def read
59
+ raise Mismatch, "#{path}: unreadable manifest" unless File.lstat(path).file?
60
+
61
+ validate(parse(slurp))
62
+ rescue SystemCallError
63
+ raise Mismatch, "#{path}: unreadable manifest"
64
+ end
65
+
66
+ def slurp
67
+ File.open(path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
68
+ text = io.read(MAX_BYTES) || +""
69
+ raise Mismatch, "#{path}: manifest too large" unless io.eof?
70
+
71
+ text
72
+ end
73
+ end
74
+
75
+ def parse(text)
76
+ raise Mismatch, "#{path}: unreadable manifest" unless text.force_encoding(Encoding::UTF_8).valid_encoding?
77
+
78
+ JSON.parse(text)
79
+ rescue JSON::ParserError
80
+ raise Mismatch, "#{path}: unreadable manifest"
81
+ end
82
+
83
+ def validate(data)
84
+ raise Mismatch, "#{path}: unsupported manifest format" unless
85
+ data.is_a?(Hash) && data.keys.sort == KEYS && data["format"] == FORMAT && bounded?(data)
86
+
87
+ data
88
+ end
89
+
90
+ def bounded?(data)
91
+ %w[identity version].all? { |key| data[key].is_a?(String) && data[key].length <= MAX_FIELD }
92
+ end
93
+
94
+ # Written to a private temp file and linked into place: link fails
95
+ # when another run created the manifest first, and then that run's
96
+ # manifest is the one compared.
97
+ def write
98
+ FileUtils.mkdir_p(@directory)
99
+ temp = "#{path}.#{Process.pid}.#{SecureRandom.hex(4)}.tmp"
100
+ File.write(temp, JSON.generate(format: FORMAT, version: VERSION, identity: @identity))
101
+ File.link(temp, path)
102
+ nil
103
+ rescue Errno::EEXIST
104
+ compare(read)
105
+ ensure
106
+ FileUtils.rm_f(temp) if temp
107
+ end
108
+ end
109
+ end
110
+ end
@@ -64,11 +64,11 @@ module Saboteur
64
64
 
65
65
  needed = per_pass - remaining
66
66
  raise BudgetPhaseTimeout,
67
- "the --max-duration budget cannot cover the coverage map build: baseline passes took about " \
68
- "#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about one " \
69
- "more full pass and keeps nothing when it is cut off. Re-run with --max-duration about " \
70
- "#{needed.round}s higher, fewer --baseline-passes, or once without --max-duration so the " \
71
- "coverage cache is built for later --incremental runs"
67
+ "re-run with --max-duration about #{needed.round}s higher, fewer --baseline-passes, or once " \
68
+ "without --max-duration so the coverage cache is built for later --incremental runs: " \
69
+ "the budget cannot cover the coverage map build - baseline passes took about " \
70
+ "#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about " \
71
+ "one more full pass and keeps nothing when it is cut off"
72
72
  end
73
73
 
74
74
  # Boot and baselines can spend the budget by themselves: stop before
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "digest"
5
5
  require "json"
6
+ require_relative "../config"
6
7
  require_relative "../text"
7
8
  require_relative "data_inputs"
8
9
 
@@ -13,7 +14,7 @@ module Saboteur
13
14
  class RunFingerprint
14
15
  FORMAT_VERSION = 1
15
16
  MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
16
- LOCATION_SETTINGS = %w[cache_dir json_report html_report score_history].freeze
17
+ LOCATION_SETTINGS = Config::OUTPUT_KEYS
17
18
  ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
18
19
  EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
19
20
  Entry = Data.define(:path, :kind, :state, :sha256, :required)
@@ -144,7 +144,8 @@ module Saboteur
144
144
  def check_sandbox_write_set(all_dirs, user_dirs)
145
145
  holder = all_dirs.find { |dir| path_contains?(dir, @root) }
146
146
  if holder
147
- message = "sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
147
+ message = "narrow the sandbox grant #{holder} so it no longer contains the project root\n " \
148
+ "sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
148
149
  raise Error, %(#{message} - refusing under c.sandbox = "strict") if @config["sandbox"] == "strict"
149
150
 
150
151
  @diagnostics.warn(message)
@@ -153,7 +154,9 @@ module Saboteur
153
154
  (all_dirs - [dir]).each do |other|
154
155
  next unless path_contains?(dir, other)
155
156
 
156
- @diagnostics.warn("sandbox grant #{dir} contains the granted dir #{other}; " \
157
+ safe_other = Text.display_safe(other)
158
+ @diagnostics.warn("drop the redundant sandbox grant #{safe_other}\n " \
159
+ "sandbox grant #{Text.display_safe(dir)} contains the granted dir #{safe_other}; " \
157
160
  "the narrower grant is redundant")
158
161
  end
159
162
  end
@@ -337,7 +337,7 @@ module Saboteur
337
337
  # Each mutant is checked separately: changing a body can change which
338
338
  # definition a test reaches, so one unreachable verdict cannot skip later work.
339
339
  def note_unreachable(plan, outcome)
340
- @diagnostics.warn("#{plan.subject.id}: #{outcome.message}")
340
+ @diagnostics.warn("#{plan.subject.id}: a mutant never ran and is not scored\n #{outcome.message}")
341
341
  end
342
342
 
343
343
  # The text a known def-time wrapper needs re-evaluated with the def.
@@ -52,6 +52,7 @@ module Saboteur
52
52
  # opt-out and would warn.
53
53
  @config = config.with({ "tests" => guess.specs }, except: ["guess_tests"])
54
54
  @guessed_tests = guess.specs
55
+ @guessed_referenced = guess.referenced
55
56
  guess.scope_message(since: config["since"])
56
57
  end
57
58
  end
@@ -0,0 +1,121 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # Compares the map a partial rebuild produced with the full map (#46,
7
+ # step 3). The full map is the one a run uses; this only says whether the
8
+ # partial path would have been right.
9
+ #
10
+ # Two things are compared, and both must match:
11
+ # - the semantic evidence: which tests exist and, per source file, which
12
+ # tests hit which lines, methods and branch arms, and which lines ran
13
+ # while the app loaded. Durations, build time and the order of ids are
14
+ # left out, as are entries with no test.
15
+ # - what the result cache would watch: for every test the source files
16
+ # it touched, plus the boot files and the loaded files nothing
17
+ # observed. Evidence that matches line by line but yields another
18
+ # dependency set is worse than a line mismatch, since the set decides
19
+ # whether an old verdict is reused.
20
+ module ShadowComparison
21
+ MAX_DIFFERENCES = 50
22
+ KINDS = %i[lines methods branches].freeze
23
+
24
+ module_function
25
+
26
+ # { result: :match | :superset | :mismatch, match: bool, total: Integer, extra: Integer,
27
+ # gained: Integer,
28
+ # differences: [{kind:, name:}] } with at most MAX_DIFFERENCES entries,
29
+ # sorted by kind and name.
30
+ #
31
+ # :superset means the partial map credits tests the full map does not,
32
+ # on lines the full map also credits, and never lacks a credit the
33
+ # full map has. Anything else that differs is :mismatch, including an
34
+ # extra credit on a line the full map calls uncovered: it would turn
35
+ # "no test covers this" into "covered". Those are counted apart, in
36
+ # gained. extra counts the other extra credits (one per file, line and
37
+ # test) in lines, methods and branches.
38
+ def call(partial, full, fallback_files: [])
39
+ maps = [partial, full]
40
+ found = evidence_differences(maps) + dependency_differences(maps, fallback_files)
41
+ counted = found.select { |item| KINDS.map(&:to_s).include?(item[:kind]) }
42
+ extra = counted.sum { |item| item[:extra] }
43
+ gained = counted.sum { |item| item[:gained] }
44
+ result = if found.empty? then :match
45
+ elsif gained.positive? || found.any? { |item| item[:missing] } then :mismatch
46
+ else :superset
47
+ end
48
+ shown = found.sort_by { |item| [item[:kind], item[:name]] }.first(MAX_DIFFERENCES)
49
+ { result: result, match: result == :match, total: found.size, extra: extra, gained: gained,
50
+ differences: shown.map { |item| item.slice(:kind, :name) } }
51
+ end
52
+
53
+ def evidence_differences(maps)
54
+ partial, full = maps
55
+ found = []
56
+ unless partial.tests.sort == full.tests.sort
57
+ found << { kind: "tests", name: "the set of test ids", missing: true }
58
+ end
59
+ KINDS.each { |kind| found.concat(hit_differences(kind.to_s, hits(partial, kind), hits(full, kind))) }
60
+ found.concat(exact_differences("load", load_hits(partial), load_hits(full)))
61
+ end
62
+
63
+ # Per name: the credits only the partial map has (extra) and the ones
64
+ # only the full map has (missing).
65
+ def hit_differences(kind, left, right)
66
+ (left.keys | right.keys).filter_map do |name|
67
+ extra, gained, missing = credits(left[name] || {}, right[name] || {})
68
+ next if extra.zero? && gained.zero? && !missing
69
+
70
+ { kind: kind, name: name, extra: extra, gained: gained, missing: missing }
71
+ end
72
+ end
73
+
74
+ def credits(left, right)
75
+ extra = gained = 0
76
+ missing = false
77
+ (left.keys | right.keys).each do |line|
78
+ mine = left[line] || []
79
+ theirs = right[line] || []
80
+ (theirs.empty? ? gained += mine.size : extra += (mine - theirs).size)
81
+ missing ||= (theirs - mine).any?
82
+ end
83
+ [extra, gained, missing]
84
+ end
85
+
86
+ def exact_differences(kind, left, right)
87
+ (left.keys | right.keys).reject { |name| left[name] == right[name] }
88
+ .map { |name| { kind: kind, name: name, missing: true } }
89
+ end
90
+
91
+ # {file => {line-or-span => sorted ids}} without empty entries.
92
+ def hits(map, kind)
93
+ map.public_send(kind).each_with_object({}) do |(file, by_line), acc|
94
+ kept = by_line.transform_values(&:sort).reject { |_line, ids| ids.empty? }
95
+ acc[file] = kept unless kept.empty?
96
+ end
97
+ end
98
+
99
+ def load_hits(map)
100
+ [map.load_lines, map.load_methods].flat_map(&:to_a).each_with_object({}) do |(file, lines), acc|
101
+ acc[file] = (acc.fetch(file, []) | lines).sort unless lines.empty?
102
+ end
103
+ end
104
+
105
+ # The files every verdict depends on must match exactly. What each
106
+ # test depends on follows from its line and method credits, which are
107
+ # compared above, so it is not compared again.
108
+ def dependency_differences(maps, fallback_files)
109
+ partial, full = maps
110
+ left = dependencies(partial, fallback_files)
111
+ right = dependencies(full, fallback_files)
112
+ left.keys.reject { |name| left[name] == right[name] }.map { |name| { kind: "deps", name: name, missing: true } }
113
+ end
114
+
115
+ def dependencies(map, fallback_files)
116
+ unobserved = (fallback_files | map.loaded_files).select { |file| map.empty_for?([file]) }.sort
117
+ { "(boot files)" => map.boot_files, "(unobserved files)" => unobserved }
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,38 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # One shadow comparison (#46, step 3): would a partial rebuild from the
7
+ # last map have matched the full map this run just built? The run uses
8
+ # the full map either way; this only produces the report.
9
+ #
10
+ # The report always carries the share threshold and, when the partial
11
+ # path was possible, the observed stale share, so the constant can be
12
+ # fitted from data later.
13
+ module ShadowRun
14
+ module_function
15
+
16
+ # base: the earlier inputs snapshot and its map. current: the inputs
17
+ # snapshot of this run. rerun.call(stale_spec_files, seed) builds a map
18
+ # over just those files. clean turns a name into a line-safe string.
19
+ def call(base:, base_map:, current:, full_map:, invalidation:, fallback_files:, rerun:, clean:)
20
+ report = { "threshold" => PartialInvalidation::THRESHOLD, "base_map" => base["map_key"] }
21
+ stale = invalidation.stale_specs(base, current)
22
+ return report.merge("result" => "skipped", "reason" => "a full rebuild is needed") if stale == :full
23
+
24
+ merged = stale.empty? ? base_map : PartialMerge.call(base_map, rerun.call(stale, base["seed"]), stale)
25
+ verdict = ShadowComparison.call(merged, full_map, fallback_files: fallback_files)
26
+ report.merge("result" => verdict[:result].to_s, "extra_credits" => verdict[:extra],
27
+ "gained_credits" => verdict[:gained],
28
+ "stale" => stale.map(&clean),
29
+ "stale_share" => invalidation.share(stale), "total" => verdict[:total],
30
+ "differences" => verdict[:differences].map do |item|
31
+ item.to_h do |key, value|
32
+ [key.to_s, clean.call(value)]
33
+ end
34
+ end)
35
+ end
36
+ end
37
+ end
38
+ end