saboteur 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +167 -0
  3. data/README.md +393 -20
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/agent_skill.rb +6 -0
  6. data/lib/saboteur/baseline/acceptance.rb +218 -0
  7. data/lib/saboteur/baseline/command.rb +80 -0
  8. data/lib/saboteur/baseline/comparison.rb +105 -0
  9. data/lib/saboteur/baseline/completeness.rb +53 -0
  10. data/lib/saboteur/baseline/evidence.rb +114 -0
  11. data/lib/saboteur/baseline/identity.rb +78 -0
  12. data/lib/saboteur/baseline/manifest.rb +164 -0
  13. data/lib/saboteur/baseline/run_evidence.rb +117 -0
  14. data/lib/saboteur/baseline/workflow.rb +121 -0
  15. data/lib/saboteur/baseline_health.rb +1 -1
  16. data/lib/saboteur/cli/exit_status.rb +93 -0
  17. data/lib/saboteur/cli/run_reporting.rb +75 -0
  18. data/lib/saboteur/cli.rb +101 -121
  19. data/lib/saboteur/config.rb +76 -6
  20. data/lib/saboteur/doctor.rb +40 -8
  21. data/lib/saboteur/engine/baseline_guard.rb +33 -11
  22. data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
  23. data/lib/saboteur/engine/canary.rb +74 -0
  24. data/lib/saboteur/engine/candidate_selection.rb +18 -5
  25. data/lib/saboteur/engine/candidates.rb +59 -7
  26. data/lib/saboteur/engine/coverage_store.rb +34 -14
  27. data/lib/saboteur/engine/diagnostics.rb +150 -4
  28. data/lib/saboteur/engine/diff_scope.rb +10 -0
  29. data/lib/saboteur/engine/distribution.rb +54 -13
  30. data/lib/saboteur/engine/result_cache.rb +10 -5
  31. data/lib/saboteur/engine/run_fingerprint.rb +6 -4
  32. data/lib/saboteur/engine/sandbox_policy.rb +108 -0
  33. data/lib/saboteur/engine/scheduler.rb +92 -20
  34. data/lib/saboteur/engine/subject_selection.rb +24 -1
  35. data/lib/saboteur/engine/test_guess.rb +52 -0
  36. data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
  37. data/lib/saboteur/engine/verdict_check.rb +7 -6
  38. data/lib/saboteur/engine.rb +192 -18
  39. data/lib/saboteur/execution/protocol.rb +2 -1
  40. data/lib/saboteur/execution/reevaluator.rb +72 -10
  41. data/lib/saboteur/execution/runner.rb +120 -17
  42. data/lib/saboteur/execution/sandbox.rb +271 -0
  43. data/lib/saboteur/execution/stale_captures.rb +43 -0
  44. data/lib/saboteur/execution/supervisor.rb +101 -9
  45. data/lib/saboteur/execution/worker_pool.rb +20 -4
  46. data/lib/saboteur/execution/wrapper.rb +45 -0
  47. data/lib/saboteur/gates.rb +83 -0
  48. data/lib/saboteur/help.rb +31 -7
  49. data/lib/saboteur/ignore_list.rb +5 -0
  50. data/lib/saboteur/inline_suppression.rb +65 -0
  51. data/lib/saboteur/integration/base.rb +1 -0
  52. data/lib/saboteur/integration/focus_guard.rb +83 -0
  53. data/lib/saboteur/integration/rspec.rb +21 -1
  54. data/lib/saboteur/mutation_source/builtin.rb +22 -3
  55. data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
  56. data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
  57. data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
  58. data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
  59. data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
  60. data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
  61. data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
  62. data/lib/saboteur/preview.rb +106 -0
  63. data/lib/saboteur/reporter/agent_commands.rb +7 -1
  64. data/lib/saboteur/reporter/agent_incomplete.rb +24 -1
  65. data/lib/saboteur/reporter/agent_summary.rb +11 -5
  66. data/lib/saboteur/reporter/findings.rb +4 -19
  67. data/lib/saboteur/reporter/history_paths.rb +38 -0
  68. data/lib/saboteur/reporter/history_record.rb +132 -0
  69. data/lib/saboteur/reporter/html.rb +12 -1
  70. data/lib/saboteur/reporter/mutation_text.rb +36 -0
  71. data/lib/saboteur/reporter/next_steps.rb +30 -0
  72. data/lib/saboteur/reporter/palette.rb +37 -0
  73. data/lib/saboteur/reporter/score_history.rb +123 -0
  74. data/lib/saboteur/reporter/score_table.rb +57 -0
  75. data/lib/saboteur/reporter/stryker.rb +15 -2
  76. data/lib/saboteur/reporter/survivor_hints.rb +68 -0
  77. data/lib/saboteur/reporter/survivor_list.rb +126 -0
  78. data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
  79. data/lib/saboteur/reporter/terminal.rb +131 -79
  80. data/lib/saboteur/reporter/test_matrix.rb +57 -0
  81. data/lib/saboteur/result.rb +41 -6
  82. data/lib/saboteur/selection/coverage_map.rb +119 -13
  83. data/lib/saboteur/selection/git_diff.rb +28 -1
  84. data/lib/saboteur/selection/recorder.rb +55 -13
  85. data/lib/saboteur/subject.rb +40 -4
  86. data/lib/saboteur/subject_finder.rb +62 -12
  87. data/lib/saboteur/version.rb +1 -1
  88. data/lib/saboteur/working_tree.rb +8 -0
  89. data/lib/saboteur.rb +6 -0
  90. metadata +43 -2
@@ -0,0 +1,61 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "prism"
5
+ require_relative "base"
6
+
7
+ module Saboteur
8
+ module MutationSource
9
+ # Swaps the selected end of an explicitly received collection.
10
+ # Only message bytes change; arguments, receiver and block stay intact.
11
+ class EndpointReplacement < Base
12
+ REPLACEMENTS = { first: "last", last: "first" }.freeze
13
+ SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
14
+
15
+ # Base inherits Object's empty initializer; there is no parent state to set.
16
+ def initialize(arid: []) # rubocop:disable Lint/MissingSuper
17
+ @arid = arid.map { |pattern| Regexp.new(pattern) }
18
+ end
19
+
20
+ def name
21
+ "endpoint_replacement"
22
+ end
23
+
24
+ def mutations_for(subject)
25
+ found = []
26
+ visit(subject.node.body, subject, found)
27
+ found
28
+ end
29
+
30
+ private
31
+
32
+ def visit(node, subject, found)
33
+ return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
34
+
35
+ if node.is_a?(Prism::StatementsNode)
36
+ node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
37
+ return
38
+ end
39
+
40
+ replace_endpoint(node, subject, found) if node.is_a?(Prism::CallNode)
41
+ node.compact_child_nodes.each { |child| visit(child, subject, found) }
42
+ end
43
+
44
+ def replace_endpoint(node, subject, found)
45
+ replacement = REPLACEMENTS[node.name]
46
+ return unless replacement && node.receiver
47
+ return if node.block
48
+ return unless node.arguments.nil?
49
+
50
+ location = node.message_loc
51
+ edit = Edit.new(file: subject.file, start_byte: location.start_offset,
52
+ end_byte: location.end_offset, replacement: replacement)
53
+ found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
54
+ end
55
+
56
+ def arid?(node)
57
+ @arid.any? { |pattern| pattern.match?(node.slice) }
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,136 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module MutationSource
6
+ # Suppresses mutants that would run a destructive call with altered
7
+ # inputs. A mutant is only kept when every edit is provably safe: either
8
+ # the subject holds no hazardous call, or the edit removes every
9
+ # hazardous call in the method AND its replacement introduces no new one
10
+ # (checked by parsing the replacement, never by substring match). Edits
11
+ # strictly inside a call, edits elsewhere in a method that contains a
12
+ # call, and edits whose replacement re-includes a hazard (for example
13
+ # removing a negation in front of `system(...)`, or swapping
14
+ # `rm_rf("build")` for `rm_rf(".")`) all leave a destructive call
15
+ # running with altered inputs and are suppressed (issue #102).
16
+ class HazardGuard
17
+ # "<Receiver>.<name>" for calls with a receiver, the bare name for
18
+ # Kernel-style calls. Bare FileUtils operations are listed too, so an
19
+ # `include FileUtils` does not disarm the guard. Any method on Open3,
20
+ # any rmtree, and backticks/%x literals are always hazardous. Replace
21
+ # or extend via c.hazard; run them anyway with --allow-hazardous.
22
+ DEFAULT_HAZARD = [
23
+ "FileUtils.rm", "FileUtils.rm_r", "FileUtils.rm_rf", "FileUtils.rm_f",
24
+ "FileUtils.remove_dir", "FileUtils.remove_entry", "FileUtils.remove_entry_secure",
25
+ "rm", "rm_r", "rm_rf", "rm_f", "remove_dir", "remove_entry", "remove_entry_secure",
26
+ "File.delete", "File.unlink",
27
+ "Dir.rmdir", "Dir.delete", "Dir.unlink",
28
+ "IO.popen",
29
+ "Process.spawn", "Process.exec",
30
+ "system", "exec", "spawn", "`"
31
+ ].freeze
32
+
33
+ OPEN3 = /\A(::)?Open3\z/
34
+ ALWAYS_NAMES = %w[rmtree].freeze
35
+ KERNEL_RECEIVERS = %w[Kernel Process].freeze
36
+
37
+ def initialize(patterns = DEFAULT_HAZARD)
38
+ @tags = Set.new(patterns.map { |pattern| pattern.to_s.sub(/\A::/, "") })
39
+ end
40
+
41
+ def name
42
+ "hazard_guard"
43
+ end
44
+
45
+ # candidates: [[mutation, mutated_text]]. Returns [kept, suppressed].
46
+ def partition(candidates, subject)
47
+ spans = hazard_spans(subject.node)
48
+ return [candidates, []] if spans.empty?
49
+
50
+ candidates.partition { |mutation, _| safe?(mutation, spans) }
51
+ end
52
+
53
+ private
54
+
55
+ # Byte spans of hazardous calls in the node tree, with the source slice
56
+ # of each. Inner calls are covered by the outer span, so the walk does
57
+ # not descend into a hazardous call.
58
+ def hazard_spans(root)
59
+ spans = []
60
+ stack = [root]
61
+ until stack.empty?
62
+ node = stack.pop
63
+ if hazardous?(node)
64
+ spans << [node.location.start_offset, node.location.end_offset, node.slice]
65
+ next
66
+ end
67
+ stack.concat(node.compact_child_nodes)
68
+ end
69
+ spans
70
+ end
71
+
72
+ def hazardous?(node)
73
+ case node
74
+ when Prism::XStringNode, Prism::InterpolatedXStringNode then true
75
+ when Prism::CallNode then hazardous_call?(node)
76
+ else false
77
+ end
78
+ end
79
+
80
+ def hazardous_call?(node)
81
+ name = node.name.to_s
82
+ receiver = node.receiver&.slice&.sub(/\A::/, "")
83
+ return true if OPEN3.match?(receiver.to_s)
84
+ return true if ALWAYS_NAMES.include?(name)
85
+
86
+ tag = receiver ? "#{receiver}.#{name}" : name
87
+ return true if @tags.include?(tag)
88
+ # Kernel.system / Kernel.spawn / Process.spawn: the explicit-receiver
89
+ # spelling of a bare process spawn is the same hazard.
90
+ return true if KERNEL_RECEIVERS.include?(receiver) && @tags.include?(name)
91
+
92
+ piped_open?(node, name, receiver)
93
+ end
94
+
95
+ # Kernel#open with a "| command" argument spawns a process.
96
+ def piped_open?(node, name, receiver)
97
+ return false unless name == "open" && (receiver.nil? || receiver == "Kernel")
98
+
99
+ first = node.arguments&.arguments&.first
100
+ case first
101
+ when Prism::StringNode then first.unescaped.start_with?("|")
102
+ when Prism::InterpolatedStringNode
103
+ part = first.parts.first
104
+ part.is_a?(Prism::StringNode) && part.unescaped.start_with?("|")
105
+ else false
106
+ end
107
+ end
108
+
109
+ # A mutation is safe only while no hazardous call survives it: every
110
+ # edit must destroy every span (cover it wholly with a hazard-free
111
+ # replacement). Anything less - touching a call, or editing other
112
+ # code in a method that still holds a call - keeps the call running
113
+ # with altered inputs, so the mutant is suppressed.
114
+ def safe?(mutation, spans)
115
+ mutation.edits.all? { |edit| destroys_all?(edit, spans) }
116
+ end
117
+
118
+ def destroys_all?(edit, spans)
119
+ spans.all? do |start_byte, end_byte, _slice|
120
+ edit.start_byte <= start_byte && edit.end_byte >= end_byte
121
+ end && replacement_clean?(edit.replacement)
122
+ end
123
+
124
+ # The replacement itself, parsed as code. A replacement that parses
125
+ # and contains no hazardous call cannot run one; anything else -
126
+ # a hazard in the new text, or text that does not parse on its own -
127
+ # is treated as hazardous.
128
+ def replacement_clean?(replacement)
129
+ result = Prism.parse(replacement.to_s)
130
+ return false unless result.success?
131
+
132
+ hazard_spans(result.value).empty?
133
+ end
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,52 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "base"
5
+
6
+ module Saboteur
7
+ module MutationSource
8
+ # Changes the arithmetic operation, without changing the assignment target.
9
+ class OperatorAssignment < Base
10
+ REPLACEMENTS = { :+ => "-=", :- => "+=", :* => "/=", :/ => "*=" }.freeze
11
+ WRITE_NODES = [Prism::LocalVariableOperatorWriteNode, Prism::InstanceVariableOperatorWriteNode,
12
+ Prism::ClassVariableOperatorWriteNode, Prism::GlobalVariableOperatorWriteNode,
13
+ Prism::ConstantOperatorWriteNode, Prism::ConstantPathOperatorWriteNode,
14
+ Prism::CallOperatorWriteNode, Prism::IndexOperatorWriteNode].freeze
15
+ SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
16
+
17
+ def initialize(arid: []) # rubocop:disable Lint/MissingSuper
18
+ @arid = arid.map { |pattern| Regexp.new(pattern) }
19
+ end
20
+
21
+ def name
22
+ "operator_assignment"
23
+ end
24
+
25
+ def mutations_for(subject)
26
+ found = []
27
+ visit(subject.node.body, subject, found)
28
+ found
29
+ end
30
+
31
+ private
32
+
33
+ def visit(node, subject, found)
34
+ return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
35
+ return if @arid.any? { |pattern| pattern.match?(node.slice) }
36
+
37
+ replace(node, subject, found) if WRITE_NODES.any? { |type| node.is_a?(type) }
38
+ node.compact_child_nodes.each { |child| visit(child, subject, found) }
39
+ end
40
+
41
+ def replace(node, subject, found)
42
+ replacement = REPLACEMENTS[node.binary_operator]
43
+ return unless replacement
44
+
45
+ location = node.binary_operator_loc
46
+ edit = Edit.new(file: subject.file, start_byte: location.start_offset,
47
+ end_byte: location.end_offset, replacement: replacement)
48
+ found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,57 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module MutationSource
6
+ # Public names used by selection, help and inline suppression validation.
7
+ module OperatorRegistry
8
+ DESCRIPTIONS = {
9
+ "operator_assignment" => "Swap arithmetic compound assignments",
10
+ "condition_negation" => "Negate if, unless and ternary predicates",
11
+ "range" => "Swap inclusive and exclusive bounded ranges",
12
+ "body_nil" => "Replace the method body with nil",
13
+ "body_true" => "Replace a predicate body with true",
14
+ "body_false" => "Replace a predicate body with false",
15
+ "body_raise" => "Replace the method body with raise",
16
+ "comparison" => "Change a comparison or replace its result",
17
+ "arithmetic" => "Swap an arithmetic operator",
18
+ "boolean_logic" => "Change a boolean connector",
19
+ "condition_true" => "Force a condition true",
20
+ "condition_false" => "Force a condition false",
21
+ "negation_removal" => "Remove a logical negation",
22
+ "literal_number" => "Change a numeric literal",
23
+ "or_assign" => "Replace ||= with assignment",
24
+ "safe_navigation" => "Replace safe navigation with a regular call",
25
+ "statement_removal" => "Replace a statement with nil",
26
+ "filter_replacement" => "Swap select and reject",
27
+ "literal_string" => "Empty a string literal",
28
+ "literal_boolean" => "Swap a boolean literal",
29
+ "argument_removal" => "Remove a call argument",
30
+ "pair_removal" => "Remove a hash pair",
31
+ "endpoint_replacement" => "Swap explicit no-argument first and last",
32
+ "chain_removal" => "Remove one call from a method chain"
33
+ }.freeze
34
+ EXTENDED = %w[range operator_assignment condition_negation].freeze
35
+ TIERS = { "core" => (DESCRIPTIONS.keys - EXTENDED).freeze, "extended" => EXTENDED,
36
+ "all" => DESCRIPTIONS.keys.freeze }.freeze
37
+
38
+ def self.list(out)
39
+ DESCRIPTIONS.each do |name, description|
40
+ out.puts("#{name} #{EXTENDED.include?(name) ? 'extended' : 'core'} #{description}")
41
+ end
42
+ nil
43
+ end
44
+
45
+ def self.resolve(names)
46
+ return TIERS.fetch("core") if names.empty?
47
+
48
+ unknown = names - DESCRIPTIONS.keys - TIERS.keys
49
+ unless unknown.empty?
50
+ raise ConfigError, "unknown operators: #{unknown.join(', ')}. Valid names: " \
51
+ "#{(DESCRIPTIONS.keys + TIERS.keys).join(', ')}"
52
+ end
53
+ names.flat_map { |name| TIERS.fetch(name) { [name] } }.uniq
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,43 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "base"
5
+
6
+ module Saboteur
7
+ module MutationSource
8
+ # Changes whether a range includes its upper endpoint.
9
+ class RangeReplacement < Base
10
+ SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
11
+
12
+ def initialize(arid: []) # rubocop:disable Lint/MissingSuper
13
+ @arid = arid.map { |pattern| Regexp.new(pattern) }
14
+ end
15
+
16
+ def name
17
+ "range"
18
+ end
19
+
20
+ def mutations_for(subject)
21
+ found = []
22
+ visit(subject.node.body, subject, found)
23
+ found
24
+ end
25
+
26
+ private
27
+
28
+ def visit(node, subject, found)
29
+ return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
30
+ return if @arid.any? { |pattern| pattern.match?(node.slice) }
31
+
32
+ if node.is_a?(Prism::RangeNode) && node.left && node.right
33
+ location = node.operator_loc
34
+ replacement = location.slice == ".." ? "..." : ".."
35
+ edit = Edit.new(file: subject.file, start_byte: location.start_offset,
36
+ end_byte: location.end_offset, replacement: replacement)
37
+ found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
38
+ end
39
+ node.compact_child_nodes.each { |child| visit(child, subject, found) }
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,106 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "engine"
5
+
6
+ module Saboteur
7
+ # Source-only planning. Never boots the host or executes its test suite.
8
+ class Preview
9
+ def initialize(config, out:, err:)
10
+ @config = config
11
+ @out = out
12
+ @diagnostics = Engine::Diagnostics.new(err)
13
+ end
14
+
15
+ def call
16
+ selection = Engine::SubjectSelection.new(@config, @diagnostics)
17
+ subjects = selection.select(@config.included_files)
18
+ scope = selection.changed_lines && Engine::DiffScope.new(selection.changed_lines)
19
+ guard = MutationSource::HazardGuard.new(@config["hazard"]) unless @config["allow_hazardous"]
20
+ candidates = Engine::Candidates.new(Engine::Candidates.build_sources(@config), @diagnostics, hazard_guard: guard)
21
+ lists = subjects.map { |subject| planned(subject, candidates, scope) }
22
+ inline_suppressions
23
+ incomplete = @diagnostics.discovery_failures.size
24
+ @out.puts("Total: #{lists.sum(&:size)} mutants; #{@diagnostics.omissions.sum(&:count)} " \
25
+ "dropped by max_mutants_per_method#{incomplete_note(incomplete)}")
26
+ incomplete.zero? ? estimate(subjects, lists) : @out.puts("no estimate: source discovery was incomplete")
27
+ @diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
28
+ nil
29
+ end
30
+
31
+ private
32
+
33
+ # Candidate filtering precedes diff restriction and the per-method cap.
34
+ # Name that scope rather than implying these rows were executed or scored.
35
+ def inline_suppressions
36
+ entries = @diagnostics.inline_suppressions
37
+ return if entries.empty?
38
+
39
+ @out.puts("Inline suppressions: #{entries.size} candidates in selected methods " \
40
+ "(before diff restriction and per-method cap; outside the planned total)")
41
+ entries.each do |entry|
42
+ path = entry[:file].delete_prefix("#{@config.root}/")
43
+ @out.puts(" #{path}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
44
+ "#{entry[:digest]}: #{entry[:reason]}")
45
+ end
46
+ end
47
+
48
+ def incomplete_note(count)
49
+ return "" if count.zero?
50
+
51
+ " - INCOMPLETE: #{count} file(s) could not be parsed, so this total is not authoritative"
52
+ end
53
+
54
+ def estimate(subjects, lists)
55
+ map = cached_map
56
+ return @out.puts("no estimate: run once to build the coverage map") unless map
57
+
58
+ seconds = work_seconds(map, subjects, lists)
59
+ return @out.puts("no estimate: cached test durations are incomplete") unless seconds
60
+
61
+ @out.puts("Estimated baseline covering-test work: #{seconds.round(2)} s (cached original-test durations " \
62
+ "times planned mutants; not a bound on mutant runtime; excludes boot and forks)")
63
+ end
64
+
65
+ # Cached data is untrusted input: every duration must be a non-negative
66
+ # number, the total must stay finite (JSON text such as 1e999 parses to
67
+ # Infinity), and anything that fails while reading it, such as a string or
68
+ # null, is no estimate. Source generation has already finished, so this
69
+ # cannot hide its errors.
70
+ def work_seconds(map, subjects, lists)
71
+ return unless map.durations.values.all? { |seconds| seconds >= 0 }
72
+
73
+ total = subjects.zip(lists).sum do |subject, list|
74
+ # nil when a covering test has no recorded duration: the rescue below.
75
+ map.recorded_runtime(map.tests_for(subject)) * list.size
76
+ end
77
+ total if total.finite?
78
+ rescue StandardError
79
+ nil
80
+ end
81
+
82
+ # Any receipt that is not exactly a current, readable pointer to a
83
+ # coverage map means "no estimate", never an exception.
84
+ def cached_map
85
+ path = File.join(@config.path("cache_dir"), "preview.json")
86
+ receipt = JSON.parse(File.read(path))
87
+ return unless receipt["fingerprint"] == Engine::RunFingerprint.new(config: @config).snapshot.digest
88
+
89
+ Selection::CoverageMap.load(receipt["path"])
90
+ rescue StandardError
91
+ # An unreadable receipt or a map of the wrong shape (IndexError, TypeError, ...) is no estimate.
92
+ nil
93
+ end
94
+
95
+ def planned(subject, candidates, scope)
96
+ list = candidates.for(subject)
97
+ list = scope.restrict(subject, list) if scope
98
+ limit = @config["max_mutants_per_method"]
99
+ @diagnostics.record_omission(subject, count: list.size - limit, limit: limit) if list.size > limit
100
+ list = list.first(limit)
101
+ path = subject.file.delete_prefix("#{@config.root}/")
102
+ @out.puts("#{path}:#{subject.start_line} #{subject.id} #{list.size} mutants")
103
+ list
104
+ end
105
+ end
106
+ end
@@ -40,12 +40,18 @@ module Saboteur
40
40
  command.bytesize <= MAX_BYTES ? command : nil
41
41
  end
42
42
 
43
+ # An explicit scope is frozen: --no-guess-tests keeps a config-file
44
+ # guess_tests from conflicting with the --test list.
45
+ def test_words(tests)
46
+ tests.empty? ? [] : ["--no-guess-tests", *tests.flat_map { |test| ["--test", test] }]
47
+ end
48
+
43
49
  def selection(provenance, replay: false)
44
50
  words = []
45
51
  words.push("--config", provenance.config_path) if provenance.config_path
46
52
  words.push("--since", provenance.ref) if provenance.ref
47
53
  provenance.subjects.each { |subject| words.push("--subject", subject) }
48
- provenance.tests.each { |test| words.push("--test", test) }
54
+ words.push(*test_words(provenance.tests))
49
55
  SETTING_FLAGS.each do |key, flag|
50
56
  words.push(flag, provenance.settings[key].to_s) if provenance.settings.key?(key)
51
57
  end
@@ -7,24 +7,47 @@ module Saboteur
7
7
  # What makes a verdict cover less than the whole selection. Each reason has a
8
8
  # stable code, a count and a plain meaning.
9
9
  module Incomplete
10
+ NO_COVERAGE = "no coverage"
11
+
10
12
  class << self
11
13
  def reasons(report)
12
14
  reasons = []
13
15
  add(reasons, "no_coverage", skipped_for_coverage(report), "subjects or mutants had no covering test")
16
+ load_only = report.counts[:ran_at_load] + report.skipped_subjects.values.count do |reason|
17
+ reason == Selection::CoverageMap::RAN_AT_LOAD
18
+ end
19
+ add(reasons, "unsupported", unsupported_subjects(report),
20
+ "subjects were skipped as unsupported (for example defined inside a block); no mutant was judged")
21
+ add(reasons, "ran_at_load", load_only, "code ran only at load; no mutant was judged")
22
+ add(reasons, "unreachable", report.counts[:unreachable], "tests did not call the replacement definition")
14
23
  add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
15
24
  add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
16
25
  add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
26
+ add(reasons, "fail_fast", report.early_stop&.unreached&.size.to_i,
27
+ "selected subjects did not run because --fail-fast stopped the run")
28
+ inline_scope(reasons, report)
17
29
  reasons
18
30
  end
19
31
 
20
32
  private
21
33
 
34
+ def inline_scope(reasons, report)
35
+ add(reasons, "inline_suppressed", report.inline_suppressions.size,
36
+ "candidates were suppressed by inline directives; see the full report audit")
37
+ end
38
+
22
39
  def add(reasons, code, count, meaning)
23
40
  reasons << { reason: code, count: count, meaning: meaning } if count.positive?
24
41
  end
25
42
 
26
43
  def skipped_for_coverage(report)
27
- report.skipped_subjects.size + report.counts[:skipped]
44
+ report.skipped_subjects.values.count(NO_COVERAGE) + report.counts[:skipped]
45
+ end
46
+
47
+ def unsupported_subjects(report)
48
+ report.skipped_subjects.values.count do |reason|
49
+ reason != NO_COVERAGE && reason != Selection::CoverageMap::RAN_AT_LOAD
50
+ end
28
51
  end
29
52
 
30
53
  def omitted_by_cap(report)
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require_relative "findings"
5
+ require_relative "survivor_hints"
5
6
  require_relative "agent_incomplete"
6
7
  require_relative "agent_clipping"
7
8
 
@@ -26,7 +27,7 @@ module Saboteur
26
27
  scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
27
28
  incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
28
29
  findings: findings(report, shown, snippet_limit),
29
- warnings: warnings(report),
30
+ warnings: report.warnings.first(WARNING_LIMIT).map { |warning| warning.to_s[0, WARNING_LENGTH] },
30
31
  next: next_steps(report, provenance)
31
32
  }
32
33
  end
@@ -34,7 +35,7 @@ module Saboteur
34
35
  private
35
36
 
36
37
  def status_of(report, incomplete)
37
- return "not_applicable" if report.results.empty? && report.skipped_subjects.empty?
38
+ return "not_applicable" if report.results.empty? && incomplete.empty?
38
39
 
39
40
  incomplete.empty? ? "complete" : "partial"
40
41
  end
@@ -87,7 +88,8 @@ module Saboteur
87
88
  {
88
89
  survivors: live, shown: shown_findings(all[:survivors], shown, snippet_limit),
89
90
  omitted: live - [shown, all[:survivors].size].min,
90
- accepted: report.survivors.size - live, ordering: all[:ordering]
91
+ accepted: report.survivors.size - live, ordering: all[:ordering],
92
+ hintCaveat: SurvivorHints::CAVEAT
91
93
  }
92
94
  end
93
95
 
@@ -97,11 +99,15 @@ module Saboteur
97
99
  .merge(path: clip_value(finding[:path], 200), subject: clip_value(finding[:subject], 200),
98
100
  original: clip(finding[:original], snippet_limit),
99
101
  replacement: clip(finding[:replacement], snippet_limit))
102
+ .merge(finding_hints(finding))
100
103
  end
101
104
  end
102
105
 
103
- def warnings(report)
104
- report.warnings.first(WARNING_LIMIT).map { |warning| warning.to_s[0, WARNING_LENGTH] }
106
+ def finding_hints(finding)
107
+ structured = SurvivorHints.structured(operator: finding[:operator], original: finding[:original],
108
+ replacement: finding[:replacement], coarse: finding[:coarse])
109
+ { hint: SurvivorHints.for(finding[:operator], coarse: finding[:coarse]) }.compact
110
+ .merge(testHint: structured)
105
111
  end
106
112
 
107
113
  def next_steps(report, provenance)
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "survivor_ranking"
5
+
4
6
  module Saboteur
5
7
  module Reporter
6
8
  # Findings for a code-review surface such as the saboteur GitHub Action:
@@ -15,12 +17,7 @@ module Saboteur
15
17
  # GitHub shows at most 10 annotations of one kind per step.
16
18
  LIMIT = 10
17
19
 
18
- # Published ordering, coarse survivors first: a surviving coarse mutant
19
- # replaced the whole method body, which hints that no test pins the
20
- # method's behavior - though it can also be a re-evaluation limit or an
21
- # equivalent mutant, so this is a heuristic, not a guarantee. Ties break
22
- # by file, line and digest, so one report always reads the same way.
23
- ORDERING = "coarse survivors first, then file, line and digest"
20
+ ORDERING = SurvivorRanking::ORDERING
24
21
 
25
22
  class << self
26
23
  def build(report)
@@ -46,20 +43,8 @@ module Saboteur
46
43
  [results.size - LIMIT, 0].max
47
44
  end
48
45
 
49
- # partition keeps the coarse group ahead; inside a group the key is
50
- # total (no two mutants share a digest), so the order is stable.
51
46
  def ordered(report, results)
52
- results.partition { |result| result.mutation.coarse }
53
- .map { |group| group.sort_by { |result| tie_break(report, result) } }
54
- .flatten
55
- end
56
-
57
- def tie_break(report, result)
58
- [path(report, result), first_line(result), result.digest]
59
- end
60
-
61
- def first_line(result)
62
- result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
47
+ SurvivorRanking.ordered(report, results)
63
48
  end
64
49
 
65
50
  def finding(report, result)
@@ -0,0 +1,38 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # Refuses linked or non-regular history/report destinations before writes.
7
+ module HistoryPaths
8
+ module_function
9
+
10
+ def validate(paths)
11
+ paths = paths.compact.map { |path| File.expand_path(path) }
12
+ identities = paths.map do |path|
13
+ reject_links(path)
14
+ next unless File.exist?(path)
15
+
16
+ stat = File.stat(path)
17
+ raise Error, "score history/report destination must be a regular file" unless stat.file?
18
+
19
+ [stat.dev, stat.ino]
20
+ end.compact
21
+ raise Error, "score history/report destinations alias the same file" unless identities.uniq == identities
22
+ rescue SystemCallError => e
23
+ raise Error, "score history/report paths cannot be verified: #{e.message}"
24
+ end
25
+
26
+ def reject_links(path)
27
+ loop do
28
+ raise Error, "score history/report paths must not contain symlinks" if File.symlink?(path)
29
+
30
+ parent = File.dirname(path)
31
+ break if parent == path
32
+
33
+ path = parent
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end