saboteur 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +399 -0
  3. data/README.md +202 -42
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/comparison.rb +1 -0
  6. data/lib/saboteur/baseline/evidence.rb +11 -1
  7. data/lib/saboteur/baseline/run_evidence.rb +1 -1
  8. data/lib/saboteur/baseline/workflow.rb +2 -1
  9. data/lib/saboteur/baseline_health.rb +40 -6
  10. data/lib/saboteur/cli/exit_status.rb +49 -4
  11. data/lib/saboteur/cli/help_options.rb +16 -0
  12. data/lib/saboteur/cli/presets.rb +36 -0
  13. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  14. data/lib/saboteur/cli/run_reporting.rb +75 -6
  15. data/lib/saboteur/cli.rb +80 -48
  16. data/lib/saboteur/config.rb +88 -13
  17. data/lib/saboteur/crash_report.rb +154 -0
  18. data/lib/saboteur/custom_map.rb +3 -1
  19. data/lib/saboteur/doctor.rb +64 -2
  20. data/lib/saboteur/engine/baseline_guard.rb +79 -5
  21. data/lib/saboteur/engine/boot.rb +12 -1
  22. data/lib/saboteur/engine/canary.rb +41 -10
  23. data/lib/saboteur/engine/candidate_selection.rb +9 -1
  24. data/lib/saboteur/engine/candidates.rb +4 -1
  25. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  26. data/lib/saboteur/engine/coverage_store.rb +48 -6
  27. data/lib/saboteur/engine/diagnostics.rb +23 -2
  28. data/lib/saboteur/engine/diff_scope.rb +24 -5
  29. data/lib/saboteur/engine/distribution.rb +124 -8
  30. data/lib/saboteur/engine/early_stop_recording.rb +46 -0
  31. data/lib/saboteur/engine/result_cache.rb +2 -1
  32. data/lib/saboteur/engine/run_budget.rb +63 -0
  33. data/lib/saboteur/engine/run_fingerprint.rb +5 -1
  34. data/lib/saboteur/engine/sandbox_policy.rb +18 -11
  35. data/lib/saboteur/engine/scheduler.rb +114 -18
  36. data/lib/saboteur/engine/subject_selection.rb +84 -5
  37. data/lib/saboteur/engine/suite_files.rb +3 -1
  38. data/lib/saboteur/engine/test_guess.rb +183 -23
  39. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  40. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  41. data/lib/saboteur/engine/verdict_check.rb +54 -18
  42. data/lib/saboteur/engine.rb +175 -96
  43. data/lib/saboteur/execution/protocol.rb +2 -1
  44. data/lib/saboteur/execution/runner.rb +179 -17
  45. data/lib/saboteur/execution/sandbox.rb +6 -1
  46. data/lib/saboteur/execution/supervisor.rb +18 -3
  47. data/lib/saboteur/execution/worker_pool.rb +87 -7
  48. data/lib/saboteur/frameworks/database_hosts.rb +30 -0
  49. data/lib/saboteur/frameworks/rails.rb +8 -10
  50. data/lib/saboteur/help.rb +55 -10
  51. data/lib/saboteur/ignore_list.rb +11 -1
  52. data/lib/saboteur/install.rb +4 -3
  53. data/lib/saboteur/integration/base.rb +7 -2
  54. data/lib/saboteur/integration/rspec.rb +40 -4
  55. data/lib/saboteur/mutation_source/builtin.rb +22 -13
  56. data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
  57. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  58. data/lib/saboteur/mutation_source/method_swap.rb +45 -0
  59. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  60. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  61. data/lib/saboteur/mutation_source/operator_examples.rb +38 -0
  62. data/lib/saboteur/mutation_source/operator_registry.rb +36 -8
  63. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  64. data/lib/saboteur/preview.rb +12 -2
  65. data/lib/saboteur/reporter/agent_commands.rb +49 -16
  66. data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
  67. data/lib/saboteur/reporter/agent_summary.rb +1 -1
  68. data/lib/saboteur/reporter/history_policy.rb +35 -0
  69. data/lib/saboteur/reporter/history_record.rb +4 -5
  70. data/lib/saboteur/reporter/next_steps.rb +14 -4
  71. data/lib/saboteur/reporter/score_history.rb +19 -3
  72. data/lib/saboteur/reporter/stryker.rb +19 -8
  73. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  74. data/lib/saboteur/reporter/terminal.rb +43 -10
  75. data/lib/saboteur/result.rb +25 -7
  76. data/lib/saboteur/selection/coverage_map.rb +2 -1
  77. data/lib/saboteur/selection/git_diff.rb +15 -5
  78. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  79. data/lib/saboteur/subject.rb +11 -0
  80. data/lib/saboteur/text.rb +42 -0
  81. data/lib/saboteur/version.rb +1 -1
  82. data/lib/saboteur.rb +16 -1
  83. metadata +20 -2
@@ -1,6 +1,9 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require "did_you_mean"
5
+ require_relative "operator_examples"
6
+
4
7
  module Saboteur
5
8
  module MutationSource
6
9
  # Public names used by selection, help and inline suppression validation.
@@ -29,26 +32,51 @@ module Saboteur
29
32
  "argument_removal" => "Remove a call argument",
30
33
  "pair_removal" => "Remove a hash pair",
31
34
  "endpoint_replacement" => "Swap explicit no-argument first and last",
32
- "chain_removal" => "Remove one call from a method chain"
35
+ "chain_removal" => "Remove one call from a method chain",
36
+ "method_swap" => "Swap a method with its semantic opposite",
37
+ "operand_replacement" => "Replace a binary operation with one operand",
38
+ "interpolation_removal" => "Remove part of an interpolated string"
33
39
  }.freeze
34
- EXTENDED = %w[range operator_assignment condition_negation].freeze
35
- TIERS = { "core" => (DESCRIPTIONS.keys - EXTENDED).freeze, "extended" => EXTENDED,
36
- "all" => DESCRIPTIONS.keys.freeze }.freeze
40
+ POLICY_VERSION = 2
41
+ TIERS = { "all" => DESCRIPTIONS.keys.freeze }.freeze
37
42
 
38
- def self.list(out)
43
+ def self.list(out, names: [], excluded: [])
44
+ active = select(names, excluded)
39
45
  DESCRIPTIONS.each do |name, description|
40
- out.puts("#{name} #{EXTENDED.include?(name) ? 'extended' : 'core'} #{description}")
46
+ out.puts("#{name} #{active.include?(name) ? 'on' : 'off'} #{description} #{OperatorExamples::EXAMPLES.fetch(name)}")
41
47
  end
42
48
  nil
43
49
  end
44
50
 
51
+ def self.policy(names, excluded)
52
+ { "version" => POLICY_VERSION, "selected" => names, "excluded" => excluded,
53
+ "effective" => select(names, excluded) }
54
+ end
55
+
56
+ # Exclusion wins over explicit selection; empty exclusions remove nothing.
57
+ def self.select(names, excluded)
58
+ selected = resolve(names)
59
+ excluded.empty? ? selected : selected - resolve(excluded)
60
+ end
61
+
62
+ def self.suggestion(name)
63
+ matches = DidYouMean::SpellChecker.new(dictionary: DESCRIPTIONS.keys).correct(name)
64
+ matches.empty? ? "" : "Did you mean #{matches.first}?"
65
+ end
66
+
45
67
  def self.resolve(names)
46
- return TIERS.fetch("core") if names.empty?
68
+ return TIERS.fetch("all") if names.empty?
47
69
 
70
+ legacy = names & %w[core extended]
71
+ unless legacy.empty?
72
+ raise ConfigError, "operator tiers #{legacy.join(', ')} were removed: use concrete names or all; " \
73
+ "for the old core set exclude range,operator_assignment,condition_negation"
74
+ end
48
75
  unknown = names - DESCRIPTIONS.keys - TIERS.keys
49
76
  unless unknown.empty?
50
77
  raise ConfigError, "unknown operators: #{unknown.join(', ')}. Valid names: " \
51
- "#{(DESCRIPTIONS.keys + TIERS.keys).join(', ')}"
78
+ "#{(DESCRIPTIONS.keys + TIERS.keys).join(', ')}. " \
79
+ "#{suggestion(unknown.first)} See --list-operators."
52
80
  end
53
81
  names.flat_map { |name| TIERS.fetch(name) { [name] } }.uniq
54
82
  end
@@ -0,0 +1,39 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "prism"
5
+
6
+ module Saboteur
7
+ module MutationSource
8
+ # The tree walk shared by the statement-driven operators: visit every
9
+ # node except nested definitions, skip arid statements, and hand each
10
+ # remaining node to the operator's mutate_node hook.
11
+ module OperatorVisit
12
+ SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
13
+
14
+ def mutations_for(subject)
15
+ found = []
16
+ visit(subject.node.body, subject, found)
17
+ found
18
+ end
19
+
20
+ private
21
+
22
+ def visit(node, subject, found)
23
+ return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
24
+
25
+ if node.is_a?(Prism::StatementsNode)
26
+ node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
27
+ return
28
+ end
29
+
30
+ mutate_node(node, subject, found)
31
+ node.compact_child_nodes.each { |child| visit(child, subject, found) }
32
+ end
33
+
34
+ def arid?(node)
35
+ @arid.any? { |pattern| pattern.match?(node.slice) }
36
+ end
37
+ end
38
+ end
39
+ end
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require_relative "engine"
5
+ require_relative "text"
5
6
 
6
7
  module Saboteur
7
8
  # Source-only planning. Never boots the host or executes its test suite.
@@ -13,6 +14,7 @@ module Saboteur
13
14
  end
14
15
 
15
16
  def call
17
+ announce_operators
16
18
  selection = Engine::SubjectSelection.new(@config, @diagnostics)
17
19
  subjects = selection.select(@config.included_files)
18
20
  scope = selection.changed_lines && Engine::DiffScope.new(selection.changed_lines)
@@ -24,12 +26,20 @@ module Saboteur
24
26
  @out.puts("Total: #{lists.sum(&:size)} mutants; #{@diagnostics.omissions.sum(&:count)} " \
25
27
  "dropped by max_mutants_per_method#{incomplete_note(incomplete)}")
26
28
  incomplete.zero? ? estimate(subjects, lists) : @out.puts("no estimate: source discovery was incomplete")
27
- @diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
29
+ print_warnings
28
30
  nil
29
31
  end
30
32
 
31
33
  private
32
34
 
35
+ def print_warnings
36
+ @diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
37
+ end
38
+
39
+ def announce_operators
40
+ @out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}")
41
+ end
42
+
33
43
  # Candidate filtering precedes diff restriction and the per-method cap.
34
44
  # Name that scope rather than implying these rows were executed or scored.
35
45
  def inline_suppressions
@@ -83,7 +93,7 @@ module Saboteur
83
93
  # coverage map means "no estimate", never an exception.
84
94
  def cached_map
85
95
  path = File.join(@config.path("cache_dir"), "preview.json")
86
- receipt = JSON.parse(File.read(path))
96
+ receipt = JSON.parse(Text.utf8_strict(File.read(path)))
87
97
  return unless receipt["fingerprint"] == Engine::RunFingerprint.new(config: @config).snapshot.digest
88
98
 
89
99
  Selection::CoverageMap.load(receipt["path"])
@@ -8,14 +8,26 @@ module Saboteur
8
8
  module Agent
9
9
  # The shell commands an agent is told to run next. Every word is escaped with
10
10
  # Shellwords, so a ref, path or subject holding shell syntax stays one literal
11
- # argument. A command is never cut short: when it is longer than MAX_BYTES it
12
- # is left out, because half a command could run something else.
11
+ # argument. A command for the agent protocol is never cut short: when it is
12
+ # longer than MAX_BYTES it is left out, because half a command could run
13
+ # something else. The text footer passes capped: false instead - there the
14
+ # promise is the complete command, however long (#261).
13
15
  module Commands
14
16
  MAX_BYTES = 1024
15
17
 
16
18
  # Options typed on the command line that change how a run behaves.
17
19
  SETTING_FLAGS = { reeval: "--reeval", integration: "--integration", jobs: "--jobs",
18
- timeout_factor: "--timeout-factor", json_report: "--json", html_report: "--html" }.freeze
20
+ timeout_factor: "--timeout-factor", json_report: "--json", html_report: "--html",
21
+ operators: "--operators", exclude_operators: "--exclude-operators",
22
+ baseline_passes: "--baseline-passes", max_duration: "--max-duration" }.freeze
23
+
24
+ # CLI-only boolean switches recorded in settings, mapped by value, so
25
+ # an explicit false (--no-max-duration-ok over a config true) survives
26
+ # into the generated commands instead of reverting to the config.
27
+ BOOLEAN_SETTING_FLAGS = { allow_hazardous: { true => "--allow-hazardous" },
28
+ allow_focus: { true => "--allow-focus" },
29
+ max_duration_ok: { true => "--max-duration-ok",
30
+ false => "--no-max-duration-ok" } }.freeze
19
31
 
20
32
  class << self
21
33
  # Run the same selection again: ref, subjects, sources, tests, config,
@@ -25,38 +37,59 @@ module Saboteur
25
37
  finish(words, provenance)
26
38
  end
27
39
 
28
- # Replay one survivor by digest in the same selection.
29
- def replay(provenance, digest)
40
+ # Replay one survivor by digest in the same selection. capped: false
41
+ # skips the protocol byte cap, for callers that print to a terminal.
42
+ def replay(provenance, digest, capped: true)
30
43
  words = ["saboteur", "run", "--id", digest, *selection(provenance, replay: true), "--seed",
31
44
  provenance.seed.to_s]
32
- finish(words, provenance)
45
+ finish(words, provenance, capped: capped)
33
46
  end
34
47
 
35
48
  private
36
49
 
37
- def finish(words, provenance)
50
+ def finish(words, provenance, capped: true)
38
51
  words += ["--", *provenance.sources] unless provenance.sources.empty?
39
52
  command = Shellwords.join(words)
40
- command.bytesize <= MAX_BYTES ? command : nil
53
+ command if !capped || command.bytesize <= MAX_BYTES
54
+ end
55
+
56
+ def mode_words(provenance)
57
+ words = []
58
+ words << "--no-cache" if provenance.cache == false
59
+ words << "--incremental" if provenance.incremental
60
+ words.concat(boolean_words(provenance))
61
+ end
62
+
63
+ def boolean_words(provenance)
64
+ BOOLEAN_SETTING_FLAGS.filter_map { |key, words| words[provenance.settings[key]] }
41
65
  end
42
66
 
43
- # An explicit scope is frozen: --no-guess-tests keeps a config-file
44
- # guess_tests from conflicting with the --test list.
67
+ # An explicit scope is frozen as --test, which wins over inferred
68
+ # selection by precedence, whatever the config file sets.
45
69
  def test_words(tests)
46
- tests.empty? ? [] : ["--no-guess-tests", *tests.flat_map { |test| ["--test", test] }]
70
+ tests.flat_map { |test| ["--test", test] }
71
+ end
72
+
73
+ def setting_words(provenance)
74
+ SETTING_FLAGS.flat_map do |key, flag|
75
+ next [] unless provenance.settings.key?(key)
76
+
77
+ value = provenance.settings[key]
78
+ [flag, value.is_a?(Array) ? value.join(",") : value.to_s]
79
+ end
47
80
  end
48
81
 
82
+ # Whole-suite scope is a bare --all, only when no tests resolved;
83
+ # --all never combines with --test.
49
84
  def selection(provenance, replay: false)
50
85
  words = []
51
86
  words.push("--config", provenance.config_path) if provenance.config_path
52
87
  words.push("--since", provenance.ref) if provenance.ref
53
88
  provenance.subjects.each { |subject| words.push("--subject", subject) }
89
+ words << "--all" if provenance.settings[:all] && provenance.tests.empty?
54
90
  words.push(*test_words(provenance.tests))
55
- SETTING_FLAGS.each do |key, flag|
56
- words.push(flag, provenance.settings[key].to_s) if provenance.settings.key?(key)
57
- end
58
- words << "--no-cache" if provenance.cache == false
59
- words << "--incremental" if provenance.incremental
91
+ words.concat(setting_words(provenance))
92
+ words.concat(mode_words(provenance))
60
93
  words.push("--id", provenance.digest) if provenance.digest && !replay
61
94
  words
62
95
  end
@@ -8,6 +8,8 @@ module Saboteur
8
8
  # stable code, a count and a plain meaning.
9
9
  module Incomplete
10
10
  NO_COVERAGE = "no coverage"
11
+ SYNTAX_FAILURE = "invalid_file_syntax"
12
+ UNREADABLE_FAILURE = "unreadable"
11
13
 
12
14
  class << self
13
15
  def reasons(report)
@@ -23,14 +25,37 @@ module Saboteur
23
25
  add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
24
26
  add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
25
27
  add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
26
- add(reasons, "fail_fast", report.early_stop&.unreached&.size.to_i,
27
- "selected subjects did not run because --fail-fast stopped the run")
28
+ add(reasons, "unverified", report.counts[:unverified],
29
+ "mutants the --max-duration budget left unverified; their kills are not confirmed")
30
+ discovery_failures(reasons, report)
31
+ early_stop_reason(reasons, report)
28
32
  inline_scope(reasons, report)
29
33
  reasons
30
34
  end
31
35
 
32
36
  private
33
37
 
38
+ # Source the selection could not read at all: its subjects never
39
+ # entered the run, so a clean or complete verdict would be wrong.
40
+ def discovery_failures(reasons, report)
41
+ failures = report.discovery_failures
42
+ add(reasons, SYNTAX_FAILURE, failures.count { |f| f.reason == SYNTAX_FAILURE },
43
+ "included files did not parse; their subjects were not discovered")
44
+ add(reasons, UNREADABLE_FAILURE, failures.count { |f| f.reason == UNREADABLE_FAILURE },
45
+ "included untracked files could not be read; their subjects were not covered")
46
+ end
47
+
48
+ def early_stop_reason(reasons, report)
49
+ stop = report.early_stop
50
+ return unless stop
51
+
52
+ label = stop.reason == "fail_fast" ? "--fail-fast stopped" : "the --max-duration budget stopped"
53
+ add(reasons, stop.reason, stop.unreached.size, "selected subjects did not run because #{label} the run")
54
+ omitted = stop.truncated.sum { |entry| entry[:omitted] }
55
+ add(reasons, "#{stop.reason}_truncated", omitted,
56
+ "mutants omitted inside subjects that stopped mid-way because #{label} the run")
57
+ end
58
+
34
59
  def inline_scope(reasons, report)
35
60
  add(reasons, "inline_suppressed", report.inline_suppressions.size,
36
61
  "candidates were suppressed by inline directives; see the full report audit")
@@ -22,7 +22,7 @@ module Saboteur
22
22
  verdict = verdict_of(report, status)
23
23
  {
24
24
  version: VERSION, status: status, verdict: verdict, exitCode: exit_code_for(verdict),
25
- provenance: provenance_hash(provenance),
25
+ provenance: provenance_hash(provenance).merge(operatorPolicy: report.operator_policy),
26
26
  counts: counts(report), score: score(report),
27
27
  scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
28
28
  incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
@@ -0,0 +1,35 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # v1 stays readable, but v2 records require versioned operator evidence.
7
+ module HistoryPolicy
8
+ def validate_header(record, fields)
9
+ version = record.is_a?(Hash) && record["version"]
10
+ raise Error, "score history contains an invalid version" unless version.is_a?(Integer) && [1,
11
+ 2].include?(version)
12
+
13
+ fields(record, version == 2 ? fields + ["operatorPolicy"] : fields)
14
+ validate_policy(record["operatorPolicy"]) if version == 2
15
+ end
16
+
17
+ def validate_policy(value)
18
+ raise Error, "score history contains invalid operator policy" unless value.is_a?(Hash)
19
+
20
+ fields(value, %w[version selected excluded effective])
21
+ raise Error, "score history contains invalid operator policy" if value["effective"].empty?
22
+
23
+ lists = value.values_at("selected", "excluded", "effective")
24
+ unless lists.all? { |list| list.is_a?(Array) && list.all?(String) }
25
+ raise Error, "score history contains invalid operator policy"
26
+ end
27
+
28
+ expected = MutationSource::OperatorRegistry.policy(value["selected"], value["excluded"])
29
+ raise Error, "score history contains invalid operator policy" unless value == expected
30
+ rescue ConfigError, NoMethodError, TypeError
31
+ raise Error, "score history contains invalid operator policy"
32
+ end
33
+ end
34
+ end
35
+ end
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "json"
5
5
  require "time"
6
+ require_relative "history_policy"
6
7
 
7
8
  module Saboteur
8
9
  module Reporter
@@ -21,6 +22,8 @@ module Saboteur
21
22
  end
22
23
  end
23
24
 
25
+ extend HistoryPolicy
26
+
24
27
  module_function
25
28
 
26
29
  def parse(line)
@@ -32,11 +35,7 @@ module Saboteur
32
35
  end
33
36
 
34
37
  def validate(record)
35
- fields(record, FIELDS)
36
- unless record.fetch("version").is_a?(Integer) && record.fetch("version") == 1
37
- raise Error,
38
- "score history contains an invalid version"
39
- end
38
+ validate_header(record, FIELDS)
40
39
 
41
40
  date(record.fetch("date"))
42
41
  commit = record.fetch("commit")
@@ -7,9 +7,14 @@ module Saboteur
7
7
  # accept one, and how to hand the run to an agent. Silent when nothing
8
8
  # survived, so a clean run stays short.
9
9
  class NextSteps
10
- def initialize(out, palette)
10
+ # replay_command: digest -> the full, copy-pasteable replay command
11
+ # carrying the run's own selection options, however long it gets
12
+ # (#261). When no builder is wired the digest-only hint (with its
13
+ # reassembly note) is the fallback.
14
+ def initialize(out, palette, replay_command: nil)
11
15
  @out = out
12
16
  @palette = palette
17
+ @replay_command = replay_command
13
18
  end
14
19
 
15
20
  def call(report)
@@ -17,9 +22,14 @@ module Saboteur
17
22
  return unless survivor
18
23
 
19
24
  @out.puts "\n#{@palette.paint(:bold, 'Next steps:')}"
20
- seed = " --seed #{report.seed}" if report.seed
21
- @out.puts " replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
22
- @out.puts " (add this run's own --config, --since, --subject, --test and source options)"
25
+ command = @replay_command&.call(survivor.digest)
26
+ if command
27
+ @out.puts " replay one survivor: #{command}"
28
+ else
29
+ seed = " --seed #{report.seed}" if report.seed
30
+ @out.puts " replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
31
+ @out.puts " (add this run's own --config, --since, --subject, --test and source options)"
32
+ end
23
33
  @out.puts " accept an equivalent one: add to your ignore file (c.ignore_file, default #{Config::IGNORE_FILE}):"
24
34
  entry = %(i.mutation "#{survivor.digest}", subject: "#{survivor.subject.id}", reason: "why it is equivalent")
25
35
  @out.puts " #{entry}"
@@ -13,6 +13,7 @@ module Saboteur
13
13
  # Opt-in append-only score observations, not a baseline or a CI gate.
14
14
  class ScoreHistory
15
15
  DIFFERENT_RUN = "score history: change unavailable (different selection/inventory or incomplete run)"
16
+ DIFFERENT_POLICY = "score history: change unavailable (different record version or operator policy)"
16
17
  MISSING_SCORE = "score history: change unavailable (no tested mutants)"
17
18
  MAX_RECORD_BYTES = 64 * 1024
18
19
  OPEN_FLAGS = File::RDWR | File::CREAT | File::NONBLOCK | File::NOFOLLOW
@@ -38,7 +39,11 @@ module Saboteur
38
39
  file.flush
39
40
  change(previous, record)
40
41
  end
41
- rescue SystemCallError, IOError, JSON::ParserError, KeyError, TypeError => e
42
+ # SystemCallError/IOError from the write itself propagate unwrapped on
43
+ # purpose: the CLI treats those as a soft artifact failure (warn and
44
+ # exit by the results), while validation, integrity, record-size and
45
+ # lock failures stay hard errors (#296).
46
+ rescue JSON::ParserError, KeyError, TypeError => e
42
47
  raise Error, "score history was not recorded: #{e.message}"
43
48
  end
44
49
 
@@ -58,13 +63,13 @@ module Saboteur
58
63
  private
59
64
 
60
65
  def build(report, head, scope)
61
- { "version" => 1, "date" => Time.now.utc.iso8601, "commit" => head,
66
+ { "version" => 2, "date" => Time.now.utc.iso8601, "commit" => head,
62
67
  "selection" => Digest::SHA256.hexdigest(JSON.generate(scope.sort_by { |key, _| key.to_s }.to_h)),
63
68
  "inventory" => Digest::SHA256.hexdigest(JSON.generate(report.results.map(&:digest).sort)),
64
69
  "score" => overall_score(report), "coveredScore" => report.score,
65
70
  "counts" => report.counts.transform_keys(&:to_s),
66
71
  "breakdown" => report.score_breakdown.to_h.transform_keys(&:to_s),
67
- "incomplete" => incompleteness(report) }
72
+ "incomplete" => incompleteness(report), "operatorPolicy" => report.operator_policy }
68
73
  end
69
74
 
70
75
  def overall_score(report)
@@ -107,11 +112,22 @@ module Saboteur
107
112
  previous
108
113
  end
109
114
 
115
+ # Only one record version and one non-null operator policy compare:
116
+ # scores never mix across v1/v2 or across policy configurations.
117
+ # A non-null previous policy forces a non-null equal current one, so
118
+ # only the previous record needs the null guard.
119
+ def comparable_policy?(previous, current)
120
+ previous["version"] == current["version"] &&
121
+ !previous["operatorPolicy"].nil? &&
122
+ previous["operatorPolicy"] == current["operatorPolicy"]
123
+ end
124
+
110
125
  def change(previous, current)
111
126
  return "score history: first record" unless previous
112
127
  return DIFFERENT_RUN unless
113
128
  previous["selection"] == current["selection"] && previous["inventory"] == current["inventory"] &&
114
129
  previous["incomplete"].empty? && current["incomplete"].empty?
130
+ return DIFFERENT_POLICY unless comparable_policy?(previous, current)
115
131
  return MISSING_SCORE if
116
132
  previous["coveredScore"].nil? || current["coveredScore"].nil?
117
133
 
@@ -40,13 +40,15 @@ module Saboteur
40
40
  warnings: report.warnings,
41
41
  framework: { name: "saboteur", version: VERSION },
42
42
  saboteur: { findings: Findings.build(report), scores: scores(report),
43
- inlineSuppressions: report.inline_suppressions.map do |entry|
44
- entry.merge(file: relative(entry[:file], report.root))
45
- end }
46
- .merge(exit_cause ? { exitCause: exit_cause } : {})
43
+ inlineSuppressions: inline_audit(report) }
44
+ .merge(operatorPolicy: report.operator_policy).merge(exit_cause ? { exitCause: exit_cause } : {})
47
45
  }.merge(report.seed ? { config: { seed: report.seed } } : {})
48
46
  end
49
47
 
48
+ def self.inline_audit(report)
49
+ report.inline_suppressions.map { |entry| entry.merge(file: relative(entry[:file], report.root)) }
50
+ end
51
+
50
52
  # Score with its assertion and crash shares, for all mutants and without
51
53
  # the whole-body ones (body_nil, body_true, body_false, body_raise).
52
54
  def self.scores(report)
@@ -56,7 +58,14 @@ module Saboteur
56
58
  # nil for a run that covered its whole selection.
57
59
  def self.early_stop(report)
58
60
  stop = report.early_stop
59
- stop && { reason: stop.reason, unreachedSubjects: stop.unreached }
61
+ return unless stop
62
+
63
+ { reason: stop.reason, unreachedSubjects: stop.unreached, truncatedSubjects: truncated_subjects(stop),
64
+ unverifiedSubjects: stop.unverified }
65
+ end
66
+
67
+ def self.truncated_subjects(stop)
68
+ stop.truncated.map { |entry| { subject: entry[:subject], omittedMutants: entry[:omitted] } }
60
69
  end
61
70
 
62
71
  def self.skipped_subjects(report)
@@ -113,8 +122,9 @@ module Saboteur
113
122
  when :error, :resource_limit then "RuntimeError"
114
123
  when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
115
124
  # The schema has no status for "ran, but no test can observe a
116
- # change here". Ignored with the reason is the honest mapping.
117
- when :ran_at_load, :unreachable then "Ignored"
125
+ # change here", nor for a verdict the budget left unchecked.
126
+ # Ignored with the reason is the honest mapping.
127
+ when :ran_at_load, :unreachable, :unverified then "Ignored"
118
128
  end
119
129
  end
120
130
 
@@ -123,7 +133,8 @@ module Saboteur
123
133
  return "resource limit: #{result.reason}" if result.status == :resource_limit
124
134
  return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
125
135
 
126
- result.reason.to_s.lines.first&.strip
136
+ reason = result.reason.to_s.strip
137
+ reason.empty? ? nil : reason
127
138
  end
128
139
 
129
140
  def self.location(edit, source)
@@ -19,6 +19,9 @@ module Saboteur
19
19
  "safe_navigation" => "try an example where the receiver is nil",
20
20
  "argument_removal" => "try have_received(...).with(...), or assert a result that depends on this argument",
21
21
  "pair_removal" => "try have_received(...).with(...), or assert the hash content under this key",
22
+ "chain_removal" => "try the shorter chain's value against the longer one; if you confirm the " \
23
+ "shorter chain is equivalent across the intended inputs (for example a duplicated " \
24
+ "column), simplify the code or ignore the mutant",
22
25
  "endpoint_replacement" => "try a collection with 2 or more elements, to tell first from last",
23
26
  "filter_replacement" => "try items that match and items that do not, to tell select from reject"
24
27
  }.freeze
@@ -17,12 +17,12 @@ module Saboteur
17
17
  MAX_ERRORS = 10
18
18
  MAX_PASSING_TESTS = SurvivorList::MAX_PASSING_TESTS
19
19
 
20
- def initialize(out, verbose: false, color: :auto)
20
+ def initialize(out, verbose: false, color: :auto, replay_command: nil)
21
21
  @out = out
22
22
  @verbose = verbose
23
23
  @palette = Palette.for(out, mode: color)
24
24
  @score_table = ScoreTable.new(out, @palette)
25
- @next_steps = NextSteps.new(out, @palette)
25
+ @next_steps = NextSteps.new(out, @palette, replay_command: replay_command)
26
26
  end
27
27
 
28
28
  def call(report)
@@ -34,11 +34,20 @@ module Saboteur
34
34
  warnings(report)
35
35
  early_stop(report)
36
36
  test_matrix(report)
37
+ operator_policy(report)
37
38
  summary(report)
38
39
  end
39
40
 
40
41
  private
41
42
 
43
+ def operator_policy(report)
44
+ policy = report.operator_policy
45
+ return unless policy && policy["excluded"].any?
46
+
47
+ @out.puts("Excluded operators: #{policy['excluded'].join(', ')} " \
48
+ "(policy #{policy['version']}; #{policy['effective'].size} enabled)")
49
+ end
50
+
42
51
  def test_matrix(report)
43
52
  return unless report.matrix?
44
53
 
@@ -100,6 +109,7 @@ module Saboteur
100
109
  "resource limit #{c[:resource_limit]} error #{c[:error]} skipped #{c[:skipped]}"
101
110
  line += " ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
102
111
  line += " unreachable #{c[:unreachable]}" if c[:unreachable].positive?
112
+ line += " unverified #{c[:unverified]}" if c[:unverified].positive?
103
113
  line
104
114
  end
105
115
 
@@ -214,17 +224,39 @@ module Saboteur
214
224
  end
215
225
  end
216
226
 
217
- # The verdict covers only the subjects that ran, so say it where the
227
+ # The verdict covers only the completed work, so say it where the
218
228
  # score is read.
219
229
  def early_stop(report)
220
230
  stop = report.early_stop
221
231
  return unless stop
222
232
 
223
- @out.puts @palette.paint(:yellow, "stopped early (--fail-fast): #{stop.unreached.size} selected " \
224
- "subject#{'s' unless stop.unreached.one?} did not run; the score below " \
225
- "covers only the subjects that ran")
226
- stop.unreached.first(MAX_ERRORS).each { |id| @out.puts " not run: #{id}" }
227
- @out.puts " ... #{stop.unreached.size - MAX_ERRORS} more" if stop.unreached.size > MAX_ERRORS
233
+ label = stop.reason == "fail_fast" ? "--fail-fast" : "--max-duration spent"
234
+ @out.puts @palette.paint(:yellow, "stopped early (#{label}): #{early_stop_summary(stop)}; the score below " \
235
+ "covers only completed work")
236
+ list(stop.unreached) { |id| " not run: #{id}" }
237
+ list(stop.truncated) { |entry| " stopped mid-way: #{entry[:subject]} (#{entry[:omitted]} mutants omitted)" }
238
+ list(stop.unverified) { |id| " unverified: #{id}" }
239
+ end
240
+
241
+ def list(entries)
242
+ entries.first(MAX_ERRORS).each { |entry| @out.puts yield(entry) }
243
+ @out.puts " ... #{entries.size - MAX_ERRORS} more" if entries.size > MAX_ERRORS
244
+ end
245
+
246
+ def early_stop_summary(stop)
247
+ parts = []
248
+ if stop.unreached.any?
249
+ parts << "#{stop.unreached.size} selected subject#{'s' unless stop.unreached.one?} did not run"
250
+ end
251
+ if stop.truncated.any?
252
+ omitted = stop.truncated.sum { |entry| entry[:omitted] }
253
+ parts << "#{stop.truncated.size} subject#{'s' unless stop.truncated.one?} stopped mid-way " \
254
+ "(#{omitted} mutants omitted)"
255
+ end
256
+ if stop.unverified.any?
257
+ parts << "#{stop.unverified.size} subject#{'s' unless stop.unverified.one?} left with unverified results"
258
+ end
259
+ parts.join("; ")
228
260
  end
229
261
 
230
262
  def warnings(report)
@@ -237,8 +269,9 @@ module Saboteur
237
269
 
238
270
  @out.puts "\nErrors (not counted as killed):"
239
271
  errored.first(MAX_ERRORS).each do |result|
240
- @out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: " \
241
- "#{result.reason.to_s.lines.first&.strip}"
272
+ lines = result.reason.to_s.lines.map(&:strip)
273
+ @out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: #{lines.first}"
274
+ lines.drop(1).each { |line| @out.puts " #{line}" }
242
275
  end
243
276
  @out.puts " ... #{errored.size - MAX_ERRORS} more" if errored.size > MAX_ERRORS
244
277
  end