saboteur 0.4.0 → 0.6.0

data/lib/saboteur/engine/diagnostics.rb

@@ -1,14 +1,22 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require "tempfile"
+
 module Saboteur
   class Engine
     # What a run has to say besides results: warnings for the report, the
     # count of mutations dropped for not parsing, and progress marks.
     class Diagnostics
       # See progress_heartbeat. Time-gated itself, so the supervisor can
-      # call tick on every poll.
+      # call tick on every poll. A phase whose child writes "N/M examples
+      # finished" lines to a log (the baseline and coverage children do) can
+      # be watched: the heartbeat then adds the count, the percentage and an
+      # estimate of the time left to its elapsed-time line.
       class ProgressHeartbeat
+        COUNT = %r{(\d+)/([1-9]\d*) examples finished}
+        TAIL_BYTES = 4096
+
         def initialize(diagnostics, label, interval: 30)
           @diagnostics = diagnostics
           @label = label
@@ -17,23 +25,66 @@ module Saboteur
           @last_print = @started
         end
 
+        # Reads progress from the end of this log on each print. Returns self.
+        def watch(log_path)
+          @log = log_path
+          self
+        end
+
         def tick
           now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
           return if now - @last_print < @interval
 
           @last_print = now
-          @diagnostics.inform("#{@label}: #{(now - @started).round} s elapsed")
+          elapsed = now - @started
+          @diagnostics.inform("#{@label}: #{elapsed.round} s elapsed#{progress_note(elapsed)}")
+        end
+
+        private
+
+        # ", 250/1000 examples (25 %), about 90 s left", or "" when the log has
+        # no count yet. The estimate assumes examples take equally long, so it
+        # is rough: slow examples late in the order make it too low.
+        def progress_note(elapsed)
+          done, total = examples_finished
+          return "" if done.nil?
+
+          note = ", #{done}/#{total} examples (#{(done * 100.0 / total).floor} %)"
+          return note if done.zero? || done >= total
+
+          "#{note}, about #{(elapsed * (total - done) / done).ceil} s left"
+        end
+
+        def examples_finished
+          return unless @log
+
+          tail = File.read(@log, TAIL_BYTES, [File.size(@log) - TAIL_BYTES, 0].max).to_s.scrub
+          tail.scan(COUNT).last&.map(&:to_i)
+        rescue SystemCallError
+          nil
         end
       end
 
       MARKS = { killed: ".", survived: "S", timeout: "T", resource_limit: "L", error: "E" }.freeze
+      LEGEND = ". killed  S survived  T timeout  L resource limit  E error"
+      LINE_WIDTH = 50
 
-      attr_reader :warnings, :invalid_mutations
+      GenerationEvent = Data.define(:file, :subject, :source, :operator, :reason)
+      Omission = Data.define(:subject, :count, :limit)
+      DiscoveryFailure = Data.define(:file, :reason)
+
+      attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
+                  :hazard_suppressions, :inline_suppressions
 
       def initialize(err)
         @err = err
         @warnings = []
         @invalid_mutations = 0
+        @generation_events = []
+        @omissions = []
+        @discovery_failures = []
+        @hazard_suppressions = 0
+        @inline_suppressions = []
       end
 
       def warn(message)
@@ -44,6 +95,36 @@ module Saboteur
         @invalid_mutations += 1
       end
 
+      def record_generation(subject, source:, reason:, operator: nil)
+        values = [subject.file, subject.id, source, operator, reason].map do |value|
+          value.nil? ? nil : value.to_s.dup.freeze
+        end
+        @generation_events << GenerationEvent.new(*values)
+      end
+
+      def record_discovery_failure(file, reason:)
+        @discovery_failures << DiscoveryFailure.new(file: file.to_s.dup.freeze, reason: reason.to_s.dup.freeze)
+      end
+
+      # A mutant the hazard guard refused to generate (issue #102): counted
+      # for the summary and recorded like any other generation event.
+      def record_hazard_suppression(subject, mutation)
+        @hazard_suppressions += 1
+        record_generation(subject, source: "hazard_guard", operator: mutation.operator, reason: "hazardous")
+      end
+
+      def record_inline_suppression(subject, mutation, directive)
+        @inline_suppressions << {
+          file: subject.file, line: directive.line, subject: subject.id,
+          operator: mutation.operator, reason: directive.reason,
+          digest: mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
+        }
+      end
+
+      def record_omission(subject, count:, limit:)
+        @omissions << Omission.new(subject: subject.id.dup.freeze, count: count, limit: limit)
+      end
+
       def inform(message)
         @err.puts("saboteur: #{message}")
       end
@@ -56,10 +137,75 @@ module Saboteur
         ProgressHeartbeat.new(self, label, interval: interval)
       end
 
+      # Call before the workers fork. total is the number of mutants the run
+      # may execute; mutants that are skipped or reused never report progress,
+      # so the counter can end below total and the ETA errs on the long side.
+      # The count lives in one file the workers inherit, so every worker adds
+      # to the same number.
+      def start_progress(total)
+        @progress_total = total
+        @progress_started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @progress_file = Tempfile.new("saboteur-progress")
+        @progress_file.binmode
+        @progress_file.sync = true
+        @err.puts("saboteur: progress #{LEGEND}") unless progress_on_terminal?
+      end
+
       def progress(result)
-        @err.print(MARKS.fetch(result.status, "?"))
+        mark = MARKS.fetch(result.status, "?")
+        return plain_progress(mark) unless @progress_file
+
+        done = record_progress(mark)
+        progress_on_terminal? ? terminal_progress(done) : ci_progress(mark, done)
+        @err.flush
+      end
+
+      def finish_progress
+        @err.puts if @progress_file && (progress_on_terminal? || @progress_file.size % LINE_WIDTH != 0)
+        @progress_file&.close!
+        @progress_file = nil
+      end
+
+      def progress_summary(done)
+        return "#{done}/#{@progress_total}" unless @progress_total.to_i.positive?
+
+        percent = (done * 100.0 / @progress_total).floor
+        "#{done}/#{@progress_total} (#{percent}%)#{eta_text(done)}"
+      end
+
+      private
+
+      def plain_progress(mark)
+        @err.print(mark)
         @err.flush
       end
+
+      def record_progress(mark)
+        @progress_file.write(mark)
+        @progress_file.size
+      end
+
+      def progress_on_terminal?
+        @err.respond_to?(:tty?) && @err.tty?
+      end
+
+      def terminal_progress(done)
+        @err.print("\r\e[Ksaboteur: #{progress_summary(done)}")
+      end
+
+      def ci_progress(mark, done)
+        @err.print(mark)
+        @err.print(" #{progress_summary(done)}\n") if (done % LINE_WIDTH).zero?
+      end
+
+      def eta_text(done)
+        remaining = @progress_total - done
+        return "" if remaining <= 0 || done.zero?
+
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @progress_started
+        seconds = (elapsed / done * remaining).round
+        " eta #{seconds / 60}m#{format('%02d', seconds % 60)}s"
+      end
     end
   end
 end

data/lib/saboteur/engine/diff_scope.rb

@@ -28,6 +28,16 @@ module Saboteur
         mutation.edits.flat_map { |edit| edited_lines(subject, edit).to_a }.uniq.select { changed.include?(_1) }.sort
       end
 
+      # Distance to the nearest changed line in this file. Unknown outside
+      # the diff is nil, not a made-up nearby location.
+      def distance_for(subject, mutation)
+        changed = @changed_lines.fetch(subject.file, Set.new)
+        mutation.edits.map do |edit|
+          first, last = edited_lines(subject, edit).minmax
+          changed.map { |line| (line - line.clamp(first, last)).abs }.min
+        end.min
+      end
+
       private
 
       def edited_lines(subject, edit)

data/lib/saboteur/engine/distribution.rb

@@ -8,19 +8,37 @@ module Saboteur
     # plain data; the parent rebuilds them against its own copy of the same
     # candidate list, so results look the same either way.
     class Distribution
-      def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil)
+      # After run: the subjects a stop_after callable left unrun (always empty
+      # without one).
+      attr_reader :unreached
+
+      # stop_after: nil, or a callable that gets one subject's results and
+      # returns true to stop before the next subject (--fail-fast). It only
+      # works on one process: parallel workers have already started the
+      # other subjects.
+      def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil, after_subject: nil,
+                     stop_after: nil)
         @jobs = jobs
+        @stop_after = stop_after
+        @unreached = []
         @on_start = on_start
         @scheduler = scheduler
         @diagnostics = diagnostics
         @cache = cache
+        @after_subject = after_subject
       end
 
       # lists: each subject's [[mutation, text]] candidates, in subject order.
       def run(subjects, lists)
-        return subjects.zip(lists).flat_map { |subject, list| @scheduler.call(subject, list) } if @jobs <= 1
+        return run_serially(subjects, lists) if @jobs <= 1
 
-        pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start)
+        # The watcher runs as each subject's row arrives, before the pool
+        # hands out more work, so a tripped canary stops the queue (jobs>1
+        # used to check only after the whole batch returned, issue #102).
+        arrival = lambda do |i, row|
+          @after_subject&.call(subjects[i], decode(row, subjects[i], lists[i]))
+        end
+        pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start, on_result: arrival)
         rows = pool.map(subjects.each_index.to_a, weights: lists.map(&:size)) do |i|
           work(subjects[i], lists[i])
         end
@@ -29,6 +47,20 @@ module Saboteur
 
       private
 
+      def run_serially(subjects, lists)
+        results = []
+        subjects.zip(lists).each_with_index do |(subject, list), index|
+          subject_results = @scheduler.call(subject, list)
+          results.concat(subject_results)
+          @after_subject&.call(subject, subject_results)
+          next unless @stop_after&.call(subject_results)
+
+          @unreached = subjects.drop(index + 1)
+          break
+        end
+        results
+      end
+
       # Runs in a worker.
       def work(subject, list)
         before = @diagnostics.warnings.size
@@ -43,29 +75,38 @@ module Saboteur
           "status" => result.status.to_s, "reason" => result.reason, "runtime" => result.runtime,
           "killed_by" => result.killed_by, "covered_by" => result.covered_by,
           "kill_cause" => result.kill_cause, "reused" => result.reused?,
-          "rejected_kill" => result.rejected_kill?
+          "rejected_kill" => result.rejected_kill?, "executed" => result.executed
         }
       end
 
       def rebuild(row, subject, list)
-        return failed(row["error"], subject, list) if row["error"]
+        if row["error"]
+          @diagnostics.warn("#{subject.id}: #{row['error']}")
+          return failed(row["error"], subject, list)
+        end
 
         @diagnostics.warnings.concat(row["warnings"])
+        decode(row, subject, list).each { |result| @cache&.record(result) unless result.reused? }
+      end
+
+      # Turns a worker row back into results, without diagnostics or caching:
+      # used both at result arrival (the canary) and at rebuild time.
+      def decode(row, subject, list)
+        return failed(row["error"], subject, list) if row["error"]
+
         row["results"].map do |data|
           mutation, text = list.fetch(data["position"])
-          result = MutantResult.new(subject: subject, mutation: mutation, status: data["status"].to_sym,
-                                    mutated_source: text, reason: data["reason"], runtime: data["runtime"],
-                                    killed_by: data["killed_by"], covered_by: data["covered_by"],
-                                    kill_cause: data["kill_cause"], reused: data["reused"],
-                                    rejected_kill: data.fetch("rejected_kill", false))
-          @cache&.record(result) unless result.reused?
-          result
+          MutantResult.new(subject: subject, mutation: mutation, status: data["status"].to_sym,
+                           mutated_source: text, reason: data["reason"], runtime: data["runtime"],
+                           killed_by: data["killed_by"], covered_by: data["covered_by"],
+                           kill_cause: data["kill_cause"], reused: data["reused"],
+                           executed: data["executed"],
+                           rejected_kill: data.fetch("rejected_kill", false))
         end
       end
 
       # The worker died (or raised) while running this subject.
       def failed(message, subject, list)
-        @diagnostics.warn("#{subject.id}: #{message}")
         list.map do |mutation, text|
           MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: text,
                            reason: message)

data/lib/saboteur/engine/result_cache.rb

@@ -11,13 +11,16 @@ module Saboteur
     # Stryker's history files.
     #
     # A verdict is reused only while everything it was computed from is the
-    # same: the method's source, the mutant, the set of covering tests, the
-    # content of those tests' files, the lockfile, the run's inputs (config,
-    # saboteur version). Only killed, survived and timeout are stored;
+    # same: the method's source, the whole file the method lives in (so a
+    # constant or helper changed elsewhere in that file is seen), the mutant,
+    # the set of covering tests, the content of those tests' files, the
+    # lockfile, the run's inputs (config, saboteur version). Only killed, survived
+    # and timeout are stored;
     # errors and skips always run again.
     #
-    # Known hole, the reason this is opt-in: a change in a method this one
-    # calls is not seen, so a stored verdict may be stale after it.
+    # Known hole, the reason this is opt-in: a change in another file that
+    # this method uses (a method it calls, a constant defined elsewhere) is
+    # not seen, so a stored verdict may be stale after it.
     class ResultCache
       FORMAT = 1
       STORED = %i[killed survived timeout].freeze
@@ -75,6 +78,8 @@ module Saboteur
       def fingerprint(subject, tests)
         digest = Digest::SHA256.new
         digest << VERSION << "\0" << @inputs << "\0" << @suite_digest << "\0" << subject.source << "\0"
+        # A hex digest has a fixed width, so it needs no separator after it.
+        digest << Digest::SHA256.hexdigest(subject.file_source)
         tests.sort.each { |test| digest << test << "\0" << file_digest(test_file(test)) << "\0" }
         LOCKFILES.each { |name| digest << file_digest(File.join(@root, name)) << "\0" }
         digest.hexdigest

data/lib/saboteur/engine/run_fingerprint.rb

@@ -12,7 +12,7 @@ module Saboteur
     class RunFingerprint
       FORMAT_VERSION = 1
       MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
-      LOCATION_SETTINGS = %w[cache_dir json_report html_report].freeze
+      LOCATION_SETTINGS = %w[cache_dir json_report html_report score_history].freeze
       ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
       EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
       Entry = Data.define(:path, :kind, :state, :sha256, :required)
@@ -127,7 +127,7 @@ module Saboteur
         issues = output_overlap_issues
         entries = inventory(issues).sort_by(&:path).freeze
         issues = issues.uniq.sort
-        settings_digest = hash_value(@config.to_h.except(*LOCATION_SETTINGS))
+        settings_digest = hash_value(@config.selection_settings.except(*LOCATION_SETTINGS))
         runtime_digest = hash_value(@runtime_identity)
         digest = hash_value(format: FORMAT_VERSION, files: entries.map(&:to_h),
                             settings: settings_digest, runtime: runtime_digest)
@@ -181,8 +181,10 @@ module Saboteur
       def output_overlap_issues
         paths = required_files.map(&:first) + MANIFESTS.map { |name| File.join(@root, name) } +
                 DataInputs.files(@root, extra: @config["fingerprint_extra"])
-        outputs = %w[json_report html_report].filter_map { |key| @config.fetch(key) }
-                                             .map { |path| canonical_path(path) }
+        # Compare physical boundaries without changing declared inventory identities.
+        paths = paths.map { |path| canonical_path(path) }.uniq
+        outputs = %w[json_report html_report score_history].filter_map { |key| @config.fetch(key) }
+                                                           .map { |path| canonical_path(path) }
         issues = outputs.filter_map do |path|
           if path.end_with?(".rb") || paths.include?(path) || File.directory?(path) || data_input?(path)
             "output overlaps input: #{display_path(path)}"
@@ -268,7 +270,7 @@ module Saboteur
 
       def excluded?(path, name)
         EXCLUDED_DIRECTORIES.include?(name) || path == canonical_path(@config["cache_dir"]) ||
-          %w[json_report html_report].filter_map { |key| @config.fetch(key) }.any? do |output|
+          %w[json_report html_report score_history].filter_map { |key| @config.fetch(key) }.any? do |output|
             path == canonical_path(output)
           end
       end

data/lib/saboteur/engine/sandbox_policy.rb

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+
+module Saboteur
+  class Engine
+    # Decides what each mutant child's Landlock sandbox grants (#152), and
+    # notes once per run when the kernel cannot honor it.
+    class SandboxPolicy
+      def initialize(config:, root:, diagnostics:)
+        @config = config
+        @root = root
+        @diagnostics = diagnostics
+      end
+
+      # The third C1 layer (#152): each mutant child makes the project tree
+      # read-only except tmp/, the cache dir and the system temp dir. The
+      # claim needs Landlock ABI 3 (Linux 6.2+): earlier ABIs cannot restrict
+      # truncate(), so the run degrades honestly - children run with the
+      # guard and canary only, noted once - unless c.sandbox = "strict",
+      # which refuses instead.
+      def hook
+        return nil unless @config["sandbox"]
+
+        abi = sandbox_abi
+        return restrict_hook if abi >= 3
+
+        if @config["sandbox"] == "strict"
+          raise Error, "sandbox is strict but #{sandbox_gap(abi)}; refusing to run without the promised " \
+                       "sandbox - set c.sandbox = true to run degraded, or false to opt out"
+        end
+        # Once, through the report's warnings only: warn + inform printed the
+        # same line twice (D's review, #178).
+        unless @sandbox_note_given
+          @sandbox_note_given = true
+          @diagnostics.warn("sandbox is on but #{sandbox_gap(abi)}; children run with the hazard guard and canary only")
+        end
+        nil
+      end
+
+      # ENOSYS/EOPNOTSUPP are a clean "no Landlock here"; any other probe
+      # error (EPERM under seccomp, ...) must not silently run unsandboxed.
+      def sandbox_abi
+        Execution::Sandbox.abi_version
+      rescue SystemCallError => e
+        raise Error, "sandbox is on but the Landlock probe failed (#{e.message}); refusing to run " \
+                     "unsandboxed - set c.sandbox = false to proceed without it"
+      end
+
+      def sandbox_gap(abi)
+        return "this kernel lacks Landlock (needs Linux 6.2+)" if abi.zero?
+
+        "this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
+      end
+
+      def restrict_hook
+        writable = sandbox_writable_dirs
+        ->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
+      end
+
+      # c.sandbox_writable adds the dirs a suite genuinely needs (db/, log/,
+      # ...): without a grant, writes there fail with EACCES in the child.
+      def sandbox_writable_dirs
+        user_dirs = @config["sandbox_writable"].map { |dir| File.expand_path(dir, @root) }
+        dirs = [File.join(@root, "tmp"), @config.path("cache_dir")] + user_dirs
+        dirs.each { |dir| FileUtils.mkdir_p(dir) }
+        real = (dirs.map { |dir| File.realpath(dir) } + [File.realpath(Dir.tmpdir)]).uniq
+        check_sandbox_write_set(real, user_dirs.map { |dir| File.realpath(dir) })
+        real
+      end
+
+      # The effective write set - every grant realpath'd, cache_dir, the
+      # project tmp and the system temp dir included - must not contain the
+      # project root: cache_dir = ".", a tmp symlink or a checkout inside the
+      # system temp dir would otherwise leave the whole tree writable without
+      # a word. c.sandbox = "strict" refuses: strict guards the effective
+      # policy, not only the kernel ABI. A user grant containing another
+      # grant only makes the narrower one redundant, so that stays a warning.
+      def check_sandbox_write_set(all_dirs, user_dirs)
+        holder = all_dirs.find { |dir| path_contains?(dir, @root) }
+        if holder
+          message = "sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
+          raise Error, %(#{message} - refusing under c.sandbox = "strict") if @config["sandbox"] == "strict"
+
+          @diagnostics.warn(message)
+        end
+        user_dirs.each do |dir|
+          (all_dirs - [dir]).each do |other|
+            next unless path_contains?(dir, other)
+
+            @diagnostics.warn("sandbox grant #{dir} contains the granted dir #{other}; " \
+                              "the narrower grant is redundant")
+          end
+        end
+      end
+
+      # Both sides are realpath'd, so a plain "#{dir}/" prefix test would miss
+      # dir == "/" (every path is under it; "//" prefixes nothing).
+      def path_contains?(dir, path)
+        return true if dir == "/"
+
+        path == dir || path.start_with?("#{dir}/")
+      end
+    end
+  end
+end