saboteur 0.2.0

data/lib/saboteur/engine/suite_files.rb

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+require "shellwords"
+
+module Saboteur
+  class Engine
+    # The files every test shares: spec helpers and support code, boot and
+    # hook files, .rspec and whatever it (or rspec_args) requires. A change
+    # to any of them can change what any test runs, so both the coverage
+    # map and the result cache depend on them. Example files (*_spec.rb)
+    # are not listed: each is tracked with its own tests.
+    class SuiteFiles
+      OPTION_FILES = %w[.rspec .rspec-local].freeze
+      REQUIRE_FLAGS = %w[--require -r].freeze
+      LOAD_PATH = %w[spec lib .].freeze
+
+      def initialize(config)
+        @config = config
+        @root = File.realpath(config.root)
+      end
+
+      def files
+        (shared_spec_files + boot_and_hook_files + option_files + required_by_options).uniq.sort
+      end
+
+      def digest
+        files.each_with_object(Digest::SHA256.new) do |file, sha|
+          sha << file << "\0" << Digest::SHA256.file(file).hexdigest << "\0"
+        end.hexdigest
+      end
+
+      private
+
+      def shared_spec_files
+        Dir.glob("spec/**/*.rb", base: @root).reject { |file| file.end_with?("_spec.rb") }
+           .map { |file| File.join(@root, file) }
+      end
+
+      def boot_and_hook_files
+        (@config["require"] + @config["hooks"]).filter_map { |file| existing(File.expand_path(file, @root)) }
+      end
+
+      def option_files
+        OPTION_FILES.map { |file| File.join(@root, file) }.select { |file| File.file?(file) }
+      end
+
+      # `--require name` resolves through RSpec's load path: spec/, lib/ and
+      # any -I directory.
+      def required_by_options
+        tokens = option_files.flat_map { |file| Shellwords.split(File.read(file)) } + @config["rspec_args"]
+        load_path = LOAD_PATH + option_values(tokens, "-I", "--include")
+        option_values(tokens, *REQUIRE_FLAGS).filter_map { |name| resolve(name, load_path) }
+      end
+
+      # Values of the given flags, written "-r x", "-rx", "--require x" or
+      # "--require=x".
+      def option_values(tokens, *flags)
+        tokens.each_with_index.filter_map do |token, index|
+          flag = flags.find { |name| token == name || token.start_with?("#{name}=") || glued?(token, name) }
+          next unless flag
+          next tokens[index + 1] if token == flag
+
+          token.delete_prefix(flag).delete_prefix("=")
+        end
+      end
+
+      def glued?(token, flag)
+        flag.length == 2 && token.start_with?(flag) && token.length > 2
+      end
+
+      def resolve(name, load_path)
+        load_path.lazy.filter_map { |dir| existing(File.expand_path(name, File.join(@root, dir))) }.first
+      end
+
+      def existing(path)
+        [path, "#{path}.rb"].find { |candidate| File.file?(candidate) }
+      end
+    end
+  end
+end

data/lib/saboteur/engine/verdict_check.rb

@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Checks each kill against the method re-evaluated unchanged: a kill
+    # stands only if the same tests, in the same order, up to and including
+    # the killer, pass without the mutant (so a test that fails only after
+    # another test left state behind cannot claim a kill). A kill with no
+    # failing test named (an exit, a suite-level failure) is checked on all
+    # the mutant's tests. A kill that fails the check becomes an error.
+    # (Timeouts are checked by the scheduler as they happen.)
+    #
+    # This re-runs the evidence, not every covering test: a method that every
+    # factory calls can have hundreds of tests, and killer-first ordering
+    # makes most kills a single test.
+    class VerdictCheck
+      # run_unchanged: runs tests against the unchanged method,
+      #   ->(test_ids, timeout) { Runner::Outcome }.
+      def initialize(run_unchanged:, baseline_timeout:)
+        @run_unchanged = run_unchanged
+        @baseline_timeout = baseline_timeout
+      end
+
+      def call(results)
+        fresh = results.reject(&:reused?)
+        failing_runs = failing_sequences(fresh.select { |result| named_kill?(result) })
+        failing = failing_claims(fresh.select { |result| unnamed_kill?(result) })
+        results.map { |result| checked(result, failing_runs, failing) }
+      end
+
+      private
+
+      def named_kill?(result)
+        result.status == :killed && result.killed_by.any?
+      end
+
+      def unnamed_kill?(result)
+        result.status == :killed && result.killed_by.empty?
+      end
+
+      # The tests the mutant ran, in order, up to and including the killer.
+      def sequence(result)
+        killer = result.killed_by.first
+        position = result.covered_by.index(killer)
+        position ? result.covered_by.first(position + 1) : [killer]
+      end
+
+      # Runs each distinct kill sequence once. Returns those that fail, each
+      # with why (the failure message, when there is one).
+      def failing_sequences(kills)
+        kills.map { |result| sequence(result) }.uniq.each_with_object({}) do |tests, failing|
+          outcome = @run_unchanged.call(tests, @baseline_timeout)
+          failing[tests] = outcome.message unless outcome.status == :survived
+        end
+      end
+
+      # Kills with no failing test named. Returns those that fail unchanged.
+      def failing_claims(claims)
+        claims.reject do |result|
+          @run_unchanged.call(result.covered_by, @baseline_timeout).status == :survived
+        end
+      end
+
+      def checked(result, failing_runs, failing)
+        if named_kill?(result) && failing_runs.key?(sequence(result))
+          why = failing_runs[sequence(result)]
+          result.as_error("killing test #{result.killed_by.first} also fails with the method unchanged" \
+                          "#{": #{why}" if why}")
+        elsif failing.include?(result)
+          result.as_error("tests also fail with the method unchanged")
+        else
+          result
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/engine.rb

@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module Saboteur
+  # One run, as a pipeline: boot the target, select subjects, fetch the
+  # coverage map, then schedule each subject's mutants. The parts live in
+  # engine/; this class only wires them together.
+  class Engine
+    # Everything besides sources and tests that can change a verdict.
+    VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources arid timeout_factor timeout_min
+                        max_memory_mb max_processes seed].freeze
+
+    attr_reader :config, :hooks
+
+    def initialize(config, err: $stderr)
+      @config = config
+      @hooks = Hooks.new
+      @diagnostics = Diagnostics.new(err)
+      @root = File.realpath(config.root)
+    end
+
+    def call
+      Boot.refuse_recursion!
+      started = monotonic
+      ignore_list = IgnoreList.load(config.path("ignore_file"))
+      boot = Boot.new(config, hooks)
+      integration = boot.load_target
+      @rails = boot.rails
+      files = config.included_files
+      subjects, scope = select_subjects(files)
+      results = subjects.empty? ? [] : mutate(subjects, files, integration, scope)
+      @diagnostics.warnings.concat(ignore_list.apply(results))
+      Report.new(results: results, subjects: subjects, wall_time: monotonic - started, root: @root,
+                 invalid_mutations: @diagnostics.invalid_mutations, warnings: @diagnostics.warnings,
+                 review: !scope.nil?, seed: config["seed"])
+    end
+
+    private
+
+    # The subjects to mutate and, in diff mode, the scope of changed lines.
+    def select_subjects(files)
+      config.unmatched_patterns.each { |pattern| @diagnostics.warn("#{pattern} matches no file") }
+      selection = SubjectSelection.new(config, @diagnostics)
+      subjects = selection.select(files)
+      [subjects, selection.changed_lines && DiffScope.new(selection.changed_lines)]
+    end
+
+    def mutate(subjects, files, integration, scope)
+      map = CoverageStore.new(config: config, integration: integration, hooks: hooks,
+                              supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).fetch(files)
+      @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
+      warn_about_threads
+      cache = result_cache
+      scheduler = scheduler_for(map, integration, cache)
+      @rails&.prepare_workers(config["jobs"])
+      results = Distribution.new(jobs: config["jobs"], scheduler: scheduler, diagnostics: @diagnostics,
+                                 cache: cache, on_start: method(:start_worker)).run(subjects, candidate_lists(subjects,
+                                                                                                              scope))
+      cache&.save
+      results
+    end
+
+    def scheduler_for(map, integration, cache)
+      reevaluator = Execution::Reevaluator.new(config["reeval"])
+      runner = Execution::Runner.new(integration: integration, reevaluator: reevaluator, hooks: hooks,
+                                     supervisor: supervisor, log_dir: fresh_log_dir, seed: config["seed"])
+      Scheduler.new(config: config, runner: runner, map: map, reevaluator: reevaluator, diagnostics: @diagnostics,
+                    cache: cache)
+    end
+
+    def result_cache
+      return nil unless config["incremental"]
+
+      inputs = JSON.generate(VERDICT_INPUTS.to_h { |key| [key, config[key]] })
+      ResultCache.new(path: File.join(config.path("cache_dir"), "results.json"), root: @root, inputs: inputs,
+                      suite_digest: SuiteFiles.new(config).digest)
+    end
+
+    # In each worker: its own database, then the user's worker_start hooks.
+    def start_worker(number)
+      @rails&.start_worker(number)
+      hooks.run(:worker_start, number)
+    end
+
+    # Each subject's candidates, narrowed to changed lines in diff mode.
+    def candidate_lists(subjects, scope)
+      candidates = Candidates.new(Candidates.build_sources(config), @diagnostics)
+      subjects.map do |subject|
+        list = candidates.for(subject)
+        scope ? scope.restrict(subject, list) : list
+      end
+    end
+
+    def supervisor(memory_factor: 1)
+      limits = Execution::Supervisor::Limits.new(max_memory_mb: config["max_memory_mb"] * memory_factor,
+                                                 max_processes: config["max_processes"])
+      Execution::Supervisor.new(limits: limits)
+    end
+
+    # The last run's child logs (only those of errors are kept).
+    def fresh_log_dir
+      File.join(config.path("cache_dir"), "logs").tap do |dir|
+        FileUtils.rm_rf(dir)
+        FileUtils.mkdir_p(dir)
+      end
+    end
+
+    def warn_about_threads
+      others = Thread.list.size - 1
+      @diagnostics.warn("#{others} other thread(s) alive at fork time; they do not run in children") if
+        others.positive?
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end
+
+require_relative "engine/diagnostics"
+require_relative "engine/boot"
+require_relative "engine/subject_selection"
+require_relative "engine/suite_files"
+require_relative "engine/coverage_store"
+require_relative "engine/candidates"
+require_relative "engine/diff_scope"
+require_relative "engine/result_cache"
+require_relative "engine/verdict_check"
+require_relative "engine/scheduler"
+require_relative "engine/distribution"

data/lib/saboteur/execution/process_table.rb

@@ -0,0 +1,177 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fiddle"
+
+module Saboteur
+  module Execution
+    # A snapshot of every process on the machine: { pid => Entry }.
+    #
+    # The supervisor reads it at least twice per mutant, so it must be cheap.
+    # Spawning `ps` costs about 30 ms; reading the kernel directly costs a
+    # few. Each backend gives the same facts:
+    #
+    #   ppid, pgid  to find descendants, even ones that left the group
+    #   rss_kb      to cap memory (compare only values from one backend)
+    #   zombie      exited but not yet reaped: nothing left to kill
+    #   started     start time, so a recycled pid is never mistaken for one
+    #               we saw earlier
+    module ProcessTable
+      Entry = Data.define(:pid, :ppid, :pgid, :rss_kb, :zombie, :started)
+
+      def self.backends
+        [Darwin, Linux, Ps].select(&:available?)
+      end
+
+      def self.backend
+        @backend ||= backends.first
+      end
+
+      # Containment must never go blind: if the fast backend fails, ask ps.
+      def self.snapshot
+        backend.snapshot
+      rescue SystemCallError, Fiddle::DLError
+        raise if backend == Ps
+
+        Ps.snapshot
+      end
+
+      # macOS: libproc, through Fiddle.
+      module Darwin
+        LISTALL_LIMIT = 16_384
+        BSDINFO = 3
+        TASKINFO = 4
+        ZOMBIE = 5 # SZOMB in pbi_status
+        # Byte offsets in struct proc_bsdinfo / proc_taskinfo (measured on
+        # macOS 14, arm64; the layout is the same on x86_64).
+        STATUS = 4
+        PID = 12
+        PPID = 16
+        PGID = 100
+        START_SECONDS = 120
+        START_MICROS = 128
+        RESIDENT = 8
+
+        module_function
+
+        def available?
+          RUBY_PLATFORM.include?("darwin") && !functions.nil?
+        end
+
+        def snapshot
+          list_all, info = functions
+          pids = all_pids(list_all)
+          bsd = Fiddle::Pointer.malloc(256, Fiddle::RUBY_FREE)
+          task = Fiddle::Pointer.malloc(256, Fiddle::RUBY_FREE)
+          pids.to_h { |pid| [pid, entry_for(pid, info, bsd, task) || exited(pid)] }
+        end
+
+        # libproc gives no details for a zombie (or a process that just
+        # ended): it is listed but has nothing left to kill.
+        def exited(pid)
+          Entry.new(pid: pid, ppid: 0, pgid: 0, rss_kb: 0, zombie: true, started: nil)
+        end
+
+        # A full buffer may mean more pids than fit: grow it and ask again.
+        def all_pids(list_all, capacity = LISTALL_LIMIT)
+          loop do
+            buffer = Fiddle::Pointer.malloc(4 * capacity, Fiddle::RUBY_FREE)
+            count = list_all.call(buffer, 4 * capacity)
+            raise SystemCallError.new("proc_listallpids", Fiddle.last_error) if count.negative?
+            return buffer[0, 4 * count].unpack("l*").reject(&:zero?) if count < capacity
+
+            capacity *= 2
+          end
+        end
+
+        def entry_for(pid, info, bsd, task)
+          size = info.call(pid, BSDINFO, 0, bsd, 256)
+          return nil if size < START_MICROS + 8
+
+          raw = bsd[0, size]
+          resident = info.call(pid, TASKINFO, 0, task, 256) >= RESIDENT + 8 ? task[RESIDENT, 8].unpack1("Q") : 0
+          Entry.new(pid: pid, ppid: raw[PPID, 4].unpack1("L"), pgid: raw[PGID, 4].unpack1("L"),
+                    rss_kb: resident / 1024, zombie: raw[STATUS, 4].unpack1("L") == ZOMBIE,
+                    started: raw[START_SECONDS, 16].unpack("QQ"))
+        end
+
+        def functions
+          return @functions if defined?(@functions)
+
+          @functions = begin
+            require "fiddle"
+            library = Fiddle.dlopen("/usr/lib/libproc.dylib")
+            [Fiddle::Function.new(library["proc_listallpids"], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT],
+                                  Fiddle::TYPE_INT),
+             Fiddle::Function.new(library["proc_pidinfo"],
+                                  [Fiddle::TYPE_INT, Fiddle::TYPE_INT, Fiddle::TYPE_LONG_LONG,
+                                   Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT], Fiddle::TYPE_INT)]
+          rescue LoadError, Fiddle::DLError
+            nil
+          end
+        end
+      end
+
+      # Linux: /proc/<pid>/stat.
+      module Linux
+        PAGE_KB = 4
+
+        module_function
+
+        def available?
+          File.exist?("/proc/self/stat")
+        end
+
+        def snapshot
+          Dir.children("/proc").each_with_object({}) do |name, table|
+            next unless name.match?(/\A\d+\z/)
+
+            entry = entry_for(name.to_i)
+            table[entry.pid] = entry if entry
+          end
+        end
+
+        # Fields after the command name, which may itself hold spaces or ")".
+        def entry_for(pid)
+          stat = File.read("/proc/#{pid}/stat")
+          fields = stat[(stat.rindex(")") + 2)..].split
+          Entry.new(pid: pid, ppid: fields[1].to_i, pgid: fields[2].to_i, rss_kb: fields[21].to_i * page_kb,
+                    zombie: fields[0] == "Z", started: fields[19].to_i)
+        rescue SystemCallError
+          nil
+        end
+
+        def page_kb
+          @page_kb ||= begin
+            require "etc"
+            Etc.sysconf(Etc::SC_PAGESIZE) / 1024
+          rescue StandardError
+            PAGE_KB
+          end
+        end
+      end
+
+      # Anywhere else: one `ps` call. Slow (it spawns a process) but portable.
+      module Ps
+        COLUMNS = "pid=,ppid=,pgid=,rss=,stat=,lstart="
+
+        module_function
+
+        def available?
+          true
+        end
+
+        def snapshot
+          out = IO.popen(["ps", "-A", "-o", COLUMNS], err: File::NULL, &:read)
+          out.each_line.with_object({}) do |line, table|
+            pid, ppid, pgid, rss, state, *started = line.split
+            next if started.empty?
+
+            table[pid.to_i] = Entry.new(pid: pid.to_i, ppid: ppid.to_i, pgid: pgid.to_i, rss_kb: rss.to_i,
+                                        zombie: state.start_with?("Z"), started: started.join(" "))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/execution/process_tree.rb

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "process_table"
+
+module Saboteur
+  module Execution
+    # Every process descended from one forked child, tracked for as long as the
+    # child is supervised.
+    #
+    # A process group is not containment: any descendant can call setpgid or
+    # setsid (a nested saboteur run does exactly that) and leave the group.
+    # Parent links survive that, so the tree is rebuilt from the process table
+    # on every poll, and every pid ever seen in it is remembered, so a
+    # descendant whose parent exits (re-parented to init) is still known.
+    #
+    # Identity is pid plus start time, so a recycled pid is never signalled.
+    class ProcessTree
+      attr_reader :root
+
+      def initialize(root)
+        @root = root
+        @known = {} # pid => start time
+      end
+
+      # The live members: not zombies, same start time as when first seen.
+      # Also starts tracking any new descendant (by parent link or by the
+      # root's process group), so call it often while the tree runs.
+      def live_members(table = ProcessTable.snapshot)
+        grow(table)
+        @known.filter_map do |pid, started|
+          entry = table[pid]
+          entry if entry && entry.started == started && !entry.zombie
+        end
+      end
+
+      # Count and resident memory of the live members. rss_baseline_kb is the
+      # supervisor's own RSS: forked children share its pages, so only growth
+      # beyond it is counted per process.
+      def usage(rss_baseline_kb, table = ProcessTable.snapshot)
+        live = live_members(table)
+        growth = live.sum { |entry| [entry.rss_kb - rss_baseline_kb, 0].max }
+        [live.size, growth]
+      end
+
+      # Stops every member (a stopped process cannot fork), rescans until no new
+      # member appears, then kills them all. Returns the pids it killed.
+      def kill_all(rounds: 20)
+        killed = []
+        rounds.times do
+          live = stop_until_stable
+          break if live.empty?
+
+          live.each { |entry| signal("KILL", entry.pid) }
+          killed.concat(live.map(&:pid))
+          sleep 0.01
+        end
+        killed.uniq
+      end
+
+      private
+
+      def grow(table)
+        @known[root] = table[root].started if @known.empty? && table[root]
+        loop do
+          added = table.values.select do |entry|
+            next false if @known.key?(entry.pid) || entry.pid == Process.pid
+
+            @known.key?(entry.ppid) || entry.pgid == root
+          end
+          break if added.empty?
+
+          added.each { |entry| @known[entry.pid] = entry.started }
+        end
+      end
+
+      def stop_until_stable
+        stopped = {}
+        5.times do
+          live = live_members
+          fresh = live.reject { |entry| stopped.key?(entry.pid) }
+          return live if fresh.empty?
+
+          fresh.each do |entry|
+            signal("STOP", entry.pid)
+            stopped[entry.pid] = true
+          end
+        end
+        live_members
+      end
+
+      def signal(name, pid)
+        Process.kill(name, pid)
+      rescue Errno::ESRCH, Errno::EPERM
+        nil
+      end
+    end
+  end
+end

data/lib/saboteur/execution/protocol.rb

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+
+module Saboteur
+  module Execution
+    # What a mutant child tells its supervisor, one JSON object per line on
+    # a pipe. Every message carries the protocol version.
+    #
+    #   started  the mutated method is in place; tests are about to run
+    #   result   passed, failed (ids), executed (ids), runtime, exited (status
+    #            when a test called exit)
+    #   error    message, backtrace: the harness failed, not the tests
+    #
+    # A child that dies after "started" without "result" crashed while its
+    # tests ran; one that dies before "started" crashed while loading.
+    module Protocol
+      VERSION = 1
+
+      module_function
+
+      def emit(io, event, **fields)
+        io.write("#{JSON.generate(v: VERSION, event: event, **fields)}\n")
+        io.flush
+      rescue IOError, SystemCallError
+        nil
+      end
+
+      # Events in order; lines that are not this protocol's are ignored.
+      def parse(payload)
+        payload.each_line.filter_map do |line|
+          message = JSON.parse(line)
+          message if message.is_a?(Hash) && message["v"] == VERSION
+        rescue JSON::ParserError
+          nil
+        end
+      end
+    end
+  end
+end