saboteur 0.5.0 → 0.7.0

data/lib/saboteur/execution/runner.rb

@@ -3,6 +3,7 @@
 
 require "json"
 require_relative "stale_captures"
+require_relative "../crash_report"
 
 module Saboteur
   module Execution
@@ -39,17 +40,42 @@ module Saboteur
         end
       end
 
+      # The child log (or its whole directory) vanished mid-job (#234).
+      # Carries the digest of the mutant whose run just ended when the job
+      # had one, so a parallel parent can prefer this report over a
+      # bystander's bare "unknown mutant" report (#285).
+      class LogVanished < Error
+        attr_reader :culprit_digest
+
+        def initialize(message, culprit_digest: nil)
+          @culprit_digest = culprit_digest
+          super(message)
+        end
+      end
+
       LOG_TAIL_BYTES = 2000
+      CRASH_LOG_BYTES = 2_000
+      CRASH_LOG_LINES = 8
+      # File mtimes follow the filesystem's clock, which can trail the
+      # process clock; a log written during this run can stamp a little
+      # before its start, so the freshness check allows that skew.
+      CRASH_LOG_CLOCK_SLACK = 2
+      # Freshness matching can attach another run's log (clock slack, two
+      # nested runs in one child), so the label must not claim exact
+      # attribution (#296).
+      CRASH_LOG_LABEL = "recent saboteur crash log (freshness-matched, may be another run's):"
 
       # log_dir: where children write their output; a child's log is kept
-      # only when it ends in an error.
+      # only when it ends in an error. crash_log: the crash log of a saboteur
+      # run nested inside a test; by default it sits next to the logs dir,
+      # where the engine's cache dir keeps it.
       # seed: every child seeds Ruby's random numbers with it before its
       # tests, so a test that uses rand sees the same values in every run.
       # sandbox: when given, a callable run first in every child (Landlock
       # makes the project tree read-only there, #152). It raises rather than
       # run half-restricted, which the child reports as an error.
       def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1,
-                     sandbox: nil)
+                     sandbox: nil, canary: nil, crash_log: nil)
         @integration = integration
         @reevaluator = reevaluator
         @hooks = hooks
@@ -57,6 +83,8 @@ module Saboteur
         @supervisor = supervisor
         @seed = seed
         @sandbox = sandbox
+        @canary = canary
+        @crash_log = crash_log || File.join(File.dirname(log_dir), CrashReport::LOG_NAME)
       end
 
       # text: method source to evaluate in the child (the unchanged source for
@@ -69,23 +97,33 @@ module Saboteur
         @sequence = @sequence.to_i + 1
         job = Job.new(subject:, text:, test_ids:, timeout:, mutation:, eval_prefix:,
                       log_path: File.join(@log_dir, "child-#{Process.pid}-#{@sequence}.log"))
-        result = @supervisor.run(timeout: timeout) { |writer| child(writer, job) }
+        log_file = open_log(job)
+        result = @supervisor.run(timeout: timeout) { |writer| child(writer, job, log_file) }
         outcome = outcome_for(result, job)
-        File.delete(job.log_path) if outcome.status != :error && File.exist?(job.log_path)
+        check_tree_still_there(job)
+        check_log_still_there(job)
+        File.delete(job.log_path) if outcome.status != :error
         outcome
+      ensure
+        log_file&.close
       end
 
       private
 
       # --- child ----------------------------------------------------------
 
-      def child(writer, job)
+      def child(writer, job, log_file)
         # Redirect before restricting: the sweep closes inherited stdio
         # redirected to a regular file outside the grants (#178), and a
         # reopened-afterwards $stdout would start from a closed stream.
-        # The log lives under the granted cache dir, so it survives.
-        redirect_output(job.log_path)
-        @sandbox&.call([writer.fileno])
+        # The parent opened the log before the fork, so the inherited
+        # descriptor survives on the keep list - the cache dir itself
+        # stays read-only inside the sandbox (#234). stdio was just
+        # reopened onto that log, so 1 and 2 point at a regular file
+        # outside the grants and the sweep would close them; keep them
+        # explicitly - keeping only the log's own fd is not enough (#283).
+        redirect_output(log_file)
+        @sandbox&.call([writer.fileno, log_file.fileno, $stdout.fileno, $stderr.fileno])
         Process.setproctitle(title(job))
         @hooks.run(:after_fork)
         # Before the evaluation: a whole-file subject runs the file's own
@@ -120,6 +158,7 @@ module Saboteur
         method = replacement_method(job) if has_captures && job.mutation
         called = method ? false : nil
         trace = TracePoint.new(:call) { called = true } if method
+        started_at = Time.now
         result = if trace
                    trace.enable(target: method) { @hooks.around(job.mutation) { run_tests(job.test_ids) } }
                  else
@@ -128,10 +167,72 @@ module Saboteur
         if result.nil?
           Protocol.emit(writer, "error", message: "around_mutant hook did not run the tests")
         else
-          Protocol.emit(writer, "result", **result, replacement_called: called)
+          Protocol.emit(writer, "result", **with_crash_detail(result, started_at), replacement_called: called)
         end
       end
 
+      # A test can run saboteur itself (the dogfood's cli_spec): the nested
+      # CLI reports its own crashes as exit 3 and logs them to
+      # internal-error.log. A log written during this run explains the
+      # failure better than the bare expectation diff, so its first lines
+      # join each failure detail; an older log is another run's (#296).
+      def with_crash_detail(result, started_at)
+        excerpt = crash_log_excerpt(started_at)
+        details = result[:failure_details]
+        return result unless excerpt && details && !details.empty?
+
+        result.merge(failure_details: details.transform_values { |d| [d, CRASH_LOG_LABEL, excerpt].join("\n") })
+      end
+
+      # The nested CLI writes its log to the cache dir, or to the pid-keyed
+      # temp fallback when the sandbox holds the cache dir read-only (#296).
+      def crash_log_excerpt(started_at)
+        excerpt = tmp_log_candidates.filter_map { |path| fresh_log_excerpt(path, started_at) }
+                                    .max_by { |mtime, _text| mtime }
+        excerpt&.last
+      rescue StandardError
+        nil
+      end
+
+      # The log paths are predictable and live in shared dirs, so read the
+      # log the way CrashReport writes it: through a no-follow, non-blocking
+      # descriptor,
+      # and validate on that same fd - a regular file, owned by this uid,
+      # with no group/other permission bits, fresh by the fd's own mtime.
+      # One open, no check-then-reopen race; a symlink, a foreign, wide or
+      # non-regular file, or any IO failure is no excerpt, never an
+      # exception (#296).
+      def fresh_log_excerpt(path, started_at)
+        # No safe open, no excerpt: without O_NOFOLLOW a symlink redirects
+        # the read, without O_NONBLOCK a no-writer FIFO blocks it (#296).
+        return unless CrashReport.safe_log_io?
+
+        flags = File::RDONLY | CrashReport.nofollow_flag | CrashReport.nonblock_flag
+        File.open(path, flags) do |file|
+          stat = file.stat
+          next unless stat.file? && stat.uid == Process.uid && stat.mode.nobits?(0o077)
+          next unless stat.mtime >= started_at - CRASH_LOG_CLOCK_SLACK
+
+          text = file.read(CRASH_LOG_BYTES)
+          # Scrub before the excerpt travels as JSON: the bounded read can
+          # end mid-character and the log can hold locale-invalid bytes, and
+          # a JSON::GeneratorError here would turn a kill into a child error
+          # (#296).
+          [stat.mtime, text ? CrashReport.scrub(text).lines.first(CRASH_LOG_LINES).join : nil]
+        end
+      rescue SystemCallError, IOError
+        nil
+      end
+
+      # The excerpt is diagnostic enrichment: when CrashReport itself is the
+      # mutated file, tmp_log_path can raise or return garbage - keep the
+      # primary candidate and never let the lookup change a verdict (#296).
+      def tmp_log_candidates
+        [@crash_log, CrashReport.tmp_log_path].compact.uniq.grep(String)
+      rescue StandardError
+        [@crash_log].compact
+      end
+
       # Targeted tracing distinguishes the new Ruby body from an old Method
       # capture, even though both have the same owner, name and source line.
       def replacement_method(job)
@@ -151,19 +252,69 @@ module Saboteur
       def run_tests(test_ids)
         result = @integration.run(test_ids)
         { passed: result.passed, failed: result.failed_ids, executed: result.executed_ids, runtime: result.runtime,
-          failure_classes: result.failure_classes, failure_messages: result.failure_messages }
+          failure_classes: result.failure_classes, failure_messages: result.failure_messages,
+          failure_details: result.failure_details }
       rescue SystemExit => e
         # A test called `exit`. The tests noticed something; count it as a failure.
         { passed: false, failed: [], executed: [], runtime: 0.0, exited: e.status }
       end
 
-      def redirect_output(log_path)
-        $stdout.reopen(log_path, "w")
+      def redirect_output(log_file)
+        $stdout.reopen(log_file)
         $stderr.reopen($stdout)
         $stdout.sync = true
         $stderr.sync = true
       end
 
+      # Names the just-run mutant when there is one (a verdict-check rerun
+      # has none): the most likely culprit, worth naming for the report.
+      def culprit(job)
+        culprit_digest(job) || "an unknown mutant"
+      end
+
+      # The 16-hex digest of the just-run mutant, or nil when the job ran
+      # none (a neutral or verdict-check rerun): a parallel parent prefers
+      # the vanished-log report that carries one (#285).
+      def culprit_digest(job)
+        return unless job.mutation && job.subject.respond_to?(:start_byte)
+
+        job.mutation.digest(subject_start: job.subject.start_byte,
+                            original_source: job.subject.file_source)
+      end
+
+      # Opened in the parent, not the sandboxed child: the cache dir is
+      # not writable inside the sandbox (#234), and a log directory that
+      # is already gone is a tool problem to report, not a mutant result.
+      def open_log(job)
+        File.open(job.log_path, "w")
+      rescue SystemCallError => e
+        raise Error, "cannot open the child log #{job.log_path} (#{e.class}: #{e.message})"
+      end
+
+      # A mutant's tests can delete saboteur's own files when the sandbox
+      # is off or degraded (#234): a vanished log makes the verdict
+      # worthless, so fail as a tool error instead of trusting the run.
+      # The mutant that just ran is the likely - not proven - cause: a
+      # leftover process from an earlier mutant could have done it.
+      # The per-mutant probe (#235): names the mutant whose run just wiped
+      # tracked files, instead of blaming the subject's last mutant when the
+      # loss surfaces at subject end. Runs before the log check, so the
+      # hazard report still wins when a mutant took both (#283).
+      def check_tree_still_there(job)
+        # culprit_digest, not the display fallback: the digest doubles as
+        # trip metadata, and "an unknown mutant" must not rank as one (#285).
+        @canary&.probe!(subject: job.subject, digest: culprit_digest(job))
+      end
+
+      def check_log_still_there(job)
+        return if File.exist?(job.log_path)
+
+        cause = "a mutant's tests may have hit saboteur's files"
+        raise LogVanished.new("the child log #{job.log_path} vanished mid-run - the mutant that just ran, " \
+                              "#{culprit(job)}, is the likely cause: #{cause}",
+                              culprit_digest: culprit_digest(job))
+      end
+
       # --- parent ---------------------------------------------------------
 
       def outcome_for(result, job)
@@ -197,7 +348,7 @@ module Saboteur
         end
       end
 
-      def classify_result(result, runtime, test_ids)
+      def classify_result(result, runtime, test_ids, verbose_failure: false)
         failed = Array(result["failed"])
         executed = Array(result["executed"])
         return Outcome.new(status: :killed, runtime: runtime, message: "test called exit") if result.key?("exited")
@@ -206,9 +357,7 @@ module Saboteur
           unexpected = failed - test_ids
           return mismatch(runtime, "failed tests not requested: #{unexpected.first(3).join(', ')}") if unexpected.any?
 
-          return Outcome.new(status: :killed, runtime: runtime, failed_ids: failed, executed_ids: executed,
-                             message: result.fetch("failure_messages", {})[failed.first],
-                             kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
+          return killed_outcome(result, runtime, failed, executed, verbose_failure)
         end
         unless result["passed"]
           return Outcome.new(status: :killed, runtime: runtime, executed_ids: executed,
@@ -222,8 +371,21 @@ module Saboteur
         Outcome.new(status: :survived, runtime: runtime, executed_ids: executed)
       end
 
+      def killed_outcome(result, runtime, failed, executed, verbose_failure)
+        Outcome.new(status: :killed, runtime: runtime, failed_ids: failed, executed_ids: executed,
+                    message: failure_message(result, failed.first, verbose_failure),
+                    kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
+      end
+
+      # A neutral check (the verdict re-run, no mutation) needs the full
+      # failure to explain why the killer fails unchanged (#296).
+      def failure_message(result, test_id, verbose_failure)
+        message = result.fetch("failure_messages", {})[test_id]
+        verbose_failure ? result.fetch("failure_details", {})[test_id] || message : message
+      end
+
       def classify_events(result, events, runtime, job)
-        outcome = classify_result(result, runtime, job.test_ids)
+        outcome = classify_result(result, runtime, job.test_ids, verbose_failure: job.mutation.nil?)
         replaced = events.find { |event| event["event"] == "replaced" }
         return replaced_by_later_definition(outcome, replaced) if never_ran?(outcome, replaced, job)
 

data/lib/saboteur/execution/sandbox.rb

@@ -160,7 +160,12 @@ module Saboteur
           return unless stat.file?
 
           granted = target.delete_suffix(" (deleted)")
-          return if writable.any? { |dir| granted == dir || granted.start_with?("#{dir}/") }
+          # readlink targets arrive tagged with default_external, so under
+          # LANG=C a non-ASCII target compared against a UTF-8 grant pattern
+          # raises Encoding::CompatibilityError before any test runs (CI #272).
+          # Compare bytes.
+          target_bytes = granted.b
+          return if writable.any? { |dir| target_bytes == dir.b || target_bytes.start_with?(dir.b + "/") } # rubocop:disable Style/StringConcatenation -- interpolation with a BINARY part raises on high bytes
 
           flags = LibC.fcntl_fn.call(descriptor, F_GETFL, 0)
           return close_fd(descriptor) if flags.negative?

data/lib/saboteur/execution/supervisor.rb

@@ -119,7 +119,7 @@ module Saboteur
       # on_tick is called from the parent's wait loop after the fork, about
       # every poll, so callers can report progress without a thread alive
       # at fork time.
-      def run(timeout:, cpu_seconds: nil, on_tick: nil, &block)
+      def run(timeout:, cpu_seconds: nil, on_tick: nil, label: "supervised child died", &block)
         depth = ENV.fetch(CHILD_ENV, "0").to_i
         raise Error, "refusing to fork: saboteur children nested #{depth} deep (limit #{MAX_DEPTH})" if
           depth >= MAX_DEPTH
@@ -131,6 +131,7 @@ module Saboteur
         started = monotonic
         @peak_growth_kb = 0
         supervisor = Process.pid
+        @child_label = label
         pid = fork { child(reader, writer, supervisor, cpu_seconds, depth + 1, &block) }
         writer.close
         contain(pid, reader, started, timeout, on_tick)
@@ -146,13 +147,27 @@ module Saboteur
         reader.close
         self.class.close_in_child.each { |io| io.close unless io.closed? }
         ENV[CHILD_ENV] = depth.to_s
-        Process.setpgid(0, 0)
+        # Best-effort like the parent-side own_group: macOS can refuse the
+        # group change (EPERM), and the child must still run (#296).
+        own_group(0)
         limit_cpu(cpu_seconds)
         trap("TERM") { exit!(143) }
         watch_parent(supervisor)
         yield writer
         exit!(0)
-      rescue Exception # rubocop:disable Lint/RescueException
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        # The pipe stays empty when the child dies before answering, so this
+        # is the only place the cause survives (#255). The warning is
+        # best-effort and the exit is ensured: a broken stderr must not let
+        # the exception unwind past the fork block into CLI code in the
+        # child, where exit handlers could emit duplicate output (#289).
+        begin
+          frames = e.backtrace ? e.backtrace.first(8).join("\n  from ") : "(no backtrace)"
+          warn("saboteur: #{@child_label}: #{e.class}: #{e.message}\n  from #{frames}")
+        rescue Exception # rubocop:disable Lint/RescueException
+          nil
+        end
+      ensure
         exit!(1)
       end
 

data/lib/saboteur/execution/worker_pool.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "json"
+require_relative "../text"
 require_relative "process_tree"
 require_relative "supervisor"
 
@@ -30,11 +31,44 @@ module Saboteur
 
       # on_start: called in each worker with its number before its first
       # item (per-worker database connections, for example).
-      def initialize(jobs:, on_start: nil, on_result: nil)
+      def initialize(jobs:, on_start: nil, on_result: nil, drain_timeout: 30)
         @jobs = jobs
         @on_start = on_start
         @on_result = on_result
         @start_failures = 0
+        @halted = false
+        @drain_timeout = drain_timeout
+      end
+
+      # Stop handing queued items out while letting in-flight items finish
+      # and report: a worker that merely noticed the damage can report
+      # before the worker whose mutant caused it, and the later report is
+      # the one that names the culprit (#285). Unlike raising from
+      # on_result, nothing in flight is killed. Unstarted items come back
+      # as nil holes in the result array.
+      #
+      # drain: true bounds the wait for a safety abort (tool error, canary):
+      # a wedged worker must not stall it, so drain_timeout seconds after
+      # the first halt the pool stops waiting, kills whatever is still
+      # running and leaves nil holes (#285 review). drain: false is the
+      # ordinary --max-duration halt: in-flight subjects always finish and
+      # report, so a nil hole only ever means "never started"; the
+      # overshoot stays bounded by each worker's own mutant deadlines.
+      def halt!(drain: true)
+        @halted = true
+        return if @drain_deadline || !drain
+
+        @drain_deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @drain_timeout
+      end
+
+      # The serialized shape of a tool error row: the message plus the
+      # attribution flags the parent needs to rank reports - a per-mutant
+      # culprit digest first, then a canary trip (#283, #285).
+      def self.tool_error_row(error)
+        row = { "tool_error" => error.message }
+        row["canary"] = true if error.is_a?(CanaryTripped)
+        row["culprit"] = error.culprit_digest if error.respond_to?(:culprit_digest) && error.culprit_digest
+        row
       end
 
       def map(items, weights: nil, &work)
@@ -64,8 +98,20 @@ module Saboteur
 
       # Waits up to a second for results, then checks for workers that died
       # without closing their pipe (a stray grandchild may still hold it).
+      # While halted the wait is bounded by the drain deadline: past it the
+      # remaining workers are retired (stop kills a wedged one) and their
+      # items come back as nil holes.
       def collect(workers, queue, results, items, &work)
-        ready = IO.select(workers.map(&:results), nil, nil, 1)&.first || []
+        timeout = 1
+        if @drain_deadline
+          remaining = @drain_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          if remaining <= 0
+            abandon_all(workers)
+            return
+          end
+          timeout = [timeout, remaining].min
+        end
+        ready = IO.select(workers.map(&:results), nil, nil, timeout)&.first || []
         ready.each do |io|
           worker = workers.find { |candidate| candidate.results == io }
           receive(worker, workers, queue, results, items, &work) if worker
@@ -82,7 +128,7 @@ module Saboteur
         take_and_report(worker, line, results) if line
         return replace(worker, workers, queue, results, items, &) if line.nil? || worker.last_words
 
-        assign(worker, queue)
+        assign(worker, queue) unless @halted
         retire(worker, workers) unless worker.current
       end
 
@@ -96,7 +142,7 @@ module Saboteur
       end
 
       def parse(line)
-        JSON.parse(line)
+        JSON.parse(Text.utf8_strict(line))
       rescue JSON::ParserError => e
         { "error" => "unreadable result from worker: #{e.message}" }
       end
@@ -127,6 +173,28 @@ module Saboteur
         nil
       end
 
+      # Past the drain deadline nothing more is coming: kill every wedged
+      # worker's tree first, then reap, so the cleanup costs one round
+      # instead of one stop grace per worker (#285 review). Items in
+      # flight come back as nil holes.
+      def abandon_all(workers)
+        wedged = workers.dup
+        wedged.each do |worker|
+          worker.commands.close unless worker.commands.closed?
+          Supervisor.live.delete(worker.pid)&.kill_all
+        end
+        # Every wedged tree dies before any wait: one cleanup round.
+        # rubocop:disable-next Style/CombinableLoops
+        wedged.each do |worker|
+          worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
+          worker.results.close unless worker.results.closed?
+        rescue Errno::ECHILD
+          nil
+        ensure
+          workers.delete(worker)
+        end
+      end
+
       def replace(worker, workers, queue, results, items, &)
         drain(worker, results)
         retire(worker, workers)
@@ -138,8 +206,11 @@ module Saboteur
           results[worker.current] = { "error" => "#{reason}, running this item" }
           @on_result&.call(worker.current, results[worker.current])
         end
-        give_up(reason) if worker.done.zero?
-        return if queue.empty?
+        # Halted pools start nothing, so a start-failure storm cannot be the
+        # run's problem then; raising it would mask the rows that carry the
+        # real abort cause (#286).
+        give_up(reason) if worker.done.zero? && !@halted
+        return if @halted || queue.empty?
 
         fresh = start(worker.number, items, workers, &)
         workers << fresh
@@ -201,7 +272,10 @@ module Saboteur
       # Runs in the worker until the parent closes its command pipe. If it
       # crashes, its last line says why.
       def serve(number, items, commands, results, &)
-        Process.setpgid(0, 0)
+        # Grouping helps tree cleanup find the worker's descendants; macOS
+        # can refuse it (EPERM), and a refused setpgid must not crash the
+        # worker - parent-link tracking still finds the tree (#296).
+        own_group(0)
         Supervisor.reset_in_worker!
         Supervisor.close_in_child.push(commands, results)
         ENV["SABOTEUR_WORKER"] = number.to_s
@@ -226,6 +300,12 @@ module Saboteur
 
       def run_one(item, &work)
         work.call(item)
+      rescue Error => e
+        # A tool error (a vanished cache dir, ...) aborts the run with
+        # exit 3; it is not a per-mutant outcome. Serialize it so the
+        # parent can re-raise it as itself (#283), with the flags the
+        # parent needs to rank reports by attribution quality (#285).
+        self.class.tool_error_row(e)
       rescue StandardError => e
         { "error" => "#{e.class}: #{e.message}" }
       end

data/lib/saboteur/frameworks/database_hosts.rb

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Frameworks
+    # Which database hosts count as local for the Rails integration's refusal
+    # to mutate a remote test database (issue #258).
+    module DatabaseHosts
+      # 0.0.0.0: a listen address, but in a URL it connects to this machine (issue #258).
+      LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]", "0.0.0.0"].freeze
+
+      module_function
+
+      def local?(host)
+        host = host.to_s
+        host.empty? || host.start_with?("/") || LOCAL_HOSTS.include?(host.downcase)
+      end
+
+      # "name (host host)" per config whose host is not local and not in
+      # extra_allowed. Shared by the boot refusal and doctor.
+      def remote(configs, extra_allowed = [])
+        allowed = extra_allowed.map(&:downcase)
+        configs.filter_map do |db|
+          host = db.configuration_hash[:host]
+          "#{db.name} (host #{host})" unless local?(host) || allowed.include?(host.downcase)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/frameworks/rails.rb

@@ -7,6 +7,8 @@ require "pathname"
 require "English"
 require "tmpdir"
 
+require_relative "database_hosts"
+
 module Saboteur
   module Frameworks
     # Opt-in Rails support (`c.rails = true` in config/saboteur.rb).
@@ -16,8 +18,6 @@ module Saboteur
     # forking, and drops database connections in the parent so each child opens
     # its own. Prints host names only, never credentials.
     class Rails
-      LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]"].freeze
-
       # How many blocking session names the template-copy error lists.
       MAX_BLOCKER_NAMES = 5
 
@@ -170,9 +170,9 @@ module Saboteur
         db.adapter.to_s.match?(/sqlite/) && db.database.to_s == ":memory:"
       end
 
-      def self.local_host?(host)
-        host.nil? || host.to_s.empty? || host.to_s.start_with?("/") || LOCAL_HOSTS.include?(host.to_s.downcase)
-      end
+      def self.local_host?(host) = DatabaseHosts.local?(host)
+
+      def self.remote_databases(configs, extra_allowed = []) = DatabaseHosts.remote(configs, extra_allowed)
 
       private
 
@@ -197,13 +197,11 @@ module Saboteur
       def check_databases
         return unless defined?(::ActiveRecord::Base)
 
-        remote = ::ActiveRecord::Base.configurations.configs_for(env_name: "test").filter_map do |db|
-          host = db.configuration_hash[:host]
-          "#{db.name} (host #{host})" unless self.class.local_host?(host)
-        end
+        remote = self.class.remote_databases(test_configs, @config["allow_db_hosts"])
         return if remote.empty?
 
-        raise Error, "rails integration refuses to run against non-local databases: #{remote.join(', ')}"
+        raise Error, "rails integration refuses to run against non-local databases: #{remote.join(', ')}; " \
+                     "if they are disposable test databases, allow them with c.allow_db_hosts in #{Config::FILE}"
       end
 
       def test_configs