rspec-hopper 0.1.0

data/lib/rspec/hopper/supervisor.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    # The `--processes N` parent. Forks N children (each a standalone worker
+    # with worker id `<worker>.<n>` and its own TEST_ENV_NUMBER), forwards INT
+    # and TERM to them, waits for all of them, and exits per the precedence
+    # rules: 2 if any child failed on infrastructure, else 4 if any child
+    # aborted a hung unit, else 0 -- or, with --report-on-exit, the report's
+    # exit code unless a child exited 2.
+    class Supervisor
+      FORWARDED_SIGNALS = %w[INT TERM].freeze
+
+      # @param worker_class [#new] `Worker` by default; must accept
+      #   `config:, queue_factory:, suite:, out:, err:` and respond to `#run`.
+      # @param suite_loader [#call] `(rspec_args, config:, out:, err:) -> suite`
+      #   used once in the parent for `--boot shared`.
+      # @param report_runner [#call] `(args, out:, err:) -> Integer`
+      # @param queue_factory_for [#call] `(config) -> lambda` building a queue.
+      # @param fork [#call] `Process.fork` (with block) replacement for tests.
+      def initialize(config:, out: $stdout, err: $stderr, worker_class: nil, suite_loader: nil,
+                     report_runner: nil, queue_factory_for: nil, fork: Process.method(:fork),
+                     fork_available: Process.respond_to?(:fork), env: ENV)
+        @config = config
+        @out = out
+        @err = err
+        @worker_class = worker_class
+        @suite_loader = suite_loader
+        @report_runner = report_runner
+        @queue_factory_for = queue_factory_for
+        @fork = fork
+        @fork_available = fork_available
+        @env = env
+        @children = {} # pid => n
+        @statuses = {} # n => Process::Status
+      end
+
+      # @return [Integer] exit code
+      def run
+        return run_inline if @config.processes == 1
+        unless @fork_available
+          raise ForkUnavailable, "--processes #{@config.processes} needs fork, which this platform lacks"
+        end
+
+        remaining, formatter_pairs = CLI::FormatterArgs.split(@config.rspec_args)
+        announce_formatter_output(formatter_pairs)
+        suite = @config.boot == :shared ? load_shared_suite(remaining) : nil
+        with_signal_forwarding do
+          1.upto(@config.processes) { |number| start_child(number, remaining, formatter_pairs, suite) }
+          wait_for_children
+        end
+        exit_code
+      rescue InfrastructureError => e
+        log(e.message)
+        e.exit_code
+      end
+
+      # Children write their formatter output to files, so nothing here prints
+      # an RSpec summary per process; the verdict comes from `report`.
+      def announce_formatter_output(formatter_pairs)
+        return unless CLI::FormatterArgs.defaults_needed?(formatter_pairs)
+
+        log("formatter output: #{CLI::FormatterArgs::DEFAULT_DIR}/ " \
+            "(the build verdict comes from `rspec-hopper report`)")
+      end
+
+      # The value of TEST_ENV_NUMBER for child `number` (1-based): "" for the
+      # first, then "2".."N", matching parallel_tests.
+      def self.env_number(number) = number == 1 ? "" : number.to_s
+
+      def child_worker_id(number) = "#{@config.worker_id}.#{number}"
+
+      # The config child `number` runs with; public so specs can assert on it.
+      def child_config(number, remaining, formatter_pairs)
+        env_number = self.class.env_number(number)
+        worker_id = child_worker_id(number)
+        child_args = CLI::FormatterArgs.for_child(formatter_pairs, env_number, label: worker_id)
+        @config.with(
+          worker_id: worker_id, supervised: true, processes: 1,
+          rspec_args: (child_args + remaining).freeze
+        )
+      end
+
+      private
+
+      def run_inline
+        worker_class.new(config: @config, queue_factory: queue_factory_for.call(@config), out: @out, err: @err).run
+      end
+
+      def load_shared_suite(remaining)
+        @env.delete("TEST_ENV_NUMBER")
+        suite = suite_loader.call(remaining, config: @config, out: @out, err: @err)
+        unless RSpec::Hopper.after_fork_hooks.any?
+          raise SharedBootWithoutHook,
+                "--boot shared needs at least one RSpec::Hopper.after_fork hook: the suite was booted once " \
+                "with TEST_ENV_NUMBER unset, so each child must re-derive every value computed from it " \
+                "(database name, Redis db, ports, paths). Register a hook or use --boot per-process."
+        end
+        RSpec::Hopper.before_fork_hooks.each(&:call)
+        suite
+      end
+
+      def start_child(number, remaining, formatter_pairs, suite)
+        config = child_config(number, remaining, formatter_pairs)
+        env_number = self.class.env_number(number)
+        pid = @fork.call { run_child(config, env_number, suite) }
+        @children[pid] = number
+        log("started #{config.worker_id} (pid #{pid}, TEST_ENV_NUMBER=#{env_number.inspect})")
+      end
+
+      def run_child(config, env_number, suite)
+        # SYSTEM_DEFAULT, not DEFAULT: Ruby's own default handling for INT and
+        # TERM raises Interrupt/SignalException, which prints a stack trace per
+        # child before re-signalling. A forwarded signal would therefore bury
+        # the run in N backtraces, and an Interrupt raised inside a unit would
+        # be caught as a non-requeueable failure rather than stopping the
+        # child. The OS default ends the child silently with the same wait
+        # status, which is what the parent reports.
+        FORWARDED_SIGNALS.each { |sig| trap(sig, "SYSTEM_DEFAULT") }
+        @env["TEST_ENV_NUMBER"] = env_number
+        RSpec::Hopper.after_fork_hooks.each { |hook| hook.call(env_number) } if @config.boot == :shared
+        kwargs = { config: config, queue_factory: queue_factory_for.call(config), out: @out, err: @err }
+        kwargs[:suite] = suite unless suite.nil?
+        # `exit`, not `exit!`: RSpec formatters may rely on at_exit handlers. An
+        # exception escaping the worker ends the child with status 1, which the
+        # parent treats as an infrastructure failure.
+        exit(worker_class.new(**kwargs).run)
+      end
+
+      def wait_for_children
+        @children.each do |pid, number|
+          _, status = Process.wait2(pid)
+          @statuses[number] = status
+          log("#{child_worker_id(number)} (pid #{pid}) #{describe(status)}")
+        rescue Errno::ECHILD
+          @statuses[number] = nil
+          log("#{child_worker_id(number)} (pid #{pid}) was already reaped; treating as failed")
+        end
+      end
+
+      def with_signal_forwarding
+        previous = FORWARDED_SIGNALS.to_h { |sig| [sig, trap(sig) { forward(sig) }] }
+        yield
+      ensure
+        previous&.each { |sig, handler| trap(sig, handler || "DEFAULT") }
+      end
+
+      def forward(sig)
+        @children.each_key do |pid|
+          next if @statuses.key?(@children[pid])
+
+          Process.kill(sig, pid)
+        rescue Errno::ESRCH
+          nil
+        end
+      end
+
+      def exit_code
+        codes = @statuses.values.map { |status| status&.exitstatus }
+        return ExitCode::INFRASTRUCTURE if codes.any? { |c| ![ExitCode::OK, ExitCode::ABORTED].include?(c) }
+
+        aborted = codes.include?(ExitCode::ABORTED)
+        return aborted ? ExitCode::ABORTED : ExitCode::OK unless @config.report_on_exit
+
+        log("a child aborted a hung unit (exit 4); the verdict comes from report") if aborted
+        report_runner.call(@config.report_args, out: @out, err: @err)
+      end
+
+      def describe(status)
+        return "exited #{status.exitstatus}" if status.exited?
+        return "killed by SIG#{Signal.signame(status.termsig)}" if status.signaled?
+
+        "stopped with #{status.inspect}"
+      end
+
+      def log(message)
+        @err.puts "[hopper #{@config.worker_id}] #{message}"
+        @err.flush if @err.respond_to?(:flush)
+      end
+
+      def worker_class = @worker_class || Worker
+
+      def suite_loader
+        @suite_loader || ->(args, config:, out:, err:) { Worker::Suite.load(args, config: config, out: out, err: err) }
+      end
+
+      def report_runner
+        @report_runner || ->(args, out:, err:) { CLI::Report.run(args, out: out, err: err) }
+      end
+
+      def queue_factory_for
+        @queue_factory_for || ->(config) { CLI::Work.queue_factory(config) }
+      end
+    end
+  end
+end

data/lib/rspec/hopper/unit.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    UNIT_TYPES = %w[file].freeze
+
+    # A schedulable unit of work. Phase 1 units are spec files; the `type`
+    # field exists so example-level units can be added without a schema change.
+    Unit = Data.define(:id, :type) do
+      def self.file(path) = new(id: path, type: "file")
+
+      def to_h = { "id" => id, "type" => type }
+    end
+  end
+end

data/lib/rspec/hopper/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    VERSION = "0.1.0"
+  end
+end

data/lib/rspec/hopper/worker/buffering_reporter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    class Worker
+      # Records every notification RSpec sends while a unit's example groups
+      # run, so that the worker can replay them to the real reporter exactly
+      # once for a final attempt or discard them for a requeued or stale one.
+      # Suite lifecycle notifications (`start`, `finish`, `close`, `report`) are
+      # deliberately not defined: they belong to the outer runner.
+      class BufferingReporter
+        BUFFERED = %i[
+          example_group_started example_group_finished
+          example_started example_finished example_passed example_failed example_pending
+          message publish notify_non_example_exception deprecation
+        ].freeze
+
+        attr_reader :events
+
+        def initialize
+          @events = []
+        end
+
+        BUFFERED.each do |name|
+          define_method(name) do |*args, **kwargs|
+            @events << [name, args, kwargs]
+            nil
+          end
+        end
+
+        # RSpec consults this after every failure; the worker never fails fast.
+        def fail_fast_limit_met? = false
+
+        def size = @events.size
+        def empty? = @events.empty?
+
+        # Sends every recorded notification to `real`, in original order.
+        def replay(real)
+          @events.each { |name, args, kwargs| real.public_send(name, *args, **kwargs) }
+          discard
+          self
+        end
+
+        def discard
+          @events = []
+          self
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/worker/heartbeat.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    class Worker
+      # Keeps a reservation alive while its unit executes. Runs in a thread
+      # that renews the reservation every `config.heartbeat_interval` seconds.
+      # After `max_unit_duration` it records `abandoned` once and stops
+      # renewing; after a further `timeout` it aborts the whole process, since
+      # the hung test thread cannot be interrupted safely.
+      class Heartbeat
+        # How long to wait before retrying a renewal that failed on a Redis
+        # connection error, capped by the normal interval.
+        RETRY_INTERVAL = 1.0
+
+        attr_reader :reservation, :abandoned_at, :renewals, :renewal_failures
+
+        # @param clock [#call] monotonic seconds
+        # @param sleeper [#call, nil] waits for the given seconds; nil uses an
+        #   interruptible wait so `stop` returns promptly
+        # @param aborter [#call] receives the exit code; defaults to `exit!`
+        def initialize(queue:, reservation:, config:, err: $stderr, clock: nil, sleeper: nil, aborter: nil,
+                       worker_id: config.worker_id)
+          @queue = queue
+          @reservation = reservation
+          @config = config
+          @err = err
+          @worker_id = worker_id
+          @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+          @sleeper = sleeper
+          @aborter = aborter || ->(code) { Kernel.exit!(code) }
+          @signal = Thread::Queue.new
+          @stopped = false
+          @stale = false
+          @abandoned_at = nil
+          @renewals = 0
+          @renewal_failures = 0
+        end
+
+        def interval = @config.heartbeat_interval
+        def stale? = @stale
+        def abandoned? = !@abandoned_at.nil?
+        def running? = !@thread.nil? && @thread.alive?
+
+        # Marks the unit as started at `now` without spawning the thread, so
+        # the loop can be driven through `tick`.
+        def prime(now = @clock.call)
+          @started_at = @last_renewal = now
+          self
+        end
+
+        def start
+          prime
+          @thread = Thread.new { run_loop }
+          @thread.name = "hopper-heartbeat" if @thread.respond_to?(:name=)
+          # Giving up on renewals re-raises through `join` in `stop`, where the
+          # worker turns it into one message and exit 2. Ruby's own report would
+          # only add a bare backtrace ahead of it.
+          @thread.report_on_exception = false
+          self
+        end
+
+        def stop
+          @stopped = true
+          @signal << :stop
+          @thread&.join
+          self
+        end
+
+        # One step of the loop, at time `now`. Public so it can be driven
+        # without a thread. Returns what happened.
+        def tick(now)
+          return :stopped if @stopped
+
+          elapsed = now - @started_at
+          if elapsed >= abort_after
+            abort!
+            :aborted
+          elsif elapsed >= @config.max_unit_duration
+            abandon!(elapsed)
+          elsif now - @last_renewal >= interval
+            renew!(now)
+          else
+            :idle
+          end
+        end
+
+        private
+
+        def abort_after = @config.max_unit_duration + @config.timeout
+
+        def run_loop
+          until @stopped
+            result = tick(@clock.call)
+            break if %i[aborted stale].include?(result)
+
+            wait(result == :retrying ? [RETRY_INTERVAL, interval].min : interval)
+          end
+        end
+
+        def wait(seconds)
+          if @sleeper
+            @sleeper.call(seconds)
+          else
+            @signal.pop(timeout: seconds)
+          end
+        end
+
+        def renew!(now)
+          @queue.heartbeat(@reservation)
+          @last_renewal = now
+          @renewals += 1
+          recovered if @renewal_failures.positive?
+          :renewed
+        rescue StaleReservation
+          @stale = true
+          :stale
+        rescue REDIS_ERROR => e
+          failed_renewal(e, now)
+        end
+
+        # A connection error is not yet a lost unit: the reservation stays this
+        # worker's until `timeout` passes without a renewal. Retry until then
+        # rather than killing a worker that is running tests fine, and give up
+        # once the entry is reclaimable, when carrying on would only let a
+        # sibling run the unit while this process still owns its output.
+        def failed_renewal(error, now)
+          @renewal_failures += 1
+          raise error if now - @last_renewal >= @config.timeout
+
+          if @renewal_failures == 1
+            warn_heartbeat "heartbeat failed (#{error.class}: #{error.message}); " \
+                           "retrying for up to #{@config.timeout}s"
+          end
+          :retrying
+        end
+
+        def recovered
+          warn_heartbeat "heartbeat recovered after #{@renewal_failures} failed " \
+                         "#{@renewal_failures == 1 ? "attempt" : "attempts"}"
+          @renewal_failures = 0
+        end
+
+        def warn_heartbeat(message)
+          @err.puts "[hopper #{@worker_id}] #{@reservation.unit_id}: #{message}"
+          @err.flush if @err.respond_to?(:flush)
+        end
+
+        # The `abandoned` event is a warning, not a terminal state, so a Redis
+        # error while recording it retries on the next tick and never ends the
+        # worker: the abort at `max_unit_duration + timeout` still fires.
+        def abandon!(elapsed)
+          return :abandoned if @abandoned_at
+
+          @queue.record_abandoned(@reservation, elapsed_ms: (elapsed * 1000).round)
+          @abandoned_at = elapsed
+          :abandoned
+        rescue StaleReservation
+          @stale = true
+          :stale
+        rescue REDIS_ERROR
+          :retrying
+        end
+
+        def abort!
+          @err.puts "[hopper #{@worker_id}] Aborting worker: #{@reservation.unit_id} exceeded " \
+                    "#{format_seconds(@config.max_unit_duration)}s"
+          @err.flush if @err.respond_to?(:flush)
+          @aborter.call(ExitCode::ABORTED)
+        end
+
+        def format_seconds(value)
+          value == value.to_i ? value.to_i : value
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/worker/requeue_policy.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    class Worker
+      # Decides, from the execution results of a unit's selected examples,
+      # whether a failed attempt may be requeued. An attempt is a requeue
+      # candidate only if every failure in it is requeueable.
+      module RequeuePolicy
+        # These never justify a retry; one anywhere in the unit makes the
+        # attempt final. They also escape `Example#run` rather than being
+        # recorded on the example, so the worker passes them in as `escaped`.
+        NON_REQUEUEABLE = [SystemExit, Interrupt, SignalException, NoMemoryError].freeze
+
+        Decision = Data.define(:status, :failed_examples, :unexecuted, :escaped) do
+          def passed? = status == :passed
+          def requeue_candidate? = status == :requeue_candidate
+          def final_failure? = status == :final_failure
+
+          def failure_count = failed_examples.size + (escaped ? 1 : 0)
+
+          def summary = RequeuePolicy.failure_summary(failure_count)
+
+          # Size-capped error payload for the attempt log.
+          def errors
+            entries = failed_examples.flat_map do |example|
+              RequeuePolicy.exceptions_of(example).map do |exception|
+                ErrorPayload.from_exception(exception, example_id: example.id, description: example.full_description)
+              end
+            end
+            entries << escaped_payload if escaped
+            ErrorPayload.cap(entries)
+          end
+
+          private
+
+          def escaped_payload
+            example = unexecuted.find { |ex| ex.execution_result.started_at }
+            ErrorPayload.from_exception(escaped, example_id: example&.id, description: example&.full_description)
+          end
+        end
+
+        module_function
+
+        # @param examples [Array<RSpec::Core::Example>] the unit's selected examples
+        # @param escaped [Exception, nil] an exception that escaped `ExampleGroup.run`
+        def decide(examples, escaped: nil)
+          failed = examples.select { |ex| ex.execution_result.status == :failed }
+          unexecuted = examples.select { |ex| ex.execution_result.status.nil? }
+          Decision.new(status: status_for(failed, escaped), failed_examples: failed, unexecuted: unexecuted,
+                       escaped: escaped)
+        end
+
+        def status_for(failed, escaped)
+          return :final_failure if escaped
+          return :passed if failed.empty?
+          return :requeue_candidate if failed.all? { |ex| exceptions_of(ex).all? { |e| requeueable?(e) } }
+
+          :final_failure
+        end
+
+        def requeueable?(exception)
+          NON_REQUEUEABLE.none? { |klass| exception.is_a?(klass) }
+        end
+
+        # Every exception behind a failed example, with aggregate errors
+        # (`MultipleExceptionError` and friends) flattened.
+        def exceptions_of(example)
+          flatten(example.execution_result.exception || example.exception)
+        end
+
+        def flatten(exception)
+          return [] if exception.nil?
+          return [exception] unless exception.respond_to?(:all_exceptions)
+
+          nested = exception.all_exceptions.flat_map { |e| flatten(e) }
+          nested.empty? ? [exception] : nested
+        end
+
+        def failure_summary(count)
+          "#{count} failure#{"s" unless count == 1}"
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/worker/runner.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+
+module RSpec
+  module Hopper
+    class Worker
+      # The RSpec runner the worker drives. `configure` is inherited unchanged;
+      # `run_specs` is replaced so that the reporter lifecycle (`start`,
+      # `finish`, `close`) and the per-process suite hooks fire exactly once
+      # around the hopper loop instead of around `ordered_example_groups`.
+      #
+      # Runner#setup is not used: its `ensure world.announce_filters` calls
+      # `reporter.abort_with` (an `exit!`) for `--only-failures` before the
+      # worker could reject the option, and load-error capture has to be
+      # scoped to `load_spec_files` alone. Suite performs those steps itself.
+      class Runner < RSpec::Core::Runner
+        # @param expected_example_count [Integer] the manifest's total examples
+        # @yield [RSpec::Core::Reporter] the real reporter, inside suite hooks
+        def run_specs(expected_example_count)
+          @configuration.reporter.report(expected_example_count) do |reporter|
+            @configuration.with_suite_hooks { yield reporter }
+          end
+        end
+      end
+    end
+  end
+end