rspec-hopper 0.1.0

data/lib/rspec/hopper/worker/suite.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require "shellwords"
+
+module RSpec
+  module Hopper
+    class Worker
+      # Loads the RSpec suite once per process, rejects unsupported options,
+      # discovers file units, computes the fingerprint and adopts the build seed.
+      # This is the only place that mutates RSpec.configuration.
+      class Suite
+        RUNNER_OPTIONS = {
+          "InitializeProject" => "--init", "PrintVersion" => "--version", "PrintHelp" => "--help",
+          "Bisect" => "--bisect", "DRbWithFallback" => "--drb"
+        }.freeze
+
+        # Captures `message` notifications while spec files load; that is how
+        # `Reporter#notify_non_example_exception` surfaces load errors.
+        class LoadErrorCapture
+          attr_reader :messages
+
+          def initialize
+            @messages = []
+            @active = false
+          end
+
+          def activate = @active = true
+          def deactivate = @active = false
+
+          def message(notification)
+            @messages << notification.message.to_s if @active
+          end
+        end
+
+        attr_reader :rspec_args, :config, :options, :runner, :units, :file_counts, :total_examples, :example_ids,
+                    :load_errors, :fingerprint, :file_args
+
+        # @return [Suite] loaded and ready; raises UnsupportedOption or BootError
+        def self.load(rspec_args, config:, out: $stdout, err: $stderr)
+          new(rspec_args, config: config, out: out, err: err).load
+        end
+
+        def initialize(rspec_args, config:, out: $stdout, err: $stderr)
+          @rspec_args = Array(rspec_args).map(&:to_s)
+          @config = config
+          @out = out
+          @err = err
+          @load_errors = []
+          @groups_by_file = {}
+        end
+
+        def configuration = RSpec.configuration
+        def world = RSpec.world
+
+        def load
+          reject_init_option!
+          @options = RSpec::Core::ConfigurationOptions.new(rspec_args)
+          reject_runner_option!
+          @runner = Runner.new(options, configuration, world)
+          runner.configure(@err, @out)
+          reject_failed_configuration!
+          reject_unsupported_configuration!
+          normalize_file_args!
+          load_spec_files
+          world.announce_filters
+          discover_units
+          @fingerprint = Fingerprint.compute(configuration: configuration, options: options, example_ids: example_ids,
+                                             file_args: file_args, revision: config.revision)
+          self
+        rescue InfrastructureError
+          raise
+        rescue StandardError, ScriptError => e
+          raise BootError, "#{e.class}: #{e.message}", e.backtrace
+        end
+
+        def unit_ids = units.map(&:id)
+
+        # Top-level groups whose file is the unit, in configured order.
+        def groups_for(unit_id) = @groups_by_file.fetch(unit_id, [])
+
+        # Selected examples of the unit, including nested groups.
+        def examples_for(unit_id) = groups_for(unit_id).flat_map(&:descendant_filtered_examples)
+
+        # Sets the build seed without changing the global ordering strategy:
+        # `Configuration#seed=` switches an unforced global ordering to random,
+        # which would silently reorder a suite that runs in defined order.
+        def adopt_seed(seed)
+          return if seed.nil?
+
+          registry = configuration.ordering_registry
+          strategy = registry.fetch(:global)
+          configuration.seed = seed
+          registry.register(:global, strategy) unless registry.fetch(:global).equal?(strategy)
+          configuration.seed
+        end
+        alias seed= adopt_seed
+
+        # Applies the `--format`/`--out` pairs of `args` to the already
+        # configured RSpec, for a suite loaded once in a shared-boot parent
+        # whose children each need their own formatter output. Other arguments
+        # are ignored: the suite already applied them. Pairing follows RSpec's
+        # parser: `--out` attaches to the preceding `--format`, or to the
+        # default progress formatter when there is none.
+        def apply_formatter_args(args)
+          _remaining, pairs = CLI::FormatterArgs.split(Array(args).map(&:to_s))
+          entries = CLI::FormatterArgs.entries(pairs).map { |formatter, out| out ? [formatter, out] : [formatter] }
+          entries.each { |entry| configuration.add_formatter(*entry) }
+          entries
+        end
+
+        def to_manifest(revision: config.revision)
+          Manifest.new(
+            total_examples: total_examples, file_counts: file_counts, file_args: file_args,
+            fingerprint: fingerprint.value, fingerprint_digests: fingerprint.digests,
+            seed: configuration.seed, revision: revision, load_errors: load_errors
+          )
+        end
+
+        private
+
+        def reject_init_option!
+          env_args = ENV["SPEC_OPTS"] ? Shellwords.split(ENV["SPEC_OPTS"]) : []
+          return unless (rspec_args + env_args).include?("--init")
+
+          raise UnsupportedOption, unsupported("--init")
+        end
+
+        def reject_runner_option!
+          runner = options.options[:runner]
+          return if runner.nil?
+
+          name = RUNNER_OPTIONS.fetch(runner.class.name.to_s.split("::").last, runner.class.name)
+          raise UnsupportedOption, unsupported(name)
+        end
+
+        # A `--require` that fails is reported by RSpec while the options are
+        # applied, before spec files load; every worker shares the arguments,
+        # so it is a boot failure rather than a spec-file load error.
+        def reject_failed_configuration!
+          return unless world.wants_to_quit || world.rspec_is_quitting
+
+          raise BootError, "RSpec reported an error while applying its options (for example a --require that " \
+                           "could not be loaded); see the RSpec output"
+        end
+
+        def reject_unsupported_configuration!
+          raise UnsupportedOption, unsupported("--bisect") if options.options[:bisect]
+          raise UnsupportedOption, unsupported("--drb") if options.options[:drb]
+          raise UnsupportedOption, unsupported("--only-failures / --next-failure") if configuration.only_failures?
+          raise UnsupportedOption, unsupported("--fail-fast") if configuration.fail_fast
+          raise UnsupportedOption, unsupported("--dry-run") if configuration.dry_run?
+        end
+
+        def unsupported(name)
+          "RSpec option #{name} is not supported by rspec-hopper: every selected unit must reach a final state"
+        end
+
+        # RSpec only falls back to `default_path` when `$0` is `rspec`; the
+        # worker applies the same default so `rspec-hopper work` with no file
+        # arguments runs the spec directory like `rspec` would.
+        def normalize_file_args!
+          files = Array(options.options[:files_or_directories_to_run]).map(&:to_s)
+          files = [configuration.default_path.to_s] if files.empty? && configuration.default_path
+          configuration.files_or_directories_to_run = files
+          @file_args = files.freeze
+        end
+
+        def load_spec_files
+          capture = LoadErrorCapture.new
+          configuration.reporter.register_listener(capture, :message)
+          capture.activate
+          begin
+            configuration.load_spec_files unless world.wants_to_quit
+          rescue SystemExit => e
+            capture.messages << "#{e.class} raised while loading spec files" if capture.messages.empty?
+          ensure
+            capture.deactivate
+          end
+          return unless world.wants_to_quit || world.rspec_is_quitting
+
+          @load_errors = capture.messages.dup.freeze
+          @load_errors = ["RSpec reported a failure while loading spec files"] if @load_errors.empty?
+        end
+
+        def discover_units
+          world.ordered_example_groups.each do |group|
+            next if group.descendant_filtered_examples.empty?
+
+            (@groups_by_file[group.metadata[:file_path]] ||= []) << group
+          end
+          @file_counts = @groups_by_file.transform_values do |groups|
+            groups.sum do |g|
+              g.descendant_filtered_examples.size
+            end
+          end
+                                        .freeze
+          @units = @file_counts.keys.map { |path| Unit.file(path) }.freeze
+          @total_examples = @file_counts.values.sum
+          @example_ids = @groups_by_file.values.flatten.flat_map(&:descendant_filtered_examples).map(&:id).sort.freeze
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/worker.rb

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "worker/runner"
+
+module RSpec
+  module Hopper
+    # The RSpec adapter: loads the suite, joins or initializes the build, pulls
+    # units, runs each through `ExampleGroup.run`, decides requeue from the
+    # execution results and forwards final attempts to the formatters.
+    class Worker
+      POLL_INTERVAL = 0.5
+
+      Outcome = Data.define(:duration_ms, :escaped, :stale)
+
+      # Internal: unwinds to `run` with an exit code after the error was recorded and printed.
+      class Abort < StandardError
+        attr_reader :code
+
+        def initialize(code)
+          @code = code
+          super("worker exiting with #{code}")
+        end
+      end
+
+      # Matches redis-rb connection failures without requiring the redis gem.
+      REDIS_ERROR = Module.new do
+        def self.===(error)
+          error.class.ancestors.any? { |a| a.name == "Redis::BaseConnectionError" }
+        end
+      end
+
+      attr_reader :config, :suite, :queue
+
+      # @param queue_factory [#call] returns the Queue; called after the suite boots
+      # @param suite [Suite, nil] a pre-loaded suite (shared boot mode); the
+      #   `--format`/`--out` pairs in `config.rspec_args` are applied to it
+      # @param clock [#call] monotonic seconds
+      # @param sleeper [#call] sleeps for the given seconds
+      # @param aborter [#call] receives an exit code; defaults to `exit!`
+      # @param ppid [#call] the parent pid, checked between units when supervised
+      def initialize(config:, queue_factory:, suite: nil, out: $stdout, err: $stderr, clock: nil, sleeper: nil,
+                     aborter: nil, ppid: nil)
+        @config = config
+        @queue_factory = queue_factory
+        @suite = suite
+        @out = out
+        @err = err
+        @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        @sleeper = sleeper || ->(seconds) { sleep(seconds) }
+        @aborter = aborter
+        @ppid = ppid || -> { Process.ppid }
+        @parent_pid = @ppid.call
+      end
+
+      def worker_id = config.worker_id
+      def build_id = config.build_id
+
+      # @return [Integer] exit code: 0 on build completion, 2 on infrastructure failure
+      def run
+        phase(:boot) do
+          if @suite
+            @suite.apply_formatter_args(config.rspec_args)
+          else
+            @suite = Suite.load(config.rspec_args, config: config, out: @out, err: @err)
+          end
+        end
+        phase(:redis) { @queue = @queue_factory.call }
+        manifest = phase(:init) { join_build }
+        phase(:init) do
+          check_fingerprint(manifest)
+          suite.adopt_seed(manifest.seed)
+        end
+        suite.runner.run_specs(manifest.total_examples) { |reporter| work_loop(reporter) }
+        ExitCode::OK
+      rescue Abort => e
+        e.code
+      end
+
+      private
+
+      # --- initialization ---------------------------------------------------
+
+      def join_build
+        deadline = @clock.call + config.init_timeout
+        loop do
+          status = queue.status
+          return joined if status.ready?
+          raise InitFailed.new("build #{build_id} failed to initialize", load_errors: recorded_load_errors(status)) if
+            status.init_failed?
+          raise PreviouslyInitialized, previously_initialized_message if status.tombstone && !status.present?
+
+          token = queue.acquire_leader(worker_id)
+          if token
+            manifest = publish(token)
+            return manifest if manifest
+          else
+            raise BuildNeverInitialized, "build never initialized (waited #{config.init_timeout}s)" if
+              @clock.call >= deadline
+
+            @sleeper.call(POLL_INTERVAL)
+          end
+        end
+      end
+
+      def joined
+        manifest = queue.manifest
+        raise BuildStateMissing, "build #{build_id} state is missing" if manifest.nil?
+
+        say "joined build #{build_id} (#{manifest.total_units} units, #{manifest.total_examples} examples)"
+        manifest
+      end
+
+      def publish(token)
+        manifest = suite.to_manifest
+        if suite.load_errors.any?
+          queue.fail_initialization(token: token, manifest: manifest)
+          raise InitFailed.new("build #{build_id} failed to initialize", load_errors: suite.load_errors)
+        end
+        queue.initialize_build(token: token, manifest: manifest, unit_ids: suite.unit_ids)
+        say "initialized build #{build_id} (#{manifest.total_units} units, #{manifest.total_examples} examples)"
+        queue.manifest
+      rescue LeaseLost, AlreadyInitialized
+        nil
+      end
+
+      def recorded_load_errors(status)
+        JSON.parse(status.meta&.fetch("load_errors", nil) || "[]")
+      rescue JSON::ParserError
+        []
+      end
+
+      def previously_initialized_message
+        "build #{build_id} was previously initialized; its state is gone. Choose a new build id."
+      end
+
+      def check_fingerprint(manifest)
+        return if manifest.fingerprint.nil? || manifest.fingerprint == suite.fingerprint.value
+
+        raise FingerprintMismatch,
+              Fingerprint::Mismatch.explain(suite.fingerprint, manifest.fingerprint, manifest.fingerprint_digests)
+      end
+
+      # --- the loop -----------------------------------------------------------
+
+      def work_loop(reporter)
+        loop do
+          phase(:execution) do
+            raise InfrastructureError, "RSpec wants to quit (a before(:suite) hook failed?)" if world.wants_to_quit
+          end
+          reservation = phase(:reserve) { queue.reclaim_lost(worker_id) || queue.reserve(worker_id) }
+          if reservation
+            execute(reservation, reporter)
+          else
+            phase(:reserve) { queue.touch_liveness(worker_id) }
+          end
+          break if phase(:reserve) { queue.complete? }
+
+          check_parent!
+        end
+      end
+
+      def check_parent!
+        return unless config.supervised
+        return if @ppid.call == @parent_pid
+
+        alert "parent gone; exiting"
+        raise Abort, ExitCode::INFRASTRUCTURE
+      end
+
+      def execute(reservation, reporter)
+        unit_id = reservation.unit_id
+        groups = suite.groups_for(unit_id)
+        examples = suite.examples_for(unit_id)
+        buffer = BufferingReporter.new
+        outcome = phase(:execution, unit_id: unit_id) do
+          raise InfrastructureError, "unit #{unit_id} is not part of this worker's suite" if groups.empty?
+
+          run_groups(groups, buffer, reservation)
+        end
+        decision = RequeuePolicy.decide(examples, escaped: outcome.escaped)
+        phase(:execution, unit_id: unit_id) do
+          if decision.unexecuted.any? && outcome.escaped.nil?
+            raise InfrastructureError, "#{decision.unexecuted.size} example(s) of #{unit_id} did not run"
+          end
+
+          settle(reservation, decision, buffer, outcome.duration_ms, reporter)
+        end
+        return unless outcome.escaped
+
+        alert "#{outcome.escaped.class} escaped an example in #{unit_id}; the unit was finalized as failed"
+      end
+
+      def run_groups(groups, buffer, reservation)
+        ExampleReset.reset(groups)
+        heartbeat = Heartbeat.new(queue: queue, reservation: reservation, config: config, err: @err, clock: @clock,
+                                  aborter: @aborter, worker_id: worker_id)
+        started = @clock.call
+        escaped = nil
+        heartbeat.start
+        begin
+          groups.each { |group| group.run(buffer) }
+        rescue *RequeuePolicy::NON_REQUEUEABLE => e
+          escaped = e
+        ensure
+          heartbeat.stop
+        end
+        Outcome.new(duration_ms: ((@clock.call - started) * 1000).round, escaped: escaped, stale: heartbeat.stale?)
+      end
+
+      def settle(reservation, decision, buffer, duration_ms, reporter)
+        unit_id = reservation.unit_id
+        operation = decision.requeue_candidate? ? :requeue : :finalize
+        begin
+          return if requeued?(reservation, decision, duration_ms)
+        rescue StaleReservation
+          buffer.discard
+          queue.record_stale_rejected(reservation, operation: operation)
+          say "reservation for #{unit_id} is stale; result discarded"
+          return
+        end
+        phase(:formatter, unit_id: unit_id) { buffer.replay(reporter) }
+      end
+
+      # Finalizes or requeues; true when the unit was requeued (buffer dropped).
+      def requeued?(reservation, decision, duration_ms)
+        unit_id = reservation.unit_id
+        if decision.passed?
+          queue.finalize(reservation, outcome: :passed, duration_ms: duration_ms)
+        elsif decision.final_failure?
+          queue.finalize(reservation, outcome: :failed, duration_ms: duration_ms, reason: "test_failure",
+                                      errors: decision.errors)
+        else
+          result = queue.requeue(reservation, duration_ms: duration_ms, failure_summary: decision.summary,
+                                              errors: decision.errors)
+          if result.requeued?
+            say "Retrying #{unit_id} (retry #{result.retry_index} of #{config.max_requeues}; " \
+                "next attempt #{result.retry_index + 1}): #{decision.summary}"
+            return true
+          end
+        end
+        false
+      end
+
+      # --- error handling -----------------------------------------------------
+
+      # Runs a block for a phase; any failure is recorded, printed and turned
+      # into an Abort with the infrastructure exit code.
+      def phase(name, unit_id: nil)
+        yield
+      rescue Abort, StaleReservation, *RequeuePolicy::NON_REQUEUEABLE
+        raise
+      rescue BuildStateMissing => e
+        fail_with(e, phase: "redis", unit_id: unit_id, message: "build state for #{build_id} is gone: #{e.message}")
+      rescue CorruptBuild => e
+        fail_with(e, phase: "redis", unit_id: unit_id, message: "build #{build_id} is corrupt: #{e.message}")
+      rescue REDIS_ERROR => e
+        error = RedisUnreachable.new("Redis unreachable: #{e.class}: #{e.message}")
+        error.set_backtrace(e.backtrace)
+        fail_with(error, phase: "redis", unit_id: unit_id, record: false)
+      rescue InitFailed => e
+        fail_with(e, phase: name.to_s, unit_id: unit_id, message: init_failed_message(e))
+      rescue InfrastructureError => e
+        fail_with(e, phase: name.to_s, unit_id: unit_id)
+      rescue StandardError => e
+        fail_with(e, phase: name.to_s, unit_id: unit_id,
+                     message: "#{e.class}: #{e.message}\n  #{Array(e.backtrace).first(5).join("\n  ")}")
+      end
+
+      def fail_with(error, phase:, unit_id: nil, message: error.message, record: true)
+        alert message
+        record_worker_error(error, phase: phase, unit_id: unit_id) if record
+        raise Abort, ExitCode::INFRASTRUCTURE
+      end
+
+      def record_worker_error(error, phase:, unit_id:)
+        return if queue.nil?
+
+        queue.record_worker_error(worker_id: worker_id, phase: phase, error: error, unit_id: unit_id)
+      rescue StandardError => e
+        alert "could not record worker error: #{e.class}: #{e.message}"
+      end
+
+      def init_failed_message(error)
+        lines = ["#{error.message}; spec files could not be loaded:"]
+        lines.concat(error.load_errors.map(&:to_s))
+        lines.join("\n")
+      end
+
+      # --- output -------------------------------------------------------------
+
+      def world = RSpec.world
+
+      def say(message) = @out.puts("[hopper #{worker_id}] #{message}")
+
+      def alert(message) = @err.puts("[hopper #{worker_id}] #{message}")
+    end
+  end
+end

data/lib/rspec/hopper.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "hopper/version"
+require_relative "hopper/errors"
+require_relative "hopper/config"
+require_relative "hopper/unit"
+require_relative "hopper/reservation"
+require_relative "hopper/keys"
+require_relative "hopper/manifest"
+require_relative "hopper/attempt_log"
+require_relative "hopper/queue"
+require_relative "hopper/queue/redis_streams"
+require_relative "hopper/fingerprint"
+require_relative "hopper/example_reset"
+require_relative "hopper/worker/buffering_reporter"
+require_relative "hopper/worker/heartbeat"
+require_relative "hopper/worker/requeue_policy"
+require_relative "hopper/worker/runner"
+require_relative "hopper/worker/suite"
+require_relative "hopper/worker"
+require_relative "hopper/report"
+require_relative "hopper/ci_env"
+require_relative "hopper/supervisor"
+require_relative "hopper/cli/report"
+require_relative "hopper/cli/formatter_args"
+require_relative "hopper/cli/work"
+require_relative "hopper/cli"
+
+module RSpec
+  # Distributes an RSpec suite across CI workers through Redis Streams.
+  module Hopper
+    @before_fork_hooks = []
+    @after_fork_hooks = []
+
+    class << self
+      # Registers a block to run in the parent, once, before any child is forked
+      # in `--boot shared` mode. Close connections opened during boot here.
+      def before_fork(&block)
+        @before_fork_hooks << block
+        block
+      end
+
+      # Registers a block to run in every child after fork in `--boot shared`
+      # mode. The block receives the child's TEST_ENV_NUMBER value ("" or "2".."N").
+      def after_fork(&block)
+        @after_fork_hooks << block
+        block
+      end
+
+      # @api private
+      attr_reader :before_fork_hooks
+
+      # @api private
+      attr_reader :after_fork_hooks
+
+      # @api private
+      def reset_hooks!
+        @before_fork_hooks = []
+        @after_fork_hooks = []
+      end
+    end
+  end
+end