rspec-hopper 0.1.0

data/lib/rspec/hopper/cli/formatter_args.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    module CLI
+      # Formatter output must be per child process under `--processes N`. The
+      # supervisor strips every `--format`/`--out` option from the RSpec
+      # arguments in the parent and re-applies them in each child: `%{n}` in an
+      # `--out` path becomes that child's TEST_ENV_NUMBER, and a formatter
+      # without an `--out` of its own is given a file under {DEFAULT_DIR}.
+      #
+      # Children therefore never write to a shared console stream. One build no
+      # longer prints one RSpec summary per process (an idle child used to
+      # announce "0 examples, 0 failures" for the whole build); the closing word
+      # belongs to `rspec-hopper report`, which is the only thing that sees
+      # every worker's results.
+      module FormatterArgs
+        FORMAT_FLAGS = %w[--format -f].freeze
+        OUT_FLAGS = %w[--out -o].freeze
+        PLACEHOLDER = "%{n}" # rubocop:disable Style/FormatStringToken
+        DEFAULT_FORMAT = "progress"
+        DEFAULT_DIR = "tmp/rspec-hopper"
+        # Extensions for the formatters that write a recognised file format;
+        # everything else (progress, documentation, a custom class) gets .txt.
+        EXTENSIONS = { "json" => "json", "j" => "json", "html" => "html", "h" => "html", "junit" => "xml" }.freeze
+        DEFAULT_EXTENSION = "txt"
+
+        module_function
+
+        # Splits the formatter options out of `rspec_args`.
+        #
+        # @return [Array(Array<String>, Array<Array(String, String)>)] the
+        #   remaining arguments in original order, and the formatter pairs, each
+        #   normalized to `["--format", value]` or `["--out", value]`, in the
+        #   order they appeared. Arguments after `--` are never touched.
+        def split(rspec_args)
+          remaining = []
+          pairs = []
+          args = rspec_args.dup
+          until args.empty?
+            arg = args.shift
+            if arg == "--"
+              remaining.push(arg, *args)
+              break
+            end
+            flag, value, needs_next = recognise(arg)
+            if flag.nil? || (needs_next && args.empty?) # unknown, or a dangling flag RSpec should report
+              remaining << arg
+            else
+              pairs << [flag, needs_next ? args.shift : value]
+            end
+          end
+          [remaining, pairs]
+        end
+
+        # Groups pairs the way RSpec's own parser does: an `--out` attaches to
+        # the preceding `--format`, or to the default progress formatter when
+        # there is none.
+        #
+        # @return [Array<Array(String, String, nil)>] `[formatter, out or nil]`
+        def entries(pairs)
+          pairs.each_with_object([]) do |(flag, value), list|
+            if flag == "--format"
+              list << [value, nil]
+            else
+              list << [DEFAULT_FORMAT, nil] if list.empty?
+              list[-1] = [list[-1][0], value]
+            end
+          end
+        end
+
+        # Whether any child formatter would be given a generated output path.
+        def defaults_needed?(pairs)
+          list = entries(pairs)
+          list.empty? || list.any? { |(_formatter, out)| out.nil? }
+        end
+
+        # The flat argument list to prepend to a child's RSpec arguments. Every
+        # formatter comes back with an explicit `--out`, so no child writes to
+        # the console.
+        #
+        # @param pairs [Array<Array(String, String)>] from {split}
+        # @param env_number [String] the child's TEST_ENV_NUMBER ("" or "2"..)
+        # @param label [String, nil] the child's worker id, used in generated names
+        # @param dir [String] where generated output files go
+        def for_child(pairs, env_number, label: nil, dir: DEFAULT_DIR)
+          list = entries(pairs)
+          list = [[DEFAULT_FORMAT, nil]] if list.empty?
+          seen = Hash.new(0)
+          list.flat_map do |formatter, out|
+            path = out ? out.gsub(PLACEHOLDER, env_number) : generated_path(dir, label, formatter, env_number, seen)
+            ["--format", formatter, "--out", path]
+          end
+        end
+
+        # `tmp/rspec-hopper/w-2-json.json`, numbered when one child names the
+        # same formatter twice.
+        def generated_path(dir, label, formatter, env_number, seen)
+          slug = slugify(formatter)
+          occurrence = (seen[slug] += 1)
+          suffix = occurrence > 1 ? "-#{occurrence}" : ""
+          base = label.to_s.empty? ? "worker#{env_number}" : label.to_s
+          File.join(dir, "#{slugify(base)}-#{slug}#{suffix}.#{EXTENSIONS.fetch(slug, DEFAULT_EXTENSION)}")
+        end
+
+        def slugify(value)
+          slug = value.to_s.downcase.gsub(/[^a-z0-9]+/, "-").gsub(/\A-+|-+\z/, "")
+          slug.empty? ? DEFAULT_FORMAT : slug
+        end
+
+        # @return [Array(String, String, Boolean), nil] normalized flag, inline
+        #   value (or nil), and whether the value is the next argument.
+        def recognise(arg)
+          FORMAT_FLAGS.each { |f| (m = match(f, arg)) and return ["--format", *m] }
+          OUT_FLAGS.each { |f| (m = match(f, arg)) and return ["--out", *m] }
+          nil
+        end
+
+        def match(flag, arg)
+          return [nil, true] if arg == flag
+          return [arg.delete_prefix("#{flag}="), false] if flag.start_with?("--") && arg.start_with?("#{flag}=")
+          return [arg.delete_prefix(flag), false] if !flag.start_with?("--") && arg.start_with?(flag) && arg.size > 2
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/cli/report.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "redis"
+
+module RSpec
+  module Hopper
+    module CLI
+      # `rspec-hopper report`: parses flags into a ReportConfig and runs Report.
+      class Report
+        # Raised by `parse` for -h/--help; `run` prints usage and returns 0.
+        class HelpRequested < StandardError; end
+
+        BANNER = "Usage: rspec-hopper report --build ID --redis URL [options]"
+
+        class << self
+          def usage = build_parser({}).to_s
+
+          # Returns a frozen ReportConfig. Build id falls back to HOPPER_BUILD_ID;
+          # the Redis URL to HOPPER_REDIS_URL then REDIS_URL.
+          def parse(argv, env: ENV)
+            opts = {}
+            rest = build_parser(opts).parse(argv)
+            raise UsageError, "unexpected argument(s): #{rest.join(" ")}" if rest.any?
+
+            opts[:build_id] ||= env["HOPPER_BUILD_ID"]
+            opts[:redis_url] ||= env["HOPPER_REDIS_URL"] || env["REDIS_URL"]
+            validate!(opts)
+            ReportConfig.build(**opts)
+          rescue OptionParser::ParseError => e
+            raise UsageError, e.message
+          end
+
+          def run(argv, out: $stdout, err: $stderr, env: ENV)
+            config = parse(argv, env: env)
+            redis = Redis.new(url: config.redis_url)
+            queue = Queue::RedisStreams.new(redis: redis, build_id: config.build_id)
+            Hopper::Report.new(config: config, queue: queue).run(out: out)
+          rescue HelpRequested
+            out.puts usage
+            ExitCode::OK
+          rescue UsageError => e
+            err.puts "rspec-hopper report: #{e.message}"
+            err.puts usage
+            ExitCode::INFRASTRUCTURE
+          rescue InfrastructureError => e
+            err.puts "rspec-hopper report: #{e.message}"
+            e.exit_code
+          ensure
+            redis&.close
+          end
+
+          private
+
+          def validate!(opts)
+            raise UsageError, "--build ID is required (or set HOPPER_BUILD_ID)" unless opts[:build_id]
+            raise UsageError, "--redis URL is required (or set HOPPER_REDIS_URL / REDIS_URL)" unless opts[:redis_url]
+
+            if opts[:allow_empty] && opts.fetch(:min_examples, 0).positive?
+              raise UsageError, "--allow-empty cannot be combined with a positive --min-examples"
+            end
+
+            %i[timeout init_timeout inactive_timeout min_examples].each do |key|
+              raise UsageError, "--#{key.to_s.tr("_", "-")} must not be negative" if opts[key]&.negative?
+            end
+          end
+
+          def build_parser(opts)
+            OptionParser.new(BANNER, 28) do |parser|
+              parser.separator ""
+              parser.separator "Waits for the build to initialize and complete, then prints the verdict."
+              parser.separator ""
+              connection_options(parser, opts)
+              wait_options(parser, opts)
+              output_options(parser, opts)
+              verdict_options(parser, opts)
+              parser.on_tail("-h", "--help", "show this help") { raise HelpRequested }
+            end
+          end
+
+          def connection_options(parser, opts)
+            parser.on("--build ID", "build id (default: $HOPPER_BUILD_ID)") { |v| opts[:build_id] = v }
+            parser.on("--redis URL", "Redis URL (default: $HOPPER_REDIS_URL, then $REDIS_URL)") do |v|
+              opts[:redis_url] = v
+            end
+          end
+
+          def wait_options(parser, opts)
+            parser.on("--timeout S", Float, "give up after S seconds (default 1080)") { |v| opts[:timeout] = v }
+            parser.on("--init-timeout S", Float, "wait S seconds for initialization (default 300)") do |v|
+              opts[:init_timeout] = v
+            end
+            parser.on("--inactive-timeout S", Float,
+                      "give up after S seconds without worker activity (default 300)") do |v|
+              opts[:inactive_timeout] = v
+            end
+          end
+
+          def output_options(parser, opts)
+            parser.on("--summary-out PATH", "write the JSON summary to PATH") { |v| opts[:summary_out] = v }
+            parser.on("--failed-out PATH", "write failed and never-finalized unit ids to PATH") do |v|
+              opts[:failed_out] = v
+            end
+          end
+
+          def verdict_options(parser, opts)
+            parser.on("--fail-on-empty", "zero selected examples is a failure (default)") { opts[:allow_empty] = false }
+            parser.on("--allow-empty", "zero selected examples may pass") { opts[:allow_empty] = true }
+            parser.on("--min-examples N", Integer, "fail unless at least N examples were selected") do |v|
+              opts[:min_examples] = v
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/cli/work/parser.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module RSpec
+  module Hopper
+    module CLI
+      module Work
+        # OptionParser for `rspec-hopper work`. Consumes the gem's own flags
+        # and passes every other argument, and everything after `--`, to RSpec
+        # in the original order.
+        class Parser
+          BOOT_MODES = { "per-process" => :per_process, "shared" => :shared }.freeze
+          POSITIVE = %i[timeout max_unit_duration ttl tombstone_ttl init_timeout].freeze
+          NON_NEGATIVE = %i[max_requeues max_reclaims].freeze
+
+          BANNER = <<~BANNER
+            Usage: rspec-hopper work --build ID --worker WID --redis URL [options] [rspec args...] -- [files...]
+
+            Every argument the gem does not recognise, and everything after `--`, is
+            passed to RSpec unchanged. --build, --worker and --redis fall back to
+            HOPPER_BUILD_ID, HOPPER_WORKER_ID and HOPPER_REDIS_URL/REDIS_URL, then to
+            CI environment variables (CircleCI, Buildkite, GitHub Actions, GitLab).
+          BANNER
+
+          # @return [WorkConfig]
+          # @raise [UsageError, HelpRequested]
+          def self.parse(argv, env: ENV) = new(env: env).parse(argv)
+
+          def initialize(env: ENV)
+            @env = env
+            @opts = {}
+          end
+
+          def parse(argv)
+            rspec_args = extract(argv)
+            resolve_ids
+            validate_ids!
+            validate_numbers!
+            WorkConfig.build(
+              **@opts, rspec_args: rspec_args.freeze,
+                       report_args: ["--build", @opts[:build_id], "--redis", @opts[:redis_url]].freeze
+            )
+          end
+
+          private
+
+          # Consumes the gem's flags from argv; returns everything else.
+          def extract(argv)
+            own, tail = split_on_double_dash(argv)
+            rest = []
+            parser = option_parser
+            begin
+              parser.order!(own) { |positional| rest << positional }
+            rescue OptionParser::InvalidOption => e
+              e.recover(own)
+              rest << own.shift
+              retry
+            rescue OptionParser::ParseError => e
+              raise UsageError, e.message
+            end
+            rest + tail
+          end
+
+          def split_on_double_dash(argv)
+            index = argv.index("--")
+            return [argv.dup, []] if index.nil?
+
+            [argv[0...index], argv[index..]]
+          end
+
+          def option_parser
+            OptionParser.new do |o|
+              o.banner = BANNER
+              identity_options(o)
+              policy_options(o)
+              process_options(o)
+              lifetime_options(o)
+              o.separator ""
+              o.on("-h", "--help", "show this help") { raise HelpRequested, o.help }
+            end
+          end
+
+          def identity_options(opt)
+            opt.separator ""
+            opt.separator "Identity:"
+            opt.on("--build ID", "build id shared by every worker of one CI run") { |v| @opts[:build_id] = v }
+            opt.on("--worker WID", "this worker's id, unique within the build") { |v| @opts[:worker_id] = v }
+            opt.on("--redis URL", "Redis URL") { |v| @opts[:redis_url] = v }
+            opt.on("--revision SHA", "revision string mixed into the suite fingerprint") { |v| @opts[:revision] = v }
+          end
+
+          def policy_options(opt)
+            opt.separator ""
+            opt.separator "Requeue and timeout policy:"
+            opt.on("--timeout SECONDS", Numeric, "missed-heartbeat window before a unit is reclaimable (180)") do |v|
+              @opts[:timeout] = v
+            end
+            opt.on("--max-unit-duration SECONDS", Numeric, "abandon a unit running longer than this (900)") do |v|
+              @opts[:max_unit_duration] = v
+            end
+            opt.on("--max-requeues N", Integer, "max retries of any one unit (0)") { |v| @opts[:max_requeues] = v }
+            opt.on("--requeue-tolerance R", Float, "fraction of units allowed to retry (0)") do |v|
+              @opts[:requeue_tolerance] = v
+            end
+            opt.on("--max-reclaims N", Integer, "reclaims from dead workers before a unit fails (3)") do |v|
+              @opts[:max_reclaims] = v
+            end
+          end
+
+          def process_options(opt)
+            opt.separator ""
+            opt.separator "Process parallelism:"
+            opt.on("--processes N", Integer, "fork N worker processes on this machine (1)") do |v|
+              @opts[:processes] = v
+            end
+            opt.on("--boot MODE", BOOT_MODES.keys, "per-process (default) or shared") do |v|
+              @opts[:boot] = BOOT_MODES[v]
+            end
+            opt.on("--report-on-exit", "parent runs `report` after all children exit") do
+              @opts[:report_on_exit] = true
+            end
+          end
+
+          def lifetime_options(opt)
+            opt.separator ""
+            opt.separator "Redis lifetimes:"
+            opt.on("--ttl SECONDS", Numeric, "inactivity TTL of live build keys (14400)") { |v| @opts[:ttl] = v }
+            opt.on("--tombstone-ttl SECONDS", Numeric, "lifetime of the build tombstone (604800)") do |v|
+              @opts[:tombstone_ttl] = v
+            end
+            opt.on("--init-timeout SECONDS", Numeric, "wait this long for build initialization (300)") do |v|
+              @opts[:init_timeout] = v
+            end
+          end
+
+          def resolve_ids
+            @opts[:build_id] ||= CIEnv.present(@env["HOPPER_BUILD_ID"]) || CIEnv.build_id(@env)
+            @opts[:worker_id] ||= CIEnv.present(@env["HOPPER_WORKER_ID"]) || CIEnv.worker_id(@env) ||
+                                  CIEnv.default_worker_id
+            @opts[:redis_url] ||= CIEnv.present(@env["HOPPER_REDIS_URL"]) || CIEnv.present(@env["REDIS_URL"])
+          end
+
+          def validate_ids!
+            raise UsageError, "--build is required (or set HOPPER_BUILD_ID)" unless @opts[:build_id]
+            raise UsageError, "--redis is required (or set HOPPER_REDIS_URL or REDIS_URL)" unless @opts[:redis_url]
+          end
+
+          def validate_numbers!
+            POSITIVE.each { |key| check(key, "must be positive", &:positive?) }
+            NON_NEGATIVE.each { |key| check(key, "must not be negative") { |v| !v.negative? } }
+            check(:processes, "must be at least 1") { |v| v >= 1 }
+            check(:requeue_tolerance, "must be between 0 and 1") { |v| (0.0..1.0).cover?(v) }
+          end
+
+          def check(key, requirement)
+            return unless @opts.key?(key)
+            return if yield(@opts[key])
+
+            raise UsageError, "--#{key.to_s.tr("_", "-")} #{requirement}, got #{@opts[key]}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/cli/work.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative "work/parser"
+
+module RSpec
+  module Hopper
+    module CLI
+      # `rspec-hopper work`: parses the gem's own flags into a frozen WorkConfig
+      # and hands everything else to RSpec, then runs one worker inline or a
+      # Supervisor for `--processes N`.
+      module Work
+        module_function
+
+        # @return [Integer] exit code
+        def run(argv, out: $stdout, err: $stderr, env: ENV, worker_class: nil, supervisor_class: nil)
+          config = parse(argv, env: env)
+          worker_class ||= Worker
+          if config.processes == 1
+            worker_class.new(config: config, queue_factory: queue_factory(config), out: out, err: err).run
+          else
+            (supervisor_class || Supervisor).new(config: config, out: out, err: err, worker_class: worker_class).run
+          end
+        rescue HelpRequested => e
+          out.puts e.text
+          ExitCode::OK
+        rescue UsageError => e
+          err.puts "rspec-hopper work: #{e.message}"
+          err.puts "Run `rspec-hopper work --help` for usage."
+          ExitCode::INFRASTRUCTURE
+        rescue Redis::BaseConnectionError => e
+          err.puts "rspec-hopper work: Redis unreachable at #{config&.redis_url}: #{e.message}"
+          ExitCode::INFRASTRUCTURE
+        rescue InfrastructureError => e
+          err.puts "rspec-hopper work: #{e.message}"
+          e.exit_code
+        end
+
+        # @param argv [Array<String>] arguments after the `work` subcommand
+        # @param env [#[]] environment for --build/--worker/--redis fallbacks
+        # @return [WorkConfig]
+        # @raise [UsageError, HelpRequested]
+        def parse(argv, env: ENV) = Parser.parse(argv, env: env)
+
+        # A zero-argument lambda opening Redis on first call, so the suite can
+        # boot before any connection exists and children open their own.
+        def queue_factory(config)
+          lambda do
+            Queue::RedisStreams.new(
+              redis: Redis.new(url: config.redis_url), build_id: config.build_id,
+              ttl: config.ttl, tombstone_ttl: config.tombstone_ttl, timeout: config.timeout,
+              max_requeues: config.max_requeues, requeue_tolerance: config.requeue_tolerance,
+              max_reclaims: config.max_reclaims
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/cli.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    # Command-line entry point: dispatches `work` and `report`.
+    module CLI
+      # Raised by a subcommand parser when `-h`/`--help` is given; carries the
+      # help text so the caller can print it to stdout and exit 0.
+      class HelpRequested < StandardError
+        attr_reader :text
+
+        def initialize(text)
+          @text = text
+          super("help requested")
+        end
+      end
+
+      USAGE = <<~USAGE
+        Usage: rspec-hopper <command> [options]
+
+        Commands:
+          work     run a worker: boot the suite, join the build, consume the queue
+          report   wait for a build to finish and print the verdict
+
+        Options:
+          -h, --help       show this help
+          -v, --version    print the version
+
+        Run `rspec-hopper work --help` or `rspec-hopper report --help` for the
+        options of each command.
+      USAGE
+
+      module_function
+
+      # @return [Integer] process exit code
+      def run(argv, out: $stdout, err: $stderr)
+        command, *rest = argv
+        case command
+        when "work" then Work.run(rest, out: out, err: err)
+        when "report" then Report.run(rest, out: out, err: err)
+        when "-v", "--version"
+          out.puts "rspec-hopper #{VERSION}"
+          ExitCode::OK
+        when "-h", "--help", "help"
+          out.puts USAGE
+          ExitCode::OK
+        else
+          err.puts(command.nil? ? "rspec-hopper: no command given" : "rspec-hopper: unknown command #{command.inspect}")
+          err.puts USAGE
+          ExitCode::INFRASTRUCTURE
+        end
+      rescue HelpRequested => e
+        out.puts e.text
+        ExitCode::OK
+      rescue UsageError => e
+        err.puts "rspec-hopper: #{e.message}"
+        ExitCode::INFRASTRUCTURE
+      end
+    end
+  end
+end

data/lib/rspec/hopper/config.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    # Frozen configuration for the `work` subcommand. Passed explicitly to every
+    # collaborator; there is no global.
+    WorkConfig = Data.define(
+      :build_id, :worker_id, :redis_url,
+      :timeout, :max_unit_duration, :max_requeues, :requeue_tolerance, :max_reclaims,
+      :processes, :boot, :report_on_exit,
+      :ttl, :tombstone_ttl, :init_timeout, :revision,
+      :rspec_args, :report_args, :supervised
+    ) do
+      def self.build(**attrs)
+        new(**Config::WORK_DEFAULTS, **attrs)
+      end
+
+      def heartbeat_interval
+        [timeout / 3.0, 30.0].min
+      end
+    end
+
+    # Frozen configuration for the `report` subcommand.
+    ReportConfig = Data.define(
+      :build_id, :redis_url, :timeout, :init_timeout, :inactive_timeout,
+      :summary_out, :failed_out, :allow_empty, :min_examples
+    ) do
+      def self.build(**attrs)
+        new(**Config::REPORT_DEFAULTS, **attrs)
+      end
+    end
+
+    module Config
+      WORK_DEFAULTS = {
+        build_id: nil, worker_id: nil, redis_url: nil,
+        timeout: 180, max_unit_duration: 900, max_requeues: 0, requeue_tolerance: 0.0, max_reclaims: 3,
+        processes: 1, boot: :per_process, report_on_exit: false,
+        ttl: 14_400, tombstone_ttl: 604_800, init_timeout: 300, revision: nil,
+        rspec_args: [], report_args: [], supervised: false
+      }.freeze
+
+      REPORT_DEFAULTS = {
+        build_id: nil, redis_url: nil, timeout: 1080, init_timeout: 300, inactive_timeout: 300,
+        summary_out: nil, failed_out: nil, allow_empty: false, min_examples: 0
+      }.freeze
+    end
+  end
+end

data/lib/rspec/hopper/errors.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    # Process exit codes. Precedence between workers is semantic, not numeric:
+    # INFRASTRUCTURE beats ABORTED beats OK.
+    module ExitCode
+      OK = 0
+      TEST_FAILURE = 1
+      INFRASTRUCTURE = 2
+      INCOMPLETE = 3
+      ABORTED = 4
+    end
+
+    class Error < StandardError; end
+
+    # Anything that makes the worker exit 2.
+    class InfrastructureError < Error
+      def exit_code = ExitCode::INFRASTRUCTURE
+    end
+
+    class BootError < InfrastructureError; end
+    class RedisUnreachable < InfrastructureError; end
+    class BuildNeverInitialized < InfrastructureError; end
+
+    class InitFailed < InfrastructureError
+      attr_reader :load_errors
+
+      def initialize(message = "build initialization failed", load_errors: [])
+        @load_errors = load_errors
+        super(message)
+      end
+    end
+
+    class PreviouslyInitialized < InfrastructureError; end
+    class AlreadyInitialized < InfrastructureError; end
+    class LeaseLost < InfrastructureError; end
+    class FingerprintMismatch < InfrastructureError; end
+    class UnsupportedOption < InfrastructureError; end
+    class SharedBootWithoutHook < InfrastructureError; end
+    class ForkUnavailable < InfrastructureError; end
+    class BuildStateMissing < InfrastructureError; end
+    class CorruptBuild < InfrastructureError; end
+    class UsageError < InfrastructureError; end
+
+    # Raised by queue mutations when ownership of the reservation has moved.
+    # Not an infrastructure failure: the worker discards its result and moves on.
+    class StaleReservation < Error; end
+
+    # Size-capped JSON error payloads recorded in the attempt log.
+    module ErrorPayload
+      MESSAGE_BYTES = 4096
+      BACKTRACE_LINES = 20
+      TOTAL_BYTES = 65_536
+
+      module_function
+
+      def from_exception(error, example_id: nil, description: nil)
+        {
+          "example_id" => example_id,
+          "description" => description,
+          "class" => error.class.name,
+          "message" => truncate(error.message.to_s, MESSAGE_BYTES),
+          "backtrace" => (error.backtrace || []).first(BACKTRACE_LINES)
+        }
+      end
+
+      # Drops trailing entries until the JSON fits, appending a marker for how
+      # many were dropped.
+      def cap(errors)
+        kept = errors.dup
+        dropped = 0
+        while kept.any? && JSON.generate(kept).bytesize > TOTAL_BYTES
+          kept.pop
+          dropped += 1
+        end
+        kept << { "truncated" => dropped } if dropped.positive?
+        kept
+      end
+
+      def truncate(string, bytes)
+        return string if string.bytesize <= bytes
+
+        "#{string.byteslice(0, bytes - 3).scrub("")}..."
+      end
+    end
+  end
+end