ruby_reactor 0.8.4 → 0.8.6

data/lib/ruby_reactor/map/step_rollback.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # How a map step rolls back (009 R-01): a fan-out map distributed, one job
+    # per started element (S-2); an inline map in process, in bounded reads
+    # (S-4). Both report the same failures (I-7). Mixed into `Step::MapStep`.
+    module StepRollback
+      private
+
+      # S-2: start the map's rollback once (its records, then the first
+      # throw), then either aggregate a settled rollback or hand the run off
+      # until the element jobs report (R-04, R-05). Re-entered by the owner's
+      # resume, it finds the records and only checks for the settle.
+      def distributed_rollback(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        created, rollback = storage.start_map_rollback(map_id, map_class, **rollback_metadata(map_id, step_name))
+        if created
+          log_rollback("started", step_name, total: rollback["total"], batch_size: rollback["batch_size"])
+          RubyReactor::Map::Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class)
+        end
+        total = rollback["total"].to_i
+        if storage.count_map_rollback_outcomes(map_id, map_class) >= total
+          return finish_rollback(step_name, total, distributed_failures(map_id, step_name))
+        end
+
+        raise RubyReactor::Error::RollbackHandedOff.new(map_id: map_id, reactor_class_name: map_class)
+      end
+
+      def rollback_metadata(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        forward = storage.retrieve_map_metadata(map_id, map_class) || {}
+        {
+          total: storage.count_map_element_context_ids(map_id, map_class),
+          batch_size: forward["batch_size"] || RubyReactor::Map::DEFAULT_BATCH_SIZE,
+          step_name: step_name.to_s,
+          owner_context_id: owner_context.context_id,
+          owner_reactor_class_name: RubyReactor.reactor_storage_name(owner_context.reactor_class),
+          reactor_class_info: forward["reactor_class_info"] || build_reactor_class_info(element_class, step_name)
+        }
+      end
+
+      # The failures the element jobs reported, in position order (newest
+      # first, as inline), read in bounded chunks (R-09).
+      def distributed_failures(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        reported = []
+        unnamed = 0
+        storage.each_map_rollback_outcome(map_id, map_class) do |position, outcome|
+          unnamed += 1 if outcome["outcome"] == "context_unavailable" && outcome["index"].nil?
+          next unless %w[failed element_in_flight].include?(outcome["outcome"])
+
+          reported << [position, ContextSerializer.deserialize_value(outcome["failures"])]
+        end
+        unseen = lambda do |indexes|
+          indexes.each_slice(1000).flat_map do |slice|
+            slice.zip(storage.map_rollback_indexes_seen(map_id, map_class, slice)).reject(&:last).map(&:first)
+          end
+        end
+        unavailable_entries(step_name, unnamed, unseen) + reported.sort_by(&:first).flat_map(&:last)
+      end
+
+      # S-4: in process, `Map::ROLLBACK_CHUNK` elements per read from the tail
+      # of the element index, never all at once.
+      def inline_rollback(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        total = storage.count_map_element_context_ids(map_id, map_class)
+        reported = []
+        seen = Set.new
+        unnamed = 0
+        log_rollback("started", step_name, total: total, batch_size: RubyReactor::Map::ROLLBACK_CHUNK)
+        (0...total).step(RubyReactor::Map::ROLLBACK_CHUNK) do |position|
+          ids = storage.retrieve_map_element_context_ids_from_tail(map_id, map_class, position,
+                                                                   RubyReactor::Map::ROLLBACK_CHUNK, total: total)
+          ids.each do |id|
+            outcome = inline_element_rollback(map_id, id, step_name)
+            outcome["index"] ? seen << outcome["index"] : unnamed += 1
+            reported.concat(ContextSerializer.deserialize_value(outcome["failures"])) if outcome["index"]
+          end
+        end
+        unseen = ->(indexes) { indexes.reject { |index| seen.include?(index) } }
+        finish_rollback(step_name, total, unavailable_entries(step_name, unnamed, unseen) + reported)
+      end
+
+      # Today's inline behavior: one wait for the element's lock, then report
+      # it in flight.
+      def inline_element_rollback(map_id, element_context_id, step_name)
+        outcome = RubyReactor::Map::ElementRollback.call(map_id: map_id, element_context_id: element_context_id,
+                                                         element_class: element_class, step_name: step_name)
+        if outcome["outcome"] == "contended"
+          outcome = { "index" => outcome["index"], "outcome" => "element_in_flight",
+                      "failures" => ContextSerializer.serialize_value(
+                        [rollback_entry(outcome["index"], :element_in_flight, "was still running at rollback time")]
+                      ) }
+        end
+        log_rollback("element", step_name, index: outcome["index"], outcome: outcome["outcome"],
+                                           failures: outcome["failures"].size)
+        outcome
+      end
+
+      # Each element whose context expired is reported once (R-09, I2). The
+      # parent's map reference counts the elements that started: set per
+      # element inline, and to the total when a fan-out map completes. With
+      # it, every started index no rollback saw is named, even when the index
+      # list itself expired; without it (a failed fan-out map skipped some
+      # indices), one unnamed entry per element whose row is gone.
+      def unavailable_entries(step_name, unnamed, unseen)
+        ref = Utils::FetchIndifferent.call(context.composed_contexts, step_name)
+        started = ref && Utils::FetchIndifferent.call(ref, :started)
+        missing = started ? unseen.call((0...started.to_i).to_a) : [nil] * unnamed
+        missing.map { |index| rollback_entry(index, :context_unavailable, "context expired") }
+      end
+
+      def finish_rollback(step_name, total, failures)
+        log_rollback("completed", step_name, total: total, failed: failures.size)
+        return RubyReactor.Success() if failures.empty?
+
+        RubyReactor.Failure("map :#{step_name} rollback incomplete", rollback_failures: failures)
+      end
+
+      def log_rollback(event, step_name, **fields)
+        RubyReactor::Map.log("ruby_reactor.map.rollback.#{event}",
+                             reactor: RubyReactor.reactor_storage_name(owner_context.reactor_class),
+                             context_id: owner_context.context_id, map_step: step_name.to_s, **fields)
+      end
+
+      def rollback_entry(index, reason, message)
+        RubyReactor::Map::ElementRollback.entry(context.current_step, index, reason, message)
+      end
+    end
+  end
+end

data/lib/ruby_reactor/map/sweeper.rb

@@ -16,6 +16,12 @@ module RubyReactor
     #     re-triggered (M2) — gated so it never fires while a collector or the
     #     parent is alive, or after the parent already collected.
     #
+    # For each distributed map rollback (009 S-5), the same signal on the
+    # rollback's own records: a claimed position with no outcome and no live
+    # element lock lost its job and is re-dispatched; with every claimed
+    # position settled but positions left, the throw's trigger was lost and
+    # the next throw is claimed. A lost owner resume is `RubyReactor::Sweeper`'s.
+    #
     # `run_once` is pure and idempotent; the host wires the cadence (same contract
     # as RubyReactor::Sweeper).
     class Sweeper
@@ -29,10 +35,11 @@ module RubyReactor
         @logger = logger || RubyReactor.configuration.logger
       end
 
-      # Returns { redispatched:, recollected: } counts.
+      # Returns { redispatched:, recollected:, rollback_redispatched: } counts.
       def run_once(limit: 1000)
         redispatched = 0
         recollected = 0
+        rollback_redispatched = 0
 
         @storage.scan_maps(count: limit).each do |meta|
           missing = missing_indices(meta)
@@ -46,11 +53,53 @@ module RubyReactor
           @logger.warn("RubyReactor::Map::Sweeper failed on map #{meta["map_id"]}: #{e.class}: #{e.message}")
         end
 
-        { redispatched: redispatched, recollected: recollected }
+        @storage.scan_map_rollbacks(count: limit).each do |meta|
+          rollback_redispatched += sweep_rollback(meta)
+        rescue StandardError => e
+          @logger.warn("RubyReactor::Map::Sweeper failed on map rollback #{meta["map_id"]}: #{e.class}: #{e.message}")
+        end
+
+        { redispatched: redispatched, recollected: recollected, rollback_redispatched: rollback_redispatched }
       end
 
       private
 
+      def sweep_rollback(meta)
+        map_id = meta["map_id"]
+        map_class = meta["parent_reactor_class_name"]
+        total = meta["total"].to_i
+        claimed = [@storage.retrieve_map_rollback_offset(map_id, map_class), total].min
+        missing = (0...claimed).to_a - @storage.stored_map_rollback_positions(map_id, map_class)
+        return redispatch_rollback(meta, missing) if missing.any?
+        return 0 if claimed >= total
+
+        RubyReactor::Map::Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class)
+      end
+
+      def redispatch_rollback(meta, positions)
+        map_id = meta["map_id"]
+        map_class = meta["parent_reactor_class_name"]
+        positions.count do |position|
+          id = @storage.retrieve_map_element_context_ids_from_tail(map_id, map_class, position, 1,
+                                                                   total: meta["total"].to_i).first
+          index = element_index(id, meta)
+          next false if index && @storage.lock_held?("map_element:#{map_id}:#{index}") # element alive
+
+          RubyReactor::Map::Dispatcher.queue_rollback_job(meta, map_id, map_class, position, id)
+          true
+        end
+      end
+
+      # From the element's row; nil once the row is gone (the job reports it).
+      def element_index(element_context_id, meta)
+        return nil unless element_context_id
+
+        element_class = RubyReactor::Map::Helpers.resolve_reactor_class(meta["reactor_class_info"])
+        data = @storage.retrieve_context(element_context_id, RubyReactor.reactor_storage_name(element_class))
+        metadata = data && ContextSerializer.deserialize_value(data["map_metadata"])
+        metadata && Utils::FetchIndifferent.call(metadata, :index)
+      end
+
       def missing_indices(meta)
         @storage.missing_map_indices(meta["map_id"], meta["count"].to_i, meta["parent_reactor_class_name"])
       end
@@ -77,22 +126,31 @@ module RubyReactor
       end
 
       # N1: a nested map's parent is a map element running under a `map_element:`
-      # lock, not an `async:` lock. Derive the right key from metadata.
+      # lock, not an `async:` lock. Derive the right key from metadata. Any
+      # other map's completion resumes its OWNER run (009 R-03), so that run's
+      # lock is the one to check; metadata from before 009 names no owner.
       def parent_live_lock?(meta)
         if meta["parent_is_map_element"]
           @storage.lock_held?("map_element:#{meta["outer_map_id"]}:#{meta["outer_index"]}")
         else
-          @storage.lock_held?("async:#{meta["parent_context_id"]}")
+          @storage.lock_held?("async:#{meta["owner_context_id"] || meta["parent_context_id"]}")
         end
       end
 
+      # The map's own context already recorded the step, or the owner run is
+      # past it: finished, or rolling back (it adopted the map).
       def parent_already_collected?(meta)
         data = @storage.retrieve_context(meta["parent_context_id"], meta["parent_reactor_class_name"])
-        return false unless data
+        return true if data && (data["intermediate_results"] || {}).key?(meta["step_name"].to_s)
+
+        owner = owner_context_data(meta) || data
+        owner && %w[completed failed halted skipped cancelled rolling_back].include?(owner["status"].to_s)
+      end
+
+      def owner_context_data(meta)
+        return nil unless meta["owner_context_id"]
 
-        results = data["intermediate_results"] || {}
-        status = data["status"].to_s
-        results.key?(meta["step_name"].to_s) || %w[completed failed halted skipped].include?(status)
+        @storage.retrieve_context(meta["owner_context_id"], meta["owner_reactor_class_name"])
       end
 
       def retrigger_collector(meta)

data/lib/ruby_reactor/map.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # Element jobs one throw enqueues when a fan-out map declares no
+    # `batch_size` (009 R-10), forward and rollback alike.
+    DEFAULT_BATCH_SIZE = 50
+    # Element contexts an inline map's rollback loads per read (009 R-01).
+    ROLLBACK_CHUNK = 100
+
+    # One structured key=value line (009 FR-011), as `Executor#log_completion`.
+    def self.log(event, **fields)
+      line = { event: event, **fields }.map { |key, value| "#{key}=#{value.inspect}" }.join(" ")
+      RubyReactor.configuration.logger.info(line)
+    end
+  end
+end

data/lib/ruby_reactor/max_retries_exhausted_failure.rb

@@ -7,12 +7,14 @@ module RubyReactor
     # rubocop:disable Metrics/ParameterLists
     def initialize(message, step:, attempts:, original_error: nil,
                    inputs: {}, backtrace: nil, redact_inputs: [],
-                   reactor_name: nil, step_arguments: {}, validation_errors: nil, rollback_failures: nil)
+                   reactor_name: nil, step_arguments: {}, validation_errors: nil, rollback_failures: nil,
+                   exception_class: nil)
       # rubocop:enable Metrics/ParameterLists
       super(message,
             step_name: step, inputs: inputs, backtrace: backtrace,
             redact_inputs: redact_inputs, reactor_name: reactor_name, step_arguments: step_arguments,
-            validation_errors: validation_errors, rollback_failures: rollback_failures)
+            validation_errors: validation_errors, rollback_failures: rollback_failures,
+            exception_class: exception_class)
       @attempts = attempts
       @original_error = original_error
     end

data/lib/ruby_reactor/reactor.rb

@@ -6,6 +6,9 @@ module RubyReactor
     include RubyReactor::Dsl::Reactor
     include RubyReactor::Dsl::Lockable
 
+    # Seconds a manual undo waits for the run's context lock (009 R-12).
+    UNDO_LOCK_WAIT = 5
+
     attr_reader :context, :result, :undo_trace, :execution_trace
 
     def self.find(id)
@@ -60,9 +63,12 @@ module RubyReactor
       reactor.cancel(reason)
     end
 
+    # A rollback that hands off at a fan-out map finishes in a worker, which
+    # applies `cancelled` itself (009 R-12).
     def self.undo(id)
       reactor = find(id)
-      reactor.undo
+      return if reactor.undo == :handed_off
+
       cancel(id: id, reason: "Undo triggered")
     end
 
@@ -153,6 +159,14 @@ module RubyReactor
               "Cannot resume: reactor has been cancelled (Reason: #{@context.cancellation_reason})"
       end
 
+      # Only a reactor paused at an interrupt takes a resume (008 FR-032): one
+      # that is executing or rolling back (`running`), finished, or `aborted`
+      # must not be run forward from its stored state.
+      unless @context.status.to_s == "paused"
+        raise Error::ValidationError,
+              "Cannot resume: the reactor is #{@context.status}, not paused at an interrupt"
+      end
+
       validate_continue_step!(step_name)
 
       if (failure = validate_continue_payload(payload, step_name))
@@ -170,6 +184,11 @@ module RubyReactor
         return @result = enqueue_background_resume
       end
 
+      # Claim the resume before running it, so a `continue` that arrives while
+      # this one executes or rolls back reads `running` and fails (FR-032).
+      @context.status = :running
+      save_context
+
       # Resume execution
       executor = Executor.new(self.class, {}, @context)
       @result = executor.resume_execution
@@ -179,19 +198,46 @@ module RubyReactor
       @execution_trace = executor.execution_trace
 
       @result
+    rescue Lock::AcquisitionError, Semaphore::AcquisitionError => e
+      # Contended at the resume's own gates: nothing ran, so the run is still
+      # paused and the caller may retry, as with `Reactor.run`. A context-lock
+      # contention means a live resume holds the run: leave it alone.
+      reopen_paused unless executor&.past_gates? || e.is_a?(Lock::ContextLockContention)
+      raise
     rescue Error::InputValidationError => e
       # This might catch other validations, but here we specifically want payload validation.
       # The block above handles payload validation explicitly.
       RubyReactor::Failure(e.message, invalid_payload: true)
     end
 
+    # Undoes every completed step, under the run's context lock so no worker
+    # resumes it meanwhile (009 R-12). Returns `:handed_off` when a fan-out
+    # map's rollback continues in its element jobs: the run is then
+    # `rolling_back`, and a worker finishes it as `cancelled`.
     def undo
+      raise Error::ValidationError, "rollback already in progress" if @context.rolling_back?
+
+      lock = acquire_undo_lock
       executor = Executor.new(self.class, {}, @context)
-      executor.undo_all
+      @context.rollback = { "trigger" => "undo", "compensated" => true, "failures" => [] }
+      begin
+        executor.undo_all
+      rescue Error::RollbackHandedOff => e
+        @context.status = :rolling_back
+        executor.save_context
+        executor.hand_off_rollback!(e)
+        return :handed_off
+      end
+      @context.rollback = nil
       executor.save_context
+    ensure
+      lock&.release
     end
 
     def cancel(reason)
+      # `cancelled` is terminal: the steps before the map would never be undone.
+      raise Error::ValidationError, "rollback in progress; cannot cancel" if @context.rolling_back?
+
       @context.cancelled = true
       @context.cancellation_reason = reason
       @context.status = "cancelled"
@@ -211,6 +257,23 @@ module RubyReactor
       RubyReactor::Configuration.instance
     end
 
+    def reopen_paused
+      @context.status = :paused
+      save_context
+    end
+
+    # Raises `Lock::AcquisitionError` while a live run or rollback holds it.
+    # Skipped in inline job-testing mode, as `Executor#acquire_context_lock`.
+    def acquire_undo_lock
+      return if defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
+
+      lock = RubyReactor::Lock.new("async:#{@context.context_id}", owner: SecureRandom.uuid,
+                                                                   ttl: configuration.context_lock_ttl,
+                                                                   wait: UNDO_LOCK_WAIT)
+      lock.acquire
+      lock
+    end
+
     def validate_steps!
       return unless self.class.steps.empty?
 

data/lib/ruby_reactor/rspec/active_job_helpers.rb

@@ -11,12 +11,16 @@ module RubyReactor
       PendingJob = Struct.new(:job_class, :raw) do
         def perform!
           ::ActiveJob::Base.queue_adapter.enqueued_jobs.delete(raw)
-          job_class.new(*raw[:args]).perform_now
+          job_class.new(*ActiveJobHelpers.arguments(raw)).perform_now
         end
 
         def args
           raw[:args]
         end
+
+        # The Sidekiq `PendingJob`'s name for it, so step-wise drains read the
+        # same on both backends (009 U4).
+        alias_method :worker_class, :job_class
       end
 
       def self.test_adapter?
@@ -37,11 +41,18 @@ module RubyReactor
 
           queue.dup.each do |job|
             queue.delete(job)
-            job[:job].new(*job[:args]).perform_now
+            job[:job].new(*arguments(job)).perform_now
           end
         end
       end
 
+      # The test adapter keeps a job's arguments serialized; a real backend
+      # hands `perform` the deserialized ones (a job requeueing its own
+      # arguments would otherwise re-serialize ActiveJob's markers).
+      def self.arguments(raw)
+        ::ActiveJob::Arguments.deserialize(raw[:args])
+      end
+
       def self.pending_async_jobs
         return [] unless test_adapter?