ruby_reactor 0.8.4 → 0.8.6

data/lib/ruby_reactor/step_worker.rb

@@ -88,7 +88,13 @@ module RubyReactor
       log(:error, "failed", error: "#{e.class}: #{e.message}")
       complete(RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name, retryable: false),
                context)
-    rescue StandardError => e
+    # Arguments raised: the body never started, so it is neither retried nor
+    # compensated (008 R-06).
+    rescue Error::ArgumentResolutionError => e
+      log(:error, "failed", error: "#{e.class}: #{e.message}")
+      complete(RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name, retryable: false,
+                                      exception_class: e.exception_class), context)
+    rescue Error::Rescuable => e
       # The unit's failure belongs in its record, where a reader can see it.
       # Raising instead would hand the job to the backend's retry machinery to
       # fail identically N more times while every reader waits out its timeout.
@@ -238,17 +244,7 @@ module RubyReactor
     end
 
     def run_step(context, step_config)
-      # A suppressed step never coordinates (FR-012) — decided before the
-      # arguments are validated or any hold is taken, exactly as
-      # `StepExecutor#execute_step_sync` orders it. `complete` persists the
-      # nil result, so the reader sees the same skipped unit a same-process
-      # step would produce.
-      unless step_config.should_run?(context)
-        log(:info, "skipped")
-        return RubyReactor.Success(nil)
-      end
-
-      arguments = resolve_arguments(step_config, context)
+      arguments = step_config.resolve_arguments(context)
       # Reactor-side `argument`/`validate_args` rules gate the step BEFORE its
       # coordination is acquired, exactly as `StepExecutor#execute_step_sync`
       # orders them — an async_step must not take a lock (or spend a rate-limit
@@ -289,9 +285,43 @@ module RubyReactor
         end
       end
 
+      compensate_unit(context, step_config, result, arguments)
       result
     end
 
+    # 008 R-09: the unit compensates ITSELF, once, here in its own job, after
+    # its final attempt failed — whether or not any step ever reads it, the
+    # way an `async_reactor` child rolls itself back. Never for a success,
+    # skip or halt, and never for a body that never started (invalid
+    # arguments), exactly the executor's rule. Same coordination re-take,
+    # middleware events and failure recording as an in-process compensate.
+    # The outcome goes on this unit's record (`complete`); the parent's
+    # context is never written here (single writer), so the in-memory trace
+    # entry the compensate appends stays local to this job.
+    def compensate_unit(context, step_config, result, arguments)
+      return unless result.is_a?(RubyReactor::Failure) && body_started?(result.error)
+
+      manager = Executor::CompensationManager.new(context)
+      outcome = manager.compensate(step_config, result.error, arguments)
+      @compensation = {
+        "status" => compensation_status(outcome),
+        "rollback_failures" => ContextSerializer.serialize_value(manager.rollback_failures),
+        "completed_at" => Time.now.iso8601
+      }
+    end
+
+    def body_started?(error)
+      return false if error.is_a?(Error::InputValidationError)
+
+      Executor::CompensationManager::NEVER_STARTED_ERROR_CLASSES.none? { |klass| error.is_a?(klass) }
+    end
+
+    def compensation_status(outcome)
+      return "failed" if outcome.is_a?(RubyReactor::Failure)
+
+      outcome.respond_to?(:skipped?) && outcome.skipped? ? "skipped" : "completed"
+    end
+
     # Same check and same structured, non-retryable shape `StepExecutor`
     # produces — the same arguments fail the same rules on every attempt.
     def validate_arguments(step_config, arguments)
@@ -330,7 +360,7 @@ module RubyReactor
                              retryable: false)
     rescue Executor::StepCoordination::Contended, Executor::StepCoordination::KeyError
       raise
-    rescue StandardError => e
+    rescue Error::Rescuable => e
       RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name)
     end
 
@@ -356,14 +386,6 @@ module RubyReactor
       RubyReactor.Success(result)
     end
 
-    def resolve_arguments(step_config, context)
-      step_config.arguments.to_h do |arg_name, arg_config|
-        value = arg_config[:source].resolve(context)
-        value = arg_config[:transform].call(value) if arg_config[:transform]
-        [arg_name, value]
-      end
-    end
-
     # Write first, publish second. The record is the answer; the signal only
     # saves the reader a fallback interval.
     def complete(result, context)
@@ -394,8 +416,10 @@ module RubyReactor
 
     # This delivery's run, once the body was reached: `started_at`,
     # `arguments` (redacted, serialized) and `attempts`. Empty before that.
+    # Plus `compensation` when the unit compensated itself.
     def run_fields
-      @run || {}
+      fields = @run || {}
+      @compensation ? fields.merge("compensation" => @compensation) : fields
     end
 
     def record_missing_parent

data/lib/ruby_reactor/storage/adapter.rb

@@ -56,6 +56,10 @@ module RubyReactor
         raise NotImplementedError
       end
 
+      def decrement_map_counter_by(map_id, amount, reactor_class_name)
+        raise NotImplementedError
+      end
+
       def subscribe(channel, &block)
         raise NotImplementedError
       end
@@ -103,6 +107,20 @@ module RubyReactor
       def retrieve_map_element_context_ids(map_id, reactor_class_name)
         raise NotImplementedError
       end
+
+      def claim_map_owner_signal(map_id, reactor_class_name)
+        raise NotImplementedError
+      end
+
+      # Map rollback records (009 DM §5); see RedisMapRollback.
+      %i[count_map_element_context_ids retrieve_map_element_context_ids_from_tail start_map_rollback
+         retrieve_map_rollback_metadata claim_map_rollback_positions retrieve_map_rollback_offset
+         store_map_rollback_outcome count_map_rollback_outcomes map_rollback_outcome_stored?
+         stored_map_rollback_positions each_map_rollback_outcome map_rollback_indexes_seen
+         mark_map_rollback_handed_off map_rollback_handed_off? claim_map_rollback_signal map_rollback_summary
+         scan_map_rollbacks].each do |method|
+        define_method(method) { |*, **| raise NotImplementedError }
+      end
     end
   end
 end

data/lib/ruby_reactor/storage/redis_adapter.rb

@@ -11,6 +11,7 @@ module RubyReactor
       include RedisStepResults
       include RedisPubSub
       include RedisReactorScan
+      include RedisMapRollback
 
       def initialize(redis_config)
         super()
@@ -70,7 +71,8 @@ module RubyReactor
       # rubocop:disable Metrics/ParameterLists
       def initialize_map_operation(map_id, count, parent_reactor_class_name, reactor_class_info:, strict_ordering: true,
                                    parent_context_id: nil, step_name: nil, parent_is_map_element: false,
-                                   outer_map_id: nil, outer_index: nil)
+                                   outer_map_id: nil, outer_index: nil, owner_context_id: nil,
+                                   owner_reactor_class_name: nil, batch_size: nil, atomic: nil)
         # Ensure counter is set
         set_map_counter(map_id, count, parent_reactor_class_name)
 
@@ -79,7 +81,9 @@ module RubyReactor
         # nested-map fields (parent_is_map_element + outer_map_id/outer_index)
         # record which liveness lock the parent actually holds (N1): a nested
         # map's parent is itself a map element running under a `map_element:` lock,
-        # not an `async:` lock.
+        # not an `async:` lock. The owner is the top-level run the collector
+        # resumes (009 R-03); `batch_size` and `atomic` let a re-dispatched
+        # element keep both (R-10).
         key = "reactor:#{parent_reactor_class_name}:map:#{map_id}:metadata"
         metadata = {
           map_id: map_id,
@@ -92,12 +96,22 @@ module RubyReactor
           parent_is_map_element: parent_is_map_element,
           outer_map_id: outer_map_id,
           outer_index: outer_index,
+          owner_context_id: owner_context_id,
+          owner_reactor_class_name: owner_reactor_class_name,
+          batch_size: batch_size,
+          atomic: atomic,
           created_at: Time.now.to_i
         }
         @redis.set(key, metadata.to_json, ex: durability_ttl)
       end
       # rubocop:enable Metrics/ParameterLists
 
+      # Claimed once, by the collector that found the map settled: only that
+      # one enqueues the owner's Worker (009 DM §3).
+      def claim_map_owner_signal(map_id, reactor_class_name) # rubocop:disable Naming/PredicateMethod
+        !!@redis.set("reactor:#{reactor_class_name}:map:#{map_id}:owner_signalled", "1", nx: true, ex: durability_ttl)
+      end
+
       # Enumerate active map operations for the map sweeper (Phase 5d). Returns
       # the parsed metadata hash for each (includes map_id, count,
       # parent_context_id, step_name, parent_reactor_class_name, and the nested-map
@@ -105,6 +119,8 @@ module RubyReactor
       def scan_maps(count: 1000)
         results = []
         @redis.scan_each(match: "reactor:*:map:*:metadata", count: 100) do |key|
+          next if key.end_with?(":rollback:metadata") # a hash, listed by scan_map_rollbacks
+
           json = @redis.get(key)
           next unless json
 
@@ -133,6 +149,15 @@ module RubyReactor
         @redis.decr(key)
       end
 
+      # Settles `amount` indices at once (a fail-fast dispatcher claiming the
+      # ones it never dispatched). Returns the count left.
+      def decrement_map_counter_by(map_id, amount, reactor_class_name)
+        key = map_counter_key(map_id, reactor_class_name)
+        left = @redis.decrby(key, amount)
+        @redis.expire(key, durability_ttl)
+        left
+      end
+
       def set_last_queued_index(map_id, index, reactor_class_name)
         key = map_last_queued_index_key(map_id, reactor_class_name)
         @redis.set(key, index, ex: durability_ttl)

data/lib/ruby_reactor/storage/redis_map_rollback.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Storage
+    # The records of one fan-out map's distributed rollback (009 DM §5), under
+    # `reactor:<P>:map:<map_id>:rollback:…`. Element rollback jobs write only
+    # these and their own element context, never a reactor context (I-1).
+    #
+    # Positions count from the tail of the element-context index, anchored at
+    # the `total` the rollback started with, so an id a late duplicate appends
+    # afterwards never shifts them.
+    module RedisMapRollback
+      # Creates the metadata only if no rollback of this map exists (I-3).
+      START_ROLLBACK_SCRIPT = <<~LUA
+        if redis.call('EXISTS', KEYS[1]) == 1 then return 0 end
+        redis.call('HSET', KEYS[1], unpack(ARGV, 2))
+        redis.call('EXPIRE', KEYS[1], ARGV[1])
+        return 1
+      LUA
+
+      FAILED_OUTCOMES = %w[failed element_in_flight context_unavailable].freeze
+
+      def count_map_element_context_ids(map_id, reactor_class_name)
+        @redis.llen(map_element_contexts_key(map_id, reactor_class_name))
+      end
+
+      # Ids at tail positions `position...position + count`, in tail order.
+      def retrieve_map_element_context_ids_from_tail(map_id, reactor_class_name, position, count, total:)
+        stop = total - 1 - position
+        return [] if stop.negative?
+
+        start = [total - position - count, 0].max
+        @redis.lrange(map_element_contexts_key(map_id, reactor_class_name), start, stop).reverse
+      end
+
+      # `[created, metadata]`: a second start finds the first one's records.
+      def start_map_rollback(map_id, reactor_class_name, **meta)
+        fields = meta.merge(map_id: map_id, parent_reactor_class_name: reactor_class_name, started_at: Time.now.to_i)
+        argv = fields.flat_map { |k, v| [k.to_s, JSON.generate(v)] }
+        created = @redis.eval(START_ROLLBACK_SCRIPT, keys: [map_rollback_key(map_id, reactor_class_name, :metadata)],
+                                                     argv: [durability_ttl, *argv])
+        [created == 1, retrieve_map_rollback_metadata(map_id, reactor_class_name)]
+      end
+
+      def retrieve_map_rollback_metadata(map_id, reactor_class_name)
+        parse_rollback_metadata(@redis.hgetall(map_rollback_key(map_id, reactor_class_name, :metadata)))
+      end
+
+      # Claims the next `count` positions (R-02), clipped to the total.
+      def claim_map_rollback_positions(map_id, reactor_class_name, count)
+        key = map_rollback_key(map_id, reactor_class_name, :offset)
+        stop = @redis.incrby(key, count)
+        @redis.expire(key, durability_ttl)
+        total = retrieve_map_rollback_metadata(map_id, reactor_class_name)&.fetch("total", 0).to_i
+        (stop - count)...[stop, total].min
+      end
+
+      def retrieve_map_rollback_offset(map_id, reactor_class_name)
+        @redis.get(map_rollback_key(map_id, reactor_class_name, :offset)).to_i
+      end
+
+      # The first outcome for a position wins: a duplicate delivery, which
+      # finds the element already undone, must not replace a `failed` outcome
+      # with its own `undone`. Returns whether this call stored it.
+      def store_map_rollback_outcome(map_id, reactor_class_name, position, outcome) # rubocop:disable Naming/PredicateMethod
+        results = map_rollback_key(map_id, reactor_class_name, :results)
+        indexes = map_rollback_key(map_id, reactor_class_name, :indexes)
+        index = outcome["index"] || outcome[:index]
+        stored, = @redis.multi do |tx|
+          tx.hsetnx(results, position.to_s, JSON.generate(outcome))
+          tx.expire(results, durability_ttl)
+          tx.sadd(indexes, index.to_s) unless index.nil?
+          tx.expire(indexes, durability_ttl)
+        end
+        [true, 1].include?(stored)
+      end
+
+      def count_map_rollback_outcomes(map_id, reactor_class_name)
+        @redis.hlen(map_rollback_key(map_id, reactor_class_name, :results))
+      end
+
+      def stored_map_rollback_positions(map_id, reactor_class_name)
+        @redis.hkeys(map_rollback_key(map_id, reactor_class_name, :results)).map(&:to_i)
+      end
+
+      def map_rollback_outcome_stored?(map_id, reactor_class_name, position)
+        @redis.hexists(map_rollback_key(map_id, reactor_class_name, :results), position.to_s)
+      end
+
+      # Yields `position, outcome` in bounded reads.
+      def each_map_rollback_outcome(map_id, reactor_class_name)
+        @redis.hscan_each(map_rollback_key(map_id, reactor_class_name, :results), count: 500) do |position, json|
+          yield position.to_i, JSON.parse(json)
+        end
+      end
+
+      # One boolean per index: whether a rollback job saw that element (R-09).
+      def map_rollback_indexes_seen(map_id, reactor_class_name, indexes)
+        key = map_rollback_key(map_id, reactor_class_name, :indexes)
+        @redis.pipelined { |pipe| indexes.each { |index| pipe.sismember(key, index.to_s) } }
+      end
+
+      def mark_map_rollback_handed_off(map_id, reactor_class_name)
+        @redis.set(map_rollback_key(map_id, reactor_class_name, :handed_off), "1", ex: durability_ttl)
+      end
+
+      def map_rollback_handed_off?(map_id, reactor_class_name)
+        @redis.exists?(map_rollback_key(map_id, reactor_class_name, :handed_off))
+      end
+
+      def claim_map_rollback_signal(map_id, reactor_class_name) # rubocop:disable Naming/PredicateMethod
+        !!@redis.set(map_rollback_key(map_id, reactor_class_name, :signalled), "1", nx: true, ex: durability_ttl)
+      end
+
+      # Progress for the dashboard (R-15), or nil when this map never rolled back.
+      def map_rollback_summary(map_id, reactor_class_name)
+        meta = retrieve_map_rollback_metadata(map_id, reactor_class_name)
+        return nil unless meta
+
+        failed = 0
+        each_map_rollback_outcome(map_id, reactor_class_name) do |_, outcome|
+          failed += 1 if FAILED_OUTCOMES.include?(outcome["outcome"])
+        end
+        total = meta["total"].to_i
+        settled = count_map_rollback_outcomes(map_id, reactor_class_name)
+        { total: total, settled: settled, outstanding: total - settled, failed: failed }
+      end
+
+      # Every started map rollback, for `Map::Sweeper` (S-5).
+      def scan_map_rollbacks(count: 1000)
+        results = []
+        @redis.scan_each(match: "reactor:*:map:*:rollback:metadata", count: 100) do |key|
+          meta = parse_rollback_metadata(@redis.hgetall(key))
+          results << meta if meta
+          return results if results.size >= count
+        end
+        results
+      end
+
+      private
+
+      def parse_rollback_metadata(raw)
+        return nil if raw.nil? || raw.empty?
+
+        raw.transform_values { |value| JSON.parse(value) }
+      end
+
+      def map_rollback_key(map_id, reactor_class_name, suffix)
+        "reactor:#{reactor_class_name}:map:#{map_id}:rollback:#{suffix}"
+      end
+    end
+  end
+end

data/lib/ruby_reactor/storage/redis_reactor_scan.rb

@@ -67,7 +67,7 @@ module RubyReactor
 
       def determine_status(data)
         status = data["status"].to_s == "skipped" ? "halted" : data["status"].to_s # "skipped" is the legacy halt name
-        return status if %w[failed paused completed running halted pending].include?(status)
+        return status if %w[failed paused completed running rolling_back halted pending aborted].include?(status)
         return "cancelled" if data["cancelled"]
         # Heuristic
         return "failed" if data["retry_count"]&.positive? && !data["current_step"].nil?

data/lib/ruby_reactor/sweeper.rb

@@ -48,7 +48,9 @@ module RubyReactor
       reenqueued = 0
 
       @storage.scan_reactors(count: limit, include_dispatched_children: true).each do |reactor|
-        next unless reactor[:status] == "running" # non-terminal only
+        # Non-terminal only. A `rolling_back` run whose owner resume was lost
+        # is re-enqueued too: unsettled, it simply hands off again (009 S-5).
+        next unless %w[running rolling_back].include?(reactor[:status])
         next if @storage.lock_held?("async:#{reactor[:id]}") # worker alive -> leave alone
 
         @async_router.perform_async(reactor[:id], reactor[:class])

data/lib/ruby_reactor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyReactor
-  VERSION = "0.8.4"
+  VERSION = "0.8.6"
 end

data/lib/ruby_reactor/web/api.rb

@@ -173,7 +173,7 @@ module RubyReactor
 
       def self.reactor_status(data)
         status = data[:status].to_s == "skipped" ? "halted" : data[:status].to_s
-        return status if %w[failed paused completed running halted pending].include?(status)
+        return status if %w[failed paused completed running rolling_back halted pending aborted].include?(status)
         return "cancelled" if data[:cancelled]
         return "running" if data[:current_step]
         return "completed" if execution_evidence?(data)
@@ -393,7 +393,17 @@ module RubyReactor
         ref_data.merge("context" => child)
       end
 
+      # A map that rolled back distributed also carries its progress, `{
+      # total, settled, outstanding, failed }` (009 R-15). The records outlive
+      # the run, so a client shows it only while the run is `rolling_back`.
       def self.hydrate_map_ref(ref_data, reactor_class_name)
+        map_id = ref_data[:map_id] || ref_data["map_id"]
+        hydrated = hydrate_map_element(ref_data, reactor_class_name)
+        rollback = map_id && RubyReactor.configuration.storage_adapter.map_rollback_summary(map_id, reactor_class_name)
+        rollback ? hydrated.merge("rollback" => rollback.transform_keys(&:to_s)) : hydrated
+      end
+
+      def self.hydrate_map_element(ref_data, reactor_class_name)
         storage = RubyReactor.configuration.storage_adapter
         map_id = ref_data[:map_id] || ref_data["map_id"]