ruby_reactor 0.5.1 → 0.5.3

data/lib/ruby_reactor/sidekiq_workers/worker.rb

@@ -17,15 +17,26 @@ module RubyReactor
         # Handle infrastructure failures (network, Redis, etc.)
       end
 
-      def perform(serialized_context, reactor_class_name = nil, snooze_count = 0)
+      # Identity-only payload: storage is the source of truth. Rehydrate the live
+      # context from storage by id, then resume. A nil read means the context was
+      # swept, expired, or already terminal-and-collected — nothing to resume.
+      def perform(context_id, reactor_class_name = nil, snooze_count = 0)
+        # Normalize so a nil/omitted name resolves to the same storage key the
+        # enqueue path wrote (always via reactor_storage_name). Without this a
+        # nil here builds "reactor::context:<id>" and misses the stored
+        # "reactor:AnonymousReactor:context:<id>", silently no-op'ing.
+        reactor_class_name ||= RubyReactor.reactor_storage_name(nil)
+        data = RubyReactor.configuration.storage_adapter.retrieve_context(context_id, reactor_class_name)
+        return if data.nil?
+
         begin
-          context = ContextSerializer.deserialize(serialized_context)
+          context = ContextSerializer.deserialize_hash(data)
         rescue RubyReactor::Error::DeserializationError,
                RubyReactor::Error::SchemaVersionError => e
-          # Permanent failures — retrying the same blob will keep failing.
-          # Mark the context as failed (best-effort) and return so Sidekiq
-          # does not burn its retry budget.
-          handle_deserialization_failure(serialized_context, reactor_class_name, e)
+          # Permanent failures — re-reading the same stored blob will keep
+          # failing. Mark the context as failed (best-effort) and return so
+          # Sidekiq does not burn its retry budget.
+          handle_deserialization_failure(context_id, reactor_class_name, e)
           return
         end
 
@@ -48,18 +59,26 @@ module RubyReactor
           # Resume execution from the failed step
           executor = Executor.new(context.reactor_class, {}, context)
           executor.resume_execution
-          executor.save_context
+          # No explicit save here: resume_execution's ensure block already persists
+          # the final root state (`save_context unless skip_context_persist?`), and
+          # in the worker the executor's context IS the root, so an extra checkpoint!
+          # would just re-write the identical blob to the identical key. The
+          # skip_context_persist? guard (stale-batch redelivery of an already-terminal
+          # context) is likewise honored there.
 
           # Return the executor (which now has the result stored in it)
           executor
         rescue RubyReactor::Lock::AcquisitionError,
                RubyReactor::Semaphore::AcquisitionError,
-               RubyReactor::RateLimit::ExceededError => e
-          # Snooze on expected concurrency or rate contention. We avoid
-          # Sidekiq's native retry path so this doesn't burn the job's retry
-          # budget or appear as an error in dashboards. After the configured
-          # cap is reached we escalate by marking the reactor as failed.
-          handle_snooze(serialized_context, reactor_class_name, context, snooze_count, e)
+               RubyReactor::RateLimit::ExceededError,
+               RubyReactor::OrderedLock::WaitError => e
+          # Snooze on expected concurrency, rate, or ordering contention.
+          # OrderedLock::WaitError carries a poison-pill-derived retry hint,
+          # consumed by compute_snooze_delay below. We avoid Sidekiq's native
+          # retry path so this doesn't burn the job's retry budget or appear
+          # as an error in dashboards. After the configured cap is reached we
+          # escalate by marking the reactor as failed.
+          handle_snooze(context_id, reactor_class_name, context, snooze_count, e)
         rescue RubyReactor::RateLimitRegistry::UnknownLimitError => e
           # Permanent configuration error — snoozing or retrying the same job
           # will keep failing. Mark the context failed immediately.
@@ -69,34 +88,64 @@ module RubyReactor
 
       private
 
-      def handle_snooze(serialized_context, reactor_class_name, context, snooze_count, error)
+      def handle_snooze(context_id, reactor_class_name, context, snooze_count, error)
         config = RubyReactor.configuration
         max = config.lock_snooze_max_attempts
 
-        if max != :infinity && snooze_count >= max
+        # OrderedLock::WaitError bypasses the snooze cap. The gate's
+        # poison_pill_timeout is the only meaningful upper bound on how long a
+        # nonce can legitimately wait; capping snoozes would either fail jobs
+        # prematurely or strand the nonce in `assigned_at` until poison_pill
+        # eventually advances past it. Snooze until the gate passes (or poison
+        # auto-advance moves the cursor past us).
+        # The per-context liveness lock (`async:<id>`) is also uncapped: a
+        # duplicate of the *same* execution may wait arbitrarily long for the
+        # live original to finish (e.g. a sweeper re-enqueue racing a slow but
+        # alive worker). Capping it would fail a legitimately-waiting duplicate.
+        capped = !(error.is_a?(RubyReactor::OrderedLock::WaitError) ||
+                   error.is_a?(RubyReactor::Lock::ContextLockContention))
+
+        if capped && max != :infinity && snooze_count >= max
           escalate_snooze(context, snooze_count, error)
           return
         end
 
         delay = compute_snooze_delay(config, error)
-        self.class.perform_in(delay, serialized_context, reactor_class_name, snooze_count + 1)
+        # Re-enqueue by id: the context is already persisted in storage, so the
+        # rescheduled job rehydrates fresh state (no stale blob).
+        self.class.perform_in(delay, context_id, reactor_class_name, snooze_count + 1)
       end
 
       # Use the error's `retry_after_seconds` hint when available
       # (RateLimit::ExceededError carries the time until the bucket rolls);
       # otherwise fall back to the configured base + jitter for lock/semaphore
       # contention which has no precise hint.
+      #
+      # OrderedLock::WaitError is deliberately excluded from the hint path: its
+      # `retry_after_seconds` is the poison-pill window (the upper bound before
+      # a *dead* blocker is force-advanced), NOT how long the *live* blocker
+      # will take — which is usually milliseconds. Snoozing for the full window
+      # would make every out-of-order nonce sleep up to poison_pill_timeout even
+      # though its blocker finishes immediately, collapsing throughput. Re-poll
+      # at the base delay instead; poison auto-advance still clears a genuinely
+      # dead blocker on a later gate.
       def compute_snooze_delay(config, error)
         jitter = config.lock_snooze_jitter.to_f
         jitter_amount = jitter.positive? ? rand(0.0..jitter) : 0.0
 
-        if error.respond_to?(:retry_after_seconds) && error.retry_after_seconds
+        if hinted_retry?(error)
           [error.retry_after_seconds.to_f, 0.1].max + jitter_amount
         else
           config.lock_snooze_base_delay.to_f + jitter_amount
         end
       end
 
+      def hinted_retry?(error)
+        return false if error.is_a?(RubyReactor::OrderedLock::WaitError)
+
+        error.respond_to?(:retry_after_seconds) && error.retry_after_seconds
+      end
+
       def escalate_snooze(context, snooze_count, error)
         RubyReactor.configuration.logger.warn(
           "RubyReactor snooze limit reached after #{snooze_count} attempts " \
@@ -111,12 +160,20 @@ module RubyReactor
         }
 
         serialized = ContextSerializer.serialize(context)
-        reactor_class_name = context.reactor_class&.name || "AnonymousReactor"
+        reactor_class_name = RubyReactor.reactor_storage_name(context.reactor_class)
         RubyReactor.configuration.storage_adapter.store_context(
           context.context_id,
           serialized,
           reactor_class_name
         )
+
+        # Escalation is a terminal Failure that never reaches the Executor's
+        # ensure path, so advance the ordered-lock cursor here. Without this
+        # the nonce stays stranded in assigned_at (successors stall for the
+        # full poison_pill_timeout) and, worse, the strict-mode chain marker
+        # is never recorded — successors would RUN instead of being skipped.
+        info = Executor::OrderedLockSupport.info_from(context)
+        Executor::OrderedLockSupport.advance_with_retry(info, failed: true) if info
       end
 
       def log_infrastructure_failure(msg, exception)
@@ -124,23 +181,22 @@ module RubyReactor
         RubyReactor.configuration.logger.error("Job details: #{msg.inspect}")
       end
 
-      def handle_deserialization_failure(serialized_context, reactor_class_name, error)
-        metadata = extract_failure_metadata(serialized_context)
-        context_id = metadata[:context_id]
-        resolved_reactor_class_name = reactor_class_name || metadata[:reactor_class_name]
-
+      # The id-only payload already carries context_id and reactor_class_name, so
+      # there is no blob to parse for metadata — just mark the stored context
+      # failed (best-effort) so the job stops retrying a permanently-broken blob.
+      def handle_deserialization_failure(context_id, reactor_class_name, error)
         RubyReactor.configuration.logger.error(
           "RubyReactor deserialization failure for context " \
           "#{context_id || "unknown"}: #{error.class.name}: #{error.message}"
         )
 
-        return unless context_id && resolved_reactor_class_name
+        return unless context_id && reactor_class_name
 
-        payload = build_failed_context_payload(context_id, resolved_reactor_class_name, error)
+        payload = build_failed_context_payload(context_id, reactor_class_name, error)
         RubyReactor.configuration.storage_adapter.store_context(
           context_id,
           payload,
-          resolved_reactor_class_name
+          reactor_class_name
         )
       rescue StandardError => e
         # Don't let a persistence failure mask the original deserialization error.
@@ -149,16 +205,6 @@ module RubyReactor
         )
       end
 
-      def extract_failure_metadata(serialized_context)
-        data = JSON.parse(serialized_context)
-        {
-          context_id: data["context_id"],
-          reactor_class_name: data["reactor_class"]
-        }
-      rescue StandardError
-        {}
-      end
-
       def build_failed_context_payload(context_id, reactor_class_name, error)
         JSON.generate(
           "schema_version" => ContextSerializer::SCHEMA_VERSION,

data/lib/ruby_reactor/step/map_step.rb

@@ -179,10 +179,25 @@ module RubyReactor
           storage = RubyReactor.configuration.storage_adapter
           storage.initialize_map_operation(
             map_id, arguments[:source].size, context.reactor_class.name,
-            strict_ordering: arguments[:strict_ordering], reactor_class_info: reactor_class_info
+            strict_ordering: arguments[:strict_ordering], reactor_class_info: reactor_class_info,
+            **map_recovery_metadata(context, arguments[:step_name] || context.current_step)
           )
         end
 
+        # Recovery metadata for the map sweeper. When this map runs inside a map
+        # element (context.map_metadata present), it is a NESTED map: its parent
+        # holds the element's `map_element:` lock, not an `async:` lock (N1).
+        def map_recovery_metadata(context, step_name)
+          outer = context.map_metadata
+          {
+            parent_context_id: context.context_id,
+            step_name: step_name.to_s,
+            parent_is_map_element: !outer.nil?,
+            outer_map_id: outer && (outer[:map_id] || outer["map_id"]),
+            outer_index: outer && (outer[:index] || outer["index"])
+          }
+        end
+
         def dispatch_async_map(map_id, arguments, context, _reactor_class_info, step_name)
           # Every async map runs through the per-element Dispatcher path. When no
           # batch_size is given we default to the full source size (one fan-out
@@ -231,7 +246,8 @@ module RubyReactor
           storage = RubyReactor.configuration.storage_adapter
           storage.initialize_map_operation(
             map_id, arguments[:source].size, context.reactor_class.name,
-            strict_ordering: arguments[:strict_ordering], reactor_class_info: reactor_class_info
+            strict_ordering: arguments[:strict_ordering], reactor_class_info: reactor_class_info,
+            **map_recovery_metadata(context, step_name)
           )
 
           limit ||= arguments[:source].size

data/lib/ruby_reactor/storage/redis_adapter.rb

@@ -7,6 +7,7 @@ module RubyReactor
   module Storage
     class RedisAdapter < Adapter
       include RedisLocking
+      include RedisOrderedLocking
 
       def initialize(redis_config)
         super()
@@ -15,8 +16,10 @@ module RubyReactor
 
       def store_context(context_id, serialized_context, reactor_class_name)
         key = context_key(context_id, reactor_class_name)
-        # Use standard SET for compatibility (ReJSON not strictly required for full docs)
-        @redis.set(key, serialized_context, ex: 86_400) # 24h TTL
+        # Use standard SET for compatibility (ReJSON not strictly required for full docs).
+        # TTL is re-stamped on every write so long-running / snoozed contexts
+        # never expire mid-flight (Phase 4).
+        @redis.set(key, serialized_context, ex: durability_ttl)
       end
 
       def retrieve_context(context_id, reactor_class_name)
@@ -27,52 +30,84 @@ module RubyReactor
         JSON.parse(json)
       end
 
+      # Durable map storage is ALWAYS index-keyed (HSET), regardless of
+      # strict_ordering. The index->slot mapping makes completion recoverable
+      # (missing = (0...count) - HKEYS) and re-dispatch idempotent (re-running
+      # index i overwrites slot i, never duplicates). strict_ordering is now only
+      # a read-order convenience, not a storage-layout switch (Phase 5).
+      # rubocop:disable Lint/UnusedMethodArgument
       def store_map_result(map_id, index, serialized_result, reactor_class_name, strict_ordering: true)
         key = map_results_key(map_id, reactor_class_name)
-
-        if strict_ordering
-          # Use Hash for strict ordering by index
-          # HSET key index serialized_result
-          @redis.hset(key, index.to_s, serialized_result.to_json)
-        else
-          # Loose ordering: just push to list
-          @redis.rpush(key, serialized_result.to_json)
-        end
-
-        @redis.expire(key, 86_400)
+        @redis.hset(key, index.to_s, serialized_result.to_json)
+        @redis.expire(key, durability_ttl)
       end
 
       def retrieve_map_results(map_id, reactor_class_name, strict_ordering: true)
+        # rubocop:enable Lint/UnusedMethodArgument
         key = map_results_key(map_id, reactor_class_name)
+        results = @redis.hgetall(key)
+        # Index-keyed for both modes; sort by index so reads are deterministic.
+        results.keys.sort_by(&:to_i).map { |k| JSON.parse(results[k]) }
+      end
 
-        if strict_ordering
-          results = @redis.hgetall(key)
-          # Sort by index (key)
-          results.keys.sort_by(&:to_i).map { |k| JSON.parse(results[k]) }
-        else
-          results = @redis.lrange(key, 0, -1)
-          results.map { |r| JSON.parse(r) }
-        end
+      # Indices that have NO stored result yet: the authoritative, idempotent
+      # signal for what the map sweeper must (re)dispatch.
+      def missing_map_indices(map_id, count, reactor_class_name)
+        key = map_results_key(map_id, reactor_class_name)
+        present = @redis.hkeys(key).map(&:to_i)
+        (0...count).to_a - present
       end
 
       def set_map_counter(map_id, count, reactor_class_name)
         key = map_counter_key(map_id, reactor_class_name)
-        @redis.set(key, count, ex: 86_400)
+        @redis.set(key, count, ex: durability_ttl)
       end
 
-      def initialize_map_operation(map_id, count, parent_reactor_class_name, reactor_class_info:, strict_ordering: true)
+      # rubocop:disable Metrics/ParameterLists
+      def initialize_map_operation(map_id, count, parent_reactor_class_name, reactor_class_info:, strict_ordering: true,
+                                   parent_context_id: nil, step_name: nil, parent_is_map_element: false,
+                                   outer_map_id: nil, outer_index: nil)
         # Ensure counter is set
         set_map_counter(map_id, count, parent_reactor_class_name)
 
-        # Store metadata
+        # Store metadata. parent_context_id/step_name let the map sweeper recover
+        # without re-deriving the map_id (which is brittle to split on ':'). The
+        # nested-map fields (parent_is_map_element + outer_map_id/outer_index)
+        # record which liveness lock the parent actually holds (N1): a nested
+        # map's parent is itself a map element running under a `map_element:` lock,
+        # not an `async:` lock.
         key = "reactor:#{parent_reactor_class_name}:map:#{map_id}:metadata"
         metadata = {
+          map_id: map_id,
           count: count,
           strict_ordering: strict_ordering,
           reactor_class_info: reactor_class_info,
+          parent_context_id: parent_context_id,
+          parent_reactor_class_name: parent_reactor_class_name,
+          step_name: step_name,
+          parent_is_map_element: parent_is_map_element,
+          outer_map_id: outer_map_id,
+          outer_index: outer_index,
           created_at: Time.now.to_i
         }
-        @redis.set(key, metadata.to_json, ex: 86_400)
+        @redis.set(key, metadata.to_json, ex: durability_ttl)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      # Enumerate active map operations for the map sweeper (Phase 5d). Returns
+      # the parsed metadata hash for each (includes map_id, count,
+      # parent_context_id, step_name, parent_reactor_class_name, and the nested-map
+      # lock fields). Bounded by `count` to keep a sweep cheap.
+      def scan_maps(count: 1000)
+        results = []
+        @redis.scan_each(match: "reactor:*:map:*:metadata", count: 100) do |key|
+          json = @redis.get(key)
+          next unless json
+
+          results << JSON.parse(json)
+          return results if results.size >= count
+        end
+        results
       end
 
       def retrieve_map_metadata(map_id, reactor_class_name)
@@ -86,7 +121,7 @@ module RubyReactor
       def increment_map_counter(map_id, reactor_class_name)
         key = map_counter_key(map_id, reactor_class_name)
         @redis.incr(key)
-        @redis.expire(key, 86_400)
+        @redis.expire(key, durability_ttl)
       end
 
       def decrement_map_counter(map_id, reactor_class_name)
@@ -96,7 +131,7 @@ module RubyReactor
 
       def set_last_queued_index(map_id, index, reactor_class_name)
         key = map_last_queued_index_key(map_id, reactor_class_name)
-        @redis.set(key, index, ex: 86_400)
+        @redis.set(key, index, ex: durability_ttl)
       end
 
       def increment_last_queued_index(map_id, reactor_class_name)
@@ -108,7 +143,7 @@ module RubyReactor
         key = correlation_id_key(correlation_id, reactor_class_name)
         # Store mapping correlation_id -> context_id
         # Try to set if not exists
-        success = @redis.set(key, context_id, nx: true, ex: 86_400) # 24h TTL
+        success = @redis.set(key, context_id, nx: true, ex: durability_ttl)
 
         return if success
 
@@ -117,7 +152,7 @@ module RubyReactor
 
         if existing_context_id == context_id
           # Refresh TTL
-          @redis.expire(key, 86_400)
+          @redis.expire(key, durability_ttl)
           return
         end
 
@@ -215,7 +250,7 @@ module RubyReactor
       def store_map_element_context_id(map_id, context_id, reactor_class_name)
         key = map_element_contexts_key(map_id, reactor_class_name)
         @redis.rpush(key, context_id)
-        @redis.expire(key, 86_400)
+        @redis.expire(key, durability_ttl)
       end
 
       def retrieve_map_element_context_ids(map_id, reactor_class_name)
@@ -231,7 +266,7 @@ module RubyReactor
       def store_map_failed_context_id(map_id, context_id, reactor_class_name)
         key = map_failed_context_key(map_id, reactor_class_name)
         # Only store the first failure (nx: true)
-        @redis.set(key, context_id, nx: true, ex: 86_400)
+        @redis.set(key, context_id, nx: true, ex: durability_ttl)
       end
 
       def retrieve_map_failed_context_id(map_id, reactor_class_name)
@@ -241,12 +276,12 @@ module RubyReactor
 
       def set_map_offset(map_id, offset, reactor_class_name)
         key = map_offset_key(map_id, reactor_class_name)
-        @redis.set(key, offset, ex: 86_400)
+        @redis.set(key, offset, ex: durability_ttl)
       end
 
       def set_map_offset_if_not_exists(map_id, offset, reactor_class_name)
         key = map_offset_key(map_id, reactor_class_name)
-        @redis.set(key, offset, nx: true, ex: 86_400)
+        @redis.set(key, offset, nx: true, ex: durability_ttl)
       end
 
       def retrieve_map_offset(map_id, reactor_class_name)
@@ -259,43 +294,32 @@ module RubyReactor
         @redis.incrby(key, increment)
       end
 
+      # rubocop:disable Lint/UnusedMethodArgument
       def retrieve_map_results_batch(map_id, reactor_class_name, offset:, limit:, strict_ordering: true)
+        # Always index-keyed now (Phase 5): HMGET the contiguous index window.
         key = map_results_key(map_id, reactor_class_name)
-
-        if strict_ordering
-          # For Hash based results (indexed), we can use HMGET if we know the keys.
-          # Since we use 0-based index keys, we can generate the keys for the batch.
-          fields = (offset...(offset + limit)).map(&:to_s)
-          results = @redis.hmget(key, *fields)
-
-          # HMGET returns nil for missing fields, compact them?
-          # Or should we respect the holes?
-          # Map results are usually dense.
-          results.compact.map { |r| JSON.parse(r) }
-        else
-          # For List based results
-          # LRANGE uses inclusive ending index
-          end_index = offset + limit - 1
-          results = @redis.lrange(key, offset, end_index)
-          results.map { |r| JSON.parse(r) }
-        end
+        fields = (offset...(offset + limit)).map(&:to_s)
+        results = @redis.hmget(key, *fields)
+        results.compact.map { |r| JSON.parse(r) }
       end
+      # rubocop:enable Lint/UnusedMethodArgument
 
       def count_map_results(map_id, reactor_class_name)
         key = map_results_key(map_id, reactor_class_name)
-        type = @redis.type(key)
-
-        if type == "hash"
-          @redis.hlen(key)
-        elsif type == "list"
-          @redis.llen(key)
-        else
-          0
-        end
+        @redis.hlen(key)
       end
 
       private
 
+      # Single source of truth for the retention window of all durability-bearing
+      # state (context blob, map results/counters/metadata/offsets, correlation
+      # ids). Map state is load-bearing for resume exactly like the context, so it
+      # must share the context's configurable TTL — a shorter map TTL would expire
+      # map results mid-flight and break recovery. Re-stamped on every write.
+      def durability_ttl
+        RubyReactor.configuration.context_ttl
+      end
+
       def fetch_and_filter_reactors(keys)
         return [] if keys.empty?
 

data/lib/ruby_reactor/storage/redis_locking.rb

@@ -210,6 +210,14 @@ module RubyReactor
       # rate-limit/period state without leaking Redis-specific calls into
       # test code.
 
+      # Liveness check for a logical lock by its BARE key (e.g. "async:<id>").
+      # Prepends the "lock:" prefix that Lock applies. True while a worker holds
+      # (and auto-extends) the lock; false once it expires — the sweeper's
+      # "worker died" signal.
+      def lock_held?(key)
+        @redis.exists?("lock:#{key}")
+      end
+
       # Returns { owner:, count: } for a held lock, or nil if free.
       # `prefixed_key` is the full key (e.g. "lock:order:42").
       def lock_info(prefixed_key)