ruby_reactor 0.2.0 → 0.3.0

data/lib/ruby_reactor/map/dispatcher.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    class Dispatcher
+      extend Helpers
+
+      def self.perform(arguments)
+        arguments = arguments.transform_keys(&:to_sym)
+        parent_reactor_class_name = arguments[:parent_reactor_class_name]
+
+        storage = RubyReactor.configuration.storage_adapter
+
+        # Load parent context to resolve source
+        parent_context = load_parent_context_from_storage(
+          arguments[:parent_context_id],
+          parent_reactor_class_name,
+          storage
+        )
+
+        # Initialize metadata if first run
+        initialize_map_metadata(arguments, storage) unless arguments[:continuation]
+
+        # Resolve Source
+        # We need to resolve the source to know what we are iterating.
+        # Strict "Array Only" rule means we expect an Array-like object or we handle the
+        # "Query Builder" result if user used it.
+        source = resolve_source(arguments, parent_context)
+
+        # Dispatch next batch
+        dispatch_batch(source, arguments, parent_context, storage)
+      end
+
+      def self.initialize_map_metadata(arguments, storage)
+        map_id = arguments[:map_id]
+        reactor_class_name = arguments[:parent_reactor_class_name]
+
+        # Reset or set initial offset. Use NX to act as a mutex/guard against duplicate initialization.
+        storage.set_map_offset_if_not_exists(map_id, 0, reactor_class_name)
+      end
+
+      def self.resolve_source(arguments, context)
+        # Arguments has :source which is a Template::Input or similar.
+        # We need to resolve it against the context.
+        source_template = arguments[:source]
+
+        # Fallback: look up from step config if missing (e.g. called from ElementExecutor)
+        if source_template.nil? && context
+          step_name = arguments[:step_name]
+          step_config = context.reactor_class.steps[step_name.to_sym]
+          source_template = step_config.arguments[:source][:source]
+        end
+
+        # If source is packaged in arguments as a value (deserialized)
+        return source_template if source_template.is_a?(Array)
+
+        # Resolve template
+        return source_template.resolve(context) if source_template.respond_to?(:resolve)
+
+        source_template
+      end
+
+      def self.dispatch_batch(source, arguments, parent_context, storage)
+        map_id = arguments[:map_id]
+        reactor_class_name = arguments[:parent_reactor_class_name]
+
+        # Fail Fast Check
+        if arguments[:fail_fast]
+          failed_context_id = storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
+          return if failed_context_id
+        end
+
+        batch_size = arguments[:batch_size] || source.size # Default to all if no batch_size (async=true only)
+
+        # Atomically reserve a batch
+        new_offset = storage.increment_map_offset(map_id, batch_size, reactor_class_name)
+        current_offset = new_offset - batch_size
+
+        batch_elements = if source.is_a?(Array)
+                           source.slice(current_offset, batch_size) || []
+                         elsif source.respond_to?(:offset) && source.respond_to?(:limit)
+                           # Optimized for ActiveRecord and similar query builders
+                           source.offset(current_offset).limit(batch_size).to_a
+                         else
+                           # Fallback for generic Enumerable
+                           # This is inefficient for huge sets if not Array, but compliant
+                           source.drop(current_offset).take(batch_size)
+                         end
+
+        return if batch_elements.empty?
+
+        # Queue Jobs
+        queue_options = {
+          map_id: map_id,
+          arguments: arguments,
+          context: parent_context,
+          reactor_class_info: resolve_reactor_class_info(arguments, parent_context),
+          step_name: arguments[:step_name]
+        }
+
+        batch_elements.each_with_index do |element, i|
+          absolute_index = current_offset + i
+          queue_element_job(element, absolute_index, queue_options)
+        end
+      end
+
+      def self.queue_element_job(element, index, options)
+        arguments = options[:arguments]
+        context = options[:context]
+
+        # Resolve mappings
+        mappings_template = arguments[:argument_mappings]
+
+        # Fallback: look up from step config if missing (e.g. called from ElementExecutor)
+        if mappings_template.nil? && context
+          step_name = options[:step_name] || arguments[:step_name]
+          step_config = context.reactor_class.steps[step_name.to_sym]
+          mappings_template = step_config.arguments[:argument_mappings]
+        end
+
+        mappings = if mappings_template.respond_to?(:resolve)
+                     mappings_template.resolve(context)
+                   else
+                     mappings_template || {}
+                   end
+
+        # Fix for weird structure observed in fallback (wrapped in :source -> Template::Value)
+        if mappings.key?(:source) && mappings[:source].respond_to?(:value) && mappings[:source].value.is_a?(Hash)
+          mappings = mappings[:source].value
+        end
+
+        mapped_inputs = build_element_inputs(mappings, context, element)
+        serialized_inputs = ContextSerializer.serialize_value(mapped_inputs)
+
+        RubyReactor.configuration.async_router.perform_map_element_async(
+          map_id: options[:map_id],
+          element_id: "#{options[:map_id]}:#{index}",
+          index: index,
+          serialized_inputs: serialized_inputs,
+          reactor_class_info: options[:reactor_class_info],
+          strict_ordering: arguments[:strict_ordering],
+          parent_context_id: context.context_id,
+          parent_reactor_class_name: context.reactor_class.name,
+          step_name: options[:step_name].to_s,
+          batch_size: arguments[:batch_size], # Passed to worker so it knows to trigger next batch?
+          fail_fast: arguments[:fail_fast]
+        )
+      end
+
+      def self.resolve_reactor_class_info(arguments, context)
+        mapped_reactor_class = arguments[:mapped_reactor_class]
+        step_name = arguments[:step_name]
+
+        if mapped_reactor_class.respond_to?(:name)
+          { "type" => "class", "name" => mapped_reactor_class.name }
+        else
+          { "type" => "inline", "parent" => context.reactor_class.name, "step" => step_name.to_s }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_reactor/map/element_executor.rb

@@ -25,6 +25,11 @@ module RubyReactor
           context = ContextSerializer.deserialize(serialized_context)
           context.map_metadata = arguments
           reactor_class = context.reactor_class
+
+          # Ensure inputs are present (fallback to serialized_inputs if missing from context)
+          if context.inputs.empty? && serialized_inputs
+            context.inputs = ContextSerializer.deserialize_value(serialized_inputs)
+          end
         else
           # Deserialize inputs
           inputs = ContextSerializer.deserialize_value(serialized_inputs)
@@ -41,6 +46,27 @@ module RubyReactor
         storage = RubyReactor.configuration.storage_adapter
         storage.store_map_element_context_id(map_id, context.context_id, parent_reactor_class_name)
 
+        # Fail Fast Check
+        if arguments[:fail_fast]
+          failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name)
+          if failed_context_id
+            # Decrement counter as we are skipping execution
+            new_count = storage.decrement_map_counter(map_id, parent_reactor_class_name)
+            return unless new_count.zero?
+
+            # Trigger collection if we are the last one (skipped or otherwise)
+            RubyReactor.configuration.async_router.perform_map_collection_async(
+              parent_context_id: parent_context_id,
+              map_id: map_id,
+              parent_reactor_class_name: parent_reactor_class_name,
+              step_name: step_name,
+              strict_ordering: strict_ordering,
+              timeout: 3600
+            )
+            return
+          end
+        end
+
         # Execute
         executor = Executor.new(reactor_class, {}, context)
 
@@ -53,27 +79,35 @@ module RubyReactor
         result = executor.result
 
         if result.is_a?(RetryQueuedResult)
-          queue_next_batch(arguments) if batch_size
+          trigger_next_batch_if_needed(arguments, index, batch_size)
           return
         end
 
         # Store result
 
-        # Store result
-
         if result.success?
-          storage.store_map_result(map_id, index, result.value, parent_reactor_class_name,
+          storage.store_map_result(map_id, index,
+                                   ContextSerializer.serialize_value(result.value),
+                                   parent_reactor_class_name,
                                    strict_ordering: strict_ordering)
         else
+          # Trigger Compensation Logic
+          executor.undo_all
+
           # Store error
           storage.store_map_result(map_id, index, { _error: result.error }, parent_reactor_class_name,
                                    strict_ordering: strict_ordering)
+
+          if arguments[:fail_fast]
+            storage.store_map_failed_context_id(map_id, context.context_id, parent_reactor_class_name)
+          end
         end
 
         # Decrement counter
         new_count = storage.decrement_map_counter(map_id, parent_reactor_class_name)
 
-        queue_next_batch(arguments) if batch_size
+        # Trigger next batch if it's the last element of the current batch
+        trigger_next_batch_if_needed(arguments, index, batch_size)
 
         return unless new_count.zero?
 
@@ -88,23 +122,6 @@ module RubyReactor
         )
       end
 
-      def self.queue_next_batch(arguments)
-        storage = RubyReactor.configuration.storage_adapter
-        map_id = arguments[:map_id]
-        reactor_class_name = arguments[:parent_reactor_class_name]
-
-        next_index = storage.increment_last_queued_index(map_id, reactor_class_name)
-        total_count = storage.retrieve_map_metadata(map_id, reactor_class_name)["count"]
-
-        return unless next_index < total_count
-
-        parent_context = load_parent_context(arguments, reactor_class_name, storage)
-        element = resolve_next_element(arguments, parent_context, next_index)
-        serialized_inputs = build_serialized_inputs(arguments, parent_context, element)
-
-        queue_element_job(arguments, map_id, next_index, serialized_inputs, reactor_class_name)
-      end
-
       def self.load_parent_context(arguments, reactor_class_name, storage)
         parent_context_data = storage.retrieve_context(arguments[:parent_context_id], reactor_class_name)
         parent_reactor_class = Object.const_get(reactor_class_name)
@@ -116,42 +133,28 @@ module RubyReactor
         parent_context
       end
 
-      def self.resolve_next_element(arguments, parent_context, next_index)
-        parent_reactor_class = parent_context.reactor_class
-        step_config = parent_reactor_class.steps[arguments[:step_name].to_sym]
-
-        source_template = step_config.arguments[:source][:source]
-        source = source_template.resolve(parent_context)
-        source[next_index]
+      # Legacy helpers resolved_next_element, build_serialized_inputs, queue_element_job
+      # are REMOVED as they are no longer used for self-queuing.
+
+      # Basic helper to build inputs for the CURRENT element (still needed for perform)
+      # Wait, perform uses `serialized_inputs` passed to it.
+      # We don't need `build_element_inputs` here?
+      # `perform` uses `params[:serialized_inputs]`.
+      # So we can remove input building helpers too?
+      # Let's check if they are used elsewhere.
+      # `resolve_reactor_class` is used in `perform`.
+      # `build_element_inputs` is likely in Helpers or mixed in?
+
+      def self.trigger_next_batch_if_needed(arguments, index, batch_size)
+        return unless batch_size && ((index + 1) % batch_size).zero?
+
+        # Trigger Dispatcher for next batch
+        next_batch_args = arguments.dup
+        next_batch_args[:continuation] = true
+        RubyReactor::Map::Dispatcher.perform(next_batch_args)
       end
 
-      def self.build_serialized_inputs(arguments, parent_context, element)
-        parent_reactor_class = parent_context.reactor_class
-        step_config = parent_reactor_class.steps[arguments[:step_name].to_sym]
-
-        mappings_template = step_config.arguments[:argument_mappings][:source]
-        mappings = mappings_template.resolve(parent_context) || {}
-
-        mapped_inputs = build_element_inputs(mappings, parent_context, element)
-        ContextSerializer.serialize_value(mapped_inputs)
-      end
-
-      def self.queue_element_job(arguments, map_id, next_index, serialized_inputs, reactor_class_name)
-        RubyReactor.configuration.async_router.perform_map_element_async(
-          map_id: map_id,
-          element_id: "#{map_id}:#{next_index}",
-          index: next_index,
-          serialized_inputs: serialized_inputs,
-          reactor_class_info: arguments[:reactor_class_info],
-          strict_ordering: arguments[:strict_ordering],
-          parent_context_id: arguments[:parent_context_id],
-          parent_reactor_class_name: reactor_class_name,
-          step_name: arguments[:step_name],
-          batch_size: arguments[:batch_size]
-        )
-      end
-      private_class_method :queue_next_batch, :load_parent_context,
-                           :resolve_next_element, :build_serialized_inputs, :queue_element_job
+      private_class_method :load_parent_context, :trigger_next_batch_if_needed
     end
   end
 end

data/lib/ruby_reactor/map/execution.rb

@@ -21,7 +21,8 @@ module RubyReactor
           storage_options: {
             map_id: arguments[:map_id], storage: storage,
             parent_reactor_class_name: arguments[:parent_reactor_class_name],
-            strict_ordering: arguments[:strict_ordering]
+            strict_ordering: arguments[:strict_ordering],
+            fail_fast: arguments[:fail_fast]
           }
         )
 
@@ -31,6 +32,12 @@ module RubyReactor
 
       def self.execute_all_elements(source:, mappings:, reactor_class:, parent_context:, storage_options:)
         source.map.with_index do |element, index|
+          if storage_options[:fail_fast]
+            failed_context_id = storage_options[:storage].retrieve_map_failed_context_id(
+              storage_options[:map_id], storage_options[:parent_reactor_class_name]
+            )
+            next if failed_context_id
+          end
           element_inputs = build_element_inputs(mappings, parent_context, element)
 
           # Manually create and link context to ensure parent_context_id is set
@@ -56,8 +63,14 @@ module RubyReactor
 
           store_result(result, index, storage_options)
 
+          if result.failure? && storage_options[:fail_fast]
+            storage_options[:storage].store_map_failed_context_id(
+              storage_options[:map_id], child_context.context_id, storage_options[:parent_reactor_class_name]
+            )
+          end
+
           result
-        end
+        end.compact
       end
 
       def self.link_contexts(child_context, parent_context)
@@ -70,7 +83,7 @@ module RubyReactor
       def self.store_result(result, index, options)
         value = result.success? ? result.value : { _error: result.error }
         options[:storage].store_map_result(
-          options[:map_id], index, value, options[:parent_reactor_class_name],
+          options[:map_id], index, ContextSerializer.serialize_value(value), options[:parent_reactor_class_name],
           strict_ordering: options[:strict_ordering]
         )
       end

data/lib/ruby_reactor/map/helpers.rb

@@ -54,9 +54,9 @@ module RubyReactor
       # Resumes parent reactor execution after map completion
       def resume_parent_execution(parent_context, step_name, final_result, storage)
         executor = RubyReactor::Executor.new(parent_context.reactor_class, {}, parent_context)
+        step_name_sym = step_name.to_sym
 
         if final_result.failure?
-          step_name_sym = step_name.to_sym
           parent_context.current_step = step_name_sym
 
           error = RubyReactor::Error::StepFailureError.new(
@@ -77,7 +77,7 @@ module RubyReactor
           # Manually update context status since we're not running executor loop
           executor.send(:update_context_status, failure_response)
         else
-          parent_context.set_result(step_name.to_sym, final_result.value)
+          parent_context.set_result(step_name_sym, final_result.value)
 
           # Manually update execution trace to reflect completion
           # This is necessary because resume_execution continues from the NEXT step

data/lib/ruby_reactor/map/result_enumerator.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    class ResultEnumerator
+      include Enumerable
+
+      DEFAULT_BATCH_SIZE = 1000
+
+      attr_reader :map_id, :reactor_class_name, :strict_ordering, :batch_size
+
+      def initialize(map_id, reactor_class_name, strict_ordering: true, batch_size: DEFAULT_BATCH_SIZE)
+        @map_id = map_id
+        @reactor_class_name = reactor_class_name
+        @strict_ordering = strict_ordering
+        @batch_size = batch_size
+        @storage = RubyReactor.configuration.storage_adapter
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+
+        if @strict_ordering
+          count.times do |i|
+            yield self[i]
+          end
+        else
+          offset = 0
+          loop do
+            results = @storage.retrieve_map_results_batch(
+              @map_id,
+              @reactor_class_name,
+              offset: offset,
+              limit: @batch_size,
+              strict_ordering: @strict_ordering
+            )
+
+            break if results.empty?
+
+            results.each { |result| yield wrap_result(result) }
+
+            offset += results.size
+            break if results.size < @batch_size
+          end
+        end
+      end
+
+      def count
+        @count ||= @storage.count_map_results(@map_id, @reactor_class_name)
+      end
+      alias size count
+      alias length count
+
+      def empty?
+        count.zero?
+      end
+
+      def any?
+        !empty?
+      end
+
+      def [](index)
+        return nil if index < 0 || index >= count
+
+        results = @storage.retrieve_map_results_batch(
+          @map_id,
+          @reactor_class_name,
+          offset: index,
+          limit: 1,
+          strict_ordering: @strict_ordering
+        )
+
+        return nil if results.empty?
+
+        wrap_result(results.first)
+      end
+
+      def first
+        self[0]
+      end
+
+      def last
+        self[count - 1]
+      end
+
+      def successes
+        lazy.select { |result| result.is_a?(RubyReactor::Success) }.map(&:value)
+      end
+
+      def failures
+        lazy.select { |result| result.is_a?(RubyReactor::Failure) }.map(&:error)
+      end
+
+      private
+
+      def wrap_result(result)
+        if result.is_a?(Hash) && result.key?("_error")
+          RubyReactor::Failure.new(result["_error"])
+        else
+          RubyReactor::Success.new(ContextSerializer.deserialize_value(result))
+        end
+      end
+    end
+  end
+end

data/lib/ruby_reactor/step/map_step.rb

@@ -28,6 +28,9 @@ module RubyReactor
           inputs = {}
 
           mappings.each do |mapped_input_name, source|
+            # Handle serialized template objects (Hashes from Sidekiq)
+            source = ContextSerializer.deserialize_value(source) if source.is_a?(Hash) && source["_type"]
+
             value = if source.is_a?(RubyReactor::Template::Element)
                       # Handle element reference
                       # For now assuming element() refers to the current map's element
@@ -133,8 +136,11 @@ module RubyReactor
             RubyReactor::Success(results)
           else
             # New behavior: extract successful values only
-            successes = results.select(&:success?).map(&:value)
-            RubyReactor::Success(successes)
+            # New behavior: extract successful values only IF fail_fast is true behavior implies only values
+            # However, if fail_fast is false, we want to return results as is, or if logic dictates otherwise.
+            # wait, if fail_fast=false, we expect Result objects so we can check if success/failure.
+            # If we return only successes, we hide failures.
+            RubyReactor::Success(results)
           end
         end
 
@@ -157,18 +163,9 @@ module RubyReactor
 
           reactor_class_info = build_reactor_class_info(arguments[:mapped_reactor_class], context, step_name)
 
-          job_id = if arguments[:batch_size]
-                     storage = RubyReactor.configuration.storage_adapter
-                     storage.set_last_queued_index(map_id, arguments[:batch_size] - 1, context.reactor_class.name)
-                     queue_fan_out(
-                       map_id: map_id, arguments: arguments, context: context,
-                       reactor_class_info: reactor_class_info, step_name: step_name,
-                       limit: arguments[:batch_size]
-                     )
-                   else
-                     queue_single_worker(map_id: map_id, arguments: arguments, context: context,
-                                         reactor_class_info: reactor_class_info, step_name: step_name)
-                   end
+          initialize_map_metadata(map_id, arguments, context, reactor_class_info)
+
+          job_id = dispatch_async_map(map_id, arguments, context, reactor_class_info, step_name)
 
           # Store reference in composed_contexts so the UI knows where to find elements
           context.composed_contexts[step_name.to_s] = {
@@ -178,7 +175,42 @@ module RubyReactor
             element_reactor_class: arguments[:mapped_reactor_class].name
           }
 
-          RubyReactor::AsyncResult.new(job_id: job_id, intermediate_results: context.intermediate_results)
+          RubyReactor::AsyncResult.new(
+            job_id: job_id,
+            intermediate_results: context.intermediate_results,
+            execution_id: context.context_id
+          )
+        end
+
+        def initialize_map_metadata(map_id, arguments, context, reactor_class_info)
+          storage = RubyReactor.configuration.storage_adapter
+          storage.initialize_map_operation(
+            map_id, arguments[:source].size, context.reactor_class.name,
+            strict_ordering: arguments[:strict_ordering], reactor_class_info: reactor_class_info
+          )
+        end
+
+        def dispatch_async_map(map_id, arguments, context, reactor_class_info, step_name)
+          if arguments[:batch_size]
+            # Use new Dispatcher with Backpressure
+            RubyReactor::Map::Dispatcher.perform(
+              map_id: map_id,
+              parent_context_id: context.context_id,
+              parent_reactor_class_name: context.reactor_class.name,
+              source: arguments[:source],
+              batch_size: arguments[:batch_size],
+              step_name: step_name,
+              argument_mappings: arguments[:argument_mappings],
+              strict_ordering: arguments[:strict_ordering],
+              mapped_reactor_class: arguments[:mapped_reactor_class],
+              fail_fast: arguments[:fail_fast]
+            )
+            queue_collector(map_id, context, step_name, arguments[:strict_ordering])
+            "map:#{map_id}"
+          else
+            queue_single_worker(map_id: map_id, arguments: arguments, context: context,
+                                reactor_class_info: reactor_class_info, step_name: step_name)
+          end
         end
 
         def prepare_async_execution(context, map_id, count)
@@ -252,7 +284,7 @@ module RubyReactor
             map_id: map_id, serialized_inputs: serialized_inputs,
             reactor_class_info: reactor_class_info, strict_ordering: arguments[:strict_ordering],
             parent_context_id: context.context_id, parent_reactor_class_name: context.reactor_class.name,
-            step_name: step_name.to_s
+            step_name: step_name.to_s, fail_fast: arguments[:fail_fast]
           )
         end
       end

data/lib/ruby_reactor/storage/redis_adapter.rb

@@ -230,6 +230,61 @@ module RubyReactor
         @redis.get(key)
       end
 
+      def set_map_offset(map_id, offset, reactor_class_name)
+        key = map_offset_key(map_id, reactor_class_name)
+        @redis.set(key, offset, ex: 86_400)
+      end
+
+      def set_map_offset_if_not_exists(map_id, offset, reactor_class_name)
+        key = map_offset_key(map_id, reactor_class_name)
+        @redis.set(key, offset, nx: true, ex: 86_400)
+      end
+
+      def retrieve_map_offset(map_id, reactor_class_name)
+        key = map_offset_key(map_id, reactor_class_name)
+        @redis.get(key)
+      end
+
+      def increment_map_offset(map_id, increment, reactor_class_name)
+        key = map_offset_key(map_id, reactor_class_name)
+        @redis.incrby(key, increment)
+      end
+
+      def retrieve_map_results_batch(map_id, reactor_class_name, offset:, limit:, strict_ordering: true)
+        key = map_results_key(map_id, reactor_class_name)
+
+        if strict_ordering
+          # For Hash based results (indexed), we can use HMGET if we know the keys.
+          # Since we use 0-based index keys, we can generate the keys for the batch.
+          fields = (offset...(offset + limit)).map(&:to_s)
+          results = @redis.hmget(key, *fields)
+
+          # HMGET returns nil for missing fields, compact them?
+          # Or should we respect the holes?
+          # Map results are usually dense.
+          results.compact.map { |r| JSON.parse(r) }
+        else
+          # For List based results
+          # LRANGE uses inclusive ending index
+          end_index = offset + limit - 1
+          results = @redis.lrange(key, offset, end_index)
+          results.map { |r| JSON.parse(r) }
+        end
+      end
+
+      def count_map_results(map_id, reactor_class_name)
+        key = map_results_key(map_id, reactor_class_name)
+        type = @redis.type(key)
+
+        if type == "hash"
+          @redis.hlen(key)
+        elsif type == "list"
+          @redis.llen(key)
+        else
+          0
+        end
+      end
+
       private
 
       def fetch_and_filter_reactors(keys)
@@ -267,6 +322,10 @@ module RubyReactor
         "reactor:#{reactor_class_name}:map:#{map_id}:last_queued_index"
       end
 
+      def map_offset_key(map_id, reactor_class_name)
+        "reactor:#{reactor_class_name}:map:#{map_id}:offset"
+      end
+
       def correlation_id_key(correlation_id, reactor_class_name)
         "reactor:#{reactor_class_name}:correlation:#{correlation_id}"
       end