ruby_reactor 0.1.0 → 0.3.0

data/lib/ruby_reactor/map/dispatcher.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    class Dispatcher
+      extend Helpers
+
+      def self.perform(arguments)
+        arguments = arguments.transform_keys(&:to_sym)
+        parent_reactor_class_name = arguments[:parent_reactor_class_name]
+
+        storage = RubyReactor.configuration.storage_adapter
+
+        # Load parent context to resolve source
+        parent_context = load_parent_context_from_storage(
+          arguments[:parent_context_id],
+          parent_reactor_class_name,
+          storage
+        )
+
+        # Initialize metadata if first run
+        initialize_map_metadata(arguments, storage) unless arguments[:continuation]
+
+        # Resolve Source
+        # We need to resolve the source to know what we are iterating.
+        # Strict "Array Only" rule means we expect an Array-like object or we handle the
+        # "Query Builder" result if user used it.
+        source = resolve_source(arguments, parent_context)
+
+        # Dispatch next batch
+        dispatch_batch(source, arguments, parent_context, storage)
+      end
+
+      def self.initialize_map_metadata(arguments, storage)
+        map_id = arguments[:map_id]
+        reactor_class_name = arguments[:parent_reactor_class_name]
+
+        # Reset or set initial offset. Use NX to act as a mutex/guard against duplicate initialization.
+        storage.set_map_offset_if_not_exists(map_id, 0, reactor_class_name)
+      end
+
+      def self.resolve_source(arguments, context)
+        # Arguments has :source which is a Template::Input or similar.
+        # We need to resolve it against the context.
+        source_template = arguments[:source]
+
+        # Fallback: look up from step config if missing (e.g. called from ElementExecutor)
+        if source_template.nil? && context
+          step_name = arguments[:step_name]
+          step_config = context.reactor_class.steps[step_name.to_sym]
+          source_template = step_config.arguments[:source][:source]
+        end
+
+        # If source is packaged in arguments as a value (deserialized)
+        return source_template if source_template.is_a?(Array)
+
+        # Resolve template
+        return source_template.resolve(context) if source_template.respond_to?(:resolve)
+
+        source_template
+      end
+
+      def self.dispatch_batch(source, arguments, parent_context, storage)
+        map_id = arguments[:map_id]
+        reactor_class_name = arguments[:parent_reactor_class_name]
+
+        # Fail Fast Check
+        if arguments[:fail_fast]
+          failed_context_id = storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
+          return if failed_context_id
+        end
+
+        batch_size = arguments[:batch_size] || source.size # Default to all if no batch_size (async=true only)
+
+        # Atomically reserve a batch
+        new_offset = storage.increment_map_offset(map_id, batch_size, reactor_class_name)
+        current_offset = new_offset - batch_size
+
+        batch_elements = if source.is_a?(Array)
+                           source.slice(current_offset, batch_size) || []
+                         elsif source.respond_to?(:offset) && source.respond_to?(:limit)
+                           # Optimized for ActiveRecord and similar query builders
+                           source.offset(current_offset).limit(batch_size).to_a
+                         else
+                           # Fallback for generic Enumerable
+                           # This is inefficient for huge sets if not Array, but compliant
+                           source.drop(current_offset).take(batch_size)
+                         end
+
+        return if batch_elements.empty?
+
+        # Queue Jobs
+        queue_options = {
+          map_id: map_id,
+          arguments: arguments,
+          context: parent_context,
+          reactor_class_info: resolve_reactor_class_info(arguments, parent_context),
+          step_name: arguments[:step_name]
+        }
+
+        batch_elements.each_with_index do |element, i|
+          absolute_index = current_offset + i
+          queue_element_job(element, absolute_index, queue_options)
+        end
+      end
+
+      def self.queue_element_job(element, index, options)
+        arguments = options[:arguments]
+        context = options[:context]
+
+        # Resolve mappings
+        mappings_template = arguments[:argument_mappings]
+
+        # Fallback: look up from step config if missing (e.g. called from ElementExecutor)
+        if mappings_template.nil? && context
+          step_name = options[:step_name] || arguments[:step_name]
+          step_config = context.reactor_class.steps[step_name.to_sym]
+          mappings_template = step_config.arguments[:argument_mappings]
+        end
+
+        mappings = if mappings_template.respond_to?(:resolve)
+                     mappings_template.resolve(context)
+                   else
+                     mappings_template || {}
+                   end
+
+        # Fix for weird structure observed in fallback (wrapped in :source -> Template::Value)
+        if mappings.key?(:source) && mappings[:source].respond_to?(:value) && mappings[:source].value.is_a?(Hash)
+          mappings = mappings[:source].value
+        end
+
+        mapped_inputs = build_element_inputs(mappings, context, element)
+        serialized_inputs = ContextSerializer.serialize_value(mapped_inputs)
+
+        RubyReactor.configuration.async_router.perform_map_element_async(
+          map_id: options[:map_id],
+          element_id: "#{options[:map_id]}:#{index}",
+          index: index,
+          serialized_inputs: serialized_inputs,
+          reactor_class_info: options[:reactor_class_info],
+          strict_ordering: arguments[:strict_ordering],
+          parent_context_id: context.context_id,
+          parent_reactor_class_name: context.reactor_class.name,
+          step_name: options[:step_name].to_s,
+          batch_size: arguments[:batch_size], # Passed to worker so it knows to trigger next batch?
+          fail_fast: arguments[:fail_fast]
+        )
+      end
+
+      def self.resolve_reactor_class_info(arguments, context)
+        mapped_reactor_class = arguments[:mapped_reactor_class]
+        step_name = arguments[:step_name]
+
+        if mapped_reactor_class.respond_to?(:name)
+          { "type" => "class", "name" => mapped_reactor_class.name }
+        else
+          { "type" => "inline", "parent" => context.reactor_class.name, "step" => step_name.to_s }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_reactor/map/element_executor.rb

@@ -25,6 +25,11 @@ module RubyReactor
           context = ContextSerializer.deserialize(serialized_context)
           context.map_metadata = arguments
           reactor_class = context.reactor_class
+
+          # Ensure inputs are present (fallback to serialized_inputs if missing from context)
+          if context.inputs.empty? && serialized_inputs
+            context.inputs = ContextSerializer.deserialize_value(serialized_inputs)
+          end
         else
           # Deserialize inputs
           inputs = ContextSerializer.deserialize_value(serialized_inputs)
@@ -34,9 +39,33 @@ module RubyReactor
 
           # Create context
           context = Context.new(inputs, reactor_class)
+          context.parent_context_id = parent_context_id
           context.map_metadata = arguments
         end
+
         storage = RubyReactor.configuration.storage_adapter
+        storage.store_map_element_context_id(map_id, context.context_id, parent_reactor_class_name)
+
+        # Fail Fast Check
+        if arguments[:fail_fast]
+          failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name)
+          if failed_context_id
+            # Decrement counter as we are skipping execution
+            new_count = storage.decrement_map_counter(map_id, parent_reactor_class_name)
+            return unless new_count.zero?
+
+            # Trigger collection if we are the last one (skipped or otherwise)
+            RubyReactor.configuration.async_router.perform_map_collection_async(
+              parent_context_id: parent_context_id,
+              map_id: map_id,
+              parent_reactor_class_name: parent_reactor_class_name,
+              step_name: step_name,
+              strict_ordering: strict_ordering,
+              timeout: 3600
+            )
+            return
+          end
+        end
 
         # Execute
         executor = Executor.new(reactor_class, {}, context)
@@ -50,27 +79,35 @@ module RubyReactor
         result = executor.result
 
         if result.is_a?(RetryQueuedResult)
-          queue_next_batch(arguments) if batch_size
+          trigger_next_batch_if_needed(arguments, index, batch_size)
           return
         end
 
         # Store result
 
-        # Store result
-
         if result.success?
-          storage.store_map_result(map_id, index, result.value, parent_reactor_class_name,
+          storage.store_map_result(map_id, index,
+                                   ContextSerializer.serialize_value(result.value),
+                                   parent_reactor_class_name,
                                    strict_ordering: strict_ordering)
         else
+          # Trigger Compensation Logic
+          executor.undo_all
+
           # Store error
           storage.store_map_result(map_id, index, { _error: result.error }, parent_reactor_class_name,
                                    strict_ordering: strict_ordering)
+
+          if arguments[:fail_fast]
+            storage.store_map_failed_context_id(map_id, context.context_id, parent_reactor_class_name)
+          end
         end
 
         # Decrement counter
         new_count = storage.decrement_map_counter(map_id, parent_reactor_class_name)
 
-        queue_next_batch(arguments) if batch_size
+        # Trigger next batch if it's the last element of the current batch
+        trigger_next_batch_if_needed(arguments, index, batch_size)
 
         return unless new_count.zero?
 
@@ -85,23 +122,6 @@ module RubyReactor
         )
       end
 
-      def self.queue_next_batch(arguments)
-        storage = RubyReactor.configuration.storage_adapter
-        map_id = arguments[:map_id]
-        reactor_class_name = arguments[:parent_reactor_class_name]
-
-        next_index = storage.increment_last_queued_index(map_id, reactor_class_name)
-        total_count = storage.retrieve_map_metadata(map_id, reactor_class_name)["count"]
-
-        return unless next_index < total_count
-
-        parent_context = load_parent_context(arguments, reactor_class_name, storage)
-        element = resolve_next_element(arguments, parent_context, next_index)
-        serialized_inputs = build_serialized_inputs(arguments, parent_context, element)
-
-        queue_element_job(arguments, map_id, next_index, serialized_inputs, reactor_class_name)
-      end
-
       def self.load_parent_context(arguments, reactor_class_name, storage)
         parent_context_data = storage.retrieve_context(arguments[:parent_context_id], reactor_class_name)
         parent_reactor_class = Object.const_get(reactor_class_name)
@@ -113,42 +133,28 @@ module RubyReactor
         parent_context
       end
 
-      def self.resolve_next_element(arguments, parent_context, next_index)
-        parent_reactor_class = parent_context.reactor_class
-        step_config = parent_reactor_class.steps[arguments[:step_name].to_sym]
-
-        source_template = step_config.arguments[:source][:source]
-        source = source_template.resolve(parent_context)
-        source[next_index]
+      # Legacy helpers resolved_next_element, build_serialized_inputs, queue_element_job
+      # are REMOVED as they are no longer used for self-queuing.
+
+      # Basic helper to build inputs for the CURRENT element (still needed for perform)
+      # Wait, perform uses `serialized_inputs` passed to it.
+      # We don't need `build_element_inputs` here?
+      # `perform` uses `params[:serialized_inputs]`.
+      # So we can remove input building helpers too?
+      # Let's check if they are used elsewhere.
+      # `resolve_reactor_class` is used in `perform`.
+      # `build_element_inputs` is likely in Helpers or mixed in?
+
+      def self.trigger_next_batch_if_needed(arguments, index, batch_size)
+        return unless batch_size && ((index + 1) % batch_size).zero?
+
+        # Trigger Dispatcher for next batch
+        next_batch_args = arguments.dup
+        next_batch_args[:continuation] = true
+        RubyReactor::Map::Dispatcher.perform(next_batch_args)
       end
 
-      def self.build_serialized_inputs(arguments, parent_context, element)
-        parent_reactor_class = parent_context.reactor_class
-        step_config = parent_reactor_class.steps[arguments[:step_name].to_sym]
-
-        mappings_template = step_config.arguments[:argument_mappings][:source]
-        mappings = mappings_template.resolve(parent_context) || {}
-
-        mapped_inputs = build_element_inputs(mappings, parent_context, element)
-        ContextSerializer.serialize_value(mapped_inputs)
-      end
-
-      def self.queue_element_job(arguments, map_id, next_index, serialized_inputs, reactor_class_name)
-        RubyReactor.configuration.async_router.perform_map_element_async(
-          map_id: map_id,
-          element_id: "#{map_id}:#{next_index}",
-          index: next_index,
-          serialized_inputs: serialized_inputs,
-          reactor_class_info: arguments[:reactor_class_info],
-          strict_ordering: arguments[:strict_ordering],
-          parent_context_id: arguments[:parent_context_id],
-          parent_reactor_class_name: reactor_class_name,
-          step_name: arguments[:step_name],
-          batch_size: arguments[:batch_size]
-        )
-      end
-      private_class_method :queue_next_batch, :load_parent_context,
-                           :resolve_next_element, :build_serialized_inputs, :queue_element_job
+      private_class_method :load_parent_context, :trigger_next_batch_if_needed
     end
   end
 end

data/lib/ruby_reactor/map/execution.rb

@@ -21,7 +21,8 @@ module RubyReactor
           storage_options: {
             map_id: arguments[:map_id], storage: storage,
             parent_reactor_class_name: arguments[:parent_reactor_class_name],
-            strict_ordering: arguments[:strict_ordering]
+            strict_ordering: arguments[:strict_ordering],
+            fail_fast: arguments[:fail_fast]
           }
         )
 
@@ -31,19 +32,58 @@ module RubyReactor
 
       def self.execute_all_elements(source:, mappings:, reactor_class:, parent_context:, storage_options:)
         source.map.with_index do |element, index|
+          if storage_options[:fail_fast]
+            failed_context_id = storage_options[:storage].retrieve_map_failed_context_id(
+              storage_options[:map_id], storage_options[:parent_reactor_class_name]
+            )
+            next if failed_context_id
+          end
           element_inputs = build_element_inputs(mappings, parent_context, element)
-          result = reactor_class.run(element_inputs)
+
+          # Manually create and link context to ensure parent_context_id is set
+          child_context = RubyReactor::Context.new(element_inputs, reactor_class)
+          link_contexts(child_context, parent_context)
+
+          # Ensure we store the element context linkage
+          storage_options[:storage].store_map_element_context_id(
+            storage_options[:map_id], child_context.context_id, storage_options[:parent_reactor_class_name]
+          )
+
+          # Set map metadata for failure handling
+          metadata = {
+            map_id: storage_options[:map_id],
+            parent_reactor_class_name: storage_options[:parent_reactor_class_name],
+            index: index
+          }
+          child_context.map_metadata = metadata
+
+          executor = RubyReactor::Executor.new(reactor_class, {}, child_context)
+          executor.execute
+          result = executor.result
 
           store_result(result, index, storage_options)
 
+          if result.failure? && storage_options[:fail_fast]
+            storage_options[:storage].store_map_failed_context_id(
+              storage_options[:map_id], child_context.context_id, storage_options[:parent_reactor_class_name]
+            )
+          end
+
           result
-        end
+        end.compact
+      end
+
+      def self.link_contexts(child_context, parent_context)
+        child_context.parent_context = parent_context
+        child_context.root_context = parent_context.root_context || parent_context
+        child_context.test_mode = parent_context.test_mode
+        child_context.inline_async_execution = parent_context.inline_async_execution
       end
 
       def self.store_result(result, index, options)
         value = result.success? ? result.value : { _error: result.error }
         options[:storage].store_map_result(
-          options[:map_id], index, value, options[:parent_reactor_class_name],
+          options[:map_id], index, ContextSerializer.serialize_value(value), options[:parent_reactor_class_name],
           strict_ordering: options[:strict_ordering]
         )
       end

data/lib/ruby_reactor/map/helpers.rb

@@ -7,7 +7,11 @@ module RubyReactor
       # Resolves the reactor class from reactor_class_info
       def resolve_reactor_class(info)
         if info["type"] == "class"
-          Object.const_get(info["name"])
+          begin
+            Object.const_get(info["name"])
+          rescue NameError
+            RubyReactor::Registry.find(info["name"])
+          end
         elsif info["type"] == "inline"
           parent_class = Object.const_get(info["parent"])
           step_config = parent_class.steps[info["step"].to_sym]
@@ -49,12 +53,46 @@ module RubyReactor
 
       # Resumes parent reactor execution after map completion
       def resume_parent_execution(parent_context, step_name, final_result, storage)
-        value = final_result.success? ? final_result.value : final_result
-        parent_context.set_result(step_name.to_sym, value)
-        parent_context.current_step = nil
-
         executor = RubyReactor::Executor.new(parent_context.reactor_class, {}, parent_context)
-        executor.resume_execution
+        step_name_sym = step_name.to_sym
+
+        if final_result.failure?
+          parent_context.current_step = step_name_sym
+
+          error = RubyReactor::Error::StepFailureError.new(
+            final_result.error,
+            step: step_name_sym,
+            context: parent_context,
+            original_error: final_result.error.is_a?(Exception) ? final_result.error : nil
+          )
+
+          # Pass backtrace if available
+          if final_result.respond_to?(:backtrace) && final_result.backtrace
+            error.set_backtrace(final_result.backtrace)
+          elsif final_result.error.respond_to?(:backtrace)
+            error.set_backtrace(final_result.error.backtrace)
+          end
+
+          failure_response = executor.result_handler.handle_execution_error(error)
+          # Manually update context status since we're not running executor loop
+          executor.send(:update_context_status, failure_response)
+        else
+          parent_context.set_result(step_name_sym, final_result.value)
+
+          # Manually update execution trace to reflect completion
+          # This is necessary because resume_execution continues from the NEXT step
+          # and the async step (which returned AsyncResult) needs to be marked as done with actual value
+          parent_context.execution_trace << {
+            type: :result,
+            step: step_name_sym,
+            timestamp: Time.now,
+            value: final_result.value,
+            status: :success
+          }
+
+          parent_context.current_step = nil
+          executor.resume_execution
+        end
 
         storage.store_context(
           parent_context.context_id,

data/lib/ruby_reactor/map/result_enumerator.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    class ResultEnumerator
+      include Enumerable
+
+      DEFAULT_BATCH_SIZE = 1000
+
+      attr_reader :map_id, :reactor_class_name, :strict_ordering, :batch_size
+
+      def initialize(map_id, reactor_class_name, strict_ordering: true, batch_size: DEFAULT_BATCH_SIZE)
+        @map_id = map_id
+        @reactor_class_name = reactor_class_name
+        @strict_ordering = strict_ordering
+        @batch_size = batch_size
+        @storage = RubyReactor.configuration.storage_adapter
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+
+        if @strict_ordering
+          count.times do |i|
+            yield self[i]
+          end
+        else
+          offset = 0
+          loop do
+            results = @storage.retrieve_map_results_batch(
+              @map_id,
+              @reactor_class_name,
+              offset: offset,
+              limit: @batch_size,
+              strict_ordering: @strict_ordering
+            )
+
+            break if results.empty?
+
+            results.each { |result| yield wrap_result(result) }
+
+            offset += results.size
+            break if results.size < @batch_size
+          end
+        end
+      end
+
+      def count
+        @count ||= @storage.count_map_results(@map_id, @reactor_class_name)
+      end
+      alias size count
+      alias length count
+
+      def empty?
+        count.zero?
+      end
+
+      def any?
+        !empty?
+      end
+
+      def [](index)
+        return nil if index < 0 || index >= count
+
+        results = @storage.retrieve_map_results_batch(
+          @map_id,
+          @reactor_class_name,
+          offset: index,
+          limit: 1,
+          strict_ordering: @strict_ordering
+        )
+
+        return nil if results.empty?
+
+        wrap_result(results.first)
+      end
+
+      def first
+        self[0]
+      end
+
+      def last
+        self[count - 1]
+      end
+
+      def successes
+        lazy.select { |result| result.is_a?(RubyReactor::Success) }.map(&:value)
+      end
+
+      def failures
+        lazy.select { |result| result.is_a?(RubyReactor::Failure) }.map(&:error)
+      end
+
+      private
+
+      def wrap_result(result)
+        if result.is_a?(Hash) && result.key?("_error")
+          RubyReactor::Failure.new(result["_error"])
+        else
+          RubyReactor::Success.new(ContextSerializer.deserialize_value(result))
+        end
+      end
+    end
+  end
+end