ruby_reactor 0.2.0 → 0.3.1

data/lib/ruby_reactor/configuration.rb

@@ -22,7 +22,7 @@ module RubyReactor
     end
 
     def async_router
-      @async_router ||= RubyReactor::AsyncRouter
+      @async_router ||= RubyReactor::SidekiqAdapter
     end
 
     def storage

data/lib/ruby_reactor/context.rb

@@ -3,7 +3,7 @@
 module RubyReactor
   class Context
     attr_accessor :inputs, :intermediate_results, :private_data, :current_step, :retry_count, :concurrency_key,
-                  :retry_context, :reactor_class, :execution_trace, :inline_async_execution, :undo_stack, :test_mode,
+                  :retry_context, :reactor_class, :execution_trace, :inline_async_execution, :undo_stack,
                   :parent_context, :root_context, :composed_contexts, :context_id, :map_operations, :map_metadata,
                   :cancelled, :cancellation_reason, :parent_context_id, :status, :failure_reason
 
@@ -23,7 +23,6 @@ module RubyReactor
       @execution_trace = []
       @inline_async_execution = false # Flag to prevent nested async calls
       @undo_stack = [] # Initialize the undo stack
-      @test_mode = false
       @cancelled = false
       @cancellation_reason = nil
       @status = "pending"
@@ -33,6 +32,14 @@ module RubyReactor
       @root_context = nil
     end
 
+    def finished?
+      %w[completed failed cancelled].include?(@status.to_s)
+    end
+
+    def failed?
+      @status.to_s == "failed"
+    end
+
     def get_input(name, path = nil)
       value = @inputs[name.to_sym] || @inputs[name.to_s]
       return nil if value.nil?
@@ -57,6 +64,10 @@ module RubyReactor
     end
     alias result get_result
 
+    def has_result?(step_name)
+      @intermediate_results.key?(step_name.to_sym) || @intermediate_results.key?(step_name.to_s)
+    end
+
     def set_result(step_name, value)
       @intermediate_results[step_name.to_sym] = value
     end
@@ -81,7 +92,6 @@ module RubyReactor
         retry_context: @retry_context,
         reactor_class: @reactor_class,
         execution_trace: @execution_trace,
-        test_mode: @test_mode,
         status: @status,
         failure_reason: @failure_reason
       }
@@ -105,7 +115,6 @@ module RubyReactor
         retry_context: @retry_context.serialize_for_retry,
         execution_trace: ContextSerializer.serialize_value(@execution_trace),
         undo_stack: serialize_undo_stack,
-        test_mode: @test_mode,
         cancelled: @cancelled,
         cancellation_reason: @cancellation_reason,
         status: @status,
@@ -130,7 +139,6 @@ module RubyReactor
       context.retry_context = RetryContext.deserialize_from_retry(data["retry_context"] || {})
       context.execution_trace = ContextSerializer.deserialize_value(data["execution_trace"]) || []
       context.undo_stack = deserialize_undo_stack(data["undo_stack"] || [], context.reactor_class)
-      context.test_mode = data["test_mode"] || false
       context.cancelled = data["cancelled"] || false
       context.cancellation_reason = data["cancellation_reason"]
       context.status = data["status"] || "pending"

data/lib/ruby_reactor/context_serializer.rb

@@ -34,9 +34,25 @@ module RubyReactor
         when RubyReactor::Success
           { "_type" => "Success", "value" => serialize_value(value.value) }
         when RubyReactor::Failure
-          { "_type" => "Failure", "error" => serialize_value(value.error), "retryable" => value.retryable }
+          {
+            "_type" => "Failure",
+            "error" => serialize_value(value.error),
+            "retryable" => value.retryable,
+            "step_name" => value.step_name,
+            "inputs" => serialize_value(value.inputs),
+            "backtrace" => value.backtrace,
+            "reactor_name" => value.reactor_name,
+            "step_arguments" => serialize_value(value.step_arguments),
+            "exception_class" => value.exception_class,
+            "file_path" => value.file_path,
+            "line_number" => value.line_number,
+            "code_snippet" => serialize_value(value.code_snippet),
+            "validation_errors" => serialize_value(value.validation_errors)
+          }
         when RubyReactor::Context
           { "_type" => "Context", "value" => value.serialize_for_retry }
+        when Symbol
+          { "_type" => "Symbol", "value" => value.to_s }
         when Time
           { "_type" => "Time", "value" => value.iso8601 }
         when BigDecimal
@@ -68,6 +84,14 @@ module RubyReactor
           { "_type" => "Template::Value", "value" => serialize_value(value.instance_variable_get(:@value)) }
         when RubyReactor::Template::Result
           { "_type" => "Template::Result", "step_name" => value.step_name.to_s, "path" => value.path }
+        when RubyReactor::Map::ResultEnumerator
+          {
+            "_type" => "Map::ResultEnumerator",
+            "map_id" => value.map_id,
+            "reactor_class_name" => value.reactor_class_name,
+            "strict_ordering" => value.strict_ordering,
+            "batch_size" => value.batch_size
+          }
         when Hash
           value.transform_keys(&:to_s).transform_values { |v| serialize_value(v) }
         when Array
@@ -86,9 +110,24 @@ module RubyReactor
             when "Success"
               RubyReactor::Success(deserialize_value(value["value"]))
             when "Failure"
-              RubyReactor::Failure(deserialize_value(value["error"]), retryable: value["retryable"])
+              RubyReactor::Failure.new(
+                deserialize_value(value["error"]),
+                retryable: value["retryable"],
+                step_name: value["step_name"],
+                inputs: deserialize_value(value["inputs"]),
+                backtrace: value["backtrace"],
+                reactor_name: value["reactor_name"],
+                step_arguments: deserialize_value(value["step_arguments"]),
+                exception_class: value["exception_class"],
+                file_path: value["file_path"],
+                line_number: value["line_number"],
+                code_snippet: deserialize_value(value["code_snippet"]),
+                validation_errors: deserialize_value(value["validation_errors"])
+              )
             when "Context"
               Context.deserialize_from_retry(value["value"])
+            when "Symbol"
+              value["value"].to_sym
             when "Time"
               Time.iso8601(value["value"])
             when "BigDecimal"
@@ -115,11 +154,20 @@ module RubyReactor
               RubyReactor::Template::Value.new(deserialize_value(value["value"]))
             when "Template::Result"
               RubyReactor::Template::Result.new(value["step_name"], value["path"])
+            when "Map::ResultEnumerator"
+              RubyReactor::Map::ResultEnumerator.new(
+                value["map_id"],
+                value["reactor_class_name"],
+                strict_ordering: value["strict_ordering"],
+                batch_size: value["batch_size"]
+              )
+
             else
-              value
+              # Unknown type wrapper, return as is (but deserialize values)
+              value.transform_values { |v| deserialize_value(v) }
             end
           else
-            # Regular hash - symbolize all keys recursively
+            # Regular Hash
             value.transform_keys(&:to_sym).transform_values { |v| deserialize_value(v) }
           end
         when Array
@@ -128,6 +176,24 @@ module RubyReactor
           value
         end
       end
+
+      # Simplifies data for public API usage (removes wrappers, flattens types)
+      def simplify_for_api(value)
+        case value
+        when Hash
+          value.each_with_object({}) do |(k, v), hash|
+            hash[k.to_s] = simplify_for_api(v)
+          end
+        when Array
+          value.map { |v| simplify_for_api(v) }
+        when Success, Failure, Context
+          simplify_for_api(value.to_h)
+        when Symbol
+          value.to_s
+        else
+          value
+        end
+      end
       # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength
 
       private

data/lib/ruby_reactor/dsl/map_builder.rb

@@ -32,8 +32,12 @@ module RubyReactor
         @argument_mappings[mapped_input_name] = source
       end
 
-      def source(enumerable)
-        @source_enumerable = enumerable
+      def source(enumerable = nil, &block)
+        @source_enumerable = if block
+                               RubyReactor::Template::DynamicSource.new(@argument_mappings, &block)
+                             else
+                               enumerable
+                             end
       end
 
       def async(async = true, batch_size: nil)

data/lib/ruby_reactor/dsl/reactor.rb

@@ -118,8 +118,9 @@ module RubyReactor
           step_config
         end
 
-        def returns(step_name)
-          @return_step = step_name
+        def returns(step_name = nil)
+          @return_step = step_name if step_name
+          @return_step
         end
 
         def middleware(middleware_class)

data/lib/ruby_reactor/error/step_failure_error.rb

@@ -3,11 +3,14 @@
 module RubyReactor
   module Error
     class StepFailureError < Base
-      attr_reader :step_arguments
+      attr_reader :step_arguments, :exception_class
 
-      def initialize(message, step: nil, context: nil, original_error: nil, step_arguments: {})
+      # rubocop:disable Metrics/ParameterLists
+      def initialize(message, step: nil, context: nil, original_error: nil, step_arguments: {}, exception_class: nil)
+        # rubocop:enable Metrics/ParameterLists
         super(message, step: step, context: context, original_error: original_error)
         @step_arguments = step_arguments
+        @exception_class = exception_class
       end
 
       def retryable?

data/lib/ruby_reactor/executor/result_handler.rb

@@ -31,7 +31,7 @@ module RubyReactor
           handle_step_failure_error(error)
         when Error::InputValidationError
           # Preserve validation errors as-is for proper error handling
-          RubyReactor.Failure(error)
+          RubyReactor.Failure(error, validation_errors: error.field_errors)
         when Error::Base
           # Other errors need rollback
           @compensation_manager.rollback_completed_steps
@@ -117,6 +117,7 @@ module RubyReactor
 
       def store_failed_map_context(context)
         return unless context.map_metadata && context.map_metadata[:map_id]
+        return unless context.map_metadata[:fail_fast]
 
         storage = RubyReactor.configuration.storage_adapter
         storage.store_map_failed_context_id(
@@ -128,7 +129,7 @@ module RubyReactor
 
       def create_failure_from_error(error, redact_inputs)
         original_error = error.original_error
-        exception_class = original_error&.class&.name
+        exception_class = resolve_exception_class(original_error, error)
         backtrace = original_error&.backtrace || error.backtrace
         file_path, line_number = extract_location(backtrace)
         code_snippet = RubyReactor::Utils::CodeExtractor.extract(file_path, line_number) if file_path
@@ -148,6 +149,12 @@ module RubyReactor
         )
       end
 
+      def resolve_exception_class(original_error, error)
+        return original_error.class.name if original_error
+
+        error.respond_to?(:exception_class) ? error.exception_class : nil
+      end
+
       def validate_step_output(step_config, value, resolved_arguments = {})
         return unless step_config.output_validator
 

data/lib/ruby_reactor/executor/retry_manager.rb

@@ -39,7 +39,17 @@ module RubyReactor
 
         # Serialize context and requeue the job
         # Use root context if available to ensure we serialize the full tree
-        context_to_serialize = @context.root_context || @context
+        # BUT for map elements (which have map_metadata), we must serialize the element context itself
+
+        context_to_serialize = if @context.map_metadata
+                                 @context
+                               else
+                                 @context.root_context || @context
+                               end
+
+        puts "SERIALIZING CONTEXT: #{context_to_serialize.reactor_class.name}"
+        puts "INPUTS KEYS: #{context_to_serialize.inputs.keys}" if context_to_serialize.respond_to?(:inputs)
+
         reactor_class_name = context_to_serialize.reactor_class.name
 
         serialized_context = ContextSerializer.serialize(context_to_serialize)
@@ -106,7 +116,8 @@ module RubyReactor
                    @context.root_context&.reactor_class&.async? ||
                    @context.inline_async_execution
 
-        if is_async && !@context.test_mode
+        # Always try async retry if configured
+        if is_async
           handle_async_retry(step_config, reactor_class, result)
         else
           handle_sync_retry(step_config, reactor_class, result)
@@ -114,12 +125,19 @@ module RubyReactor
       end
 
       def handle_async_retry(step_config, reactor_class, result)
-        requeue_job_for_step_retry(step_config, result.error, reactor_class)
-        RetryQueuedResult.new(
-          step_config.name,
-          @context.retry_context.attempts_for_step(step_config.name),
-          @context.retry_context.next_retry_at
-        )
+        requeue_result = requeue_job_for_step_retry(step_config, result.error, reactor_class)
+
+        # If it returned an AsyncResult, we are truly async.
+        # Otherwise, it ran inline and we should return the result of that execution.
+        if requeue_result.is_a?(RubyReactor::AsyncResult)
+          RetryQueuedResult.new(
+            step_config.name,
+            @context.retry_context.attempts_for_step(step_config.name),
+            @context.retry_context.next_retry_at
+          )
+        else
+          requeue_result
+        end
       end
 
       def handle_sync_retry(step_config, reactor_class, result)

data/lib/ruby_reactor/executor/step_executor.rb

@@ -13,7 +13,7 @@ module RubyReactor
       end
 
       def execute_all_steps
-        until @dependency_graph.all_completed?
+        until @dependency_graph.all_completed? || @context.finished?
           ready_steps = @dependency_graph.ready_steps
 
           if ready_steps.empty?
@@ -65,53 +65,29 @@ module RubyReactor
         end
       end
 
-      def merge_executor_state(other_executor)
-        # Merge the state from the async-executed executor back into ours
-        # We need to update our context IN PLACE, not replace the reference,
-        # because the Executor also holds a reference to the same context object
-
-        # Update intermediate results
-        other_executor.context.intermediate_results.each do |step_name, value|
-          @context.set_result(step_name, value)
-        end
-
-        # Append execution trace from the async execution
-        # The Worker's execution will have ALL steps including ones we already executed,
-        # but we only want to add the NEW entries (from current_step onwards)
-        current_trace_length = @context.execution_trace.length
-        new_trace_entries = other_executor.context.execution_trace[current_trace_length..] || []
-
-        @context.execution_trace.concat(new_trace_entries)
-
-        # Update retry context
-        @context.retry_context = other_executor.context.retry_context
-
-        # Update current_step:
-        # If the other executor has a current_step, it means it paused/interrupted there. We should adopt it.
-        # If it's nil, it means it completed successfully, so we clear our current_step (which was the async step).
-        @context.current_step = other_executor.context.current_step
-
-        # Update our dependency graph to reflect completed steps
-        other_executor.context.intermediate_results.each_key do |step_name|
-          @dependency_graph.complete_step(step_name)
-        end
-
-        # Also mark the current_step as completed if it exists (for failed steps that don't have results)
-        @dependency_graph.complete_step(other_executor.context.current_step) if other_executor.context.current_step
-
-        # Merge any undo stack items
-        other_executor.undo_stack.each do |item|
-          # Avoid duplicates by checking if this step is already in the undo stack
-          # Use string comparison for step names to avoid symbol/string mismatch issues
-          unless @compensation_manager.undo_stack.any? { |existing| existing[:step].name.to_s == item[:step].name.to_s }
-            @compensation_manager.add_to_undo_stack(item)
-          end
-        end
+      private
 
-        # Merge undo trace from the other executor
-        other_executor.undo_trace.each do |trace_entry|
-          @compensation_manager.undo_trace << trace_entry
-        end
+      def reconstruct_failure(data)
+        return data if data.is_a?(RubyReactor::Failure)
+        return nil unless data.is_a?(Hash)
+
+        # Helper for hash access with string/symbol keys
+        get = ->(key) { data[key] || data[key.to_s] }
+
+        RubyReactor::Failure.new(
+          get.call(:message),
+          step_name: get.call(:step_name),
+          inputs: get.call(:inputs),
+          redact_inputs: get.call(:redact_inputs) || [],
+          backtrace: get.call(:backtrace),
+          reactor_name: get.call(:reactor_name),
+          step_arguments: get.call(:step_arguments),
+          exception_class: get.call(:exception_class),
+          file_path: get.call(:file_path),
+          line_number: get.call(:line_number),
+          code_snippet: get.call(:code_snippet),
+          validation_errors: get.call(:validation_errors)
+        )
       end
 
       def execute_step_with_retry(step_config)
@@ -190,8 +166,6 @@ module RubyReactor
         end
       end
 
-      private
-
       def handle_async_step(step_config)
         # Step-level async: hand off execution to worker
 
@@ -204,60 +178,11 @@ module RubyReactor
 
         serialized_context = ContextSerializer.serialize(context_to_serialize)
 
-        result = configuration.async_router.perform_async(
+        configuration.async_router.perform_async(
           serialized_context,
           reactor_class_name,
           intermediate_results: @context.intermediate_results
         )
-
-        # Handle different result types from async router
-        case result
-        when RubyReactor::AsyncResult
-          # Production behavior: return async result to caller
-
-          result
-        when Executor
-          handle_inline_executor_result(result)
-        else
-          # Unexpected result type, treat as error
-          raise Error::ValidationError.new(
-            "Unexpected result type from async router: #{result.class}",
-            context: @context
-          )
-        end
-      end
-
-      def handle_inline_executor_result(result)
-        # Worker executed inline and returned an executor.
-        # This happens when running in test mode or when perform_async returns an executor.
-        # We need to merge the state back into our current executor.
-        #
-        # If we are a child reactor, the worker executed the root reactor, so the result
-        # will be a Root executor. We handle this mismatch below by finding our
-        # corresponding child context within the root result.
-        if @context.root_context && (result.context.reactor_class != @reactor_class)
-          # We are a child, and result is root.
-          # We need to find ourselves in the root result using context_id.
-          matching_context = find_context_by_id(result.context, @context.context_id)
-
-          if matching_context
-            # Replace the result's context with the matching child context
-            # so merge_executor_state works correctly
-            result.instance_variable_set(:@context, matching_context)
-          else
-            # Fallback: if we can't find it (shouldn't happen), we might be in trouble.
-            # But let's try to proceed, maybe it's not nested?
-            # For now, raise an error to be explicit
-            raise Error::ValidationError.new(
-              "Could not find child context with ID #{@context.context_id} in root result",
-              context: @context
-            )
-          end
-        end
-
-        merge_executor_state(result)
-
-        result.result
       end
 
       def handle_interrupt_step(step_config)

data/lib/ruby_reactor/executor.rb

@@ -40,6 +40,7 @@ module RubyReactor
       input_validator = InputValidator.new(@reactor_class, @context)
       input_validator.validate!
 
+      @context.status = :running
       save_context
 
       graph_manager = GraphManager.new(@reactor_class, @dependency_graph, @context)
@@ -59,6 +60,7 @@ module RubyReactor
     end
 
     def resume_execution
+      @context.status = :running
       prepare_for_resume
       save_context
 
@@ -118,19 +120,7 @@ module RubyReactor
         @context.status = :completed
       when RubyReactor::Failure
         @context.status = :failed
-        @context.failure_reason = {
-          message: result.error.is_a?(Exception) ? result.error.message : result.error.to_s,
-          step_name: result.step_name,
-          inputs: result.inputs,
-          backtrace: result.backtrace,
-          reactor_name: result.reactor_name,
-          step_arguments: result.step_arguments,
-          exception_class: result.exception_class,
-          file_path: result.file_path,
-          line_number: result.line_number,
-          code_snippet: result.code_snippet,
-          validation_errors: result.validation_errors
-        }
+        @context.failure_reason = result
       when RubyReactor::InterruptResult
         @context.status = :paused
       end

data/lib/ruby_reactor/map/collector.rb

@@ -7,56 +7,95 @@ module RubyReactor
 
       def self.perform(arguments)
         arguments = arguments.transform_keys(&:to_sym)
-        parent_context_id = arguments[:parent_context_id]
         map_id = arguments[:map_id]
+        parent_context_id = arguments[:parent_context_id]
         parent_reactor_class_name = arguments[:parent_reactor_class_name]
         step_name = arguments[:step_name]
         strict_ordering = arguments[:strict_ordering]
+        # timeout = arguments[:timeout]
 
         storage = RubyReactor.configuration.storage_adapter
+        parent_context_data = storage.retrieve_context(parent_context_id, parent_reactor_class_name)
+        parent_context = RubyReactor::Context.deserialize_from_retry(parent_context_data)
+
+        # Check if all tasks are completed
+        metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
+        total_count = metadata ? metadata["count"].to_i : 0
 
-        # Retrieve parent context
-        parent_context = load_parent_context_from_storage(
-          parent_context_id,
+        results_count = storage.count_map_results(map_id, parent_reactor_class_name)
+
+        # Not done yet, requeue or wait?
+        # Actually Collector currently assumes we only call it when we expect completion or check progress
+        # Since map_offset tracks dispatching progress and might exceed count due to batching reservation,
+        # we must strictly check against the total count of elements.
+        # Check for fail_fast failure FIRST
+        if (failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name))
+          handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
+          return
+        end
+
+        return if results_count < total_count
+
+        # Retrieve results lazily
+        results = RubyReactor::Map::ResultEnumerator.new(
+          map_id,
           parent_reactor_class_name,
-          storage
+          strict_ordering: strict_ordering
         )
 
-        # Retrieve results
-        serialized_results = storage.retrieve_map_results(map_id, parent_reactor_class_name,
-                                                          strict_ordering: strict_ordering)
+        # Apply collect block (or default collection)
+        step_config = parent_context.reactor_class.steps[step_name.to_sym]
 
-        results = serialized_results.map do |r|
-          if r.is_a?(Hash) && r.key?("_error")
-            RubyReactor::Failure(r["_error"])
-          else
-            RubyReactor::Success(r)
+        begin
+          final_result = apply_collect_block(results, step_config)
+
+          if final_result.failure?
+            # Optionally log failure internally or just rely on context status update
           end
+        rescue StandardError => e
+          final_result = RubyReactor::Failure(e)
         end
 
-        # Get step config to check for collect block and other options
-        parent_class = Object.const_get(parent_reactor_class_name)
-        step_config = parent_class.steps[step_name.to_sym]
+        # Resume parent execution
+        resume_parent_execution(parent_context, step_name, final_result, storage)
+      rescue StandardError => e
+        puts "COLLECTOR CRASH: #{e.message}"
+        puts e.backtrace
+        raise e
+      end
 
-        collect_block = step_config.arguments[:collect_block][:source].value
+      def self.apply_collect_block(results, step_config)
+        collect_block = step_config.arguments[:collect_block][:source].value if step_config.arguments[:collect_block]
         # TODO: Check allow_partial_failure option
 
-        final_result = if collect_block
-                         begin
-                           # Pass all results (Success and Failure) to collect block
-                           collected = collect_block.call(results)
-                           RubyReactor::Success(collected)
-                         rescue StandardError => e
-                           RubyReactor::Failure(e)
-                         end
-                       else
-                         # Default behavior: fail if any failure
-                         first_failure = results.find(&:failure?)
-                         first_failure || RubyReactor::Success(results.map(&:value))
-                       end
-
-        # Resume execution
-        resume_parent_execution(parent_context, step_name, final_result, storage)
+        if collect_block
+          begin
+            # Pass Enumerator to collect block
+            collected = collect_block.call(results)
+            RubyReactor::Success(collected)
+          rescue StandardError => e
+            puts "COLLECTOR INNER EXCEPTION: #{e.message}"
+            puts e.backtrace
+            RubyReactor::Failure(e)
+          end
+        else
+          # Default behavior: Return Success(Enumerator).
+          # Logic for checking failures is deferred to the consumer of the enumerator.
+          RubyReactor::Success(results)
+        end
+      end
+
+      def self.handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
+        # Resolve the class of the mapped reactor to retrieve its context
+        reactor_class = resolve_reactor_class(metadata["reactor_class_info"])
+        failed_context_data = storage.retrieve_context(failed_context_id, reactor_class.name)
+
+        return unless failed_context_data
+
+        failed_context = RubyReactor::Context.deserialize_from_retry(failed_context_data)
+        reason = failed_context.failure_reason
+        result = reason.is_a?(RubyReactor::Failure) ? reason : RubyReactor::Failure(reason)
+        resume_parent_execution(parent_context, step_name, result, storage)
       end
     end
   end