ruby_reactor 0.1.0

data/documentation/getting_started.md

@@ -0,0 +1,224 @@
+# Getting Started with RubyReactor
+
+This guide will help you get started with RubyReactor, from installation to your first reactor.
+
+## Installation
+
+Add RubyReactor to your Gemfile:
+
+```ruby
+gem 'ruby_reactor'
+```
+
+Or install directly:
+
+```bash
+gem install ruby_reactor
+```
+
+## Your First Reactor
+
+Let's create a simple order processing reactor:
+
+```ruby
+require 'ruby_reactor'
+
+class OrderProcessingReactor < RubyReactor::Reactor
+  step :validate_order do
+    validate_args do
+      required(:order_id).filled(:string)
+    end
+
+    run do |order_id|
+      # Validate the order exists and is in correct state
+      order = Order.find(order_id)
+      raise "Order not found" unless order
+      raise "Order already processed" if order.processed?
+
+      Success({ order: order })
+    end
+  end
+
+  step :process_payment do
+    run do |order:, **|
+      # Process payment for the order
+      payment_result = PaymentService.charge(order.total, order.customer.card_token)
+      raise "Payment failed" unless payment_result.success?
+
+      Success({ payment_id: payment_result.id })
+    end
+  end
+
+  step :update_inventory do
+    run do |order:, **|
+      # Update inventory for each item
+      order.items.each do |item|
+        InventoryService.decrement(item.product_id, item.quantity)
+      end
+
+      Success({ inventory_updated: true })
+    end
+  end
+
+  step :send_confirmation do
+    run do |order:, payment_id:, **|
+      # Send confirmation email
+      email_result = EmailService.send_confirmation(
+        order.customer.email,
+        order_id: order.id,
+        payment_id: payment_id
+      )
+
+      Success({ confirmation_sent: email_result.success? })
+    end
+  end
+end
+```
+
+## Executing a Reactor
+
+### Synchronous Execution
+
+```ruby
+# Run the reactor synchronously
+result = OrderProcessingReactor.run(order_id: 123)
+
+if result.success?
+  puts "Order processed successfully!"
+  puts "Results: #{result.step_results}"
+else
+  puts "Order processing failed: #{result.error}"
+end
+```
+
+### Asynchronous Execution
+
+For async execution, you need Sidekiq configured. Mark the reactor as async:
+
+```ruby
+class OrderProcessingReactor < RubyReactor::Reactor
+  async true  # Enable full reactor async
+
+  # ... steps defined above
+end
+
+# Run asynchronously
+async_result = OrderProcessingReactor.run(order_id: 123)
+# Returns immediately with AsyncResult
+# Check status later with async_result.status
+```
+
+## Understanding Results
+
+RubyReactor returns detailed execution results:
+
+```ruby
+result = OrderProcessingReactor.run(order_id: 123)
+
+# Check overall success
+result.success?  # => true/false
+
+# Access step results
+result.step_results[:validate_order]  # => { order: #<Order> }
+result.step_results[:process_payment] # => { payment_id: "pay_123" }
+
+# Access intermediate results
+result.intermediate_results  # => Hash of all step outputs
+
+# Check completed steps
+result.completed_steps  # => Set of completed step names
+
+# Error information (if failed)
+result.error  # => Exception that caused failure
+```
+
+## Step Dependencies
+
+Steps can depend on each other using the `argument` method with `result()`:
+
+```ruby
+class ComplexReactor < RubyReactor::Reactor
+  step :validate_order do
+    run { validate_order_logic }
+  end
+
+  step :check_inventory do
+    argument :order, result(:validate_order)
+
+    run do |args, _context|
+      check_inventory_for_order(args[:order])
+    end
+  end
+
+  step :process_payment do
+    argument :order, result(:check_inventory)
+
+    run do |args, _context|
+      process_payment_for_order(args[:order])
+    end
+  end
+end
+```
+
+## Error Handling and Compensation
+
+RubyReactor automatically handles errors and provides compensation:
+
+```ruby
+class OrderProcessingReactor < RubyReactor::Reactor
+  step :validate_order do
+    run { validate_order_logic }
+  end
+
+  step :process_payment do
+    run { process_payment_logic }
+
+    undo do |payment_id:, **|
+      # Undo the payment if something fails later
+      PaymentService.refund(payment_id)
+    end
+  end
+
+  step :update_inventory do
+    run { update_inventory_logic }
+
+    compensate do |order:, **|
+      # Restore inventory if something fails later
+      order.items.each do |item|
+        InventoryService.increment(item.product_id, item.quantity)
+      end
+    end
+  end
+end
+```
+
+If `update_inventory` fails, RubyReactor will:
+1. Run the `update_inventory` compensate block
+2. Run the `process_payment` undo block
+3. Return a failure result
+
+## Configuration
+
+### Sidekiq Setup (for Async)
+
+Add to your Sidekiq configuration:
+
+```ruby
+# config/sidekiq.rb
+require 'ruby_reactor/worker'
+
+# Configure RubyReactor
+RubyReactor.configure do |config|
+  config.sidekiq_queue = :default
+  config.sidekiq_retry_count = 3
+  config.logger = Logger.new('log/ruby_reactor.log')
+end
+```
+
+
+## Next Steps
+
+- Learn about [async reactors](async_reactors.md)
+- Configure [retry policies](retry_configuration.md)
+- See [examples](examples/) for more patterns
+- Check the [API reference](api_reference.md) for detailed documentation

data/documentation/retry_configuration.md

@@ -0,0 +1,357 @@
+# Retry Configuration
+
+RubyReactor provides flexible, non-blocking retry mechanisms that requeue jobs instead of blocking worker threads. Retry policies can be configured at both reactor and step levels.
+
+## Overview
+
+The retry system offers:
+
+- **Non-blocking retries on Async**: Jobs are requeued with calculated delays
+- **Multiple backoff strategies**: Exponential, linear, and fixed delays
+- **Step-level control**: Different retry policies for different steps
+- **Full observability**: Complete visibility into retry attempts
+
+### Retry Flow Architecture
+
+```mermaid
+graph TD
+    A[Step Execution] --> B{Step<br/>Succeeds?}
+    B -->|Yes| C[Continue to Next Step]
+    B -->|No| D{Can Retry?<br/>attempts < max_attempts}
+    D -->|No| E[Final Failure<br/>Run Compensation]
+    D -->|Yes| F[Calculate Backoff Delay<br/>exponential/linear/fixed]
+    F --> G[Serialize Context<br/>with Retry State]
+    G --> H[Queue Job for Retry<br/>with Calculated Delay]
+    H --> I[Worker Freed<br/>No Thread Blocking]
+    I --> J[Delay Elapses]
+    J --> K[Worker Picks Up<br/>Retry Job]
+    K --> L[Deserialize Context]
+    L --> M[Resume Execution<br/>from Failed Step]
+    M --> A
+```
+
+## Basic Retry Configuration
+
+### Step-Level Retry
+
+```ruby
+class PaymentReactor < RubyReactor::Reactor
+  async true
+
+  step :charge_card do
+    retries max_attempts: 3, backoff: :exponential, base_delay: 5.seconds
+    run { PaymentService.charge(card_token, amount) }
+  end
+end
+```
+
+### Reactor-Level Defaults
+
+```ruby
+class PaymentReactor < RubyReactor::Reactor
+  async true
+
+  # All steps inherit these defaults
+  retry_defaults max_attempts: 3, backoff: :exponential, base_delay: 2.seconds
+
+  step :validate_card do
+    # Uses reactor defaults
+    run { validate_card_details }
+  end
+
+  step :charge_card do
+    # Override for this specific step
+    retries max_attempts: 5, backoff: :linear, base_delay: 10.seconds
+    run { PaymentService.charge(card_token, amount) }
+  end
+end
+```
+
+## Retry Parameters
+
+### max_attempts
+Maximum number of execution attempts (including the initial attempt).
+
+```ruby
+retries max_attempts: 5  # 1 initial + 4 retries = 5 total attempts
+```
+
+### backoff
+The backoff strategy for calculating delays between retry attempts.
+
+**Options:**
+- `:exponential` (default): Delay doubles with each attempt
+- `:linear`: Delay increases linearly
+- `:fixed`: Same delay for each attempt
+
+### base_delay
+The base delay for retry calculations. Can be a number (seconds) or ActiveSupport duration.
+
+```ruby
+retry base_delay: 5.seconds
+retry base_delay: 300  # 5 minutes in seconds
+```
+
+## Backoff Strategies
+
+### Exponential Backoff
+
+Delay doubles with each retry attempt. Best for external services that may be temporarily overloaded.
+
+```ruby
+retries max_attempts: 4, backoff: :exponential, base_delay: 1.second
+# Delays: 1s, 2s, 4s (total: 7 seconds)
+```
+
+**Use cases:**
+- API rate limiting
+- Temporary service unavailability
+- Network timeouts
+
+### Linear Backoff
+
+Delay increases linearly with each attempt. Provides predictable, gradually increasing delays.
+
+```ruby
+retries max_attempts: 4, backoff: :linear, base_delay: 5.seconds
+# Delays: 5s, 10s, 15s (total: 30 seconds)
+```
+
+**Use cases:**
+- Database connection issues
+- Resource contention
+- Gradual backpressure
+
+### Fixed Backoff
+
+Same delay between each retry attempt. Simplest strategy with predictable timing.
+
+```ruby
+retries max_attempts: 4, backoff: :fixed, base_delay: 10.seconds
+# Delays: 10s, 10s, 10s (total: 30 seconds)
+```
+
+**Use cases:**
+- Simple retry scenarios
+- When timing precision matters
+- Testing environments
+
+## Idempotency
+
+### What is Idempotency?
+
+An operation is idempotent if executing it multiple times produces the same result as executing it once.
+
+### Idempotent vs Non-Idempotent Operations
+
+**Idempotent operations (safe to retry):**
+- Reading data
+- Updating records with same values
+- Sending notifications (with deduplication)
+- Idempotent API calls
+
+**Non-idempotent operations (unsafe to retry):**
+- Creating new records
+- Charging payments (without deduplication)
+- Sending unique messages
+- File system operations
+
+### Best Practices
+
+1. **Design for idempotency**: Structure operations to be safely retryable
+2. **Use idempotency keys**: For payments, orders, etc.
+3. **Test thoroughly**: Verify retry behavior doesn't cause issues
+
+## Advanced Configuration
+
+### Complex Retry Scenarios
+
+```ruby
+class OrderProcessingReactor < RubyReactor::Reactor
+  async true
+
+  retry_defaults max_attempts: 3, backoff: :exponential, base_delay: 2.seconds
+
+  step :validate_order do
+    # Quick validation - no retry needed
+    run { validate_order_exists(order_id) }
+  end
+
+  step :check_inventory do
+    # Inventory checks can be retried
+    retries max_attempts: 5, backoff: :linear, base_delay: 1.second
+    run { check_inventory_availability(order) }
+  end
+
+  step :reserve_inventory do
+    # Inventory reservation - must be idempotent
+    retries max_attempts: 3, backoff: :fixed, base_delay: 5.seconds
+    run { InventoryService.reserve_items(order.items) }
+
+    compensate do
+      # Release reservation on failure
+      InventoryService.release_reservation(order.items)
+    end
+  end
+
+  step :process_payment do
+    # Payment processing - critical, fewer retries
+    retries max_attempts: 2, backoff: :exponential, base_delay: 10.seconds
+    run { PaymentService.charge(order.total, order.card_token) }
+
+    compensate do |payment_id:|
+      # Refund on failure
+      PaymentService.refund(payment_id) if payment_id
+    end
+  end
+
+  step :confirm_order do
+    # Final confirmation - must succeed
+    run { OrderService.mark_completed(order_id) }
+  end
+end
+```
+
+### Conditional Retry Logic
+
+For more complex retry logic, you can implement custom retry handlers:
+
+```ruby
+class CustomRetryReactor < RubyReactor::Reactor
+  async true
+
+  step :call_external_api do
+    retries max_attempts: 5, backoff: :exponential, base_delay: 1.second
+    run do
+      result = ExternalAPI.call
+      # Raise specific errors based on response
+      case result.status
+      when 429  # Rate limited
+        Failure(RateLimitError.new(result) retryable: true)
+      when 500  # Server error
+        Failure(ServerError.new(result) retryable: true)
+      when 400  # Bad request
+        Failure(ValidationError.new(result) retryable: false)
+      else
+        result
+      end
+    end
+  end
+end
+```
+
+## Monitoring and Observability
+
+### Retry Metrics
+
+Track these important metrics:
+
+```ruby
+# In your monitoring system
+retry_attempt_count(step_name)
+retry_success_rate(step_name)
+average_retry_delay(step_name)
+retry_timeout_count(step_name)
+```
+
+
+### Sidekiq Web UI
+
+Retry jobs are visible in Sidekiq web interface with:
+- Step name and attempt number
+- Failure reason and stack trace
+- Scheduled retry time
+- Job arguments and context
+
+## Performance Considerations
+
+### Retry Storm Prevention
+
+Avoid retry storms by:
+
+1. **Reasonable delays**: Don't use very short base delays
+2. **Limited attempts**: Set appropriate max_attempts limits
+3. **Circuit breakers**: Implement circuit breaker patterns for external services
+4. **Rate limiting**: Consider rate limiting at the application level
+
+### Resource Usage
+
+- **Worker threads**: Retries don't block workers, improving utilization
+- **Memory**: Context serialization adds memory overhead
+- **Redis**: Job storage and queue management
+- **Database**: Potential increased load from idempotent operations
+
+### Tuning Guidelines
+
+```ruby
+# Fast-retry scenario (API calls)
+retries max_attempts: 3, backoff: :exponential, base_delay: 1.second
+
+# Slow-retry scenario (batch processing)
+retries max_attempts: 5, backoff: :linear, base_delay: 5.minutes
+
+# Critical operations (payments)
+retries max_attempts: 2, backoff: :fixed, base_delay: 30.seconds
+```
+
+## Error Types and Handling
+
+### Retryable Errors
+
+```ruby
+class NetworkTimeoutError < StandardError
+  def retryable?
+    true
+  end
+end
+
+class ValidationError < StandardError
+  def retryable?
+    false  # Don't retry validation errors
+  end
+end
+```
+
+## Testing Retry Behavior
+
+### Unit Testing
+
+```ruby
+RSpec.describe PaymentReactor do
+  it "retries failed payment with exponential backoff" do
+    allow(PaymentService).to receive(:charge)
+      .and_raise(NetworkError.new("Timeout"))
+      .and_raise(NetworkError.new("Timeout"))
+      .and_return(payment_result)
+
+    expect(PaymentService).to receive(:charge).exactly(3).times
+
+    result = PaymentReactor.run(card_token: "tok_123", amount: 100)
+
+    expect(result).to be_success
+    expect(result.step_results[:charge_card][:payment_id]).to eq("pay_123")
+  end
+end
+```
+
+### Integration Testing
+
+```ruby
+describe "Retry integration" do
+  it "handles real Sidekiq retry scenarios" do
+    # Test with actual Sidekiq worker
+    Sidekiq::Testing.fake! do
+      result = FailingReactor.run(input: "test")
+
+      # Verify job was queued for retry
+      expect(RubyReactor::SidekiqWorkers::Worker.jobs.size).to eq(1)
+
+      # Process the retry
+      RubyReactor::SidekiqWorkers::Worker.drain
+
+      # Verify final success
+      expect(result).to be_success
+    end
+  end
+end
+```

data/lib/ruby_reactor/async_router.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  class AsyncRouter
+    def self.perform_async(serialized_context, reactor_class_name = nil, intermediate_results: {})
+      job_id = SidekiqWorkers::Worker.perform_async(serialized_context, reactor_class_name)
+      RubyReactor::AsyncResult.new(job_id: job_id, intermediate_results: intermediate_results)
+    end
+
+    def self.perform_in(delay, serialized_context, reactor_class_name = nil, intermediate_results: {})
+      job_id = SidekiqWorkers::Worker.perform_in(delay, serialized_context, reactor_class_name)
+      RubyReactor::AsyncResult.new(job_id: job_id, intermediate_results: intermediate_results)
+    end
+
+    # rubocop:disable Metrics/ParameterLists
+    def self.perform_map_element_async(map_id:, element_id:, index:, serialized_inputs:, reactor_class_info:,
+                                       strict_ordering:, parent_context_id:, parent_reactor_class_name:, step_name:,
+                                       batch_size: nil, serialized_context: nil)
+      RubyReactor::SidekiqWorkers::MapElementWorker.perform_async(
+        {
+          "map_id" => map_id,
+          "element_id" => element_id,
+          "index" => index,
+          "serialized_inputs" => serialized_inputs,
+          "reactor_class_info" => reactor_class_info,
+          "strict_ordering" => strict_ordering,
+          "parent_context_id" => parent_context_id,
+          "parent_reactor_class_name" => parent_reactor_class_name,
+          "step_name" => step_name,
+          "batch_size" => batch_size,
+          "serialized_context" => serialized_context
+        }
+      )
+    end
+
+    def self.perform_map_element_in(delay, map_id:, element_id:, index:, serialized_inputs:, reactor_class_info:,
+                                    strict_ordering:, parent_context_id:, parent_reactor_class_name:, step_name:,
+                                    batch_size: nil, serialized_context: nil)
+      RubyReactor::SidekiqWorkers::MapElementWorker.perform_in(
+        delay,
+        {
+          "map_id" => map_id,
+          "element_id" => element_id,
+          "index" => index,
+          "serialized_inputs" => serialized_inputs,
+          "reactor_class_info" => reactor_class_info,
+          "strict_ordering" => strict_ordering,
+          "parent_context_id" => parent_context_id,
+          "parent_reactor_class_name" => parent_reactor_class_name,
+          "step_name" => step_name,
+          "batch_size" => batch_size,
+          "serialized_context" => serialized_context
+        }
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    # rubocop:disable Metrics/ParameterLists
+    def self.perform_map_collection_async(parent_context_id:, map_id:, parent_reactor_class_name:, step_name:,
+                                          strict_ordering:, timeout:)
+      RubyReactor::SidekiqWorkers::MapCollectorWorker.perform_async(
+        {
+          "parent_context_id" => parent_context_id,
+          "map_id" => map_id,
+          "parent_reactor_class_name" => parent_reactor_class_name,
+          "step_name" => step_name,
+          "strict_ordering" => strict_ordering,
+          "timeout" => timeout
+        }
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    # rubocop:disable Metrics/ParameterLists
+    def self.perform_map_execution_async(map_id:, serialized_inputs:, reactor_class_info:, strict_ordering:,
+                                         parent_context_id:, parent_reactor_class_name:, step_name:)
+      RubyReactor::SidekiqWorkers::MapExecutionWorker.perform_async(
+        {
+          "map_id" => map_id,
+          "serialized_inputs" => serialized_inputs,
+          "reactor_class_info" => reactor_class_info,
+          "strict_ordering" => strict_ordering,
+          "parent_context_id" => parent_context_id,
+          "parent_reactor_class_name" => parent_reactor_class_name,
+          "step_name" => step_name
+        }
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists
+  end
+end

data/lib/ruby_reactor/configuration.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module RubyReactor
+  # Configuration class for RubyReactor settings
+  class Configuration
+    include Singleton
+
+    attr_writer :sidekiq_queue, :sidekiq_retry_count, :logger, :async_router
+
+    def sidekiq_queue
+      @sidekiq_queue ||= :default
+    end
+
+    def sidekiq_retry_count
+      @sidekiq_retry_count ||= 3
+    end
+
+    def logger
+      @logger ||= Logger.new($stderr)
+    end
+
+    def async_router
+      @async_router ||= RubyReactor::AsyncRouter
+    end
+
+    def storage
+      @storage ||= RubyReactor::Storage::Configuration.new
+    end
+
+    def storage_adapter
+      @storage_adapter ||= case storage.adapter
+                           when :redis
+                             RubyReactor::Storage::RedisAdapter.new(url: storage.redis_url, **storage.redis_options)
+                           else
+                             raise "Unknown storage adapter: #{storage.adapter}"
+                           end
+    end
+  end
+end