fractor 0.1.6 → 0.1.7

data/lib/fractor/workflow/circuit_breaker_orchestrator.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require_relative "circuit_breaker"
+
+module Fractor
+  class Workflow
+    # Orchestrates circuit breaker logic for workflow job execution.
+    # Wraps a CircuitBreaker and provides workflow-specific integration.
+    #
+    # @example Basic usage
+    #   orchestrator = CircuitBreakerOrchestrator.new(threshold: 5, timeout: 60)
+    #   result = orchestrator.execute_with_breaker(job) { |job| execute_job(job) }
+    class CircuitBreakerOrchestrator
+      attr_reader :breaker, :debug, :job_name
+
+      # Initialize a new circuit breaker orchestrator.
+      #
+      # @param threshold [Integer] Number of failures before opening circuit
+      # @param timeout [Integer] Seconds to wait before trying half-open
+      # @param half_open_calls [Integer] Number of test calls in half-open
+      # @param job_name [String] Optional job name for logging
+      # @param debug [Boolean] Whether to enable debug logging
+      def initialize(threshold: 5, timeout: 60, half_open_calls: 3,
+job_name: nil, debug: false)
+        @breaker = CircuitBreaker.new(
+          threshold: threshold,
+          timeout: timeout,
+          half_open_calls: half_open_calls,
+        )
+        @job_name = job_name
+        @debug = debug
+        @execution_count = 0
+        @success_count = 0
+        @blocked_count = 0
+      end
+
+      # Execute a job with circuit breaker protection.
+      #
+      # @param job [Job] The job to execute
+      # @yield [Job] Block that executes the job
+      # @return [Object] The execution result
+      # @raise [CircuitOpenError] If circuit is open
+      def execute_with_breaker(job, &)
+        @execution_count += 1
+
+        log_debug "Executing job '#{job.name}' with circuit breaker protection"
+
+        check_and_call_breaker(job, &)
+      rescue CircuitOpenError => e
+        @blocked_count += 1
+        log_debug "Job '#{job.name}' blocked by circuit breaker: #{e.message}"
+        raise
+      rescue StandardError => e
+        log_debug "Job '#{job.name}' failed with #{e.class}"
+        raise
+      end
+
+      # Check if the circuit is currently open.
+      #
+      # @return [Boolean] true if circuit is open
+      def open?
+        @breaker.open?
+      end
+
+      # Check if the circuit is currently closed.
+      #
+      # @return [Boolean] true if circuit is closed
+      def closed?
+        @breaker.closed?
+      end
+
+      # Check if the circuit is currently half-open.
+      #
+      # @return [Boolean] true if circuit is half-open
+      def half_open?
+        @breaker.half_open?
+      end
+
+      # Get the current circuit breaker state.
+      #
+      # @return [Symbol] The state (:closed, :open, :half_open)
+      def state
+        @breaker.state
+      end
+
+      # Get the failure count.
+      #
+      # @return [Integer] Number of failures recorded
+      def failure_count
+        @breaker.failure_count
+      end
+
+      # Get the last failure time.
+      #
+      # @return [Time, nil] Last failure time or nil
+      def last_failure_time
+        @breaker.last_failure_time
+      end
+
+      # Reset the circuit breaker to closed state.
+      def reset!
+        @breaker.reset
+        @execution_count = 0
+        @success_count = 0
+        @blocked_count = 0
+        log_debug "Circuit breaker reset for job '#{@job_name}'"
+      end
+
+      # Get circuit breaker statistics including orchestrator metrics.
+      #
+      # @return [Hash] Statistics and metrics
+      def stats
+        @breaker.stats.merge(
+          execution_count: @execution_count,
+          success_count: @success_count,
+          blocked_count: @blocked_count,
+        )
+      end
+
+      # Get the current state as a human-readable string.
+      #
+      # @return [String] State description
+      def state_description
+        case state
+        when CircuitBreaker::STATE_CLOSED
+          "CLOSED (normal operation)"
+        when CircuitBreaker::STATE_OPEN
+          "OPEN (blocking requests, #{failure_count}/#{@breaker.threshold} failures)"
+        when CircuitBreaker::STATE_HALF_OPEN
+          "HALF_OPEN (testing recovery, #{@breaker.instance_variable_get(:@success_count)}/#{@breaker.half_open_calls} successes)"
+        else
+          "UNKNOWN"
+        end
+      end
+
+      # Try to execute the job regardless of circuit state.
+      # This bypasses the circuit breaker but still tracks results.
+      #
+      # @param job [Job] The job to execute
+      # @yield [Job] Block that executes the job
+      # @return [Object] The execution result
+      def execute_bypassing_breaker(job)
+        @execution_count += 1
+
+        log_debug "Executing job '#{job.name}' bypassing circuit breaker"
+
+        result = yield(job)
+        @success_count += 1
+        result
+      rescue StandardError => e
+        log_debug "Bypassed execution failed: #{e.class}"
+        raise
+      end
+
+      # Manually open the circuit (for testing or emergency).
+      def open_circuit!
+        @breaker.instance_variable_get(:@mutex).synchronize do
+          @breaker.instance_variable_set(:@state, CircuitBreaker::STATE_OPEN)
+          @breaker.instance_variable_set(:@failure_count, @breaker.threshold)
+          @breaker.instance_variable_set(:@last_failure_time, Time.now)
+        end
+        log_debug "Circuit manually opened for job '#{@job_name}'"
+      end
+
+      # Manually close the circuit (for testing or recovery).
+      def close_circuit!
+        @breaker.reset
+        log_debug "Circuit manually closed for job '#{@job_name}'"
+      end
+
+      private
+
+      # Check circuit state and call the breaker.
+      #
+      # @param job [Job] The job to execute
+      # @yield [Job] Block that executes the job
+      # @return [Object] The execution result
+      def check_and_call_breaker(job, &)
+        result = @breaker.call(&)
+
+        @success_count += 1
+        log_success(job) if @debug
+
+        result
+      end
+
+      # Log a successful execution.
+      #
+      # @param job [Job] The job that succeeded
+      def log_success(job)
+        state_info = case state
+                     when CircuitBreaker::STATE_CLOSED then "(closed)"
+                     when CircuitBreaker::STATE_HALF_OPEN then "(half-open, recovering)"
+                     else ""
+                     end
+
+        puts "[CircuitBreakerOrchestrator] Job '#{job.name}' succeeded #{state_info}" if @debug
+      end
+
+      # Log a debug message.
+      #
+      # @param message [String] The message to log
+      def log_debug(message)
+        puts "[CircuitBreakerOrchestrator] #{message}" if @debug
+      end
+    end
+  end
+end

data/lib/fractor/workflow/circuit_breaker_registry.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require_relative "circuit_breaker_orchestrator"
+
+module Fractor
+  class Workflow
+    # Registry for managing circuit breakers across jobs
+    #
+    # Provides centralized circuit breaker management, allowing multiple
+    # jobs to share circuit breakers or have isolated ones.
+    #
+    # @example Shared circuit breaker
+    #   registry = CircuitBreakerRegistry.new
+    #   breaker = registry.get_or_create("api", threshold: 5)
+    #
+    # @example Per-job circuit breaker
+    #   registry = CircuitBreakerRegistry.new
+    #   breaker = registry.get_or_create("job_123", threshold: 3)
+    #
+    # @example Circuit breaker orchestrator
+    #   registry = CircuitBreakerRegistry.new
+    #   orchestrator = registry.get_or_create_orchestrator("api", threshold: 5, job_name: "my_job")
+    class CircuitBreakerRegistry
+      def initialize
+        @breakers = {}
+        @orchestrators = {}
+        @mutex = Mutex.new
+      end
+
+      # Get or create a circuit breaker
+      #
+      # @param key [String] Unique identifier for the circuit breaker
+      # @param options [Hash] Circuit breaker options
+      # @option options [Integer] :threshold Failure threshold
+      # @option options [Integer] :timeout Timeout in seconds
+      # @option options [Integer] :half_open_calls Test calls in half-open
+      # @return [CircuitBreaker] The circuit breaker
+      def get_or_create(key, **options)
+        @mutex.synchronize do
+          @breakers[key] ||= CircuitBreaker.new(**options)
+        end
+      end
+
+      # Get or create a circuit breaker orchestrator
+      #
+      # @param key [String] Unique identifier for the circuit breaker
+      # @param options [Hash] Circuit breaker orchestrator options
+      # @option options [Integer] :threshold Failure threshold
+      # @option options [Integer] :timeout Timeout in seconds
+      # @option options [Integer] :half_open_calls Test calls in half-open
+      # @option options [String] :job_name Job name for logging
+      # @option options [Boolean] :debug Debug logging flag
+      # @return [CircuitBreakerOrchestrator] The circuit breaker orchestrator
+      def get_or_create_orchestrator(key, **options)
+        @mutex.synchronize do
+          @orchestrators[key] ||= CircuitBreakerOrchestrator.new(**options)
+        end
+      end
+
+      # Get an existing circuit breaker
+      #
+      # @param key [String] Unique identifier for the circuit breaker
+      # @return [CircuitBreaker, nil] The circuit breaker or nil
+      def get(key)
+        @breakers[key]
+      end
+
+      # Get an existing circuit breaker orchestrator
+      #
+      # @param key [String] Unique identifier for the orchestrator
+      # @return [CircuitBreakerOrchestrator, nil] The orchestrator or nil
+      def get_orchestrator(key)
+        @orchestrators[key]
+      end
+
+      # Remove a circuit breaker
+      #
+      # @param key [String] Unique identifier for the circuit breaker
+      # @return [CircuitBreaker, CircuitBreakerOrchestrator, nil] The removed object or nil
+      def remove(key)
+        @mutex.synchronize do
+          @breakers.delete(key) || @orchestrators.delete(key)
+        end
+      end
+
+      # Reset all circuit breakers and orchestrators
+      def reset_all
+        @mutex.synchronize do
+          @breakers.each_value(&:reset)
+          @orchestrators.each_value(&:reset!)
+        end
+      end
+
+      # Get statistics for all circuit breakers and orchestrators
+      #
+      # @return [Hash] Map of key to circuit breaker/orchestrator statistics
+      def all_stats
+        breakers_stats = @breakers.transform_values(&:stats)
+        orchestrators_stats = @orchestrators.transform_values(&:stats)
+        breakers_stats.merge(orchestrators_stats)
+      end
+
+      # Clear all circuit breakers and orchestrators
+      def clear
+        @mutex.synchronize do
+          @breakers.clear
+          @orchestrators.clear
+        end
+      end
+    end
+  end
+end

data/lib/fractor/workflow/dead_letter_queue.rb

@@ -0,0 +1,334 @@
+# frozen_string_literal: true
+
+module Fractor
+  class Workflow
+    # Dead Letter Queue for capturing permanently failed work
+    #
+    # The dead letter queue captures work items that have exhausted all
+    # retry attempts and cannot be processed successfully. This provides
+    # a mechanism for:
+    #
+    # - Preventing data loss for failed items
+    # - Enabling manual inspection and reprocessing
+    # - Supporting different persistence strategies
+    # - Providing visibility into failure patterns
+    #
+    # @example Basic usage
+    #   dlq = DeadLetterQueue.new(max_size: 1000)
+    #   dlq.add(work, error, context)
+    #   failed_items = dlq.all
+    #
+    # @example With handler
+    #   dlq = DeadLetterQueue.new
+    #   dlq.on_add do |entry|
+    #     Logger.error("Dead letter: #{entry.error}")
+    #   end
+    class DeadLetterQueue
+      # Entry in the dead letter queue
+      class Entry
+        attr_reader :work, :error, :context, :timestamp, :metadata
+
+        def initialize(work:, error:, context: nil, metadata: {})
+          @work = work
+          @error = error
+          @context = context
+          @timestamp = Time.now
+          @metadata = metadata
+        end
+
+        # Convert entry to hash for serialization
+        #
+        # @return [Hash] Entry data as hash
+        def to_h
+          {
+            work: work,
+            error: error.to_s,
+            error_class: error.class.name,
+            context: context&.to_h,
+            timestamp: timestamp.iso8601,
+            metadata: metadata,
+          }
+        end
+      end
+
+      attr_reader :max_size, :entries
+
+      # Initialize a new dead letter queue
+      #
+      # @param max_size [Integer] Maximum queue size (nil for unlimited)
+      # @param persistence [Symbol] Persistence strategy (:memory, :file, :redis, :database)
+      # @param persistence_options [Hash] Options for persistence backend
+      def initialize(max_size: nil, persistence: :memory, **persistence_options)
+        @max_size = max_size
+        @entries = []
+        @handlers = []
+        @mutex = Mutex.new
+        @persistence = persistence
+        @persistence_options = persistence_options
+        @persister = create_persister(persistence, persistence_options)
+      end
+
+      # Add a failed work item to the dead letter queue
+      #
+      # @param work [Fractor::Work] The failed work item
+      # @param error [Exception] The error that caused failure
+      # @param context [Hash] Additional context about the failure
+      # @param metadata [Hash] Additional metadata
+      # @return [Entry] The created entry
+      def add(work, error, context: nil, metadata: {})
+        entry = Entry.new(
+          work: work,
+          error: error,
+          context: context,
+          metadata: metadata,
+        )
+
+        @mutex.synchronize do
+          # Enforce max size if set
+          if max_size && @entries.size >= max_size
+            # Remove oldest entry
+            removed = @entries.shift
+            @persister&.remove(removed)
+          end
+
+          @entries << entry
+          @persister&.persist(entry)
+        end
+
+        # Notify handlers
+        notify_handlers(entry)
+
+        entry
+      end
+
+      # Enqueue a failed work item to the dead letter queue (alias for add)
+      # Standardized API method name for consistency across queue implementations
+      #
+      # @param work [Fractor::Work] The failed work item
+      # @param error [Exception] The error that caused failure
+      # @param context [Hash] Additional context about the failure
+      # @param metadata [Hash] Additional metadata
+      # @return [Entry] The created entry
+      def enqueue(work, error, context: nil, metadata: {})
+        add(work, error, context: context, metadata: metadata)
+      end
+
+      # Register a handler to be called when items are added
+      #
+      # @yield [Entry] Block to execute when item is added
+      def on_add(&block)
+        @handlers << block if block
+      end
+
+      # Get all entries in the queue
+      #
+      # @return [Array<Entry>] All entries
+      def all
+        @mutex.synchronize { @entries.dup }
+      end
+
+      # Get entries matching a filter
+      #
+      # @yield [Entry] Block to filter entries
+      # @return [Array<Entry>] Filtered entries
+      def filter(&block)
+        @mutex.synchronize { @entries.select(&block) }
+      end
+
+      # Get entries for a specific error class
+      #
+      # @param error_class [Class] Error class to filter by
+      # @return [Array<Entry>] Matching entries
+      def by_error_class(error_class)
+        filter { |entry| entry.error.is_a?(error_class) }
+      end
+
+      # Get entries within a time range
+      #
+      # @param start_time [Time] Start of time range
+      # @param end_time [Time] End of time range (defaults to now)
+      # @return [Array<Entry>] Entries in time range
+      def by_time_range(start_time, end_time = Time.now)
+        filter do |entry|
+          entry.timestamp >= start_time && entry.timestamp <= end_time
+        end
+      end
+
+      # Remove an entry from the queue
+      #
+      # @param entry [Entry] Entry to remove
+      # @return [Entry, nil] Removed entry or nil
+      def remove(entry)
+        @mutex.synchronize do
+          if @entries.delete(entry)
+            @persister&.remove(entry)
+            entry
+          end
+        end
+      end
+
+      # Clear all entries from the queue
+      #
+      # @return [Integer] Number of entries cleared
+      def clear
+        @mutex.synchronize do
+          count = @entries.size
+          @entries.clear
+          @persister&.clear
+          count
+        end
+      end
+
+      # Get the current size of the queue
+      #
+      # @return [Integer] Number of entries
+      def size
+        @mutex.synchronize { @entries.size }
+      end
+
+      # Check if queue is empty
+      #
+      # @return [Boolean] True if empty
+      def empty?
+        size.zero?
+      end
+
+      # Check if queue is at capacity
+      #
+      # @return [Boolean] True if at max_size
+      def full?
+        return false unless max_size
+
+        size >= max_size
+      end
+
+      # Get statistics about the queue
+      #
+      # @return [Hash] Queue statistics
+      def stats
+        entries_copy = all
+
+        {
+          size: entries_copy.size,
+          max_size: max_size,
+          full: full?,
+          oldest_timestamp: entries_copy.first&.timestamp,
+          newest_timestamp: entries_copy.last&.timestamp,
+          error_classes: entries_copy.map { |e| e.error.class.name }.uniq,
+          persistence: @persistence,
+        }
+      end
+
+      # Retry a specific entry
+      #
+      # @param entry [Entry] Entry to retry
+      # @yield [Work] Block to process the work
+      # @return [Boolean] True if retry succeeded
+      def retry_entry(entry, &block)
+        return false unless block
+
+        begin
+          yield(entry.work)
+          remove(entry)
+          true
+        rescue StandardError => e
+          # Add back to queue with new error
+          remove(entry)
+          add(entry.work, e, context: entry.context,
+                             metadata: entry.metadata.merge(retried: true))
+          false
+        end
+      end
+
+      # Retry all entries
+      #
+      # @yield [Work] Block to process each work item
+      # @return [Hash] Results with :success and :failed counts
+      def retry_all(&block)
+        return { success: 0, failed: 0 } unless block
+
+        results = { success: 0, failed: 0 }
+        entries_to_retry = all
+
+        entries_to_retry.each do |entry|
+          if retry_entry(entry, &block)
+            results[:success] += 1
+          else
+            results[:failed] += 1
+          end
+        end
+
+        results
+      end
+
+      private
+
+      def notify_handlers(entry)
+        @handlers.each do |handler|
+          handler.call(entry)
+        rescue StandardError => e
+          warn "Dead letter queue handler error: #{e.message}"
+        end
+      end
+
+      def create_persister(persistence, options)
+        case persistence
+        when :memory
+          nil # No persistence needed
+        when :file
+          FilePersister.new(**options)
+        when :redis
+          RedisPersister.new(**options) if defined?(Redis)
+        when :database
+          DatabasePersister.new(**options)
+        else
+          raise ArgumentError, "Unknown persistence strategy: #{persistence}"
+        end
+      end
+    end
+
+    # File-based persistence for dead letter queue
+    class FilePersister
+      def initialize(file_path: "dead_letter_queue.json")
+        @file_path = file_path
+        @mutex = Mutex.new
+      end
+
+      def persist(entry)
+        @mutex.synchronize do
+          entries = load_entries
+          entries << entry.to_h
+          save_entries(entries)
+        end
+      end
+
+      def remove(entry)
+        @mutex.synchronize do
+          entries = load_entries
+          entries.reject! { |e| e[:timestamp] == entry.timestamp.iso8601 }
+          save_entries(entries)
+        end
+      end
+
+      def clear
+        @mutex.synchronize do
+          File.delete(@file_path) if File.exist?(@file_path)
+        end
+      end
+
+      private
+
+      def load_entries
+        return [] unless File.exist?(@file_path)
+
+        JSON.parse(File.read(@file_path), symbolize_names: true)
+      rescue JSON::ParserError
+        []
+      end
+
+      def save_entries(entries)
+        File.write(@file_path, JSON.pretty_generate(entries))
+      end
+    end
+  end
+end

data/lib/fractor/workflow/execution_hooks.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Fractor
+  class Workflow
+    # Manages lifecycle hooks for workflow execution.
+    # Allows registering callbacks for workflow/job lifecycle events.
+    class ExecutionHooks
+      def initialize
+        @hooks = Hash.new { |h, k| h[k] = [] }
+      end
+
+      # Register a callback for a specific event.
+      #
+      # @param event [Symbol] The event to hook into
+      # @yield [Object] Block to execute when event is triggered
+      #
+      # @example Register a workflow start hook
+      #   hooks.register(:workflow_start) do |workflow|
+      #     puts "Workflow starting: #{workflow.class.workflow_name}"
+      #   end
+      def register(event, &block)
+        @hooks[event] << block
+      end
+
+      # Trigger all callbacks registered for an event.
+      #
+      # @param event [Symbol] The event to trigger
+      # @param args [Array] Arguments to pass to the callbacks
+      #
+      # @example Trigger workflow completion
+      #   hooks.trigger(:workflow_complete, result)
+      def trigger(event, *args)
+        @hooks[event].each do |hook|
+          hook.call(*args)
+        end
+      end
+    end
+  end
+end