fractor 0.1.6 → 0.1.8

data/lib/fractor/workflow/retry_config.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative "retry_strategy"
+
+module Fractor
+  class Workflow
+    # Configuration for job retry behavior
+    class RetryConfig
+      attr_reader :strategy, :timeout, :retryable_errors
+
+      def initialize(
+        strategy: NoRetry.new,
+        timeout: nil,
+        retryable_errors: [StandardError]
+      )
+        @strategy = strategy
+        @timeout = timeout
+        @retryable_errors = Array(retryable_errors)
+      end
+
+      # Check if an error should trigger a retry
+      # @param error [Exception] The error to check
+      # @return [Boolean] true if error should be retried
+      def retryable?(error)
+        retryable_errors.any? { |err_class| error.is_a?(err_class) }
+      end
+
+      # Get maximum number of retry attempts
+      # @return [Integer] Maximum attempts
+      def max_attempts
+        strategy.max_attempts
+      end
+
+      # Calculate delay for a given attempt
+      # @param attempt [Integer] The attempt number
+      # @return [Numeric] Delay in seconds
+      def delay_for(attempt)
+        strategy.delay_for(attempt)
+      end
+
+      # Create a retry config from a hash of options
+      # @param options [Hash] Configuration options
+      # @option options [Symbol] :backoff Strategy type (:exponential, :linear, :constant, :none)
+      # @option options [Integer] :max_attempts Maximum retry attempts
+      # @option options [Numeric] :initial_delay Initial delay in seconds
+      # @option options [Numeric] :max_delay Maximum delay in seconds
+      # @option options [Numeric] :timeout Job timeout in seconds
+      # @option options [Array<Class>] :retryable_errors List of retryable error classes
+      # @return [RetryConfig] New retry configuration
+      def self.from_options(**options)
+        strategy = create_strategy(**options)
+        new(
+          strategy: strategy,
+          timeout: options[:timeout],
+          retryable_errors: options[:retryable_errors] || [StandardError],
+        )
+      end
+
+      # Create a retry strategy from options
+      # @param options [Hash] Strategy options
+      # @return [RetryStrategy] A retry strategy instance
+      def self.create_strategy(**options)
+        backoff = options[:backoff] || :exponential
+        max_attempts = options[:max_attempts] || 3
+        max_delay = options[:max_delay]
+
+        case backoff
+        when :exponential
+          ExponentialBackoff.new(
+            initial_delay: options[:initial_delay] || 1,
+            multiplier: options[:multiplier] || 2,
+            max_attempts: max_attempts,
+            max_delay: max_delay,
+          )
+        when :linear
+          LinearBackoff.new(
+            initial_delay: options[:initial_delay] || 1,
+            increment: options[:increment] || 1,
+            max_attempts: max_attempts,
+            max_delay: max_delay,
+          )
+        when :constant
+          ConstantDelay.new(
+            delay: options[:delay] || 1,
+            max_attempts: max_attempts,
+            max_delay: max_delay,
+          )
+        when :none, false
+          NoRetry.new
+        else
+          raise ArgumentError, "Unknown backoff strategy: #{backoff}"
+        end
+      end
+    end
+
+    # Tracks retry state for a job execution
+    class RetryState
+      attr_reader :job_name, :attempt, :errors, :started_at
+
+      def initialize(job_name)
+        @job_name = job_name
+        @attempt = 1
+        @errors = []
+        @started_at = Time.now
+      end
+
+      # Record a failed attempt
+      # @param error [Exception] The error that occurred
+      def record_failure(error)
+        @errors << {
+          attempt: @attempt,
+          error: error,
+          timestamp: Time.now,
+        }
+        @attempt += 1
+      end
+
+      # Check if retry attempts have been exhausted
+      # @param max_attempts [Integer] Maximum allowed attempts
+      # @return [Boolean] true if attempts exhausted
+      def exhausted?(max_attempts)
+        @attempt > max_attempts
+      end
+
+      # Get the last error that occurred
+      # @return [Exception, nil] The last error or nil
+      def last_error
+        @errors.last&.dig(:error)
+      end
+
+      # Get total execution time across all attempts
+      # @return [Numeric] Total time in seconds
+      def total_time
+        Time.now - @started_at
+      end
+
+      # Get a summary of all retry attempts
+      # @return [Hash] Summary data
+      def summary
+        {
+          job_name: @job_name,
+          total_attempts: @attempt - 1,
+          total_time: total_time,
+          errors: @errors.map do |err|
+            {
+              attempt: err[:attempt],
+              error_class: err[:error].class.name,
+              error_message: err[:error].message,
+              timestamp: err[:timestamp],
+            }
+          end,
+        }
+      end
+    end
+  end
+end

data/lib/fractor/workflow/retry_orchestrator.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require_relative "retry_config"
+
+module Fractor
+  class Workflow
+    # Orchestrates retry logic for workflow job execution.
+    # Handles retry strategies, backoff calculations, and attempt tracking.
+    #
+    # @example Basic usage
+    #   config = RetryConfig.from_options(backoff: :exponential, max_attempts: 3)
+    #   orchestrator = RetryOrchestrator.new(config, debug: true)
+    #   result = orchestrator.execute_with_retry(job) { |job| execute_job(job) }
+    class RetryOrchestrator
+      attr_reader :retry_config, :debug, :attempts
+
+      # Initialize a new retry orchestrator.
+      #
+      # @param retry_config [RetryConfig] The retry configuration
+      # @param debug [Boolean] Whether to enable debug logging
+      def initialize(retry_config, debug: false)
+        @retry_config = retry_config
+        @debug = debug
+        @attempts = 0
+        @last_error = nil
+        @all_errors = []
+        @started_at = nil
+      end
+
+      # Execute a job with retry logic.
+      # Retries the job execution according to the retry strategy configuration.
+      #
+      # @param job [Job] The job to execute
+      # @yield [Job] Block that executes the job
+      # @return [Object] The execution result
+      # @raise [StandardError] If all retries are exhausted
+      def execute_with_retry(job)
+        reset!
+
+        @started_at = Time.now
+
+        loop do
+          @attempts += 1
+
+          log_debug "Executing job '#{job.name}', attempt #{@attempts}"
+
+          result = yield job
+
+          # If we got here without error, execution succeeded
+          log_retry_success(job) if @attempts > 1
+          return result
+        rescue StandardError => e
+          @last_error = e
+
+          # Track all errors for DLQ entry
+          @all_errors << {
+            attempt: @attempts,
+            error: e,
+            timestamp: Time.now,
+          }
+
+          # Check if error is retryable
+          unless @retry_config.retryable?(e)
+            log_debug "Error #{e.class} is not retryable, failing immediately"
+            raise e
+          end
+
+          # Record the failure and check if we've exhausted retries
+          if exhausted?(@retry_config.max_attempts)
+            log_retry_exhausted(job)
+            raise e
+          end
+
+          # Calculate delay for this attempt
+          delay = calculate_delay(@attempts)
+
+          # Log retry attempt
+          log_retry_attempt(job, delay)
+
+          # Wait before retrying
+          sleep(delay) if delay.positive?
+        end
+      end
+
+      # Check if a retry should be attempted.
+      #
+      # @param attempt [Integer] The current attempt number
+      # @param error [Exception] The error that occurred
+      # @return [Boolean] true if retry should be attempted
+      def should_retry?(_attempt, error)
+        return false if exhausted?(@retry_config.max_attempts)
+
+        @retry_config.retryable?(error)
+      end
+
+      # Calculate the delay before the next retry attempt.
+      #
+      # @param attempt [Integer] The attempt number
+      # @return [Numeric] The delay in seconds
+      def calculate_delay(attempt)
+        @retry_config.delay_for(attempt)
+      end
+
+      # Get the last error that occurred during retry.
+      #
+      # @return [Exception, nil] The last error or nil
+      def last_error
+        @last_error
+      end
+
+      # Check if all retry attempts are exhausted.
+      #
+      # @param max_attempts [Integer] Maximum number of attempts
+      # @return [Boolean] true if retries are exhausted
+      def exhausted?(max_attempts)
+        @attempts >= max_attempts
+      end
+
+      # Reset the attempt counter and state.
+      def reset!
+        @attempts = 0
+        @last_error = nil
+        @all_errors = []
+        @started_at = nil
+      end
+
+      # Get the current retry state information.
+      #
+      # @return [Hash] Retry state details
+      def state
+        {
+          attempts: @attempts,
+          max_attempts: @retry_config.max_attempts,
+          last_error: @last_error&.class&.name,
+          exhausted: exhausted?(@retry_config.max_attempts),
+          all_errors: @all_errors.map do |err|
+            {
+              attempt: err[:attempt],
+              error_class: err[:error].class.name,
+              error_message: err[:error].message,
+              timestamp: err[:timestamp],
+            }
+          end,
+          total_time: @started_at ? Time.now - @started_at : 0,
+        }
+      end
+
+      private
+
+      # Log a successful retry.
+      #
+      # @param job [Job] The job that succeeded
+      def log_retry_success(job)
+        puts "[RetryOrchestrator] Job '#{job.name}' succeeded on attempt #{@attempts}" if @debug
+      end
+
+      # Log that retries are exhausted.
+      #
+      # @param job [Job] The job that failed
+      def log_retry_exhausted(job)
+        puts "[RetryOrchestrator] Job '#{job.name}' retries exhausted after #{@attempts} attempts" if @debug
+      end
+
+      # Log a retry attempt.
+      #
+      # @param job [Job] The job being retried
+      # @param delay [Numeric] The delay before next attempt
+      def log_retry_attempt(job, delay)
+        message = "[RetryOrchestrator] Retrying job '#{job.name}' (attempt #{@attempts + 1}"
+        message += " after #{delay}s delay" if delay.positive?
+        message += ")"
+
+        puts message if @debug
+      end
+
+      # Log a debug message.
+      #
+      # @param message [String] The message to log
+      def log_debug(message)
+        puts "[RetryOrchestrator] #{message}" if @debug
+      end
+    end
+  end
+end

data/lib/fractor/workflow/retry_strategy.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Fractor
+  class Workflow
+    # Base class for retry strategies
+    class RetryStrategy
+      attr_reader :max_attempts, :max_delay
+
+      def initialize(max_attempts: 3, max_delay: nil)
+        @max_attempts = max_attempts
+        @max_delay = max_delay
+      end
+
+      # Calculate delay for the given attempt number
+      # @param attempt [Integer] The attempt number (1-based)
+      # @return [Numeric] Delay in seconds
+      def delay_for(attempt)
+        raise NotImplementedError, "Subclasses must implement delay_for"
+      end
+
+      protected
+
+      def cap_delay(delay)
+        return delay unless max_delay
+
+        [delay, max_delay].min
+      end
+    end
+
+    # Exponential backoff retry strategy
+    class ExponentialBackoff < RetryStrategy
+      attr_reader :initial_delay, :multiplier
+
+      def initialize(initial_delay: 1, multiplier: 2, **options)
+        super(**options)
+        @initial_delay = initial_delay
+        @multiplier = multiplier
+      end
+
+      def delay_for(attempt)
+        return 0 if attempt <= 1
+
+        delay = initial_delay * (multiplier**(attempt - 2))
+        cap_delay(delay)
+      end
+    end
+
+    # Linear backoff retry strategy
+    class LinearBackoff < RetryStrategy
+      attr_reader :initial_delay, :increment
+
+      def initialize(initial_delay: 1, increment: 1, **options)
+        super(**options)
+        @initial_delay = initial_delay
+        @increment = increment
+      end
+
+      def delay_for(attempt)
+        return 0 if attempt <= 1
+
+        delay = initial_delay + (increment * (attempt - 2))
+        cap_delay(delay)
+      end
+    end
+
+    # Constant delay retry strategy
+    class ConstantDelay < RetryStrategy
+      attr_reader :delay
+
+      def initialize(delay: 1, **options)
+        super(**options)
+        @delay = delay
+      end
+
+      def delay_for(attempt)
+        return 0 if attempt <= 1
+
+        cap_delay(delay)
+      end
+    end
+
+    # No retry strategy
+    class NoRetry < RetryStrategy
+      def initialize
+        super(max_attempts: 1)
+      end
+
+      def delay_for(_attempt)
+        0
+      end
+    end
+  end
+end

data/lib/fractor/workflow/structured_logger.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "logger"
+require "json"
+
+module Fractor
+  class Workflow
+    # Structured logger that outputs JSON-formatted logs.
+    # Useful for log aggregation systems like ELK, Splunk, CloudWatch, etc.
+    class StructuredLogger < WorkflowLogger
+      def initialize(logger: nil, correlation_id: nil, format: :json)
+        super(logger: logger, correlation_id: correlation_id)
+        @format = format
+      end
+
+      private
+
+      def format_message(log_data)
+        case @format
+        when :json
+          log_data.to_json
+        when :pretty_json
+          JSON.pretty_generate(log_data)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/fractor/workflow/type_compatibility_validator.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module Fractor
+  class Workflow
+    # Validates type compatibility between jobs in workflows.
+    # Ensures input/output types are properly declared and compatible.
+    #
+    # This validator helps catch type mismatches before workflow execution.
+    class TypeCompatibilityValidator
+      # Error raised when type validation fails.
+      class TypeError < StandardError; end
+
+      def initialize(jobs)
+        @jobs = jobs
+      end
+
+      # Validate all job type declarations.
+      # Raises TypeError if any validation fails.
+      #
+      # @raise [TypeError] if validation fails
+      # @return [true] if validation passes
+      def validate!
+        @jobs.each do |job|
+          check_job_compatibility(job)
+        end
+        true
+      end
+
+      # Check that a job's type declarations are valid.
+      #
+      # @param job [Job] The job to check
+      # @raise [TypeError] if job has invalid type declarations
+      # @return [true] if job is valid
+      def check_job_compatibility(job)
+        # Check if worker has input type declared
+        if job.input_type
+          check_type_declaration(job, :input, job.input_type)
+        end
+
+        # Check if worker has output type declared
+        if job.output_type
+          check_type_declaration(job, :output, job.output_type)
+        end
+
+        true
+      end
+
+      # Check that a type declaration is valid.
+      #
+      # @param job [Job] The job with the type declaration
+      # @param direction [Symbol] :input or :output
+      # @param type [Class] The type class to validate
+      # @raise [TypeError] if type declaration is invalid
+      # @return [true] if type is valid
+      def check_type_declaration(job, direction, type)
+        # Check if type is a class
+        unless type.is_a?(Class)
+          raise TypeError, type_declaration_error(job, direction,
+                                                  "#{type.inspect} is not a class",
+                                                  "Use a class like String or Integer")
+        end
+
+        # Check if type is not Object (too generic)
+        if type == Object
+          warn "Job '#{job.name}' has #{direction}_type Object, which is too generic. " \
+               "Consider using a more specific type for better validation."
+        end
+
+        # Check if type is BasicObject (even more generic)
+        if type == BasicObject
+          raise TypeError, type_declaration_error(job, direction,
+                                                  "#{type} is too generic to be useful",
+                                                  "Use a specific class like String or Hash")
+        end
+
+        true
+      end
+
+      # Check type compatibility between connected jobs.
+      # Validates that output type of producer matches input type of consumer.
+      # Skips jobs with multiple dependencies (using inputs_from_multiple).
+      # Skips jobs using inputs_from_workflow (they use workflow input, not dependency output).
+      #
+      # @return [Hash] Compatibility report with any issues found
+      def check_compatibility_between_jobs
+        issues = []
+
+        @jobs.each do |consumer_job|
+          # Skip type checking for jobs with multiple dependencies
+          # These jobs use inputs_from_multiple to explicitly map outputs
+          next if consumer_job.dependencies.size > 1
+
+          # Skip type checking for jobs using inputs_from_workflow
+          # These jobs use the workflow's input type, not their dependency's output type
+          next if consumer_job.input_mappings.key?(:workflow)
+
+          consumer_job.dependencies.each do |producer_name|
+            producer_job = find_job(producer_name)
+            next unless producer_job
+
+            # Check if both have type declarations
+            # Check if types are compatible
+            if producer_job.output_type && consumer_job.input_type && !types_compatible?(producer_job.output_type,
+                                                                                         consumer_job.input_type)
+              issues << {
+                producer: producer_job.name,
+                consumer: consumer_job.name,
+                producer_type: producer_job.output_type,
+                consumer_type: consumer_job.input_type,
+                suggestion: suggest_type_fix(producer_job.output_type,
+                                             consumer_job.input_type),
+              }
+            end
+          end
+        end
+
+        issues
+      end
+
+      private
+
+      # Find a job by name.
+      #
+      # @param name [String] Job name
+      # @return [Job, nil] The job or nil if not found
+      def find_job(name)
+        @jobs.find { |j| j.name == name }
+      end
+
+      # Check if two types are compatible.
+      # For now, we use a simple check: output type should be a subclass of input type
+      # or they should be the same class.
+      #
+      # @param output_type [Class] The producer's output type
+      # @param input_type [Class] The consumer's input type
+      # @return [Boolean] true if types are compatible
+      def types_compatible?(output_type, input_type)
+        # Same type is always compatible
+        return true if output_type == input_type
+
+        # Output type is a subclass of input type (covariance)
+        return true if output_type < input_type
+
+        # Input type is Object (accepts anything)
+        return true if input_type == Object
+
+        # Special case: Numeric and Integer/Float are compatible
+        return true if numeric_compatibility?(output_type, input_type)
+
+        false
+      end
+
+      # Check for numeric type compatibility.
+      #
+      # @param output_type [Class] The producer's output type
+      # @param input_type [Class] The consumer's input type
+      # @return [Boolean] true if numerically compatible
+      def numeric_compatibility?(output_type, input_type)
+        # Integer is compatible with Numeric
+        return true if output_type == Integer && input_type == Numeric
+
+        # Float is compatible with Numeric
+        return true if output_type == Float && input_type == Numeric
+
+        false
+      end
+
+      # Suggest a fix for type incompatibility.
+      #
+      # @param output_type [Class] The producer's output type
+      # @param input_type [Class] The consumer's input type
+      # @return [String] Suggestion message
+      def suggest_type_fix(output_type, input_type)
+        # Find common ancestor
+        common_ancestor = find_common_ancestor(output_type, input_type)
+
+        if common_ancestor
+          "Consider using #{common_ancestor.name} as the input type for the consumer, " \
+          "or ensure the producer outputs #{input_type.name} instead of #{output_type.name}"
+        else
+          "The producer's output type (#{output_type.name}) is not compatible with " \
+          "the consumer's input type (#{input_type.name}). " \
+          "Ensure the producer outputs data that the consumer can process."
+        end
+      end
+
+      # Find the common ancestor class of two types.
+      #
+      # @param type1 [Class] First type
+      # @param type2 [Class] Second type
+      # @return [Class, nil] Common ancestor class or nil
+      def find_common_ancestor(type1, type2)
+        return type1 if type2 == Object
+        return type2 if type1 == Object
+
+        # Get ancestry chains
+        type1_ancestors = type1.ancestors
+        type2_ancestors = type2.ancestors
+
+        # Find common ancestor
+        type1_ancestors.each do |ancestor|
+          return ancestor if type2_ancestors.include?(ancestor)
+        end
+
+        nil
+      end
+
+      # Build a formatted error message for type declaration issues.
+      #
+      # @param job [Job] The job with the issue
+      # @param direction [Symbol] :input or :output
+      # @param problem [String] Description of the problem
+      # @param suggestion [String] Suggestion for fixing
+      # @return [String] Formatted error message
+      def type_declaration_error(job, direction, problem, suggestion)
+        "Job '#{job.name}' has invalid #{direction}_type declaration:\n" \
+        "  Problem: #{problem}\n" \
+        "  Suggestion: #{suggestion}"
+      end
+    end
+  end
+end