fractor 0.1.6 → 0.1.8

data/lib/fractor/workflow/workflow_result.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Fractor
+  class Workflow
+    # Represents the result of a workflow execution.
+    # Contains information about completed jobs, failed jobs, execution time, and output.
+    class WorkflowResult
+      attr_reader :workflow_name, :output, :completed_jobs, :failed_jobs,
+                  :execution_time, :success, :trace, :correlation_id
+
+      # Initialize a new workflow result.
+      #
+      # @param workflow_name [String] The name of the workflow
+      # @param output [Object] The workflow output
+      # @param completed_jobs [Array<String>] List of completed job names
+      # @param failed_jobs [Array<String>] List of failed job names
+      # @param execution_time [Float] Execution time in seconds
+      # @param success [Boolean] Whether the workflow succeeded
+      # @param trace [ExecutionTrace, nil] Optional execution trace
+      # @param correlation_id [String, nil] Optional correlation ID
+      def initialize(workflow_name:, output:, completed_jobs:, failed_jobs:,
+                     execution_time:, success:, trace: nil, correlation_id: nil)
+        @workflow_name = workflow_name
+        @output = output
+        @completed_jobs = completed_jobs
+        @failed_jobs = failed_jobs
+        @execution_time = execution_time
+        @success = success
+        @trace = trace
+        @correlation_id = correlation_id
+      end
+
+      # Check if the workflow succeeded.
+      #
+      # @return [Boolean] true if successful
+      def success?
+        @success
+      end
+
+      # Check if the workflow failed.
+      #
+      # @return [Boolean] true if failed
+      def failed?
+        !@success
+      end
+
+      # Get execution time in milliseconds.
+      #
+      # @return [Float] Execution time in milliseconds
+      def execution_time_ms
+        (@execution_time * 1000).round(2)
+      end
+    end
+  end
+end

data/lib/fractor/workflow/workflow_validator.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Fractor
+  class Workflow
+    # Validates workflow structure and configuration.
+    # Checks for cycles, missing dependencies, type compatibility, and proper entry/exit points.
+    #
+    # This validator integrates JobDependencyValidator and TypeCompatibilityValidator
+    # to provide comprehensive validation with detailed error messages.
+    class WorkflowValidator
+      attr_reader :workflow_class
+
+      def initialize(workflow_class)
+        @workflow_class = workflow_class
+      end
+
+      # Validate the workflow structure.
+      # Raises appropriate errors if validation fails.
+      def validate!
+        validate_basic_structure!
+        apply_smart_defaults!
+        auto_wire_job_inputs!
+
+        # Use new validators for better error messages
+        validate_dependencies_with_new_validator!
+        validate_type_compatibility!
+
+        validate_entry_exit_points! unless continuous_mode?
+        validate_job_workers!
+        validate_input_mappings!
+      end
+
+      private
+
+      def continuous_mode?
+        @workflow_class.workflow_mode == :continuous
+      end
+
+      def validate_basic_structure!
+        if @workflow_class.jobs.empty?
+          raise WorkflowError,
+                "Workflow '#{@workflow_class.workflow_name}' has no jobs defined.\n\n" \
+                "A workflow must define at least one job using the `job` DSL method:\n\n" \
+                "  workflow '#{@workflow_class.workflow_name}' do\n" \
+                "    job 'process' do\n" \
+                "      runs_with MyWorker\n" \
+                "      inputs_from_workflow\n" \
+                "      outputs_to_workflow\n" \
+                "    end\n" \
+                "  end"
+        end
+      end
+
+      # Apply smart defaults for start/end jobs if not explicitly configured
+      def apply_smart_defaults!
+        return if continuous_mode?
+
+        # Auto-detect start job if not specified
+        unless @workflow_class.start_job_name
+          start_jobs = @workflow_class.jobs.values.select do |job|
+            job.dependencies.empty?
+          end
+
+          if start_jobs.size == 1
+            @workflow_class.instance_variable_set(:@start_job_name,
+                                                  start_jobs.first.name)
+          end
+        end
+
+        # Auto-detect end jobs if not specified
+        if @workflow_class.end_job_names.empty?
+          # Find jobs with no dependents (leaf jobs)
+          all_dependencies = @workflow_class.jobs.values.flat_map(&:dependencies).to_set
+          end_job_candidates = @workflow_class.jobs.keys.reject do |job_name|
+            all_dependencies.include?(job_name)
+          end
+
+          end_job_candidates.each do |job_name|
+            job = @workflow_class.jobs[job_name]
+            job.outputs_to_workflow
+            job.terminates_workflow
+            @workflow_class.end_job_names << { name: job_name,
+                                               condition: :success }
+          end
+        end
+      end
+
+      # Auto-wire job inputs based on dependencies
+      def auto_wire_job_inputs!
+        @workflow_class.jobs.each_value(&:auto_wire_inputs!)
+      end
+
+      # Validate dependencies using JobDependencyValidator for better error messages.
+      def validate_dependencies_with_new_validator!
+        jobs = @workflow_class.jobs.values
+        validator = JobDependencyValidator.new(jobs)
+
+        begin
+          validator.validate!
+        rescue JobDependencyValidator::DependencyError => e
+          # Convert to WorkflowError with additional context
+          raise WorkflowError,
+                "Workflow '#{@workflow_class.workflow_name}' has dependency issues:\n\n" \
+                "#{e.message}\n\n" \
+                "Fix: Ensure all job dependencies exist and there are no circular dependencies."
+        end
+      end
+
+      # Validate type compatibility between connected jobs.
+      def validate_type_compatibility!
+        jobs = @workflow_class.jobs.values
+        validator = TypeCompatibilityValidator.new(jobs)
+
+        issues = validator.check_compatibility_between_jobs
+        return if issues.empty?
+
+        # Build detailed error message
+        error_lines = ["Workflow '#{@workflow_class.workflow_name}' has type compatibility issues:\n"]
+
+        issues.each do |issue|
+          error_lines << "  Job '#{issue[:consumer]}' depends on '#{issue[:producer]}'"
+          error_lines << "    Producer output type: #{issue[:producer_type]}"
+          error_lines << "    Consumer input type: #{issue[:consumer_type]}"
+          error_lines << "    Suggestion: #{issue[:suggestion]}"
+          error_lines << ""
+        end
+
+        error_lines << "Fix: Ensure compatible types between connected jobs."
+
+        raise WorkflowError, error_lines.join("\n")
+      end
+
+      def validate_entry_exit_points!
+        # Pipeline mode requires start_with and end_with
+        unless @workflow_class.start_job_name
+          raise WorkflowError,
+                "Pipeline workflow '#{@workflow_class.workflow_name}' must define start_with.\n\n" \
+                "Add a start job to your workflow:\n\n" \
+                "  workflow '#{@workflow_class.workflow_name}' do\n" \
+                "    start_with 'process'  # Define the starting job\n" \
+                "    job 'process' do\n" \
+                "      runs_with MyWorker\n" \
+                "      # ...\n" \
+                "    end\n" \
+                "  end"
+        end
+
+        if @workflow_class.end_job_names.empty?
+          raise WorkflowError,
+                "Pipeline workflow '#{@workflow_class.workflow_name}' must define at least one end_with.\n\n" \
+                "Add an end job to your workflow:\n\n" \
+                "  workflow '#{@workflow_class.workflow_name}' do\n" \
+                "    # ...\n" \
+                "    end_with 'finalize'  # Define the ending job\n" \
+                "    job 'finalize' do\n" \
+                "      runs_with FinalizeWorker\n" \
+                "      outputs_to_workflow\n" \
+                "      terminates_workflow\n" \
+                "    end\n" \
+                "  end"
+        end
+
+        # Verify start job exists
+        unless @workflow_class.jobs.key?(@workflow_class.start_job_name)
+          raise WorkflowError,
+                "Start job '#{@workflow_class.start_job_name}' not defined in workflow.\n\n" \
+                "Available jobs: #{@workflow_class.jobs.keys.join(', ')}\n\n" \
+                "Fix: Define the missing job or correct the start_with name."
+        end
+
+        # Verify end jobs exist
+        @workflow_class.end_job_names.each do |end_job_spec|
+          job_name = end_job_spec[:name]
+          unless @workflow_class.jobs.key?(job_name)
+            raise WorkflowError,
+                  "End job '#{job_name}' not defined in workflow.\n\n" \
+                  "Available jobs: #{@workflow_class.jobs.keys.join(', ')}\n\n" \
+                  "Fix: Define the missing job or correct the end_with name."
+          end
+        end
+
+        # Verify all jobs are reachable from start
+        validate_reachability!
+      end
+
+      def validate_job_workers!
+        @workflow_class.jobs.each do |name, job|
+          unless job.worker_class
+            raise WorkflowError,
+                  "Job '#{name}' does not specify a worker class.\n\n" \
+                  "Add a worker using runs_with:\n\n" \
+                  "  job '#{name}' do\n" \
+                  "    runs_with MyWorker  # Specify the worker class\n" \
+                  "  end"
+          end
+
+          unless job.input_type
+            raise WorkflowError,
+                  "Job '#{name}' worker '#{job.worker_class}' does not declare input_type.\n\n" \
+                  "Add input_type to your worker:\n\n" \
+                  "  class #{job.worker_class} < Fractor::Worker\n" \
+                  "    input_type MyInputClass\n" \
+                  "    output_type MyOutputClass\n" \
+                  "  end"
+          end
+
+          unless job.output_type
+            raise WorkflowError,
+                  "Job '#{name}' worker '#{job.worker_class}' does not declare output_type.\n\n" \
+                  "Add output_type to your worker:\n\n" \
+                  "  class #{job.worker_class} < Fractor::Worker\n" \
+                  "    input_type MyInputClass\n" \
+                  "    output_type MyOutputClass\n" \
+                  "  end"
+          end
+        end
+      end
+
+      def validate_input_mappings!
+        @workflow_class.jobs.each do |name, job|
+          # After auto-wiring, all jobs should have input mappings
+          if job.input_mappings.empty?
+            if job.dependencies.size > 1
+              raise WorkflowError,
+                    "Job '#{name}' has multiple dependencies (#{job.dependencies.join(', ')}). " \
+                    "Please explicitly configure inputs using inputs_from_job or inputs_from_multiple"
+            else
+              raise WorkflowError,
+                    "Job '#{name}' has no input mappings configured"
+            end
+          end
+
+          # Validate source jobs exist in mappings
+          job.input_mappings.each_key do |source|
+            next if source == :workflow
+
+            unless @workflow_class.jobs.key?(source)
+              raise WorkflowError,
+                    "Job '#{name}' maps inputs from '#{source}' which is not defined"
+            end
+          end
+        end
+      end
+
+      def validate_reachability!
+        start_job = @workflow_class.start_job_name
+        reachable = compute_reachable_jobs(start_job)
+
+        unreachable = @workflow_class.jobs.keys.to_set - reachable
+        return if unreachable.empty?
+
+        raise WorkflowError,
+              "Unreachable jobs detected: #{unreachable.to_a.join(', ')}. " \
+              "All jobs must be reachable from start_with job '#{start_job}'"
+      end
+
+      def compute_reachable_jobs(start_job)
+        reachable = Set.new
+        queue = [start_job]
+
+        until queue.empty?
+          current = queue.shift
+          next if reachable.include?(current)
+
+          reachable.add(current)
+
+          # Find jobs that depend on current job
+          @workflow_class.jobs.each do |name, job|
+            # Follow explicit dependencies
+            if job.dependencies.include?(current) && !reachable.include?(name)
+              queue << name
+            end
+
+            # Follow fallback relationships from current job
+            if current == name && job.fallback_job && !reachable.include?(job.fallback_job)
+              queue << job.fallback_job
+            end
+          end
+        end
+
+        reachable
+      end
+    end
+  end
+
+  # Custom error classes
+  class WorkflowError < StandardError; end
+  class WorkflowCycleError < WorkflowError; end
+  class WorkflowValidationError < WorkflowError; end
+  class InputMismatchError < WorkflowError; end
+  class OutputMismatchError < WorkflowError; end
+  class WorkflowExecutionError < WorkflowError; end
+end

data/lib/fractor/workflow.rb

@@ -0,0 +1,333 @@
+# frozen_string_literal: true
+
+require_relative "workflow/job"
+require_relative "workflow/workflow_context"
+require_relative "workflow/workflow_executor"
+require_relative "workflow/workflow_validator"
+require_relative "workflow/job_dependency_validator"
+require_relative "workflow/type_compatibility_validator"
+require_relative "workflow/execution_strategy"
+require_relative "workflow/builder"
+require_relative "workflow/chain_builder"
+require_relative "workflow/helpers"
+require_relative "workflow/logger"
+require_relative "workflow/structured_logger"
+require_relative "workflow/execution_trace"
+require_relative "workflow/visualizer"
+require_relative "workflow/dead_letter_queue"
+require_relative "workflow/pre_execution_context"
+require_relative "workflow/retry_orchestrator"
+require_relative "workflow/circuit_breaker_orchestrator"
+
+module Fractor
+  # Base class for defining workflows using a declarative DSL.
+  # Workflows coordinate multiple jobs with dependencies, type-safe data flow,
+  # and support both pipeline and continuous execution modes.
+  class Workflow
+    class << self
+      attr_reader :workflow_name, :workflow_mode, :jobs, :start_job_name,
+                  :end_job_names, :input_model_class, :output_model_class,
+                  :dlq_config
+
+      # Create a workflow class without inheritance.
+      # This is a convenience method that creates an anonymous workflow class.
+      #
+      # @param name [String] The workflow name
+      # @param mode [Symbol] :pipeline (default) or :continuous
+      # @yield Block containing job definitions using workflow DSL
+      # @return [Class] A new Workflow subclass
+      #
+      # @example
+      #   workflow = Fractor::Workflow.define("my-workflow") do
+      #     job "step1", Worker1
+      #     job "step2", Worker2, needs: "step1"
+      #     job "step3", Worker3, needs: "step2"
+      #   end
+      #   instance = workflow.new
+      #   result = instance.execute(input: data)
+      def define(name, mode: :pipeline, &block)
+        Class.new(Workflow) do
+          workflow(name, mode: mode, &block)
+        end
+      end
+
+      # Create a linear chain workflow for sequential processing.
+      # This provides a fluent API for simple pipelines.
+      #
+      # @param name [String] The workflow name
+      # @return [ChainBuilder] A builder for constructing the chain
+      #
+      # @example
+      #   workflow = Fractor::Workflow.chain("pipeline")
+      #     .step("uppercase", UppercaseWorker)
+      #     .step("reverse", ReverseWorker)
+      #     .step("finalize", FinalizeWorker)
+      #     .build
+      def chain(name)
+        ChainBuilder.new(name)
+      end
+
+      # Define a workflow with the given name and optional mode.
+      #
+      # @param name [String] The workflow name
+      # @param mode [Symbol] :pipeline (default) or :continuous
+      # @yield Block containing job definitions
+      def workflow(name, mode: :pipeline, &block)
+        @workflow_name = name
+        @workflow_mode = mode
+        @jobs = {}
+        @start_job_name = nil
+        @end_job_names = []
+        @input_model_class = nil
+        @output_model_class = nil
+        @dlq_config = nil
+
+        instance_eval(&block) if block
+
+        validate_workflow!
+      end
+
+      # Declare the workflow's input type.
+      #
+      # @param klass [Class] A Lutaml::Model::Serializable subclass
+      def input_type(klass)
+        validate_model_class!(klass, "input_type")
+        @input_model_class = klass
+      end
+
+      # Declare the workflow's output type.
+      #
+      # @param klass [Class] A Lutaml::Model::Serializable subclass
+      def output_type(klass)
+        validate_model_class!(klass, "output_type")
+        @output_model_class = klass
+      end
+
+      # Define the starting job for pipeline mode.
+      #
+      # @param job_name [String, Symbol] The name of the start job
+      def start_with(job_name)
+        @start_job_name = job_name.to_s
+      end
+
+      # Define an ending job for pipeline mode.
+      #
+      # @param job_name [String, Symbol] The name of the end job
+      # @param on [Symbol] Condition: :success (default), :failure, :cancellation
+      def end_with(job_name, on: :success)
+        @end_job_names << { name: job_name.to_s, condition: on }
+      end
+
+      # Configure the Dead Letter Queue for failed work.
+      #
+      # @param max_size [Integer] Maximum number of entries to retain
+      # @param persister [Object] Optional persistence strategy
+      # @param on_add [Proc] Optional callback when entry is added
+      def configure_dead_letter_queue(max_size: 1000, persister: nil,
+on_add: nil)
+        @dlq_config = {
+          max_size: max_size,
+          persister: persister,
+          on_add: on_add,
+        }
+      end
+
+      # Define a job in the workflow.
+      #
+      # @param name [String, Symbol] The job name
+      # @param worker_class [Class] Optional worker class (shorthand syntax)
+      # @param needs [String, Symbol, Array] Optional dependencies (shorthand)
+      # @param inputs [Symbol, String, Hash] Optional input configuration (shorthand)
+      # @param outputs [Symbol] Optional :workflow to mark outputs (shorthand)
+      # @param workers [Integer] Optional parallel worker count (shorthand)
+      # @param condition [Proc] Optional conditional execution (shorthand)
+      # @yield Block containing job configuration (DSL syntax)
+      #
+      # @example DSL syntax (original)
+      #   job "process" do
+      #     runs_with ProcessWorker
+      #     needs "validate"
+      #   end
+      #
+      # @example Shorthand syntax (simplified)
+      #   job "process", ProcessWorker, needs: "validate"
+      #
+      # @example Shorthand with multiple options
+      #   job "process", ProcessWorker, needs: "validate", outputs: :workflow
+      def job(name, worker_class = nil, needs: nil, inputs: nil, outputs: nil,
+              workers: nil, condition: nil, &block)
+        job_name = name.to_s
+        if @jobs.key?(job_name)
+          raise ArgumentError,
+                "Job '#{job_name}' already defined"
+        end
+
+        job_obj = Job.new(job_name, self)
+
+        # Apply shorthand parameters if provided
+        if worker_class
+          job_obj.runs_with(worker_class)
+        end
+
+        if needs
+          needs_array = needs.is_a?(Array) ? needs : [needs]
+          job_obj.needs(*needs_array)
+        end
+
+        if inputs
+          case inputs
+          when :workflow, "workflow"
+            job_obj.inputs_from_workflow
+          when String, Symbol
+            job_obj.inputs_from_job(inputs.to_s)
+          when Hash
+            job_obj.inputs_from_multiple(inputs)
+          end
+        end
+
+        if outputs == :workflow
+          job_obj.outputs_to_workflow
+        end
+
+        if workers
+          job_obj.parallel_workers(workers)
+        end
+
+        if condition
+          job_obj.if_condition(condition)
+        end
+
+        # Apply DSL block if provided
+        job_obj.instance_eval(&block) if block
+
+        @jobs[job_name] = job_obj
+      end
+
+      # Generate a Mermaid flowchart diagram of the workflow
+      #
+      # @return [String] Mermaid diagram syntax
+      def to_mermaid
+        Visualizer.new(self).to_mermaid
+      end
+
+      # Generate a DOT/Graphviz diagram of the workflow
+      #
+      # @return [String] DOT diagram syntax
+      def to_dot
+        Visualizer.new(self).to_dot
+      end
+
+      # Generate an ASCII art diagram of the workflow
+      #
+      # @return [String] ASCII art representation
+      def to_ascii
+        Visualizer.new(self).to_ascii
+      end
+
+      # Print ASCII diagram to stdout
+      def print_diagram
+        Visualizer.new(self).print
+      end
+
+      private
+
+      def validate_model_class!(klass, method_name)
+        # Allow any class - in production you may want stricter validation
+        return if klass.is_a?(Class)
+
+        raise ArgumentError, "#{method_name} must be a Class"
+      end
+
+      def validate_workflow!
+        validator = WorkflowValidator.new(self)
+        validator.validate!
+      end
+    end
+
+    # Create a new workflow instance.
+    def initialize(input = nil)
+      unless self.class.workflow_name
+        raise "Workflow not defined. Use 'workflow \"name\" do ... end' in class definition"
+      end
+
+      @workflow_input = input
+      @dead_letter_queue = initialize_dead_letter_queue
+    end
+
+    # Access the Dead Letter Queue for this workflow.
+    #
+    # @return [DeadLetterQueue, nil] The DLQ instance or nil if not configured
+    def dead_letter_queue
+      @dead_letter_queue
+    end
+
+    # Execute the workflow with the given input.
+    #
+    # @param input [Lutaml::Model::Serializable, nil] The workflow input (optional if provided to initialize)
+    # @param correlation_id [String] Optional correlation ID for tracking
+    # @param logger [Logger] Optional logger instance
+    # @param trace [Boolean] Whether to generate execution trace
+    # @yield [WorkflowExecutor] Optional block for registering hooks
+    # @return [WorkflowResult] The execution result
+    def execute(input: nil, correlation_id: nil, logger: nil, trace: false,
+&block)
+      # Use provided input or fall back to initialization input
+      workflow_input = input || @workflow_input
+      validate_input!(workflow_input)
+
+      executor = WorkflowExecutor.new(
+        self,
+        workflow_input,
+        correlation_id: correlation_id,
+        logger: logger,
+        trace: trace,
+        dead_letter_queue: @dead_letter_queue,
+      )
+
+      # Allow block to register hooks
+      block&.call(executor)
+
+      executor.execute
+    end
+
+    # Run the workflow in continuous mode with a work queue.
+    #
+    # @param work_queue [WorkQueue] The queue to receive workflow inputs
+    def run_continuous(work_queue:)
+      unless self.class.workflow_mode == :continuous
+        raise "Workflow '#{self.class.workflow_name}' is not configured for continuous mode"
+      end
+
+      # Continuous mode implementation will be added
+      raise NotImplementedError, "Continuous mode coming soon"
+    end
+
+    private
+
+    def initialize_dead_letter_queue
+      config = self.class.dlq_config
+      return nil unless config
+
+      dlq = DeadLetterQueue.new(
+        max_size: config[:max_size],
+        persister: config[:persister],
+      )
+
+      # Register callback if provided
+      dlq.on_add(&config[:on_add]) if config[:on_add]
+
+      dlq
+    end
+
+    def validate_input!(input)
+      expected_type = self.class.input_model_class
+      return unless expected_type
+
+      unless input.is_a?(expected_type)
+        raise TypeError,
+              "Workflow '#{self.class.workflow_name}' expects input of type " \
+              "#{expected_type}, got #{input.class}"
+      end
+    end
+  end
+end