tasker-engine 1.0.0

data/lib/tasker/orchestration/task_initializer.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require_relative '../concerns/idempotent_state_transitions'
+require_relative '../concerns/event_publisher'
+require 'tasker/events/event_payload_builder'
+
+module Tasker
+  module Orchestration
+    # TaskInitializer handles task creation and initialization logic
+    #
+    # This component is responsible for:
+    # - Creating tasks from TaskRequest objects
+    # - Validating task context against schemas
+    # - Starting task execution (transitioning to in_progress)
+    # - Enqueuing tasks for processing
+    class TaskInitializer
+      include Tasker::Concerns::IdempotentStateTransitions
+      include Tasker::Concerns::EventPublisher
+
+      class << self
+        # Initialize a new task from a task request
+        #
+        # Creates a task record, validates the context against the schema,
+        # and enqueues the task for processing.
+        #
+        # @param task_request [Tasker::Types::TaskRequest] The task request
+        # @param task_handler [Object] The task handler instance for schema validation
+        # @return [Tasker::Task] The created task
+        delegate :initialize_task!, to: :new
+
+        # Start a task's execution
+        #
+        # Updates the task status to IN_PROGRESS and fires the appropriate event.
+        #
+        # @param task [Tasker::Task] The task to start
+        # @return [Boolean] True if the task was started successfully
+        delegate :start_task!, to: :new
+      end
+
+      # Initialize a new task from a task request
+      #
+      # @param task_request [Tasker::Types::TaskRequest] The task request
+      # @param task_handler [Object] The task handler instance for schema validation
+      # @return [Tasker::Task] The created task
+      def initialize_task!(task_request, task_handler)
+        task = nil
+        context_errors = validate_context_with_handler(task_request.context, task_handler)
+
+        if context_errors.length.positive?
+          task = Tasker::Task.from_task_request(task_request)
+          context_errors.each do |error|
+            task.errors.add(:context, error)
+          end
+
+          # Use clean API for task initialization failure
+          publish_task_failed(
+            task,
+            error_message: context_errors.join(', '),
+            initialization_failed: true
+          )
+          return task
+        end
+
+        Tasker::Task.transaction do
+          task = Tasker::Task.create_with_defaults!(task_request)
+          # Get sequence and establish dependencies
+          StepSequenceFactory.create_sequence_for_task!(task, task_handler)
+        end
+
+        # Use clean API for task initialization success
+        publish_task_started(
+          task,
+          step_count: task.workflow_steps.count
+        )
+
+        enqueue_task(task)
+        task
+      end
+
+      # Start a task's execution
+      #
+      # @param task [Tasker::Task] The task to start
+      # @return [Boolean] True if the task was started successfully
+      def start_task!(task)
+        raise(Tasker::ProceduralError, "task already complete for task #{task.task_id}") if task.complete
+
+        unless task.status == Tasker::Constants::TaskStatuses::PENDING
+          raise(Tasker::ProceduralError,
+                "task is not pending for task #{task.task_id}, status is #{task.status}")
+        end
+
+        task.context = ActiveSupport::HashWithIndifferentAccess.new(task.context)
+
+        # Use state machine to transition task to in_progress
+        unless safe_transition_to(task, Tasker::Constants::TaskStatuses::IN_PROGRESS, {
+                                    initialization_completed: true,
+                                    step_dependencies_established: task.workflow_steps.count
+                                  })
+
+          # Use clean API for task start failure
+          publish_task_failed(
+            task,
+            error_message: 'Failed to transition to in_progress',
+            initialization_failed: true
+          )
+
+          return false
+        end
+
+        # Use clean API for task start success
+        publish_task_started(task, task_context: task.context)
+
+        true
+      end
+
+      private
+
+      # Validate a task context against the handler's schema
+      #
+      # @param context [Hash] The context to validate
+      # @param task_handler [Object] The task handler with schema
+      # @return [Array<String>] Validation errors, if any
+      def validate_context_with_handler(context, task_handler)
+        return [] unless task_handler.respond_to?(:schema) && task_handler.schema
+
+        data = context.to_hash.deep_symbolize_keys
+        JSON::Validator.fully_validate(task_handler.schema, data, strict: true, insert_defaults: true)
+      end
+
+      # Enqueue a task for processing
+      #
+      # @param task [Tasker::Task] The task to enqueue
+      def enqueue_task(task)
+        publish_task_enqueue(task)
+
+        Tasker::TaskRunnerJob.perform_later(task.task_id)
+      end
+    end
+  end
+end

data/lib/tasker/orchestration/task_reenqueuer.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative '../concerns/idempotent_state_transitions'
+require_relative '../concerns/event_publisher'
+
+module Tasker
+  module Orchestration
+    # TaskReenqueuer handles the mechanics of re-enqueueing tasks for continued processing
+    #
+    # This class provides implementation for task re-enqueueing logic while firing
+    # lifecycle events for observability. Separates the decision to re-enqueue
+    # from the mechanics of how re-enqueueing works.
+    class TaskReenqueuer
+      include Tasker::Concerns::IdempotentStateTransitions
+      include Tasker::Concerns::EventPublisher
+
+      # Re-enqueue a task for continued processing
+      #
+      # @param task [Tasker::Task] The task to re-enqueue
+      # @param reason [String] The reason for re-enqueueing (for observability)
+      # @return [Boolean] True if re-enqueueing was successful
+      def reenqueue_task(task, reason: Constants::TaskFinalization::ReenqueueReasons::PENDING_STEPS_REMAINING)
+        # Fire re-enqueue started event
+        publish_task_reenqueue_started(task, reason: reason)
+
+        # Transition task back to pending state for clarity
+        if safe_transition_to(task, Tasker::Constants::TaskStatuses::PENDING)
+          Rails.logger.debug { "TaskReenqueuer: Task #{task.task_id} transitioned back to pending" }
+        end
+
+        # Enqueue the task for processing
+        Tasker::TaskRunnerJob.perform_later(task.task_id)
+
+        # Fire re-enqueue completed event
+        publish_task_reenqueue_requested(task, reason: reason)
+
+        Rails.logger.debug { "TaskReenqueuer: Task #{task.task_id} re-enqueued due to #{reason}" }
+        true
+      rescue StandardError => e
+        # Fire re-enqueue failed event
+        publish_task_reenqueue_failed(task, reason: reason, error: e.message)
+
+        Rails.logger.error("TaskReenqueuer: Failed to re-enqueue task #{task.task_id}: #{e.message}")
+        false
+      end
+
+      # Schedule a delayed re-enqueue (for retry scenarios)
+      #
+      # @param task [Tasker::Task] The task to re-enqueue
+      # @param delay_seconds [Integer] Number of seconds to delay
+      # @param reason [String] The reason for delayed re-enqueueing
+      # @return [Boolean] True if scheduling was successful
+      def reenqueue_task_delayed(task, delay_seconds:,
+                                 reason: Constants::TaskFinalization::ReenqueueReasons::RETRY_BACKOFF)
+        # Fire delayed re-enqueue started event
+        publish_task_reenqueue_delayed(task, delay_seconds: delay_seconds, reason: reason)
+
+        # Schedule the delayed job
+        Tasker::TaskRunnerJob.set(wait: delay_seconds.seconds).perform_later(task.task_id)
+
+        Rails.logger.debug do
+          "TaskReenqueuer: Task #{task.task_id} scheduled for re-enqueue in #{delay_seconds} seconds"
+        end
+        true
+      rescue StandardError => e
+        Rails.logger.error("TaskReenqueuer: Failed to schedule delayed re-enqueue for task #{task.task_id}: #{e.message}")
+        false
+      end
+    end
+  end
+end

data/lib/tasker/orchestration/viable_step_discovery.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../concerns/idempotent_state_transitions'
+require_relative '../concerns/event_publisher'
+require_relative '../types/step_sequence'
+
+module Tasker
+  module Orchestration
+    # ViableStepDiscovery provides implementation for finding steps ready for execution
+    #
+    # This class is a simple implementation provider that wraps the existing
+    # WorkflowStep.get_viable_steps logic while firing lifecycle events
+    # for observability. No complex event subscriptions needed.
+    class ViableStepDiscovery
+      include Tasker::Concerns::EventPublisher
+
+      # Find viable steps for execution
+      #
+      # This is just a clean wrapper around the existing WorkflowStep.get_viable_steps
+      # method that fires observability events.
+      #
+      # @param task [Tasker::Task] The task to find steps for
+      # @param sequence [Tasker::Types::StepSequence] The step sequence
+      # @return [Array<Tasker::WorkflowStep>] Array of viable steps
+      def find_viable_steps(task, sequence)
+        # Use the existing proven logic - this is where the real work happens
+        viable_steps = Tasker::WorkflowStep.get_viable_steps(task, sequence)
+
+        # Fire appropriate discovery event based on results through orchestrator
+        if viable_steps.any?
+          publish_viable_steps_discovered(
+            task.task_id,
+            viable_steps.map(&:workflow_step_id),
+            processing_mode: 'concurrent',
+            viable_count: viable_steps.size,
+            step_names: viable_steps.map(&:name)
+          )
+        else
+          publish_no_viable_steps(
+            task.task_id,
+            reason: 'No steps ready for execution',
+            total_steps_checked: sequence.steps.size
+          )
+        end
+
+        Rails.logger.debug { "ViableStepDiscovery: Found #{viable_steps.size} viable steps for task #{task.task_id}" }
+        viable_steps
+      end
+
+      # Discover viable steps for a task by task ID
+      #
+      # Convenience method for testing and external usage that loads a task
+      # and gets its sequence before calling find_viable_steps.
+      #
+      # @param task_id [String] The task ID to discover steps for
+      # @return [Array<Tasker::WorkflowStep>] Array of viable steps
+      def discover_steps_for_task(task_id)
+        task = Tasker::Task.find(task_id)
+        task_handler = Tasker::HandlerFactory.instance.get(task.name)
+        sequence = Tasker::Orchestration::StepSequenceFactory.get_sequence(task, task_handler)
+        find_viable_steps(task, sequence)
+      end
+    end
+  end
+end

data/lib/tasker/orchestration/workflow_coordinator.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+
+module Tasker
+  module Orchestration
+    # WorkflowCoordinator handles the main task execution loop
+    #
+    # This coordinator extracts the proven loop-based execution logic from TaskHandler
+    # and provides a strategy pattern for composition with different reenqueuer strategies.
+    # This enables proper testing of the complete workflow execution path.
+    #
+    # Enhanced with structured logging and performance monitoring for production observability.
+    class WorkflowCoordinator
+      include Tasker::Concerns::StructuredLogging
+      include Tasker::Concerns::EventPublisher
+
+      attr_reader :reenqueuer_strategy
+
+      # Initialize coordinator with reenqueuer strategy
+      #
+      # @param reenqueuer_strategy [Object] Strategy for handling task reenqueuing
+      def initialize(reenqueuer_strategy: nil)
+        @reenqueuer_strategy = reenqueuer_strategy || default_reenqueuer_strategy
+      end
+
+      # Execute the complete workflow for a task
+      #
+      # This method contains the proven loop logic extracted from TaskHandler#handle
+      # and delegates to orchestration components for implementation details.
+      #
+      # Enhanced with correlation ID propagation and structured logging.
+      #
+      # @param task [Tasker::Task] The task to execute
+      # @param task_handler [Object] The task handler instance for delegation
+      # @return [void]
+      def execute_workflow(task, task_handler)
+        # Establish correlation ID for the entire workflow execution
+        workflow_correlation_id = correlation_id
+
+        with_correlation_id(workflow_correlation_id) do
+          log_orchestration_event('workflow_execution', :started,
+                                  task_id: task.task_id,
+                                  task_name: task.name,
+                                  correlation_id: workflow_correlation_id)
+
+          # Publish workflow started event
+          publish_workflow_task_started(task.task_id, correlation_id: workflow_correlation_id)
+
+          # Execute the main workflow loop with performance monitoring
+          execute_workflow_with_monitoring(task, task_handler)
+        end
+      rescue StandardError => e
+        log_exception(e, context: {
+                        task_id: task.task_id,
+                        operation: 'workflow_execution',
+                        correlation_id: workflow_correlation_id
+                      })
+        raise
+      end
+
+      private
+
+      # Execute workflow with performance monitoring
+      #
+      # @param task [Tasker::Task] The task to execute
+      # @param task_handler [Object] The task handler instance
+      # @return [void]
+      def execute_workflow_with_monitoring(task, task_handler)
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        all_processed_steps = []
+        loop_iteration = 0
+
+        log_performance_event('workflow_execution', 0,
+                              task_id: task.task_id,
+                              operation: 'workflow_start')
+
+        # PROVEN APPROACH: Process steps iteratively until completion or error
+        loop do
+          break if execute_workflow_iteration(task, task_handler, all_processed_steps, loop_iteration)
+
+          loop_iteration += 1
+        end
+
+        # Calculate total execution time
+        total_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+
+        log_performance_event('workflow_execution', total_duration,
+                              task_id: task.task_id,
+                              total_iterations: loop_iteration,
+                              total_steps_processed: all_processed_steps.size,
+                              operation: 'workflow_complete')
+
+        # DELEGATE: Finalize via TaskFinalizer (fires events internally)
+        finalize_task_with_logging(task, task_handler.get_sequence(task), all_processed_steps, task_handler)
+      end
+
+      # Execute a single workflow iteration
+      #
+      # @param task [Tasker::Task] The task being processed
+      # @param task_handler [Object] The task handler instance
+      # @param all_processed_steps [Array] Accumulator for processed steps
+      # @param iteration [Integer] Current iteration number
+      # @return [Boolean] True if workflow should exit, false to continue
+      def execute_workflow_iteration(task, task_handler, all_processed_steps, iteration)
+        task.reload
+        sequence = task_handler.get_sequence(task)
+
+        log_orchestration_event('workflow_iteration', :started,
+                                task_id: task.task_id,
+                                iteration: iteration,
+                                current_status: task.status)
+
+        # Find viable steps with performance monitoring
+        viable_steps = find_viable_steps_with_monitoring(task, sequence, task_handler)
+
+        if viable_steps.empty?
+          log_orchestration_event('workflow_iteration', :completed,
+                                  task_id: task.task_id,
+                                  iteration: iteration,
+                                  result: 'no_viable_steps',
+                                  total_processed: all_processed_steps.size)
+          return true # Exit loop
+        end
+
+        # Execute viable steps with monitoring
+        processed_steps = handle_viable_steps_with_monitoring(task, sequence, viable_steps, task_handler)
+        all_processed_steps.concat(processed_steps)
+
+        # Check if blocked by errors
+        if blocked_by_errors_with_monitoring?(task, sequence, processed_steps, task_handler)
+          log_orchestration_event('workflow_iteration', :completed,
+                                  task_id: task.task_id,
+                                  iteration: iteration,
+                                  result: 'blocked_by_errors',
+                                  total_processed: all_processed_steps.size)
+          return true # Exit loop
+        end
+
+        log_orchestration_event('workflow_iteration', :completed,
+                                task_id: task.task_id,
+                                iteration: iteration,
+                                result: 'continue',
+                                steps_processed: processed_steps.size,
+                                total_processed: all_processed_steps.size)
+
+        false # Continue loop
+      end
+
+      # Get default reenqueuer strategy for production use
+      #
+      # @return [Tasker::Orchestration::TaskReenqueuer] Production reenqueuer
+      def default_reenqueuer_strategy
+        Tasker::Orchestration::TaskReenqueuer.new
+      end
+
+      # Find steps that are ready for execution with performance monitoring
+      #
+      # @param task [Tasker::Task] The task being processed
+      # @param sequence [Tasker::Types::StepSequence] The step sequence
+      # @param task_handler [Object] The task handler for delegation
+      # @return [Array<Tasker::WorkflowStep>] Steps ready for execution
+      def find_viable_steps_with_monitoring(task, sequence, _task_handler)
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        # Delegate to the existing orchestration component
+        viable_steps = Orchestration::ViableStepDiscovery.new.find_viable_steps(task, sequence)
+
+        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+
+        log_performance_event('viable_step_discovery', duration,
+                              task_id: task.task_id,
+                              step_count: viable_steps.size,
+                              sequence_length: sequence.steps.size)
+
+        if viable_steps.any?
+          publish_viable_steps_discovered(task.task_id, viable_steps.map(&:workflow_step_id))
+        else
+          publish_no_viable_steps(task.task_id, reason: 'No steps ready for execution')
+        end
+
+        viable_steps
+      end
+
+      # Handle execution of viable steps with monitoring
+      #
+      # @param task [Tasker::Task] The task being processed
+      # @param sequence [Tasker::Types::StepSequence] The step sequence
+      # @param viable_steps [Array<Tasker::WorkflowStep>] Steps ready for execution
+      # @param task_handler [Object] The task handler for delegation
+      # @return [Array<Tasker::WorkflowStep>] Processed steps
+      def handle_viable_steps_with_monitoring(task, sequence, viable_steps, task_handler)
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        log_orchestration_event('step_batch_execution', :started,
+                                task_id: task.task_id,
+                                step_count: viable_steps.size,
+                                step_names: viable_steps.map(&:name))
+
+        # Delegate to task handler's step executor
+        processed_steps = task_handler.send(:step_executor).execute_steps(task, sequence, viable_steps, task_handler)
+
+        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+        successful_count = processed_steps.count { |s| s&.status == Tasker::Constants::WorkflowStepStatuses::COMPLETE }
+
+        log_performance_event('step_batch_execution', duration,
+                              task_id: task.task_id,
+                              step_count: viable_steps.size,
+                              processed_count: processed_steps.size,
+                              successful_count: successful_count,
+                              failure_count: processed_steps.size - successful_count)
+
+        log_orchestration_event('step_batch_execution', :completed,
+                                task_id: task.task_id,
+                                processed_count: processed_steps.size,
+                                successful_count: successful_count,
+                                duration_ms: (duration * 1000).round(2))
+
+        processed_steps
+      end
+
+      # Check if task is blocked by errors with monitoring
+      #
+      # @param task [Tasker::Task] The task to check
+      # @param sequence [Tasker::Types::StepSequence] The step sequence
+      # @param processed_steps [Array<Tasker::WorkflowStep>] Recently processed steps
+      # @param task_handler [Object] The task handler for delegation
+      # @return [Boolean] True if blocked by errors
+      def blocked_by_errors_with_monitoring?(task, sequence, processed_steps, task_handler)
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        # Delegate to task handler's task finalizer
+        is_blocked = task_handler.send(:task_finalizer).blocked_by_errors?(task, sequence, processed_steps)
+
+        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+
+        log_performance_event('error_blocking_check', duration,
+                              task_id: task.task_id,
+                              is_blocked: is_blocked,
+                              processed_steps_count: processed_steps.size)
+
+        if is_blocked
+          log_orchestration_event('workflow_blocking', :detected,
+                                  task_id: task.task_id,
+                                  reason: 'blocked_by_errors',
+                                  failed_steps: processed_steps.select do |s|
+                                    s&.status == Tasker::Constants::WorkflowStepStatuses::ERROR
+                                  end.map(&:name))
+        end
+
+        is_blocked
+      end
+
+      # Finalize the task with structured logging
+      #
+      # @param task [Tasker::Task] The task to finalize
+      # @param sequence [Tasker::Types::StepSequence] The final step sequence
+      # @param processed_steps [Array<Tasker::WorkflowStep>] All processed steps
+      # @param task_handler [Object] The task handler for delegation
+      def finalize_task_with_logging(task, sequence, processed_steps, task_handler)
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        log_orchestration_event('task_finalization', :started,
+                                task_id: task.task_id,
+                                processed_steps_count: processed_steps.size)
+
+        # Publish finalization started event
+        publish_task_finalization_started(task, processed_steps_count: processed_steps.size)
+
+        # Call update_annotations hook before finalizing
+        if task_handler.respond_to?(:update_annotations)
+          task_handler.update_annotations(task, sequence, processed_steps)
+        end
+
+        # Delegate to task handler's task finalizer
+        task_handler.send(:task_finalizer).finalize_task_with_steps(task, sequence, processed_steps)
+
+        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+        task.reload # Get final status
+
+        log_performance_event('task_finalization', duration,
+                              task_id: task.task_id,
+                              final_status: task.status,
+                              processed_steps_count: processed_steps.size)
+
+        # Publish finalization completed event
+        publish_task_finalization_completed(task, processed_steps_count: processed_steps.size)
+
+        log_orchestration_event('task_finalization', :completed,
+                                task_id: task.task_id,
+                                final_status: task.status,
+                                duration_ms: (duration * 1000).round(2))
+      end
+    end
+  end
+end

data/lib/tasker/orchestration.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Tasker::Orchestration namespace for event-driven workflow orchestration
+#
+# This module provides a declarative, event-driven alternative to the imperative
+# TaskHandler workflow loop. It coordinates workflow execution through state machine
+# events and publisher/subscriber patterns.
+#
+# Components:
+# - Events::Publisher: Unified event publisher for all workflow events (singleton)
+# - ViableStepDiscovery: Discovers which steps can be executed based on workflow state
+# - StepExecutor: Handles the execution of workflow steps via state machine transitions
+# - TaskFinalizer: Handles task completion and finalization logic
+# - Coordinator: Manages the setup and initialization of the event-driven workflow system
+#
+# Usage:
+#   # Initialize the workflow orchestration system (do this once at startup)
+#   Tasker::Orchestration::Coordinator.initialize!
+#
+#   # Access the unified event publisher
+#   publisher = Tasker::Events::Publisher.instance
+#
+# Benefits:
+# - Declarative: Steps declare what events they respond to
+# - Testable: Each component can be tested in isolation
+# - Extensible: Easy to add new behaviors via event subscribers
+# - Observable: Rich event stream for debugging and monitoring
+# - Concurrent-Safe: State machines handle race conditions
+# - Audit Trail: Complete history of all state transitions
+
+# Explicitly require orchestration components for predictable loading
+require_relative 'orchestration/viable_step_discovery'
+require_relative 'orchestration/step_executor'
+require_relative 'orchestration/task_finalizer'
+require_relative 'orchestration/task_initializer'
+require_relative 'orchestration/task_reenqueuer'
+require_relative 'orchestration/step_sequence_factory'
+require_relative 'orchestration/coordinator'
+
+module Tasker
+  module Orchestration
+    # All orchestration components are now explicitly loaded above
+    # This provides predictable loading order and avoids autoload complexity
+  end
+end

data/lib/tasker/railtie.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Tasker
+  class Railtie < ::Rails::Railtie
+    initializer 'tasker.load_observability', after: :load_config_initializers do
+      # Legacy instrumentation removed - using TelemetrySubscriber instead
+    end
+  end
+end